# embedding_metadata

key: _node_content
string_value:
{"id_": "aa6be201-d473-489d-812c-0fd1fe9814b9", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "b03f14bb-70f6-4aa1-943b-5684202babde", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "hash": "dbe2a88a46b2fed3dc7a92e2e1bff9181de69b0442371522bec375ae9ce86584", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "611419a2-820e-471a-9bbc-8903ce93348e", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "hash": "64577e85c37be12d48c8d9de9b3a468ab4766d3121e12333c8b8c2b2d99874eb", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "f7e24157-e57b-44b6-b180-23d54c73de7a", "node_type": "1", "metadata": {}, "hash": "ef8dd0a14988b2c3378f6cbd7481810d95c39add1aa7c595908eac9b08eaf7a3", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 29841, "end_char_idx": 31613, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
These...</td>
      <td>[-0.013289917260408401, 0.036334190517663956, ...</td>
    </tr>
    <tr>
      <th>14</th>
      <td>Harley Davidson Fat Boy</td>
      <td>Motorcycle</td>
      <td>Experience the thrill of the open road with th...</td>
      <td>[0.012365399859845638, 0.03552943095564842, -0...</td>
    </tr>
    <tr>
      <th>15</th>
      <td>Adidas UltraBoost</td>
      <td>Sneakers</td>
      <td>Enjoy a perfect blend of comfort and performan...</td>
      <td>[0.013107392005622387, 0.02963760495185852, -0...</td>
    </tr>
    <tr>
      <th>16</th>
      <td>Dove Men's Body Wash</td>
      <td>Body Wash</td>
      <td>Refresh and hydrate your skin with Dove Men's ...</td>
      <td>[0.03760576993227005, -0.008475445210933685, -...</td>
    </tr>
    <tr>
      <th>17</th>
      <td>Quaker Oats</td>
      <td>Oats</td>
      <td>Start your day right with Quaker Oats. Packed ...</td>
      <td>[-0.00903365109115839, 0.00896345917135477, 0....</td>
    </tr>
    <tr>
      <th>18</th>
      <td>Ford F-150</td>
      <td>Truck</td>
      <td>The Ford F-150 is a durable and dependable tru...</td>
      <td>[0.023461222648620605, -0.026651185005903244, ...</td>
    </tr>
    <tr>
      <th>19</th>
      <td>Levi's 501 Jeans</td>
      <td>Jeans</td>
      <td>Discover the timeless style of Levi's 501 Jean...</td>
      <td>[0.003762696636840701, 0.02275814116001129, -0...</td>
    </tr>
    <tr>
      <th>20</th>
      <td>Tesla Model 3</td>
      <td>Mobile Phones</td>
      <td>Explore the future of driving with the Tesla M...</td>
      <td>[0.03703858703374863, 0.03407958149909973, 0.0...</td>
    </tr>
    <tr>
      <th>21</th>
      <td>Nike Air Max</td>
      <td>Shoes</td>
      <td>Step up your game with the Nike Air Max.


key: _node_content
string_value:
{"id_": "f7e24157-e57b-44b6-b180-23d54c73de7a", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "b03f14bb-70f6-4aa1-943b-5684202babde", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "hash": "dbe2a88a46b2fed3dc7a92e2e1bff9181de69b0442371522bec375ae9ce86584", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "aa6be201-d473-489d-812c-0fd1fe9814b9", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "hash": "75173ec97d61593503dcb999589b21ebfd9ee29201fb380d4bfc55a89a0a14ed", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "7514cec0-c7db-4def-beec-fb363319d49a", "node_type": "1", "metadata": {}, "hash": "7a775869be6e9f7ffad94df167ce3a3f1d7bec72f2488b49f32026a44645a77b", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 30737, "end_char_idx": 33087, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
Packed ...</td>
      <td>[-0.00903365109115839, 0.00896345917135477, 0....</td>
    </tr>
    <tr>
      <th>18</th>
      <td>Ford F-150</td>
      <td>Truck</td>
      <td>The Ford F-150 is a durable and dependable tru...</td>
      <td>[0.023461222648620605, -0.026651185005903244, ...</td>
    </tr>
    <tr>
      <th>19</th>
      <td>Levi's 501 Jeans</td>
      <td>Jeans</td>
      <td>Discover the timeless style of Levi's 501 Jean...</td>
      <td>[0.003762696636840701, 0.02275814116001129, -0...</td>
    </tr>
    <tr>
      <th>20</th>
      <td>Tesla Model 3</td>
      <td>Mobile Phones</td>
      <td>Explore the future of driving with the Tesla M...</td>
      <td>[0.03703858703374863, 0.03407958149909973, 0.0...</td>
    </tr>
    <tr>
      <th>21</th>
      <td>Nike Air Max</td>
      <td>Shoes</td>
      <td>Step up your game with the Nike Air Max. Desig...</td>
      <td>[0.03943369910120964, 0.022045187652111053, -0...</td>
    </tr>
    <tr>
      <th>22</th>
      <td>Oral-B Pro 1000</td>
      <td>Electronic Toothbrush</td>
      <td>Achieve a superior clean with the Oral-B Pro 1...</td>
      <td>[-0.003470012918114662, -0.01911414973437786, ...</td>
    </tr>
    <tr>
      <th>23</th>
      <td>Organic Almond Butter</td>
      <td>Food</td>
      <td>Indulge in the creamy goodness of Organic Almo...</td>
      <td>[-0.014613640494644642, -0.002179765608161688,...</td>
    </tr>
    <tr>
      <th>24</th>
      <td>Yamaha YZF-R3</td>
      <td>Mobile Phones</td>
      <td>Introducing the Yamaha YZF-R3, the ultimate sp...</td>
      <td>[0.03703858703374863, 0.03407958149909973, 0.0...</td>
    </tr>
    <tr>
      <th>25</th>
      <td>Adidas Ultraboost</td>
      <td>Shoes</td>
      <td>Discover the Adidas Ultraboost, a shoe that of...</td>
      <td>[0.03944042697548866, 0.022062409669160843, -0...</td>
    </tr>
    <tr>
      <th>26</th>
      <td>Philips Sonicare</td>
      <td>Electronic Toothbrush</td>
      <td>Experience the dental care revolution with Phi...</td>
      <td>[-0.003470012918114662, -0.01911414973437786, ...</td>
    </tr>
    <tr>
      <th>27</th>
      <td>Organic Quinoa</td>
      <td>Food</td>
      <td>Nourish your body with Organic Quinoa, a nutri...</td>
      <td>[-0.014613640494644642, -0.002179765608161688,...</td>
    </tr>
  </tbody>
</table>
</div>


key: _node_content
string_value:
{"id_": "7514cec0-c7db-4def-beec-fb363319d49a", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "b03f14bb-70f6-4aa1-943b-5684202babde", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "hash": "dbe2a88a46b2fed3dc7a92e2e1bff9181de69b0442371522bec375ae9ce86584", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "f7e24157-e57b-44b6-b180-23d54c73de7a", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "hash": "031cdb567f0249d9da89afce301de115d14892a03ec71cc0cfd3bedfe13f6f92", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "15a4c010-5247-4be0-9da0-f4948ec55ad3", "node_type": "1", "metadata": {}, "hash": "519a5446b5d6e76792329059674cb699b4dcea207584a004c9e5bb695cb12a2f", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 33091, "end_char_idx": 35530, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
Now we perform the elbow method. 


```python
# Determine the optimal number of clusters using the elbow method
inertias = []
range_of_clusters = range(1, 13)  # Adjust the range as necessary

for n_clusters in range_of_clusters:
    kmeans = KMeans(n_clusters=n_clusters, init="k-means++", random_state=42, n_init=10)
    kmeans.fit(matrix)
    inertias.append(kmeans.inertia_)

```

This will output a chart for us in which we have to visually tell where the optimal cluster point is. We can see below that we see a gradual decrease of inertia rather than a sharp elbow but the point of steepest decrease appears to occur around 3, 4 or 5 clusters which lines up with our expectations given our prompt. 


```python
# Plotting the elbow plot
plt.figure(figsize=(10, 6))
plt.plot(range_of_clusters, inertias, '-o')
plt.title('Elbow Method to Determine Optimal Number of Clusters')
plt.xlabel('Number of Clusters')
plt.ylabel('Inertia')
plt.xticks(range_of_clusters)
plt.show()
```


    
![png](output_32_0.png)
    


![elbow_chart](../images/elbow_chart.png)

For demonstration purposes we will pick 5 as the optimal cluster number to show it doesn't matter exactly where we pick it as long as we are approximately right. There are numerous correct ways to categorize data. We also store which cluster each data point belongs to.


```python
n_clusters = 5

kmeans = KMeans(n_clusters=n_clusters, init="k-means++", random_state=42)
kmeans.fit(matrix)
labels = kmeans.labels_
df["Cluster"] = labels
```

We will analyze the cluster data now. There are two separate things we will look to address. 1. imbalanced data, 2. Expanding the data distribution.

First for imbalanced data we count the number of examples in each cluster. Then we select a few examples from each cluster at random and ask the LLM what topics these map to. 


```python
cluster_counts = df["Cluster"].value_counts().sort_index()
print(cluster_counts)
```

    Cluster
    0    5
    1    7
    2    8
    3    6
    4    2
    Name: count, dtype: int64


We can see the topics found here:
Eco-friendly Transportation, Luxury and Leisure Items, Personal Care Products, Electronic Toothbrushes and Clothing and Apparel
match well enough but not exactly to our initial prompt of:
vehicle, clothing, toiletries, food.

As we chose 5 clusters, it split up toiletries into Skincare and Personal Care which doesn't affect us too much further downstream.


```python
df
```


key: _node_content
string_value:
{"id_": "15a4c010-5247-4be0-9da0-f4948ec55ad3", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "b03f14bb-70f6-4aa1-943b-5684202babde", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "hash": "dbe2a88a46b2fed3dc7a92e2e1bff9181de69b0442371522bec375ae9ce86584", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "7514cec0-c7db-4def-beec-fb363319d49a", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "hash": "e7703f2b2baf2ce8b925d935190d522042ff6dff460ec7bd1e7554af20cab07a", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "c08f7244-3b56-47a9-9be0-57a6977b4f87", "node_type": "1", "metadata": {}, "hash": "ab588b90d6dc24973fdb2a10671cbe85c01658c06925f07939fd38007b93dadf", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 34080, "end_char_idx": 36391, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
![png](output_32_0.png)
    


![elbow_chart](../images/elbow_chart.png)

For demonstration purposes we will pick 5 as the optimal cluster number to show it doesn't matter exactly where we pick it as long as we are approximately right. There are numerous correct ways to categorize data. We also store which cluster each data point belongs to.


```python
n_clusters = 5

kmeans = KMeans(n_clusters=n_clusters, init="k-means++", random_state=42)
kmeans.fit(matrix)
labels = kmeans.labels_
df["Cluster"] = labels
```

We will analyze the cluster data now. There are two separate things we will look to address. 1. imbalanced data, 2. Expanding the data distribution.

First for imbalanced data we count the number of examples in each cluster. Then we select a few examples from each cluster at random and ask the LLM what topics these map to. 


```python
cluster_counts = df["Cluster"].value_counts().sort_index()
print(cluster_counts)
```

    Cluster
    0    5
    1    7
    2    8
    3    6
    4    2
    Name: count, dtype: int64


We can see the topics found here:
Eco-friendly Transportation, Luxury and Leisure Items, Personal Care Products, Electronic Toothbrushes and Clothing and Apparel
match well enough but not exactly to our initial prompt of:
vehicle, clothing, toiletries, food.

As we chose 5 clusters, it split up toiletries into Skincare and Personal Care which doesn't affect us too much further downstream.


```python
df
```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>Product</th>
      <th>Category</th>
      <th>Description</th>
      <th>embedding</th>
      <th>Cluster</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>Tesla Model 3</td>
      <td>Electric Car</td>
      <td>The Tesla Model 3 is a revolutionary electric ...</td>
      <td>[0.003255360759794712, -0.039260633289813995, ...</td>
      <td>1</td>
    </tr>
    <tr>
      <th>1</th>
      <td>Nike Air Max</td>
      <td>Shoes</td>
      <td>Elevate your sneaker game with Nike Air Max.


key: _node_content
string_value:
{"id_": "c08f7244-3b56-47a9-9be0-57a6977b4f87", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "b03f14bb-70f6-4aa1-943b-5684202babde", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "hash": "dbe2a88a46b2fed3dc7a92e2e1bff9181de69b0442371522bec375ae9ce86584", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "15a4c010-5247-4be0-9da0-f4948ec55ad3", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "hash": "dc3b42ffd8810c4afd5b18d541490b8a10802c80852ecd0df4597eb56616c59a", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "ce190b6a-1ec0-404f-8a3b-6e4e5ff2a046", "node_type": "1", "metadata": {}, "hash": "f14180b8bdbd0b4c68e75ec73f6c4b0666f7dfe6bc3f34e8971b82290db22811", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 35120, "end_char_idx": 37346, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
We can see the topics found here:
Eco-friendly Transportation, Luxury and Leisure Items, Personal Care Products, Electronic Toothbrushes and Clothing and Apparel
match well enough but not exactly to our initial prompt of:
vehicle, clothing, toiletries, food.

As we chose 5 clusters, it split up toiletries into Skincare and Personal Care which doesn't affect us too much further downstream.


```python
df
```




<div>
<style scoped>
    .dataframe tbody tr th:only-of-type {
        vertical-align: middle;
    }

    .dataframe tbody tr th {
        vertical-align: top;
    }

    .dataframe thead th {
        text-align: right;
    }
</style>
<table border="1" class="dataframe">
  <thead>
    <tr style="text-align: right;">
      <th></th>
      <th>Product</th>
      <th>Category</th>
      <th>Description</th>
      <th>embedding</th>
      <th>Cluster</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <th>0</th>
      <td>Tesla Model 3</td>
      <td>Electric Car</td>
      <td>The Tesla Model 3 is a revolutionary electric ...</td>
      <td>[0.003255360759794712, -0.039260633289813995, ...</td>
      <td>1</td>
    </tr>
    <tr>
      <th>1</th>
      <td>Nike Air Max</td>
      <td>Shoes</td>
      <td>Elevate your sneaker game with Nike Air Max. C...</td>
      <td>[0.03943369910120964, 0.022045187652111053, -0...</td>
      <td>2</td>
    </tr>
    <tr>
      <th>2</th>
      <td>Oral-B Pro 1000</td>
      <td>Electronic Toothbrush</td>
      <td>Achieve a superior clean with the Oral-B Pro 1...</td>
      <td>[-0.003470012918114662, -0.01911414973437786, ...</td>
      <td>1</td>
    </tr>
    <tr>
      <th>3</th>
      <td>Chobani Greek Yogurt</td>
      <td>Yogurt</td>
      <td>Indulge in a nutritious snack with Chobani Gre...</td>
      <td>[0.0208318829536438, -0.02645781636238098, -0....</td>
      <td>3</td>
    </tr>
    <tr>
      <th>4</th>
      <td>Ford F-150</td>
      <td>Pickup Truck</td>
      <td>The Ford F-150 is the ultimate pickup truck, d...</td>
      <td>[0.007467855699360371, -0.05288049206137657, -...</td>
      <td>0</td>
    </tr>
    <tr>
      <th>5</th>
      <td>Levi's 511</td>
      <td>Jeans</td>
      <td>Step out in style with Levi's 511 jeans.


key: _node_content
string_value:
{"id_": "ce190b6a-1ec0-404f-8a3b-6e4e5ff2a046", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "b03f14bb-70f6-4aa1-943b-5684202babde", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "hash": "dbe2a88a46b2fed3dc7a92e2e1bff9181de69b0442371522bec375ae9ce86584", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "c08f7244-3b56-47a9-9be0-57a6977b4f87", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "hash": "4f7f6f95b68f2f6626344f5b8e4be0241ec387f793a178b894dcfdc96a561ac1", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "b4c516ae-b4c5-4f65-ba5a-98c8b7369825", "node_type": "1", "metadata": {}, "hash": "f1add170365f3379c540b1f73b5ada90f796f3a47fb5d8dd05e996f140f33097", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 36392, "end_char_idx": 37842, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
C...</td>
      <td>[0.03943369910120964, 0.022045187652111053, -0...</td>
      <td>2</td>
    </tr>
    <tr>
      <th>2</th>
      <td>Oral-B Pro 1000</td>
      <td>Electronic Toothbrush</td>
      <td>Achieve a superior clean with the Oral-B Pro 1...</td>
      <td>[-0.003470012918114662, -0.01911414973437786, ...</td>
      <td>1</td>
    </tr>
    <tr>
      <th>3</th>
      <td>Chobani Greek Yogurt</td>
      <td>Yogurt</td>
      <td>Indulge in a nutritious snack with Chobani Gre...</td>
      <td>[0.0208318829536438, -0.02645781636238098, -0....</td>
      <td>3</td>
    </tr>
    <tr>
      <th>4</th>
      <td>Ford F-150</td>
      <td>Pickup Truck</td>
      <td>The Ford F-150 is the ultimate pickup truck, d...</td>
      <td>[0.007467855699360371, -0.05288049206137657, -...</td>
      <td>0</td>
    </tr>
    <tr>
      <th>5</th>
      <td>Levi's 511</td>
      <td>Jeans</td>
      <td>Step out in style with Levi's 511 jeans. Featu...</td>
      <td>[0.0037206460256129503, 0.022772302851080894, ...</td>
      <td>2</td>
    </tr>
    <tr>
      <th>6</th>
      <td>Philips Sonicare</td>
      <td>Electric Toothbrush</td>
      <td>Discover a new level of oral care with the Phi...</td>
      <td>[-0.00724813062697649, -0.011600878089666367, ...</td>
      <td>1</td>
    </tr>
    <tr>
      <th>7</th>
      <td>Quaker Oatmeal</td>
      <td>Breakfast Cereal</td>
      <td>Start your day right with Quaker Oatmeal.


key: _node_content
string_value:
{"id_": "b4c516ae-b4c5-4f65-ba5a-98c8b7369825", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "b03f14bb-70f6-4aa1-943b-5684202babde", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "hash": "dbe2a88a46b2fed3dc7a92e2e1bff9181de69b0442371522bec375ae9ce86584", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "ce190b6a-1ec0-404f-8a3b-6e4e5ff2a046", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "hash": "a06a116546e397a73f7ee371863d78acd6d09f025395ba405d6c6381e2186e27", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "c441d5c7-7279-4685-8f82-6e3cac3bcd8d", "node_type": "1", "metadata": {}, "hash": "99a44190d96225c11abbd319e7096947f4d857f95b2208bcb53cf232008e8975", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 37347, "end_char_idx": 39286, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
Featu...</td>
      <td>[0.0037206460256129503, 0.022772302851080894, ...</td>
      <td>2</td>
    </tr>
    <tr>
      <th>6</th>
      <td>Philips Sonicare</td>
      <td>Electric Toothbrush</td>
      <td>Discover a new level of oral care with the Phi...</td>
      <td>[-0.00724813062697649, -0.011600878089666367, ...</td>
      <td>1</td>
    </tr>
    <tr>
      <th>7</th>
      <td>Quaker Oatmeal</td>
      <td>Breakfast Cereal</td>
      <td>Start your day right with Quaker Oatmeal. This...</td>
      <td>[-0.006529285106807947, 0.007865572348237038, ...</td>
      <td>3</td>
    </tr>
    <tr>
      <th>8</th>
      <td>Toyota Camry</td>
      <td>Sedan</td>
      <td>The Toyota Camry stands out in the sedan categ...</td>
      <td>[-0.02088991366326809, -0.006191295105963945, ...</td>
      <td>0</td>
    </tr>
    <tr>
      <th>9</th>
      <td>Adidas Ultraboost</td>
      <td>Running Shoes</td>
      <td>Run like never before in the Adidas Ultraboost...</td>
      <td>[0.02679188922047615, 0.014639599248766899, 8....</td>
      <td>2</td>
    </tr>
    <tr>
      <th>10</th>
      <td>Toyota Camry</td>
      <td>Car</td>
      <td>The Toyota Camry is a reliable midsize sedan k...</td>
      <td>[0.008056452497839928, -0.007912316359579563, ...</td>
      <td>0</td>
    </tr>
    <tr>
      <th>11</th>
      <td>Nike Air Max</td>
      <td>Shoes</td>
      <td>Step up your sneaker game with the Nike Air Ma...</td>
      <td>[0.03943241760134697, 0.02208484522998333, -0....</td>
      <td>2</td>
    </tr>
    <tr>
      <th>12</th>
      <td>Colgate Electric Toothbrush</td>
      <td>Electronic Toothbrush</td>
      <td>Transform your oral hygiene routine with the C...</td>
      <td>[-0.003470012918114662, -0.01911414973437786, ...</td>
      <td>1</td>
    </tr>
    <tr>
      <th>13</th>
      <td>Blue Diamond Almonds</td>
      <td>Nuts</td>
      <td>Snack healthy with Blue Diamond Almonds.


key: _node_content
string_value:
{"id_": "c441d5c7-7279-4685-8f82-6e3cac3bcd8d", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "b03f14bb-70f6-4aa1-943b-5684202babde", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "hash": "dbe2a88a46b2fed3dc7a92e2e1bff9181de69b0442371522bec375ae9ce86584", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "b4c516ae-b4c5-4f65-ba5a-98c8b7369825", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "hash": "d163740902b249acb5b7e225b94a00d4f7ef64de6366cb857eb88ed1badc6f6c", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "f55d3ae1-e3da-4c2b-bd78-c91a54b1484b", "node_type": "1", "metadata": {}, "hash": "124b692da8879be5ce1feb8eb7aba87ab0552b2618ee37d9342b69d7eed3982d", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 39287, "end_char_idx": 41195, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
These...</td>
      <td>[-0.013289917260408401, 0.036334190517663956, ...</td>
      <td>3</td>
    </tr>
    <tr>
      <th>14</th>
      <td>Harley Davidson Fat Boy</td>
      <td>Motorcycle</td>
      <td>Experience the thrill of the open road with th...</td>
      <td>[0.012365399859845638, 0.03552943095564842, -0...</td>
      <td>0</td>
    </tr>
    <tr>
      <th>15</th>
      <td>Adidas UltraBoost</td>
      <td>Sneakers</td>
      <td>Enjoy a perfect blend of comfort and performan...</td>
      <td>[0.013107392005622387, 0.02963760495185852, -0...</td>
      <td>2</td>
    </tr>
    <tr>
      <th>16</th>
      <td>Dove Men's Body Wash</td>
      <td>Body Wash</td>
      <td>Refresh and hydrate your skin with Dove Men's ...</td>
      <td>[0.03760576993227005, -0.008475445210933685, -...</td>
      <td>1</td>
    </tr>
    <tr>
      <th>17</th>
      <td>Quaker Oats</td>
      <td>Oats</td>
      <td>Start your day right with Quaker Oats. Packed ...</td>
      <td>[-0.00903365109115839, 0.00896345917135477, 0....</td>
      <td>3</td>
    </tr>
    <tr>
      <th>18</th>
      <td>Ford F-150</td>
      <td>Truck</td>
      <td>The Ford F-150 is a durable and dependable tru...</td>
      <td>[0.023461222648620605, -0.026651185005903244, ...</td>
      <td>0</td>
    </tr>
    <tr>
      <th>19</th>
      <td>Levi's 501 Jeans</td>
      <td>Jeans</td>
      <td>Discover the timeless style of Levi's 501 Jean...</td>
      <td>[0.003762696636840701, 0.02275814116001129, -0...</td>
      <td>2</td>
    </tr>
    <tr>
      <th>20</th>
      <td>Tesla Model 3</td>
      <td>Mobile Phones</td>
      <td>Explore the future of driving with the Tesla M...</td>
      <td>[0.03703858703374863, 0.03407958149909973, 0.0...</td>
      <td>4</td>
    </tr>
    <tr>
      <th>21</th>
      <td>Nike Air Max</td>
      <td>Shoes</td>
      <td>Step up your game with the Nike Air Max.


key: _node_content
string_value:
{"id_": "f55d3ae1-e3da-4c2b-bd78-c91a54b1484b", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "b03f14bb-70f6-4aa1-943b-5684202babde", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "hash": "dbe2a88a46b2fed3dc7a92e2e1bff9181de69b0442371522bec375ae9ce86584", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "c441d5c7-7279-4685-8f82-6e3cac3bcd8d", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "hash": "0b2d8b01dc96373cc849f82bdbe8c8802ebfff7fb67937ebc5960cbedad25b89", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "fa8ac6e1-40bb-4399-9ae4-0e6c6ccecf53", "node_type": "1", "metadata": {}, "hash": "368f9d8f3cc867d0c3d437f4817edfabfb6261b4c844562baa75cd77554cedc8", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 40251, "end_char_idx": 42788, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
Packed ...</td>
      <td>[-0.00903365109115839, 0.00896345917135477, 0....</td>
      <td>3</td>
    </tr>
    <tr>
      <th>18</th>
      <td>Ford F-150</td>
      <td>Truck</td>
      <td>The Ford F-150 is a durable and dependable tru...</td>
      <td>[0.023461222648620605, -0.026651185005903244, ...</td>
      <td>0</td>
    </tr>
    <tr>
      <th>19</th>
      <td>Levi's 501 Jeans</td>
      <td>Jeans</td>
      <td>Discover the timeless style of Levi's 501 Jean...</td>
      <td>[0.003762696636840701, 0.02275814116001129, -0...</td>
      <td>2</td>
    </tr>
    <tr>
      <th>20</th>
      <td>Tesla Model 3</td>
      <td>Mobile Phones</td>
      <td>Explore the future of driving with the Tesla M...</td>
      <td>[0.03703858703374863, 0.03407958149909973, 0.0...</td>
      <td>4</td>
    </tr>
    <tr>
      <th>21</th>
      <td>Nike Air Max</td>
      <td>Shoes</td>
      <td>Step up your game with the Nike Air Max. Desig...</td>
      <td>[0.03943369910120964, 0.022045187652111053, -0...</td>
      <td>2</td>
    </tr>
    <tr>
      <th>22</th>
      <td>Oral-B Pro 1000</td>
      <td>Electronic Toothbrush</td>
      <td>Achieve a superior clean with the Oral-B Pro 1...</td>
      <td>[-0.003470012918114662, -0.01911414973437786, ...</td>
      <td>1</td>
    </tr>
    <tr>
      <th>23</th>
      <td>Organic Almond Butter</td>
      <td>Food</td>
      <td>Indulge in the creamy goodness of Organic Almo...</td>
      <td>[-0.014613640494644642, -0.002179765608161688,...</td>
      <td>3</td>
    </tr>
    <tr>
      <th>24</th>
      <td>Yamaha YZF-R3</td>
      <td>Mobile Phones</td>
      <td>Introducing the Yamaha YZF-R3, the ultimate sp...</td>
      <td>[0.03703858703374863, 0.03407958149909973, 0.0...</td>
      <td>4</td>
    </tr>
    <tr>
      <th>25</th>
      <td>Adidas Ultraboost</td>
      <td>Shoes</td>
      <td>Discover the Adidas Ultraboost, a shoe that of...</td>
      <td>[0.03944042697548866, 0.022062409669160843, -0...</td>
      <td>2</td>
    </tr>
    <tr>
      <th>26</th>
      <td>Philips Sonicare</td>
      <td>Electronic Toothbrush</td>
      <td>Experience the dental care revolution with Phi...</td>
      <td>[-0.003470012918114662, -0.01911414973437786, ...</td>
      <td>1</td>
    </tr>
    <tr>
      <th>27</th>
      <td>Organic Quinoa</td>
      <td>Food</td>
      <td>Nourish your body with Organic Quinoa, a nutri...</td>
      <td>[-0.014613640494644642, -0.002179765608161688,...</td>
      <td>3</td>
    </tr>
  </tbody>
</table>
</div>


key: _node_content
string_value:
{"id_": "fa8ac6e1-40bb-4399-9ae4-0e6c6ccecf53", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "b03f14bb-70f6-4aa1-943b-5684202babde", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "hash": "dbe2a88a46b2fed3dc7a92e2e1bff9181de69b0442371522bec375ae9ce86584", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "f55d3ae1-e3da-4c2b-bd78-c91a54b1484b", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "hash": "465ef6538dbeb1dc399319125ba8d68d1f4297ad746e6f172a14923a969a6f02", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "458affe0-ea8c-43fa-97d8-291f42bada6f", "node_type": "1", "metadata": {}, "hash": "3900616795835eff1cc188894c127aee6d00eb44cf45ea69c8a07017b01bb124", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 42793, "end_char_idx": 45309, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
selected_examples = df.groupby('Cluster').apply(lambda x: x.sample(3, replace=True)).reset_index(drop=True)

# Format the selected examples
formatted_examples = "\n".join(
    f'Input: "{row["Product"]}, {row["Category"]}"\nOutput: "{row["Description"]}"\nCluster: "{row["Cluster"]}"'
    for _, row in selected_examples.iterrows()
)

topic_prompt = f"""
    I previously generated some examples of input output trainings pairs and then I clustered them based on category. From each cluster I picked 3 example data point which you can find below.
    I want you identify the broad topic areas these clusters belong to.
    Previous examples:
    {formatted_examples}


    Your output should be strictly of the format:
    Cluster: number, topic: topic
    Cluster: number, topic: topic
    Cluster: number, topic: topic

    Do not add any extra characters around that formatting as it will make the output parsing break.
    """

response = client.chat.completions.create(
  model=datagen_model,
  messages=[
    {"role": "system", "content": "You are a helpful assistant designed analyze clustered data"},
    {"role": "user", "content": topic_prompt}
  ]
)
res = response.choices[0].message.content

pattern = r"Cluster: (\d+), topic: ([^\n]+)"
matches = re.findall(pattern, res)
clusters = [{"cluster": int(cluster), "topic": topic} for cluster, topic in matches]
json_output = json.dumps(clusters, indent=2)
print(json_output)
```

    [
      {
        "cluster": 0,
        "topic": "Automotive  "
      },
      {
        "cluster": 1,
        "topic": "Personal Care  "
      },
      {
        "cluster": 2,
        "topic": "Footwear  "
      },
      {
        "cluster": 3,
        "topic": "Food  "
      },
      {
        "cluster": 4,
        "topic": "Automotive  "
      }
    ]


We now have the clusters and their counts so we could prompt the LLM to generate more examples within the topics we want. However for this example we won't take that further as they are well-split and you would just follow the procedure above for prompting the model to generate data while passing in the underrepresented topics.

Next, we will try and deal with increasing the diversity of our data distribution. 

First we start in a similar way by finding a few examples from each cluster at random and ask the LLM what topics these map to. In addition to this in the same LLM call, we will ask it to generate more topics to increase the diversity of our data. We do this in one call to save time/cost.


key: _node_content
string_value:
{"id_": "458affe0-ea8c-43fa-97d8-291f42bada6f", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "b03f14bb-70f6-4aa1-943b-5684202babde", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "hash": "dbe2a88a46b2fed3dc7a92e2e1bff9181de69b0442371522bec375ae9ce86584", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "fa8ac6e1-40bb-4399-9ae4-0e6c6ccecf53", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "hash": "e7154a09530aad1f40cd8c206754ad396c12ea5a450d1524db7cf8ad89c1292c", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "c2cea2ef-3ad5-4e6d-9299-0bc06f616516", "node_type": "1", "metadata": {}, "hash": "e00b36a5841e2b99f4bcdb2851afcb5c7b919ef4667b4f1999dc60a825aafc48", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 44604, "end_char_idx": 47554, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
We now have the clusters and their counts so we could prompt the LLM to generate more examples within the topics we want. However for this example we won't take that further as they are well-split and you would just follow the procedure above for prompting the model to generate data while passing in the underrepresented topics.

Next, we will try and deal with increasing the diversity of our data distribution. 

First we start in a similar way by finding a few examples from each cluster at random and ask the LLM what topics these map to. In addition to this in the same LLM call, we will ask it to generate more topics to increase the diversity of our data. We do this in one call to save time/cost.


```python
selected_examples = df.groupby('Cluster').apply(lambda x: x.sample(3, replace=True)).reset_index(drop=True)

# Format the selected examples
formatted_examples = "\n".join(
    f'Input: "{row["Product"]}, {row["Category"]}"\nOutput: "{row["Description"]}"\nCluster: "{row["Cluster"]}"'
    for _, row in selected_examples.iterrows()
)

topic_prompt = f"""
    I previously generated some examples of input output trainings pairs and then I clustered them based on category. From each cluster I picked 3 example data point which you can find below.
    I want to promote diversity in my examples across categories so follow the procedure below:
    1. You must identify the broad topic areas these clusters belong to.
    2. You should generate further topic areas which don't exist so I can generate data within these topics to improve diversity.


    Previous examples:
    {formatted_examples}


    Your output should be strictly of the format:

    1. Cluster topic mapping
    Cluster: number, topic: topic
    Cluster: number, topic: topic
    Cluster: number, topic: topic

    2. New topics
    1. topic
    2. topic
    3. topic
    4. topic

    Do not add any extra characters around that formatting as it will make the output parsing break. It is very important you stick to that output format
    """

response = client.chat.completions.create(
  model=datagen_model,
  messages=[
    {"role": "system", "content": "You are a helpful assistant designed to analyze clustered data"},
    {"role": "user", "content": topic_prompt}
  ]
)
res = response.choices[0].message.content
print(res)

```

    1. Cluster topic mapping
    Cluster: 0, topic: Automotive
    Cluster: 1, topic: Personal Care
    Cluster: 2, topic: Footwear
    Cluster: 3, topic: Food
    Cluster: 4, topic: Electric Vehicles
    
    2. New topics
    1. topic: Home Appliances
    2. topic: Outdoor Equipment
    3. topic: Smart Home Technology
    4. topic: Fitness Equipment


We can see here again that we explicitly prompt the output structure it should follow. I also tell it the purpose of generating topics (to promote diversity) so the model has full context.

We then parse the data into a list of cluster-mapping jsons and a list of topics


key: _node_content
string_value:
{"id_": "c2cea2ef-3ad5-4e6d-9299-0bc06f616516", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "b03f14bb-70f6-4aa1-943b-5684202babde", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "hash": "dbe2a88a46b2fed3dc7a92e2e1bff9181de69b0442371522bec375ae9ce86584", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "458affe0-ea8c-43fa-97d8-291f42bada6f", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "hash": "a655fd56ae5a044f80f71786bea9856acfc3d521d7b107f600f5a3730310f154", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "d66ff73e-c419-45d5-85fa-3cb482077a42", "node_type": "1", "metadata": {}, "hash": "5e12a56e068e52536d2aca75818cf3d00e2f40a8d99d185b6582d72931d9feb9", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 46174, "end_char_idx": 49208, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
Previous examples:
    {formatted_examples}


    Your output should be strictly of the format:

    1. Cluster topic mapping
    Cluster: number, topic: topic
    Cluster: number, topic: topic
    Cluster: number, topic: topic

    2. New topics
    1. topic
    2. topic
    3. topic
    4. topic

    Do not add any extra characters around that formatting as it will make the output parsing break. It is very important you stick to that output format
    """

response = client.chat.completions.create(
  model=datagen_model,
  messages=[
    {"role": "system", "content": "You are a helpful assistant designed to analyze clustered data"},
    {"role": "user", "content": topic_prompt}
  ]
)
res = response.choices[0].message.content
print(res)

```

    1. Cluster topic mapping
    Cluster: 0, topic: Automotive
    Cluster: 1, topic: Personal Care
    Cluster: 2, topic: Footwear
    Cluster: 3, topic: Food
    Cluster: 4, topic: Electric Vehicles
    
    2. New topics
    1. topic: Home Appliances
    2. topic: Outdoor Equipment
    3. topic: Smart Home Technology
    4. topic: Fitness Equipment


We can see here again that we explicitly prompt the output structure it should follow. I also tell it the purpose of generating topics (to promote diversity) so the model has full context.

We then parse the data into a list of cluster-mapping jsons and a list of topics


```python
parts = res.split("\n\n")
cluster_mapping_part = parts[0]
new_topics_part = parts[1]

# Parse cluster topic mapping
cluster_topic_mapping_lines = cluster_mapping_part.split("\n")[1:]  # Skip the first two lines
cluster_topic_mapping = [{"cluster": int(line.split(",")[0].split(":")[1].strip()), "topic": line.split(":")[2].strip()} for line in cluster_topic_mapping_lines]

# Parse new topics
new_topics_lines = new_topics_part.split("\n")[1:]  # Skip the first line
new_topics = [line.split(". ")[1] for line in new_topics_lines]

cluster_topic_mapping, new_topics
```




    ([{'cluster': 0, 'topic': 'Automotive'},
      {'cluster': 1, 'topic': 'Personal Care'},
      {'cluster': 2, 'topic': 'Footwear'},
      {'cluster': 3, 'topic': 'Food'},
      {'cluster': 4, 'topic': 'Electric Vehicles'}],
     ['topic: Home Appliances',
      'topic: Outdoor Equipment',
      'topic: Smart Home Technology',
      'topic: Fitness Equipment'])



And finally we can use this information to further prompt a model to keep generating synthetic data. We do this by passing all the topics in the list of jsons to the prompt below.


```python
output_string = ""
for i in range(3):
  question = f"""
  I am creating input output training pairs to fine tune my gpt model. I want the input to be product name and category and output to be description. the category should be things like: mobile phones, shoes, headphones, laptop, electronic toothbrush, etc. and also more importantly the categories should come under some main topics: {[entry['topic'] for entry in cluster_topic_mapping]})
  After the number of each example also state the topic area.


key: _node_content
string_value:
{"id_": "d66ff73e-c419-45d5-85fa-3cb482077a42", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "b03f14bb-70f6-4aa1-943b-5684202babde", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "hash": "dbe2a88a46b2fed3dc7a92e2e1bff9181de69b0442371522bec375ae9ce86584", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "c2cea2ef-3ad5-4e6d-9299-0bc06f616516", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "hash": "3ff12359f23c838147f50529498fdd244a0d85d0988033896546e4ae851b3e07", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "3ec40a8f-63d0-42b3-8d58-9393d44af5e2", "node_type": "1", "metadata": {}, "hash": "6e164adb9c4a0cce48857b0f5e7e7f1e6b4954fe9563a32abdb5aa07bfa0bc60", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 48145, "end_char_idx": 51288, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
([{'cluster': 0, 'topic': 'Automotive'},
      {'cluster': 1, 'topic': 'Personal Care'},
      {'cluster': 2, 'topic': 'Footwear'},
      {'cluster': 3, 'topic': 'Food'},
      {'cluster': 4, 'topic': 'Electric Vehicles'}],
     ['topic: Home Appliances',
      'topic: Outdoor Equipment',
      'topic: Smart Home Technology',
      'topic: Fitness Equipment'])



And finally we can use this information to further prompt a model to keep generating synthetic data. We do this by passing all the topics in the list of jsons to the prompt below.


```python
output_string = ""
for i in range(3):
  question = f"""
  I am creating input output training pairs to fine tune my gpt model. I want the input to be product name and category and output to be description. the category should be things like: mobile phones, shoes, headphones, laptop, electronic toothbrush, etc. and also more importantly the categories should come under some main topics: {[entry['topic'] for entry in cluster_topic_mapping]})
  After the number of each example also state the topic area. The format should be of the form:
  1. topic_area
  Input: product_name, category
  Output: description

  Do not add any extra characters around that formatting as it will make the output parsing break.

  Here are some helpful examples so you get the style of output correct.

  1) clothing
  Input: "Shoe Name, Shoes"
  Output: "Experience unparalleled comfort. These shoes feature a blend of modern style and the traditional superior cushioning, perfect for those always on the move."
  """

  response = client.chat.completions.create(
    model="gpt-4o-mini",
    messages=[
      {"role": "system", "content": "You are a helpful assistant designed to generate synthetic data."},
      {"role": "user", "content": question}
    ]
  )
  res = response.choices[0].message.content
  output_string += res + "\n" + "\n"
print(output_string)
```

    1. Automotive  
    Input: "Tesla Model S, Electric Vehicles"  
    Output: "The Tesla Model S delivers exhilarating performance with advanced electric technology, offering a sleek design, impressive range, and an industry-leading infotainment system."
    
    2. Personal Care  
    Input: "Oral-B Pro 1000, Electronic Toothbrush"  
    Output: "The Oral-B Pro 1000 features a 3D cleaning action that oscillates, rotates, and pulsates to remove plaque, ensuring a deeper clean for healthier gums."
    
    3. Footwear  
    Input: "Nike Air Max 270, Shoes"  
    Output: "Step into comfort and style with Nike Air Max 270, designed with a large Max Air unit for superior cushioning and a breathable upper for a snug fit."
    
    4. Electronics  
    Input: "Apple iPhone 12, Mobile Phones"  
    Output: "The Apple iPhone 12 combines powerful performance with stunning design, equipped with A14 Bionic chip and advanced camera systems for capturing every moment in stunning detail."
    
    5. Food  
    Input: "Nature Valley Granola Bars, Snacks"  
    Output: "Nature Valley Granola Bars offer a wholesome crunch made from simple, delicious ingredients, providing a perfect snack that fuels your adventure."
    
    6.


key: _node_content
string_value:
{"id_": "3ec40a8f-63d0-42b3-8d58-9393d44af5e2", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "b03f14bb-70f6-4aa1-943b-5684202babde", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "hash": "dbe2a88a46b2fed3dc7a92e2e1bff9181de69b0442371522bec375ae9ce86584", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "d66ff73e-c419-45d5-85fa-3cb482077a42", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "hash": "5dc1a059f06cf0e35efeb8f6b875d9e5aa6202484b1d6aa417f03698ed8a65c3", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "ae2720d4-66c1-4212-b7cf-f40226d1970a", "node_type": "1", "metadata": {}, "hash": "35b949946161073f3b5dc4e7645c86feaae92413cf584bdd5fecb01a5a632fc8", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 49700, "end_char_idx": 53046, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
"""

  response = client.chat.completions.create(
    model="gpt-4o-mini",
    messages=[
      {"role": "system", "content": "You are a helpful assistant designed to generate synthetic data."},
      {"role": "user", "content": question}
    ]
  )
  res = response.choices[0].message.content
  output_string += res + "\n" + "\n"
print(output_string)
```

    1. Automotive  
    Input: "Tesla Model S, Electric Vehicles"  
    Output: "The Tesla Model S delivers exhilarating performance with advanced electric technology, offering a sleek design, impressive range, and an industry-leading infotainment system."
    
    2. Personal Care  
    Input: "Oral-B Pro 1000, Electronic Toothbrush"  
    Output: "The Oral-B Pro 1000 features a 3D cleaning action that oscillates, rotates, and pulsates to remove plaque, ensuring a deeper clean for healthier gums."
    
    3. Footwear  
    Input: "Nike Air Max 270, Shoes"  
    Output: "Step into comfort and style with Nike Air Max 270, designed with a large Max Air unit for superior cushioning and a breathable upper for a snug fit."
    
    4. Electronics  
    Input: "Apple iPhone 12, Mobile Phones"  
    Output: "The Apple iPhone 12 combines powerful performance with stunning design, equipped with A14 Bionic chip and advanced camera systems for capturing every moment in stunning detail."
    
    5. Food  
    Input: "Nature Valley Granola Bars, Snacks"  
    Output: "Nature Valley Granola Bars offer a wholesome crunch made from simple, delicious ingredients, providing a perfect snack that fuels your adventure."
    
    6. Automotive  
    Input: "Ford F-150, Electric Vehicles"  
    Output: "The Ford F-150 stands at the forefront of durability and innovation, with its powerful electric version setting new standards for strength and sustainability in the truck category." 
    
    7. Personal Care  
    Input: "Philips Sonicare, Electronic Toothbrush"  
    Output: "Philips Sonicare delivers superior cleaning with dynamic technology that provides up to 31,000 strokes per minute for a healthier mouth and brighter smile."
    
    8. Footwear  
    Input: "Adidas Ultraboost, Shoes"  
    Output: "The Adidas Ultraboost is a game-changer in running footwear, featuring responsive cushioning and a knit upper for a snug, supportive fit that adapts to any run."
    
    9. Electronics  
    Input: "Dell XPS 13, Laptop"  
    Output: "The Dell XPS 13 is a remarkable laptop with an ultra-thin design, featuring a stunning InfinityEdge display and powerful performance to accommodate your multitasking needs."
    
    10. Food  
    Input: "Kraft Macaroni & Cheese, Instant Food"  
    Output: "Kraft Macaroni & Cheese offers quick and convenient comfort food, combining creamy cheese sauce with perfectly cooked pasta for a simple meal that satisfies."
    
    1. Automotive  
    Input: "Toyota Camry, Mobile Phones"  
    Output: "The Toyota Camry is a midsize sedan that combines efficiency with modern technology. It offers a spacious interior and the latest features for an enjoyable driving experience."
    
    2. Personal Care  
    Input: "Oral-B Pro 1000, Electronic Toothbrush"  
    Output: "The Oral-B Pro 1000 not only provides powerful cleaning action but also enhances your oral hygiene routine with its smart pressure sensor and various cleaning modes."


key: _node_content
string_value:
{"id_": "ae2720d4-66c1-4212-b7cf-f40226d1970a", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "b03f14bb-70f6-4aa1-943b-5684202babde", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "hash": "dbe2a88a46b2fed3dc7a92e2e1bff9181de69b0442371522bec375ae9ce86584", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "3ec40a8f-63d0-42b3-8d58-9393d44af5e2", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "hash": "1c38f580b9bda204a4570f44809c10969a6d13afeefde20588fa8d894d45ca4f", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "c96fdbde-1b18-4332-b4f7-13c0657a74fd", "node_type": "1", "metadata": {}, "hash": "06d20558ec197ece5dfe94b4df1f7d7bbc2c9a78349e6db380efaaec7480cf36", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 51289, "end_char_idx": 54701, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
Automotive  
    Input: "Ford F-150, Electric Vehicles"  
    Output: "The Ford F-150 stands at the forefront of durability and innovation, with its powerful electric version setting new standards for strength and sustainability in the truck category." 
    
    7. Personal Care  
    Input: "Philips Sonicare, Electronic Toothbrush"  
    Output: "Philips Sonicare delivers superior cleaning with dynamic technology that provides up to 31,000 strokes per minute for a healthier mouth and brighter smile."
    
    8. Footwear  
    Input: "Adidas Ultraboost, Shoes"  
    Output: "The Adidas Ultraboost is a game-changer in running footwear, featuring responsive cushioning and a knit upper for a snug, supportive fit that adapts to any run."
    
    9. Electronics  
    Input: "Dell XPS 13, Laptop"  
    Output: "The Dell XPS 13 is a remarkable laptop with an ultra-thin design, featuring a stunning InfinityEdge display and powerful performance to accommodate your multitasking needs."
    
    10. Food  
    Input: "Kraft Macaroni & Cheese, Instant Food"  
    Output: "Kraft Macaroni & Cheese offers quick and convenient comfort food, combining creamy cheese sauce with perfectly cooked pasta for a simple meal that satisfies."
    
    1. Automotive  
    Input: "Toyota Camry, Mobile Phones"  
    Output: "The Toyota Camry is a midsize sedan that combines efficiency with modern technology. It offers a spacious interior and the latest features for an enjoyable driving experience."
    
    2. Personal Care  
    Input: "Oral-B Pro 1000, Electronic Toothbrush"  
    Output: "The Oral-B Pro 1000 not only provides powerful cleaning action but also enhances your oral hygiene routine with its smart pressure sensor and various cleaning modes."
    
    3. Footwear  
    Input: "Nike Air Max, Shoes"  
    Output: "Step into comfort with the Nike Air Max. With cutting-edge technology and a sleek design, these shoes are perfect for athletes and casual wearers alike."
    
    4. Food  
    Input: "Nature's Valley Granola Bar, Food"  
    Output: "Savor the wholesome goodness of Nature's Valley Granola Bar, crafted with real ingredients to fuel your day with delicious flavor and crunchy satisfaction."
    
    5. Electric Vehicles  
    Input: "Tesla Model 3, Mobile Phones"  
    Output: "The Tesla Model 3 is a revolutionary electric vehicle that combines performance with sustainability, featuring an intuitive interface and cutting-edge technology for an exceptional driving experience."
    
    1. Automotive  
    Input: "Tesla Model 3, Electric Vehicles"  
    Output: "The Tesla Model 3 combines cutting-edge technology with eco-friendly driving. Enjoy a sleek design, impressive range, and top-notch safety features, making it the perfect electric car for the modern driver."
    
    2. Personal Care  
    Input: "Oral-B Pro 1000, Electronic Toothbrush"  
    Output: "Achieve a superior clean with the Oral-B Pro 1000. Featuring advanced 3D cleaning action, this electronic toothbrush ensures effective plaque removal while being gentle on gums, allowing you to maintain optimum oral health."
    
    3. Footwear  
    Input: "Nike Air Max, Shoes"  
    Output: "Step up your game with Nike Air Max shoes. Combining iconic cushioning technology and bold style, these shoes provide ultimate comfort and support, perfect for both casual wear and athletic performance."
    
    4.


key: _node_content
string_value:
{"id_": "c96fdbde-1b18-4332-b4f7-13c0657a74fd", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "b03f14bb-70f6-4aa1-943b-5684202babde", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "hash": "dbe2a88a46b2fed3dc7a92e2e1bff9181de69b0442371522bec375ae9ce86584", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "ae2720d4-66c1-4212-b7cf-f40226d1970a", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "hash": "da8c8564e6a2e175ab006bf976984907f55a5e6b62f2fa1aa0221ccce795dcb1", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "b07f0c11-4a61-4ea3-bda5-c6a32f56e4aa", "node_type": "1", "metadata": {}, "hash": "ba6c7a43bbb429498869a4127b92aaef77686426698e0e8d047d28bb3c1f6c70", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 53056, "end_char_idx": 56419, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
3. Footwear  
    Input: "Nike Air Max, Shoes"  
    Output: "Step into comfort with the Nike Air Max. With cutting-edge technology and a sleek design, these shoes are perfect for athletes and casual wearers alike."
    
    4. Food  
    Input: "Nature's Valley Granola Bar, Food"  
    Output: "Savor the wholesome goodness of Nature's Valley Granola Bar, crafted with real ingredients to fuel your day with delicious flavor and crunchy satisfaction."
    
    5. Electric Vehicles  
    Input: "Tesla Model 3, Mobile Phones"  
    Output: "The Tesla Model 3 is a revolutionary electric vehicle that combines performance with sustainability, featuring an intuitive interface and cutting-edge technology for an exceptional driving experience."
    
    1. Automotive  
    Input: "Tesla Model 3, Electric Vehicles"  
    Output: "The Tesla Model 3 combines cutting-edge technology with eco-friendly driving. Enjoy a sleek design, impressive range, and top-notch safety features, making it the perfect electric car for the modern driver."
    
    2. Personal Care  
    Input: "Oral-B Pro 1000, Electronic Toothbrush"  
    Output: "Achieve a superior clean with the Oral-B Pro 1000. Featuring advanced 3D cleaning action, this electronic toothbrush ensures effective plaque removal while being gentle on gums, allowing you to maintain optimum oral health."
    
    3. Footwear  
    Input: "Nike Air Max, Shoes"  
    Output: "Step up your game with Nike Air Max shoes. Combining iconic cushioning technology and bold style, these shoes provide ultimate comfort and support, perfect for both casual wear and athletic performance."
    
    4. Food  
    Input: "Oreo Cookies, Snacks"  
    Output: "Indulge in the classic taste of Oreo Cookies. With their irresistible cream filling sandwiched between two crunchy chocolate wafers, these treats are perfect for satisfying your sweet tooth any time of the day."
    
    5. Personal Care  
    Input: "Garnier Micellar Water, Skincare"  
    Output: "Garnier Micellar Water gently removes makeup and impurities while hydrating the skin. This soothing formula is suitable for all skin types, making it a must-have in your daily skincare routine."
    
    6. Automotive  
    Input: "Ford F-150, Trucks"  
    Output: "The Ford F-150 is the quintessential pickup truck, combining power, reliability, and innovative technology. Equipped with advanced towing capabilities and a spacious interior, it's designed for both work and play."
    
    7. Electronics  
    Input: "Samsung Galaxy S21, Mobile Phones"  
    Output: "Experience the future of mobile technology with the Samsung Galaxy S21. This smartphone features a stunning display, powerful processor, and multiple camera options, perfect for capturing life's moments in high definition."
    
    8. Footwear  
    Input: "Adidas Ultraboost, Shoes"  
    Output: "Run in style with Adidas Ultraboost shoes. Known for their comfort and performance, these shoes utilize responsive cushioning to provide unmatched energy return with every step you take." 
    
    9. Electronics  
    Input: "Dell XPS 13, Laptops"  
    Output: "The Dell XPS 13 redefines the laptop experience with its stunning InfinityEdge display, powerful performance, and sleek design. Ideal for both professionals and students looking for portability and functionality."
    
    10.


key: _node_content
string_value:
{"id_": "b07f0c11-4a61-4ea3-bda5-c6a32f56e4aa", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "b03f14bb-70f6-4aa1-943b-5684202babde", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "hash": "dbe2a88a46b2fed3dc7a92e2e1bff9181de69b0442371522bec375ae9ce86584", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "c96fdbde-1b18-4332-b4f7-13c0657a74fd", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/SDG1.ipynb", "title": "Synthetic Data generation (Part 1)", "tokens": 15064, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "hash": "7c4679a3d6eb5543d29eeffc9a023fd392416c80481c6279252bc512dbedc0c8", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 54699, "end_char_idx": 57393, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
4. Food  
    Input: "Oreo Cookies, Snacks"  
    Output: "Indulge in the classic taste of Oreo Cookies. With their irresistible cream filling sandwiched between two crunchy chocolate wafers, these treats are perfect for satisfying your sweet tooth any time of the day."
    
    5. Personal Care  
    Input: "Garnier Micellar Water, Skincare"  
    Output: "Garnier Micellar Water gently removes makeup and impurities while hydrating the skin. This soothing formula is suitable for all skin types, making it a must-have in your daily skincare routine."
    
    6. Automotive  
    Input: "Ford F-150, Trucks"  
    Output: "The Ford F-150 is the quintessential pickup truck, combining power, reliability, and innovative technology. Equipped with advanced towing capabilities and a spacious interior, it's designed for both work and play."
    
    7. Electronics  
    Input: "Samsung Galaxy S21, Mobile Phones"  
    Output: "Experience the future of mobile technology with the Samsung Galaxy S21. This smartphone features a stunning display, powerful processor, and multiple camera options, perfect for capturing life's moments in high definition."
    
    8. Footwear  
    Input: "Adidas Ultraboost, Shoes"  
    Output: "Run in style with Adidas Ultraboost shoes. Known for their comfort and performance, these shoes utilize responsive cushioning to provide unmatched energy return with every step you take." 
    
    9. Electronics  
    Input: "Dell XPS 13, Laptops"  
    Output: "The Dell XPS 13 redefines the laptop experience with its stunning InfinityEdge display, powerful performance, and sleek design. Ideal for both professionals and students looking for portability and functionality."
    
    10. Personal Care  
    Input: "Philips Sonicare, Electronic Toothbrush"  
    Output: "Philips Sonicare's electronic toothbrush guarantees a superior cleaning experience with its advanced sonic technology. This toothbrush not only helps remove plaque but also promotes healthier gums for a brighter smile."
    
    


You can run this in a loop to append to your previous data and in this way you can keep generating more textual synthetic data to train another GPT model while making sure that we cater to imbalanced datasets and generating a diversity of data.

You have now completed part 1 of the synthetic data generation tutorial where we have gone through:
*   CSV with a structured prompt
*   CSV with a Python program
*   Multitable CSV with a python program
*   Simply creating textual data
*   Dealing with imbalanced or non-diverse textual data

In part 2 you will find find out techniques for better prompting an LLM to enhance textual synthetic data generation.


key: _node_content
string_value:
{"id_": "011d961c-9f08-4d57-b1ae-763d07ee3704", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_use_guardrails.ipynb", "title": "How to use guardrails", "tokens": 3850, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "b4b67675-fabc-4721-8a96-142e6b48e9a9", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_use_guardrails.ipynb", "title": "How to use guardrails", "tokens": 3850, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "703dd637c257fe499d65bbd45e6b13711bfe254c78df1e92c225173f262b1017", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "bb32d8b3-91ab-4437-93d5-4dcf29864c33", "node_type": "1", "metadata": {}, "hash": "dfa65dae1b5e3305b0aa3cbd7ee1abad2219399c26154c7797fc2df3cf616163", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 0, "end_char_idx": 3512, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
# How to use guardrails

In this notebook we share examples of how to implement guardrails for your LLM applications. A guardrail is a generic term for **detective controls** that aim to steer your application. Greater steerability is a common requirement given the inherent randomness of LLMs, and so creating effective guardrails has become one of the most common areas of performance optimization when pushing an LLM from prototype to production. 

Guardrails are incredibly [diverse](https://github.com/NVIDIA/NeMo-Guardrails/blob/main/examples/README.md) and can be deployed to virtually any context you can imagine something going wrong with LLMs. This notebook aims to give simple examples that can be extended to meet your unique use case, as well as outlining the trade-offs to consider when deciding whether to implement a guardrail, and how to do it.

This notebook will focus on:
1. **Input guardrails** that flag inappropriate content before it gets to your LLM
2. **Output guardrails** that validate what your LLM has produced before it gets to the customer

**Note:** This notebook tackles guardrails as a generic term for detective controls around an LLM - for the official libraries that provide distributions of pre-built guardrails frameworks, please check out the following:
- [NeMo Guardrails](https://github.com/NVIDIA/NeMo-Guardrails/tree/main)
- [Guardrails AI](https://github.com/ShreyaR/guardrails)



```python
import openai

GPT_MODEL = 'gpt-4o-mini'
```

## 1. Input guardrails

Input guardrails aim to prevent inappropriate content getting to the LLM in the first place - some common use cases are:
- **Topical guardrails:** Identify when a user asks an off-topic question and give them advice on what topics the LLM can help them with.
- **Jailbreaking:** Detect when a user is trying to hijack the LLM and override its prompting.
- **Prompt injection:** Pick up instances of prompt injection where users try to hide malicious code that will be executed in any downstream functions the LLM executes. 

In all of these they act as a preventative control, running either before or in parallel with the LLM, and triggering your application to behave differently if one of these criteria are met.

### Designing a guardrail

When designing guardrails it is important to consider the trade-off between **accuracy**, **latency** and **cost**, where you try to achieve maximum accuracy for the least impact to your bottom line and the user's experience. 

We'll begin with a simple **topical guardrail** which aims to detect off-topic questions and prevent the LLM from answering if triggered. This guardrail consists of a simple prompt and uses `gpt-4o-mini`, maximising latency/cost holding a good enough accuracy, but if we wanted to optimize further we could consider:
- **Accuracy:** You could consider fine-tuning `gpt-4o-mini` or few-shot examples to increase the accuracy. RAG can also be effective if you have a corpus of information that can help determine whether a piece of content is allowed or not.
- **Latency/Cost:** You could try fine-tuning smaller models, such as `babbage-002` or open-source offerings like Llama, which can perform quite well when given enough training examples. When using open-source offerings you can also tune the machines you are using for inference to maximize either cost or latency reduction.

This simple guardrail aims to ensure the LLM only answers to a predefined set of topics, and responds to out-of-bounds queries with a canned message.


key: _node_content
string_value:
{"id_": "bb32d8b3-91ab-4437-93d5-4dcf29864c33", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_use_guardrails.ipynb", "title": "How to use guardrails", "tokens": 3850, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "b4b67675-fabc-4721-8a96-142e6b48e9a9", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_use_guardrails.ipynb", "title": "How to use guardrails", "tokens": 3850, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "703dd637c257fe499d65bbd45e6b13711bfe254c78df1e92c225173f262b1017", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "011d961c-9f08-4d57-b1ae-763d07ee3704", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_use_guardrails.ipynb", "title": "How to use guardrails", "tokens": 3850, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "5b464dc740696ca0f05dc31b1c3cf9066ebc7b02d5237b5c5aa23f27a2895b7d", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "9059495b-9f66-4745-b7b6-1d661514f889", "node_type": "1", "metadata": {}, "hash": "01d5271694fc3c539a2161ccddec0c0bfd6a285d726e60280e6bb0578aa7bac3", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 1767, "end_char_idx": 5400, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
- **Jailbreaking:** Detect when a user is trying to hijack the LLM and override its prompting.
- **Prompt injection:** Pick up instances of prompt injection where users try to hide malicious code that will be executed in any downstream functions the LLM executes. 

In all of these they act as a preventative control, running either before or in parallel with the LLM, and triggering your application to behave differently if one of these criteria are met.

### Designing a guardrail

When designing guardrails it is important to consider the trade-off between **accuracy**, **latency** and **cost**, where you try to achieve maximum accuracy for the least impact to your bottom line and the user's experience. 

We'll begin with a simple **topical guardrail** which aims to detect off-topic questions and prevent the LLM from answering if triggered. This guardrail consists of a simple prompt and uses `gpt-4o-mini`, maximising latency/cost holding a good enough accuracy, but if we wanted to optimize further we could consider:
- **Accuracy:** You could consider fine-tuning `gpt-4o-mini` or few-shot examples to increase the accuracy. RAG can also be effective if you have a corpus of information that can help determine whether a piece of content is allowed or not.
- **Latency/Cost:** You could try fine-tuning smaller models, such as `babbage-002` or open-source offerings like Llama, which can perform quite well when given enough training examples. When using open-source offerings you can also tune the machines you are using for inference to maximize either cost or latency reduction.

This simple guardrail aims to ensure the LLM only answers to a predefined set of topics, and responds to out-of-bounds queries with a canned message.

### Embrace async

A common design to minimize latency is to send your guardrails asynchronously along with your main LLM call. If your guardrails get triggered you send back their response, otherwise send back the LLM response.

We'll use this approach, creating an `execute_chat_with_guardrails` function that will run our LLM's `get_chat_response` and the `topical_guardrail` guardrail in parallel, and return the LLM response only if the guardrail returns `allowed`.

### Limitations

You should always consider the limitations of guardrails when developing your design. A few of the key ones to be aware of are:
- When using LLMs as a guardrail, be aware that they have the same vulnerabilities as your base LLM call itself. For example, a **prompt injection** attempt could be successful in evading both your guardrail and your actual LLM call.
- As conversations get longer, LLMs are more susceptible to **jailbreaking** as your instructions become diluted by the extra text.
- Guardrails can harm the user experience if you make them overly restrictive to compensate for the issues noted above. This manifests as **over-refusals**, where your guardrails reject innocuous user requests because there are similarities with prompt injection or jailbreaking attempts.

### Mitigations

If you can combine guardrails with rules-based or more traditional machine learning models for detection this can mitigate some of these risks. We've also seen customers have guardrails that only ever consider the latest message, to alleviate the risks of the model being confused by a long conversation.

We would also recommend doing a gradual roll-out with active monitoring of conversations so you can pick up instances of prompt injection or jailbreaking, and either add more guardrails to cover these new types of behaviour, or include them as training examples to your existing guardrails.


key: _node_content
string_value:
{"id_": "9059495b-9f66-4745-b7b6-1d661514f889", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_use_guardrails.ipynb", "title": "How to use guardrails", "tokens": 3850, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "b4b67675-fabc-4721-8a96-142e6b48e9a9", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_use_guardrails.ipynb", "title": "How to use guardrails", "tokens": 3850, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "703dd637c257fe499d65bbd45e6b13711bfe254c78df1e92c225173f262b1017", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "bb32d8b3-91ab-4437-93d5-4dcf29864c33", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_use_guardrails.ipynb", "title": "How to use guardrails", "tokens": 3850, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "c07c805da5ec04bc207f2a4ec842394ed5539e079f38e543889ef5eb12c49e9a", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "21f1ec72-421f-4134-b9f0-57aa1bbb2f95", "node_type": "1", "metadata": {}, "hash": "c137f74e2e1f1c0e6efe9c3dcddcf768eacee5a4b4a986a276e7d955329100e2", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 3514, "end_char_idx": 6613, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
### Embrace async

A common design to minimize latency is to send your guardrails asynchronously along with your main LLM call. If your guardrails get triggered you send back their response, otherwise send back the LLM response.

We'll use this approach, creating an `execute_chat_with_guardrails` function that will run our LLM's `get_chat_response` and the `topical_guardrail` guardrail in parallel, and return the LLM response only if the guardrail returns `allowed`.

### Limitations

You should always consider the limitations of guardrails when developing your design. A few of the key ones to be aware of are:
- When using LLMs as a guardrail, be aware that they have the same vulnerabilities as your base LLM call itself. For example, a **prompt injection** attempt could be successful in evading both your guardrail and your actual LLM call.
- As conversations get longer, LLMs are more susceptible to **jailbreaking** as your instructions become diluted by the extra text.
- Guardrails can harm the user experience if you make them overly restrictive to compensate for the issues noted above. This manifests as **over-refusals**, where your guardrails reject innocuous user requests because there are similarities with prompt injection or jailbreaking attempts.

### Mitigations

If you can combine guardrails with rules-based or more traditional machine learning models for detection this can mitigate some of these risks. We've also seen customers have guardrails that only ever consider the latest message, to alleviate the risks of the model being confused by a long conversation.

We would also recommend doing a gradual roll-out with active monitoring of conversations so you can pick up instances of prompt injection or jailbreaking, and either add more guardrails to cover these new types of behaviour, or include them as training examples to your existing guardrails.


```python
system_prompt = "You are a helpful assistant."

bad_request = "I want to talk about horses"
good_request = "What are the best breeds of dog for people that like cats?"
```


```python
import asyncio


async def get_chat_response(user_request):
    print("Getting LLM response")
    messages = [
        {"role": "system", "content": system_prompt},
        {"role": "user", "content": user_request},
    ]
    response = openai.chat.completions.create(
        model=GPT_MODEL, messages=messages, temperature=0.5
    )
    print("Got LLM response")

    return response.choices[0].message.content


async def topical_guardrail(user_request):
    print("Checking topical guardrail")
    messages = [
        {
            "role": "system",
            "content": "Your role is to assess whether the user question is allowed or not. The allowed topics are cats and dogs. If the topic is allowed, say 'allowed' otherwise say 'not_allowed'",
        },
        {"role": "user", "content": user_request},
    ]
    response = openai.chat.completions.create(
        model=GPT_MODEL, messages=messages, temperature=0
    )

    print("Got guardrail response")
    return response.choices[0].message.content


key: _node_content
string_value:
{"id_": "21f1ec72-421f-4134-b9f0-57aa1bbb2f95", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_use_guardrails.ipynb", "title": "How to use guardrails", "tokens": 3850, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "b4b67675-fabc-4721-8a96-142e6b48e9a9", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_use_guardrails.ipynb", "title": "How to use guardrails", "tokens": 3850, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "703dd637c257fe499d65bbd45e6b13711bfe254c78df1e92c225173f262b1017", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "9059495b-9f66-4745-b7b6-1d661514f889", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_use_guardrails.ipynb", "title": "How to use guardrails", "tokens": 3850, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "26f1fbf2c6824ccb7e8377ec910516e871fc2a8e83f6bad638a198be097af332", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "5da0b5d5-1f92-42c3-81a4-1be4e4d413f9", "node_type": "1", "metadata": {}, "hash": "a992f8e782c8497123bc919d39f4f2136c90c581ce80031cad2dd46a8e96fda6", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 4787, "end_char_idx": 7543, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
### Mitigations

If you can combine guardrails with rules-based or more traditional machine learning models for detection this can mitigate some of these risks. We've also seen customers have guardrails that only ever consider the latest message, to alleviate the risks of the model being confused by a long conversation.

We would also recommend doing a gradual roll-out with active monitoring of conversations so you can pick up instances of prompt injection or jailbreaking, and either add more guardrails to cover these new types of behaviour, or include them as training examples to your existing guardrails.


```python
system_prompt = "You are a helpful assistant."

bad_request = "I want to talk about horses"
good_request = "What are the best breeds of dog for people that like cats?"
```


```python
import asyncio


async def get_chat_response(user_request):
    print("Getting LLM response")
    messages = [
        {"role": "system", "content": system_prompt},
        {"role": "user", "content": user_request},
    ]
    response = openai.chat.completions.create(
        model=GPT_MODEL, messages=messages, temperature=0.5
    )
    print("Got LLM response")

    return response.choices[0].message.content


async def topical_guardrail(user_request):
    print("Checking topical guardrail")
    messages = [
        {
            "role": "system",
            "content": "Your role is to assess whether the user question is allowed or not. The allowed topics are cats and dogs. If the topic is allowed, say 'allowed' otherwise say 'not_allowed'",
        },
        {"role": "user", "content": user_request},
    ]
    response = openai.chat.completions.create(
        model=GPT_MODEL, messages=messages, temperature=0
    )

    print("Got guardrail response")
    return response.choices[0].message.content


async def execute_chat_with_guardrail(user_request):
    topical_guardrail_task = asyncio.create_task(topical_guardrail(user_request))
    chat_task = asyncio.create_task(get_chat_response(user_request))

    while True:
        done, _ = await asyncio.wait(
            [topical_guardrail_task, chat_task], return_when=asyncio.FIRST_COMPLETED
        )
        if topical_guardrail_task in done:
            guardrail_response = topical_guardrail_task.result()
            if guardrail_response == "not_allowed":
                chat_task.cancel()
                print("Topical guardrail triggered")
                return "I can only talk about cats and dogs, the best animals that ever lived."
            elif chat_task in done:
                chat_response = chat_task.result()
                return chat_response
        else:
            await asyncio.sleep(0.1)  # sleep for a bit before checking the tasks again
```


key: _node_content
string_value:
{"id_": "5da0b5d5-1f92-42c3-81a4-1be4e4d413f9", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_use_guardrails.ipynb", "title": "How to use guardrails", "tokens": 3850, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "b4b67675-fabc-4721-8a96-142e6b48e9a9", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_use_guardrails.ipynb", "title": "How to use guardrails", "tokens": 3850, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "703dd637c257fe499d65bbd45e6b13711bfe254c78df1e92c225173f262b1017", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "21f1ec72-421f-4134-b9f0-57aa1bbb2f95", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_use_guardrails.ipynb", "title": "How to use guardrails", "tokens": 3850, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "f2b02df6ee64f125531df9e8f78eea3f9188410e17907fedfeb17becb6c9ca1e", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "c0c83785-cc40-4b6c-b08d-33dcd1b57d66", "node_type": "1", "metadata": {}, "hash": "9e19fcc41be81fb1393d3127fe432725d65d4c932c0d8ab4711c3128c69285f2", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 6012, "end_char_idx": 9566, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
async def topical_guardrail(user_request):
    print("Checking topical guardrail")
    messages = [
        {
            "role": "system",
            "content": "Your role is to assess whether the user question is allowed or not. The allowed topics are cats and dogs. If the topic is allowed, say 'allowed' otherwise say 'not_allowed'",
        },
        {"role": "user", "content": user_request},
    ]
    response = openai.chat.completions.create(
        model=GPT_MODEL, messages=messages, temperature=0
    )

    print("Got guardrail response")
    return response.choices[0].message.content


async def execute_chat_with_guardrail(user_request):
    topical_guardrail_task = asyncio.create_task(topical_guardrail(user_request))
    chat_task = asyncio.create_task(get_chat_response(user_request))

    while True:
        done, _ = await asyncio.wait(
            [topical_guardrail_task, chat_task], return_when=asyncio.FIRST_COMPLETED
        )
        if topical_guardrail_task in done:
            guardrail_response = topical_guardrail_task.result()
            if guardrail_response == "not_allowed":
                chat_task.cancel()
                print("Topical guardrail triggered")
                return "I can only talk about cats and dogs, the best animals that ever lived."
            elif chat_task in done:
                chat_response = chat_task.result()
                return chat_response
        else:
            await asyncio.sleep(0.1)  # sleep for a bit before checking the tasks again
```


```python
# Call the main function with the good request - this should go through
response = await execute_chat_with_guardrail(good_request)
print(response)
```

    Checking topical guardrail
    Got guardrail response
    Getting LLM response
    Got LLM response
    If you like cats and are considering getting a dog, there are several breeds known for their compatibility with feline friends. Here are some of the best dog breeds that tend to get along well with cats:
    
    1. **Golden Retriever**: Friendly and tolerant, Golden Retrievers often get along well with other animals, including cats.
    
    2. **Labrador Retriever**: Similar to Golden Retrievers, Labs are social and friendly, making them good companions for cats.
    
    3. **Cavalier King Charles Spaniel**: This breed is gentle and affectionate, often forming strong bonds with other pets.
    
    4. **Basset Hound**: Basset Hounds are laid-back and generally have a calm demeanor, which can help them coexist peacefully with cats.
    
    5. **Beagle**: Beagles are friendly and sociable, and they often enjoy the company of other animals, including cats.
    
    6. **Pug**: Pugs are known for their playful and friendly nature, which can make them good companions for cats.
    
    7. **Shih Tzu**: Shih Tzus are typically friendly and adaptable, often getting along well with other pets.
    
    8. **Collie**: Collies are known for their gentle and protective nature, which can extend to their relationships with cats.
    
    9. **Newfoundland**: These gentle giants are known for their calm demeanor and often get along well with other animals.
    
    10. **Cocker Spaniel**: Cocker Spaniels are friendly and affectionate dogs that can get along well with cats if introduced properly.
    
    When introducing a dog to a cat, it's important to do so gradually and supervise their interactions to ensure a positive relationship. Each dog's personality can vary, so individual temperament is key in determining compatibility.


key: _node_content
string_value:
{"id_": "c0c83785-cc40-4b6c-b08d-33dcd1b57d66", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_use_guardrails.ipynb", "title": "How to use guardrails", "tokens": 3850, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "b4b67675-fabc-4721-8a96-142e6b48e9a9", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_use_guardrails.ipynb", "title": "How to use guardrails", "tokens": 3850, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "703dd637c257fe499d65bbd45e6b13711bfe254c78df1e92c225173f262b1017", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "5da0b5d5-1f92-42c3-81a4-1be4e4d413f9", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_use_guardrails.ipynb", "title": "How to use guardrails", "tokens": 3850, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "60da01c459bd98031fba6800e7eb7c34228d1579fe4be971c77de847c24a9320", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "80ae9ddc-ec06-46c6-a6a1-9d225a07abca", "node_type": "1", "metadata": {}, "hash": "3bccc3721b0040e85aa8c9c7dd1bc3611d817a2b5bc9067077ca12d3934b65dd", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 9570, "end_char_idx": 13031, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
# Call the main function with the good request - this should get blocked
response = await execute_chat_with_guardrail(bad_request)
print(response)
```

    Checking topical guardrail
    Got guardrail response
    Getting LLM response
    Got LLM response
    Topical guardrail triggered
    I can only talk about cats and dogs, the best animals that ever lived.


Looks like our guardrail worked - the first question was allowed through, but the second was blocked for being off-topic. Now we'll extend this concept to moderate the response we get from the LLM as well.

## 2. Output guardrails

Output guardrails govern what the LLM comes back with. These can take many forms, with some of the most common being:
- **Hallucination/fact-checking guardrails:** Using a corpus of ground truth information or a training set of hallucinated responses to  block hallucinated responses.
- **Moderation guardrails:** Applying brand and corporate guidelines to moderate the LLM's results, and either blocking or rewriting its response if it breaches them.
- **Syntax checks:** Structured outputs from LLMs can be returned corrupt or unable to be parsed - these guardrails detect those and either retry or fail gracefully, preventing failures in downstream applications.
    - This is a common control to apply with function calling, ensuring that the expected schema is returned in the `arguments` when the LLM returns a `function_call`.
    
### Moderation guardrail

Here we implement a **moderation guardrail** that uses a version of the [G-Eval](https://arxiv.org/abs/2303.16634) evaluation method to score the presence of unwanted content in the LLM's response. This method is demonstrated in more detail in of our other [notebooks](https://github.com/openai/openai-cookbook/blob/main/examples/evaluation/How_to_eval_abstractive_summarization.ipynb).

To accomplish this we will make an extensible framework for moderating content that takes in a `domain` and applies `criteria` to a piece of `content` using a set of `steps`:
1. We set a domain name, which describes the type of content we're going to moderate.
2. We provide criteria, which outline clearly what the content should and should not contain.
3. Step-by-step instructions are provided for the LLM to grade the content.
4. The LLM returns a discrete score from 1-5.

### Setting guardrail thresholds

Our output guardrail will assess the LLM's response and block anything scoring a 3 or higher. Setting this threshold is a common area for optimization - we recommend building an evaluation set and grading the results using a confusion matrix to set the right tolerance for your guardrail. The trade-off here is generally:
- More false positives leads to a fractured user experience, where customers get annoyed and the assistant seems less helpful.
- More false negatives can cause lasting harm to your business, as people get the assistant to answer inappropriate questions, or prompt inject/jailbreak it.

For example, for jailbreaking you may want to have a very low threshold, as the risk to your business if your LLM is hijacked and used to produce dangerous content that ends up on social media is very high. However, for our use case we're willing to accept a few false negatives, as the worst that could happen is someone ends up with a Bichon Frise who might have been better suited to a Labrador, which though sad will probably not cause lasting damage to our business (we hope).


key: _node_content
string_value:
{"id_": "80ae9ddc-ec06-46c6-a6a1-9d225a07abca", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_use_guardrails.ipynb", "title": "How to use guardrails", "tokens": 3850, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "b4b67675-fabc-4721-8a96-142e6b48e9a9", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_use_guardrails.ipynb", "title": "How to use guardrails", "tokens": 3850, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "703dd637c257fe499d65bbd45e6b13711bfe254c78df1e92c225173f262b1017", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "c0c83785-cc40-4b6c-b08d-33dcd1b57d66", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_use_guardrails.ipynb", "title": "How to use guardrails", "tokens": 3850, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "308bdee20be87989257839adf7a9044a9350d803b46c171eb6aff962bbed4bd7", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "d992a66d-723c-47de-8b7b-989a2a1dfb97", "node_type": "1", "metadata": {}, "hash": "25c0f8fe37a8db4a0612673a2f03e06b745cdda81076f13d32462e7426f1bc2f", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 13034, "end_char_idx": 16059, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
domain = "animal breed recommendation"

animal_advice_criteria = """
Assess the presence of explicit recommendation of cat or dog breeds in the content.
The content should contain only general advice about cats and dogs, not specific breeds to purchase."""

animal_advice_steps = """
1. Read the content and the criteria carefully.
2. Assess how much explicit recommendation of cat or dog breeds is contained in the content.
3. Assign an animal advice score from 1 to 5, with 1 being no explicit cat or dog breed advice, and 5 being multiple named cat or dog breeds.
"""

moderation_system_prompt = """
You are a moderation assistant. Your role is to detect content about {domain} in the text provided, and mark the severity of that content.

## {domain}

### Criteria

{scoring_criteria}

### Instructions

{scoring_steps}

### Content

{content}

### Evaluation (score only!)
"""
```


```python
async def moderation_guardrail(chat_response):
    print("Checking moderation guardrail")
    mod_messages = [
        {"role": "user", "content": moderation_system_prompt.format(
            domain=domain,
            scoring_criteria=animal_advice_criteria,
            scoring_steps=animal_advice_steps,
            content=chat_response
        )},
    ]
    response = openai.chat.completions.create(
        model=GPT_MODEL, messages=mod_messages, temperature=0
    )
    print("Got moderation response")
    return response.choices[0].message.content
    
    
async def execute_all_guardrails(user_request):
    topical_guardrail_task = asyncio.create_task(topical_guardrail(user_request))
    chat_task = asyncio.create_task(get_chat_response(user_request))

    while True:
        done, _ = await asyncio.wait(
            [topical_guardrail_task, chat_task], return_when=asyncio.FIRST_COMPLETED
        )
        if topical_guardrail_task in done:
            guardrail_response = topical_guardrail_task.result()
            if guardrail_response == "not_allowed":
                chat_task.cancel()
                print("Topical guardrail triggered")
                return "I can only talk about cats and dogs, the best animals that ever lived."
            elif chat_task in done:
                chat_response = chat_task.result()
                moderation_response = await moderation_guardrail(chat_response)

                if int(moderation_response) >= 3:
                    print(f"Moderation guardrail flagged with a score of {int(moderation_response)}")
                    return "Sorry, we're not permitted to give animal breed advice. I can help you with any general queries you might have."

                else:
                    print('Passed moderation')
                    return chat_response
        else:
            await asyncio.sleep(0.1)  # sleep for a bit before checking the tasks again
```


```python
# Adding a request that should pass both our topical guardrail and our moderation guardrail
great_request = 'What is some advice you can give to a new dog owner?'
```


key: _node_content
string_value:
{"id_": "d992a66d-723c-47de-8b7b-989a2a1dfb97", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_use_guardrails.ipynb", "title": "How to use guardrails", "tokens": 3850, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "b4b67675-fabc-4721-8a96-142e6b48e9a9", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_use_guardrails.ipynb", "title": "How to use guardrails", "tokens": 3850, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "703dd637c257fe499d65bbd45e6b13711bfe254c78df1e92c225173f262b1017", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "80ae9ddc-ec06-46c6-a6a1-9d225a07abca", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_use_guardrails.ipynb", "title": "How to use guardrails", "tokens": 3850, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "5fdd33a6a13dae303d4cbc42242fd7a97a3a61c18c16b974f026595b208dbdb6", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 15883, "end_char_idx": 18240, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
# Adding a request that should pass both our topical guardrail and our moderation guardrail
great_request = 'What is some advice you can give to a new dog owner?'
```


```python
tests = [good_request,bad_request,great_request]

for test in tests:
    result = await execute_all_guardrails(test)
    print(result)
    print('\n\n')
    
```

    Checking topical guardrail
    Got guardrail response
    Getting LLM response
    Got LLM response
    Checking moderation guardrail
    Got moderation response
    Moderation guardrail flagged with a score of 5
    Sorry, we're not permitted to give animal breed advice. I can help you with any general queries you might have.
    
    
    
    Checking topical guardrail
    Got guardrail response
    Getting LLM response
    Got LLM response
    Topical guardrail triggered
    I can only talk about cats and dogs, the best animals that ever lived.
    
    
    
    Checking topical guardrail
    Got guardrail response
    Getting LLM response
    Got LLM response
    Checking moderation guardrail
    Got moderation response
    Moderation guardrail flagged with a score of 3
    Sorry, we're not permitted to give animal breed advice. I can help you with any general queries you might have.
    
    
    


## Conclusion

Guardrails are a vibrant and evolving topic in LLMs, and we hope this notebook has given you an effective introduction to the core concepts around guardrails. To recap:
- Guardrails are detective controls that aim to prevent harmful content getting to your applications and your users, and add steerability to your LLM in production.
- They can take the form of input guardrails, which target content before it gets to the LLM, and output guardrails, which control the LLM's response.
- Designing guardrails and setting their thresholds is a trade-off between accuracy, latency, and cost. Your decision should be based on clear evaluations of the performance of your guardrails, and an understanding of what the cost of a false negative and false positive are for your business.
- By embracing asynchronous design principles, you can scale guardrails horizontally to minimize the impact to the user as your guardrails increase in number and scope.

We look forward to seeing how you take this forward, and how thinking on guardrails evolves as the ecosystem matures.


key: _node_content
string_value:
{"id_": "fc1bd18d-f697-45dd-a9ca-3976af527d7d", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Using_embeddings.ipynb", "title": "Negative example (slow and rate-limited)", "tokens": 380, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "7c89964a-f852-45ab-9a08-9ae67919b1bc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Using_embeddings.ipynb", "title": "Negative example (slow and rate-limited)", "tokens": 380, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "ecfa61d3c81a47dd37b07dec77955a725494d724930a992f02de00e2ac72bcb0", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 0, "end_char_idx": 1581, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
## Using embeddings

This notebook contains some helpful snippets you can use to embed text with the `text-embedding-3-small` model via the OpenAI API.


```python
from openai import OpenAI
client = OpenAI()

embedding = client.embeddings.create(
    input="Your text goes here", model="text-embedding-3-small"
).data[0].embedding
len(embedding)

```




    1536



It's recommended to use the 'tenacity' package or another exponential backoff implementation to better manage API rate limits, as hitting the API too much too fast can trigger rate limits. Using the following function ensures you get your embeddings as fast as possible.


```python
# Negative example (slow and rate-limited)
from openai import OpenAI
client = OpenAI()

num_embeddings = 10000 # Some large number
for i in range(num_embeddings):
    embedding = client.embeddings.create(
        input="Your text goes here", model="text-embedding-3-small"
    ).data[0].embedding
    print(len(embedding))
```


```python
# Best practice
from tenacity import retry, wait_random_exponential, stop_after_attempt
from openai import OpenAI
client = OpenAI()

# Retry up to 6 times with exponential backoff, starting at 1 second and maxing out at 20 seconds delay
@retry(wait=wait_random_exponential(min=1, max=20), stop=stop_after_attempt(6))
def get_embedding(text: str, model="text-embedding-3-small") -> list[float]:
    return client.embeddings.create(input=[text], model=model).data[0].embedding

embedding = get_embedding("Your text goes here", model="text-embedding-3-small")
print(len(embedding))
```

    1536


key: url
string_value:
https://github.com/openai/openai-cookbook/blob/main/examples/Embedding_Wikipedia_articles_for_search.ipynb


key: _node_content
string_value:
{"id_": "25ee58e1-4d9f-4dee-afa5-b6955791803a", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Embedding_Wikipedia_articles_for_search.ipynb", "title": "Embedding Wikipedia articles for search", "tokens": 4092, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "ad35521b-dea3-4465-8104-7425091d5da0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Embedding_Wikipedia_articles_for_search.ipynb", "title": "Embedding Wikipedia articles for search", "tokens": 4092, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "092a5d71318519b610b43cf482903767e5ce1886ad354365508f58cb2cc05f13", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "9b1892c0-f56a-401c-b4fd-be92a1adafa9", "node_type": "1", "metadata": {}, "hash": "2aac946d934a339e30acde13fdfe37b5d6578b90ead86bf76aaaae023c17087c", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 0, "end_char_idx": 2956, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
# Embedding Wikipedia articles for search

This notebook shows how we prepared a dataset of Wikipedia articles for search, used in [Question_answering_using_embeddings.ipynb](Question_answering_using_embeddings.ipynb).

Procedure:

0. Prerequisites: Import libraries, set API key (if needed)
1. Collect: We download a few hundred Wikipedia articles about the 2022 Olympics
2. Chunk: Documents are split into short, semi-self-contained sections to be embedded
3. Embed: Each section is embedded with the OpenAI API
4. Store: Embeddings are saved in a CSV file (for large datasets, use a vector database)

## 0. Prerequisites

### Import libraries


```python
# imports
import mwclient  # for downloading example Wikipedia articles
import mwparserfromhell  # for splitting Wikipedia articles into sections
import openai  # for generating embeddings
import os  # for environment variables
import pandas as pd  # for DataFrames to store article sections and embeddings
import re  # for cutting <ref> links out of Wikipedia articles
import tiktoken  # for counting tokens

client = openai.OpenAI(api_key=os.environ.get("OPENAI_API_KEY", "<your OpenAI API key if not set as env var>"))
```

Install any missing libraries with `pip install` in your terminal. E.g.,

```zsh
pip install openai
```

(You can also do this in a notebook cell with `!pip install openai`.)

If you install any libraries, be sure to restart the notebook kernel.

### Set API key (if needed)

Note that the OpenAI library will try to read your API key from the `OPENAI_API_KEY` environment variable. If you haven't already, set this environment variable by following [these instructions](https://help.openai.com/en/articles/5112595-best-practices-for-api-key-safety).

## 1. Collect documents

In this example, we'll download a few hundred Wikipedia articles related to the 2022 Winter Olympics.


```python
# get Wikipedia pages about the 2022 Winter Olympics

CATEGORY_TITLE = "Category:2022 Winter Olympics"
WIKI_SITE = "en.wikipedia.org"


def titles_from_category(
    category: mwclient.listing.Category, max_depth: int
) -> set[str]:
    """Return a set of page titles in a given Wiki category and its subcategories."""
    titles = set()
    for cm in category.members():
        if type(cm) == mwclient.page.Page:
            # ^type() used instead of isinstance() to catch match w/ no inheritance
            titles.add(cm.name)
        elif isinstance(cm, mwclient.listing.Category) and max_depth > 0:
            deeper_titles = titles_from_category(cm, max_depth=max_depth - 1)
            titles.update(deeper_titles)
    return titles


site = mwclient.Site(WIKI_SITE)
category_page = site.pages[CATEGORY_TITLE]
titles = titles_from_category(category_page, max_depth=1)
# ^note: max_depth=1 means we go one level deep in the category tree
print(f"Found {len(titles)} article titles in {CATEGORY_TITLE}.")

```

    Found 731 article titles in Category:2022 Winter Olympics.


key: _node_content
string_value:
{"id_": "9b1892c0-f56a-401c-b4fd-be92a1adafa9", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Embedding_Wikipedia_articles_for_search.ipynb", "title": "Embedding Wikipedia articles for search", "tokens": 4092, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "ad35521b-dea3-4465-8104-7425091d5da0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Embedding_Wikipedia_articles_for_search.ipynb", "title": "Embedding Wikipedia articles for search", "tokens": 4092, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "092a5d71318519b610b43cf482903767e5ce1886ad354365508f58cb2cc05f13", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "25ee58e1-4d9f-4dee-afa5-b6955791803a", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Embedding_Wikipedia_articles_for_search.ipynb", "title": "Embedding Wikipedia articles for search", "tokens": 4092, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "a1f7d7568b22f19a99938e7955da5ed3aea20adc6e0c08e763d6c6bb251c8803", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "b59db87b-9dc9-4cdc-ba1f-b5924437a77e", "node_type": "1", "metadata": {}, "hash": "534b7fa4171e7091d90093be9202722096521ebfbd5eb8bcd2f4ad43f2f76c74", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 1866, "end_char_idx": 4140, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
# get Wikipedia pages about the 2022 Winter Olympics

CATEGORY_TITLE = "Category:2022 Winter Olympics"
WIKI_SITE = "en.wikipedia.org"


def titles_from_category(
    category: mwclient.listing.Category, max_depth: int
) -> set[str]:
    """Return a set of page titles in a given Wiki category and its subcategories."""
    titles = set()
    for cm in category.members():
        if type(cm) == mwclient.page.Page:
            # ^type() used instead of isinstance() to catch match w/ no inheritance
            titles.add(cm.name)
        elif isinstance(cm, mwclient.listing.Category) and max_depth > 0:
            deeper_titles = titles_from_category(cm, max_depth=max_depth - 1)
            titles.update(deeper_titles)
    return titles


site = mwclient.Site(WIKI_SITE)
category_page = site.pages[CATEGORY_TITLE]
titles = titles_from_category(category_page, max_depth=1)
# ^note: max_depth=1 means we go one level deep in the category tree
print(f"Found {len(titles)} article titles in {CATEGORY_TITLE}.")

```

    Found 731 article titles in Category:2022 Winter Olympics.


## 2. Chunk documents

Now that we have our reference documents, we need to prepare them for search.

Because GPT can only read a limited amount of text at once, we'll split each document into chunks short enough to be read.

For this specific example on Wikipedia articles, we'll:
- Discard less relevant-looking sections like External Links and Footnotes
- Clean up the text by removing reference tags (e.g., <ref>), whitespace, and super short sections
- Split each article into sections
- Prepend titles and subtitles to each section's text, to help GPT understand the context
- If a section is long (say, > 1,600 tokens), we'll recursively split it into smaller sections, trying to split along semantic boundaries like paragraphs


```python
# define functions to split Wikipedia pages into sections

SECTIONS_TO_IGNORE = [
    "See also",
    "References",
    "External links",
    "Further reading",
    "Footnotes",
    "Bibliography",
    "Sources",
    "Citations",
    "Literature",
    "Footnotes",
    "Notes and references",
    "Photo gallery",
    "Works cited",
    "Photos",
    "Gallery",
    "Notes",
    "References and sources",
    "References and notes",
]


key: _node_content
string_value:
{"id_": "b59db87b-9dc9-4cdc-ba1f-b5924437a77e", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Embedding_Wikipedia_articles_for_search.ipynb", "title": "Embedding Wikipedia articles for search", "tokens": 4092, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "ad35521b-dea3-4465-8104-7425091d5da0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Embedding_Wikipedia_articles_for_search.ipynb", "title": "Embedding Wikipedia articles for search", "tokens": 4092, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "092a5d71318519b610b43cf482903767e5ce1886ad354365508f58cb2cc05f13", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "9b1892c0-f56a-401c-b4fd-be92a1adafa9", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Embedding_Wikipedia_articles_for_search.ipynb", "title": "Embedding Wikipedia articles for search", "tokens": 4092, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "d0a278678f18c5ce34d86c7adc952d5ab95dd0dc7043ace578c1559ae0ac9ede", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "247c1288-e42d-4731-a66a-86207023ee57", "node_type": "1", "metadata": {}, "hash": "ab6de8e84580e2e728db30e92221ead9e96c6ed9b28725d473f06200dbe7e0a3", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 2620, "end_char_idx": 5364, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
site = mwclient.Site(WIKI_SITE)
category_page = site.pages[CATEGORY_TITLE]
titles = titles_from_category(category_page, max_depth=1)
# ^note: max_depth=1 means we go one level deep in the category tree
print(f"Found {len(titles)} article titles in {CATEGORY_TITLE}.")

```

    Found 731 article titles in Category:2022 Winter Olympics.


## 2. Chunk documents

Now that we have our reference documents, we need to prepare them for search.

Because GPT can only read a limited amount of text at once, we'll split each document into chunks short enough to be read.

For this specific example on Wikipedia articles, we'll:
- Discard less relevant-looking sections like External Links and Footnotes
- Clean up the text by removing reference tags (e.g., <ref>), whitespace, and super short sections
- Split each article into sections
- Prepend titles and subtitles to each section's text, to help GPT understand the context
- If a section is long (say, > 1,600 tokens), we'll recursively split it into smaller sections, trying to split along semantic boundaries like paragraphs


```python
# define functions to split Wikipedia pages into sections

SECTIONS_TO_IGNORE = [
    "See also",
    "References",
    "External links",
    "Further reading",
    "Footnotes",
    "Bibliography",
    "Sources",
    "Citations",
    "Literature",
    "Footnotes",
    "Notes and references",
    "Photo gallery",
    "Works cited",
    "Photos",
    "Gallery",
    "Notes",
    "References and sources",
    "References and notes",
]


def all_subsections_from_section(
    section: mwparserfromhell.wikicode.Wikicode,
    parent_titles: list[str],
    sections_to_ignore: set[str],
) -> list[tuple[list[str], str]]:
    """
    From a Wikipedia section, return a flattened list of all nested subsections.
    Each subsection is a tuple, where:
        - the first element is a list of parent subtitles, starting with the page title
        - the second element is the text of the subsection (but not any children)
    """
    headings = [str(h) for h in section.filter_headings()]
    title = headings[0]
    if title.strip("=" + " ") in sections_to_ignore:
        # ^wiki headings are wrapped like "== Heading =="
        return []
    titles = parent_titles + [title]
    full_text = str(section)
    section_text = full_text.split(title)[1]
    if len(headings) == 1:
        return [(titles, section_text)]
    else:
        first_subtitle = headings[1]
        section_text = section_text.split(first_subtitle)[0]
        results = [(titles, section_text)]
        for subsection in section.get_sections(levels=[len(titles) + 1]):
            results.extend(all_subsections_from_section(subsection, titles, sections_to_ignore))
        return results


key: _node_content
string_value:
{"id_": "247c1288-e42d-4731-a66a-86207023ee57", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Embedding_Wikipedia_articles_for_search.ipynb", "title": "Embedding Wikipedia articles for search", "tokens": 4092, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "ad35521b-dea3-4465-8104-7425091d5da0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Embedding_Wikipedia_articles_for_search.ipynb", "title": "Embedding Wikipedia articles for search", "tokens": 4092, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "092a5d71318519b610b43cf482903767e5ce1886ad354365508f58cb2cc05f13", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "b59db87b-9dc9-4cdc-ba1f-b5924437a77e", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Embedding_Wikipedia_articles_for_search.ipynb", "title": "Embedding Wikipedia articles for search", "tokens": 4092, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "51bd9fe17085c541e18bf2b8cd2898d8743f801b0fc07538e2d5ff5f8d56c5ea", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "7930ba24-0bea-43f5-a30f-f15079133172", "node_type": "1", "metadata": {}, "hash": "fef1b0557444b23cb6c3158919f1093d088e727cbf6883239f7dfb73d78a5f9b", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 3696, "end_char_idx": 6691, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
# define functions to split Wikipedia pages into sections

SECTIONS_TO_IGNORE = [
    "See also",
    "References",
    "External links",
    "Further reading",
    "Footnotes",
    "Bibliography",
    "Sources",
    "Citations",
    "Literature",
    "Footnotes",
    "Notes and references",
    "Photo gallery",
    "Works cited",
    "Photos",
    "Gallery",
    "Notes",
    "References and sources",
    "References and notes",
]


def all_subsections_from_section(
    section: mwparserfromhell.wikicode.Wikicode,
    parent_titles: list[str],
    sections_to_ignore: set[str],
) -> list[tuple[list[str], str]]:
    """
    From a Wikipedia section, return a flattened list of all nested subsections.
    Each subsection is a tuple, where:
        - the first element is a list of parent subtitles, starting with the page title
        - the second element is the text of the subsection (but not any children)
    """
    headings = [str(h) for h in section.filter_headings()]
    title = headings[0]
    if title.strip("=" + " ") in sections_to_ignore:
        # ^wiki headings are wrapped like "== Heading =="
        return []
    titles = parent_titles + [title]
    full_text = str(section)
    section_text = full_text.split(title)[1]
    if len(headings) == 1:
        return [(titles, section_text)]
    else:
        first_subtitle = headings[1]
        section_text = section_text.split(first_subtitle)[0]
        results = [(titles, section_text)]
        for subsection in section.get_sections(levels=[len(titles) + 1]):
            results.extend(all_subsections_from_section(subsection, titles, sections_to_ignore))
        return results


def all_subsections_from_title(
    title: str,
    sections_to_ignore: set[str] = SECTIONS_TO_IGNORE,
    site_name: str = WIKI_SITE,
) -> list[tuple[list[str], str]]:
    """From a Wikipedia page title, return a flattened list of all nested subsections.
    Each subsection is a tuple, where:
        - the first element is a list of parent subtitles, starting with the page title
        - the second element is the text of the subsection (but not any children)
    """
    site = mwclient.Site(site_name)
    page = site.pages[title]
    text = page.text()
    parsed_text = mwparserfromhell.parse(text)
    headings = [str(h) for h in parsed_text.filter_headings()]
    if headings:
        summary_text = str(parsed_text).split(headings[0])[0]
    else:
        summary_text = str(parsed_text)
    results = [([title], summary_text)]
    for subsection in parsed_text.get_sections(levels=[2]):
        results.extend(all_subsections_from_section(subsection, [title], sections_to_ignore))
    return results

```


```python
# split pages into sections
# may take ~1 minute per 100 articles
wikipedia_sections = []
for title in titles:
    wikipedia_sections.extend(all_subsections_from_title(title))
print(f"Found {len(wikipedia_sections)} sections in {len(titles)} pages.")

```

    Found 5730 sections in 731 pages.


key: _node_content
string_value:
{"id_": "7930ba24-0bea-43f5-a30f-f15079133172", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Embedding_Wikipedia_articles_for_search.ipynb", "title": "Embedding Wikipedia articles for search", "tokens": 4092, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "ad35521b-dea3-4465-8104-7425091d5da0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Embedding_Wikipedia_articles_for_search.ipynb", "title": "Embedding Wikipedia articles for search", "tokens": 4092, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "092a5d71318519b610b43cf482903767e5ce1886ad354365508f58cb2cc05f13", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "247c1288-e42d-4731-a66a-86207023ee57", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Embedding_Wikipedia_articles_for_search.ipynb", "title": "Embedding Wikipedia articles for search", "tokens": 4092, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "fff54f242562bf20b07a247664c17e3b36b373a77988cdb44833e1c092077319", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "173c0183-bcab-4cab-84ba-5f8ee8c1aaf9", "node_type": "1", "metadata": {}, "hash": "823da9c5677e327c9be9e55a9f0fc7e220f97c0062c24d17da43f28ff7bff32f", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 5367, "end_char_idx": 8384, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
def all_subsections_from_title(
    title: str,
    sections_to_ignore: set[str] = SECTIONS_TO_IGNORE,
    site_name: str = WIKI_SITE,
) -> list[tuple[list[str], str]]:
    """From a Wikipedia page title, return a flattened list of all nested subsections.
    Each subsection is a tuple, where:
        - the first element is a list of parent subtitles, starting with the page title
        - the second element is the text of the subsection (but not any children)
    """
    site = mwclient.Site(site_name)
    page = site.pages[title]
    text = page.text()
    parsed_text = mwparserfromhell.parse(text)
    headings = [str(h) for h in parsed_text.filter_headings()]
    if headings:
        summary_text = str(parsed_text).split(headings[0])[0]
    else:
        summary_text = str(parsed_text)
    results = [([title], summary_text)]
    for subsection in parsed_text.get_sections(levels=[2]):
        results.extend(all_subsections_from_section(subsection, [title], sections_to_ignore))
    return results

```


```python
# split pages into sections
# may take ~1 minute per 100 articles
wikipedia_sections = []
for title in titles:
    wikipedia_sections.extend(all_subsections_from_title(title))
print(f"Found {len(wikipedia_sections)} sections in {len(titles)} pages.")

```

    Found 5730 sections in 731 pages.



```python
# clean text
def clean_section(section: tuple[list[str], str]) -> tuple[list[str], str]:
    """
    Return a cleaned up section with:
        - <ref>xyz</ref> patterns removed
        - leading/trailing whitespace removed
    """
    titles, text = section
    text = re.sub(r"<ref.*?</ref>", "", text)
    text = text.strip()
    return (titles, text)


wikipedia_sections = [clean_section(ws) for ws in wikipedia_sections]

# filter out short/blank sections
def keep_section(section: tuple[list[str], str]) -> bool:
    """Return True if the section should be kept, False otherwise."""
    titles, text = section
    if len(text) < 16:
        return False
    else:
        return True


original_num_sections = len(wikipedia_sections)
wikipedia_sections = [ws for ws in wikipedia_sections if keep_section(ws)]
print(f"Filtered out {original_num_sections-len(wikipedia_sections)} sections, leaving {len(wikipedia_sections)} sections.")

```

    Filtered out 530 sections, leaving 5200 sections.



```python
# print example data
for ws in wikipedia_sections[:5]:
    print(ws[0])
    display(ws[1][:77] + "...")
    print()

```

    ['Lviv bid for the 2022 Winter Olympics']



    '{{Olympic bid|2022|Winter|\n| Paralympics = yes\n| logo = Lviv 2022 Winter Olym...'


    
    ['Lviv bid for the 2022 Winter Olympics', '==History==']



    '[[Image:Lwów - Rynek 01.JPG|thumb|right|200px|View of Rynok Square in Lviv]]\n...'


    
    ['Lviv bid for the 2022 Winter Olympics', '==Venues==']



    '{{Location map+\n|Ukraine\n|border =\n|caption = Venue areas\n|float = left\n|widt...'


    
    ['Lviv bid for the 2022 Winter Olympics', '==Venues==', '===City zone===']


key: _node_content
string_value:
{"id_": "173c0183-bcab-4cab-84ba-5f8ee8c1aaf9", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Embedding_Wikipedia_articles_for_search.ipynb", "title": "Embedding Wikipedia articles for search", "tokens": 4092, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "ad35521b-dea3-4465-8104-7425091d5da0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Embedding_Wikipedia_articles_for_search.ipynb", "title": "Embedding Wikipedia articles for search", "tokens": 4092, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "092a5d71318519b610b43cf482903767e5ce1886ad354365508f58cb2cc05f13", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "7930ba24-0bea-43f5-a30f-f15079133172", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Embedding_Wikipedia_articles_for_search.ipynb", "title": "Embedding Wikipedia articles for search", "tokens": 4092, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "4a37653d4e3068f5a9954118cda702ca55dad101cc0138a21dbb03d37c6397cd", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "b9163845-f908-4910-ae9c-c005481c3bab", "node_type": "1", "metadata": {}, "hash": "54965f447fbdbf6114035709ce5edac391c4994da6e4253b4960c3d23448b7ab", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 7061, "end_char_idx": 9824, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
wikipedia_sections = [clean_section(ws) for ws in wikipedia_sections]

# filter out short/blank sections
def keep_section(section: tuple[list[str], str]) -> bool:
    """Return True if the section should be kept, False otherwise."""
    titles, text = section
    if len(text) < 16:
        return False
    else:
        return True


original_num_sections = len(wikipedia_sections)
wikipedia_sections = [ws for ws in wikipedia_sections if keep_section(ws)]
print(f"Filtered out {original_num_sections-len(wikipedia_sections)} sections, leaving {len(wikipedia_sections)} sections.")

```

    Filtered out 530 sections, leaving 5200 sections.



```python
# print example data
for ws in wikipedia_sections[:5]:
    print(ws[0])
    display(ws[1][:77] + "...")
    print()

```

    ['Lviv bid for the 2022 Winter Olympics']



    '{{Olympic bid|2022|Winter|\n| Paralympics = yes\n| logo = Lviv 2022 Winter Olym...'


    
    ['Lviv bid for the 2022 Winter Olympics', '==History==']



    '[[Image:Lwów - Rynek 01.JPG|thumb|right|200px|View of Rynok Square in Lviv]]\n...'


    
    ['Lviv bid for the 2022 Winter Olympics', '==Venues==']



    '{{Location map+\n|Ukraine\n|border =\n|caption = Venue areas\n|float = left\n|widt...'


    
    ['Lviv bid for the 2022 Winter Olympics', '==Venues==', '===City zone===']



    'The main Olympic Park would be centered around the [[Arena Lviv]], hosting th...'


    
    ['Lviv bid for the 2022 Winter Olympics', '==Venues==', '===Mountain zone===', '====Venue cluster Tysovets-Panasivka====']



    'An existing military ski training facility in [[Tysovets, Skole Raion|Tysovet...'


    


Next, we'll recursively split long sections into smaller sections.

There's no perfect recipe for splitting text into sections.

Some tradeoffs include:
- Longer sections may be better for questions that require more context
- Longer sections may be worse for retrieval, as they may have more topics muddled together
- Shorter sections are better for reducing costs (which are proportional to the number of tokens)
- Shorter sections allow more sections to be retrieved, which may help with recall
- Overlapping sections may help prevent answers from being cut by section boundaries

Here, we'll use a simple approach and limit sections to 1,600 tokens each, recursively halving any sections that are too long. To avoid cutting in the middle of useful sentences, we'll split along paragraph boundaries when possible.


```python
GPT_MODEL = "gpt-3.5-turbo"  # only matters insofar as it selects which tokenizer to use


def num_tokens(text: str, model: str = GPT_MODEL) -> int:
    """Return the number of tokens in a string."""
    encoding = tiktoken.encoding_for_model(model)
    return len(encoding.encode(text))


key: _node_content
string_value:
{"id_": "b9163845-f908-4910-ae9c-c005481c3bab", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Embedding_Wikipedia_articles_for_search.ipynb", "title": "Embedding Wikipedia articles for search", "tokens": 4092, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "ad35521b-dea3-4465-8104-7425091d5da0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Embedding_Wikipedia_articles_for_search.ipynb", "title": "Embedding Wikipedia articles for search", "tokens": 4092, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "092a5d71318519b610b43cf482903767e5ce1886ad354365508f58cb2cc05f13", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "173c0183-bcab-4cab-84ba-5f8ee8c1aaf9", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Embedding_Wikipedia_articles_for_search.ipynb", "title": "Embedding Wikipedia articles for search", "tokens": 4092, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "1bfde80667db87a95b5d674e8ece041db0b73d3d04ed44fbc43035ba693bc218", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "ef31f890-3e09-418d-a8af-abaf73951ddc", "node_type": "1", "metadata": {}, "hash": "9262974da17f999f2b938a2262e342d664fbc8a3f904ca70d63b3cc816ab6c41", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 8211, "end_char_idx": 11244, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
'{{Location map+\n|Ukraine\n|border =\n|caption = Venue areas\n|float = left\n|widt...'


    
    ['Lviv bid for the 2022 Winter Olympics', '==Venues==', '===City zone===']



    'The main Olympic Park would be centered around the [[Arena Lviv]], hosting th...'


    
    ['Lviv bid for the 2022 Winter Olympics', '==Venues==', '===Mountain zone===', '====Venue cluster Tysovets-Panasivka====']



    'An existing military ski training facility in [[Tysovets, Skole Raion|Tysovet...'


    


Next, we'll recursively split long sections into smaller sections.

There's no perfect recipe for splitting text into sections.

Some tradeoffs include:
- Longer sections may be better for questions that require more context
- Longer sections may be worse for retrieval, as they may have more topics muddled together
- Shorter sections are better for reducing costs (which are proportional to the number of tokens)
- Shorter sections allow more sections to be retrieved, which may help with recall
- Overlapping sections may help prevent answers from being cut by section boundaries

Here, we'll use a simple approach and limit sections to 1,600 tokens each, recursively halving any sections that are too long. To avoid cutting in the middle of useful sentences, we'll split along paragraph boundaries when possible.


```python
GPT_MODEL = "gpt-3.5-turbo"  # only matters insofar as it selects which tokenizer to use


def num_tokens(text: str, model: str = GPT_MODEL) -> int:
    """Return the number of tokens in a string."""
    encoding = tiktoken.encoding_for_model(model)
    return len(encoding.encode(text))


def halved_by_delimiter(string: str, delimiter: str = "\n") -> list[str, str]:
    """Split a string in two, on a delimiter, trying to balance tokens on each side."""
    chunks = string.split(delimiter)
    if len(chunks) == 1:
        return [string, ""]  # no delimiter found
    elif len(chunks) == 2:
        return chunks  # no need to search for halfway point
    else:
        total_tokens = num_tokens(string)
        halfway = total_tokens // 2
        best_diff = halfway
        for i, chunk in enumerate(chunks):
            left = delimiter.join(chunks[: i + 1])
            left_tokens = num_tokens(left)
            diff = abs(halfway - left_tokens)
            if diff >= best_diff:
                break
            else:
                best_diff = diff
        left = delimiter.join(chunks[:i])
        right = delimiter.join(chunks[i:])
        return [left, right]


def truncated_string(
    string: str,
    model: str,
    max_tokens: int,
    print_warning: bool = True,
) -> str:
    """Truncate a string to a maximum number of tokens."""
    encoding = tiktoken.encoding_for_model(model)
    encoded_string = encoding.encode(string)
    truncated_string = encoding.decode(encoded_string[:max_tokens])
    if print_warning and len(encoded_string) > max_tokens:
        print(f"Warning: Truncated string from {len(encoded_string)} tokens to {max_tokens} tokens.")
    return truncated_string


key: _node_content
string_value:
{"id_": "ef31f890-3e09-418d-a8af-abaf73951ddc", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Embedding_Wikipedia_articles_for_search.ipynb", "title": "Embedding Wikipedia articles for search", "tokens": 4092, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "ad35521b-dea3-4465-8104-7425091d5da0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Embedding_Wikipedia_articles_for_search.ipynb", "title": "Embedding Wikipedia articles for search", "tokens": 4092, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "092a5d71318519b610b43cf482903767e5ce1886ad354365508f58cb2cc05f13", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "b9163845-f908-4910-ae9c-c005481c3bab", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Embedding_Wikipedia_articles_for_search.ipynb", "title": "Embedding Wikipedia articles for search", "tokens": 4092, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "87e761e83248f5317afd252d40220fae262112a91200398c6b2caad63ce4bd18", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "22658c43-ca7b-4f80-96ae-5e8338df91a0", "node_type": "1", "metadata": {}, "hash": "d0600a2d7fadbeb388d13462889003f391808220cbccc08090e5ee7ced8e6cc6", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 9628, "end_char_idx": 13049, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
def num_tokens(text: str, model: str = GPT_MODEL) -> int:
    """Return the number of tokens in a string."""
    encoding = tiktoken.encoding_for_model(model)
    return len(encoding.encode(text))


def halved_by_delimiter(string: str, delimiter: str = "\n") -> list[str, str]:
    """Split a string in two, on a delimiter, trying to balance tokens on each side."""
    chunks = string.split(delimiter)
    if len(chunks) == 1:
        return [string, ""]  # no delimiter found
    elif len(chunks) == 2:
        return chunks  # no need to search for halfway point
    else:
        total_tokens = num_tokens(string)
        halfway = total_tokens // 2
        best_diff = halfway
        for i, chunk in enumerate(chunks):
            left = delimiter.join(chunks[: i + 1])
            left_tokens = num_tokens(left)
            diff = abs(halfway - left_tokens)
            if diff >= best_diff:
                break
            else:
                best_diff = diff
        left = delimiter.join(chunks[:i])
        right = delimiter.join(chunks[i:])
        return [left, right]


def truncated_string(
    string: str,
    model: str,
    max_tokens: int,
    print_warning: bool = True,
) -> str:
    """Truncate a string to a maximum number of tokens."""
    encoding = tiktoken.encoding_for_model(model)
    encoded_string = encoding.encode(string)
    truncated_string = encoding.decode(encoded_string[:max_tokens])
    if print_warning and len(encoded_string) > max_tokens:
        print(f"Warning: Truncated string from {len(encoded_string)} tokens to {max_tokens} tokens.")
    return truncated_string


def split_strings_from_subsection(
    subsection: tuple[list[str], str],
    max_tokens: int = 1000,
    model: str = GPT_MODEL,
    max_recursion: int = 5,
) -> list[str]:
    """
    Split a subsection into a list of subsections, each with no more than max_tokens.
    Each subsection is a tuple of parent titles [H1, H2, ...] and text (str).
    """
    titles, text = subsection
    string = "\n\n".join(titles + [text])
    num_tokens_in_string = num_tokens(string)
    # if length is fine, return string
    if num_tokens_in_string <= max_tokens:
        return [string]
    # if recursion hasn't found a split after X iterations, just truncate
    elif max_recursion == 0:
        return [truncated_string(string, model=model, max_tokens=max_tokens)]
    # otherwise, split in half and recurse
    else:
        titles, text = subsection
        for delimiter in ["\n\n", "\n", ". "]:
            left, right = halved_by_delimiter(text, delimiter=delimiter)
            if left == "" or right == "":
                # if either half is empty, retry with a more fine-grained delimiter
                continue
            else:
                # recurse on each half
                results = []
                for half in [left, right]:
                    half_subsection = (titles, half)
                    half_strings = split_strings_from_subsection(
                        half_subsection,
                        max_tokens=max_tokens,
                        model=model,
                        max_recursion=max_recursion - 1,
                    )
                    results.extend(half_strings)
                return results
    # otherwise no split was found, so just truncate (should be very rare)
    return [truncated_string(string, model=model, max_tokens=max_tokens)]

```


key: _node_content
string_value:
{"id_": "22658c43-ca7b-4f80-96ae-5e8338df91a0", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Embedding_Wikipedia_articles_for_search.ipynb", "title": "Embedding Wikipedia articles for search", "tokens": 4092, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "ad35521b-dea3-4465-8104-7425091d5da0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Embedding_Wikipedia_articles_for_search.ipynb", "title": "Embedding Wikipedia articles for search", "tokens": 4092, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "092a5d71318519b610b43cf482903767e5ce1886ad354365508f58cb2cc05f13", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "ef31f890-3e09-418d-a8af-abaf73951ddc", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Embedding_Wikipedia_articles_for_search.ipynb", "title": "Embedding Wikipedia articles for search", "tokens": 4092, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "b9cd6edab1ef76df1b20349c425a0d071b548f4449c24138383f3428468eacfb", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "6eaf1dcb-bac4-4222-93e3-86f4a4d2ff3c", "node_type": "1", "metadata": {}, "hash": "03a3f10f5abeb9397fa17a2edcedc1a05eed3edeaadde150b4c31f3e2fbd6173", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 11247, "end_char_idx": 13418, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
def split_strings_from_subsection(
    subsection: tuple[list[str], str],
    max_tokens: int = 1000,
    model: str = GPT_MODEL,
    max_recursion: int = 5,
) -> list[str]:
    """
    Split a subsection into a list of subsections, each with no more than max_tokens.
    Each subsection is a tuple of parent titles [H1, H2, ...] and text (str).
    """
    titles, text = subsection
    string = "\n\n".join(titles + [text])
    num_tokens_in_string = num_tokens(string)
    # if length is fine, return string
    if num_tokens_in_string <= max_tokens:
        return [string]
    # if recursion hasn't found a split after X iterations, just truncate
    elif max_recursion == 0:
        return [truncated_string(string, model=model, max_tokens=max_tokens)]
    # otherwise, split in half and recurse
    else:
        titles, text = subsection
        for delimiter in ["\n\n", "\n", ". "]:
            left, right = halved_by_delimiter(text, delimiter=delimiter)
            if left == "" or right == "":
                # if either half is empty, retry with a more fine-grained delimiter
                continue
            else:
                # recurse on each half
                results = []
                for half in [left, right]:
                    half_subsection = (titles, half)
                    half_strings = split_strings_from_subsection(
                        half_subsection,
                        max_tokens=max_tokens,
                        model=model,
                        max_recursion=max_recursion - 1,
                    )
                    results.extend(half_strings)
                return results
    # otherwise no split was found, so just truncate (should be very rare)
    return [truncated_string(string, model=model, max_tokens=max_tokens)]

```


```python
# split sections into chunks
MAX_TOKENS = 1600
wikipedia_strings = []
for section in wikipedia_sections:
    wikipedia_strings.extend(split_strings_from_subsection(section, max_tokens=MAX_TOKENS))

print(f"{len(wikipedia_sections)} Wikipedia sections split into {len(wikipedia_strings)} strings.")

```

    5200 Wikipedia sections split into 6059 strings.


key: _node_content
string_value:
{"id_": "6eaf1dcb-bac4-4222-93e3-86f4a4d2ff3c", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Embedding_Wikipedia_articles_for_search.ipynb", "title": "Embedding Wikipedia articles for search", "tokens": 4092, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "ad35521b-dea3-4465-8104-7425091d5da0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Embedding_Wikipedia_articles_for_search.ipynb", "title": "Embedding Wikipedia articles for search", "tokens": 4092, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "092a5d71318519b610b43cf482903767e5ce1886ad354365508f58cb2cc05f13", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "22658c43-ca7b-4f80-96ae-5e8338df91a0", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Embedding_Wikipedia_articles_for_search.ipynb", "title": "Embedding Wikipedia articles for search", "tokens": 4092, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "ee12140feeb22703ebcbd5a80822c537401570e844e703686d40c6d0b0926d84", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "46889feb-f266-481a-8e59-9ac89daa24f8", "node_type": "1", "metadata": {}, "hash": "718d3a26e40cb050b114ec7bfa6bc261ad029897cdb8a6345beb434b43436e3a", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 13052, "end_char_idx": 16232, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
# split sections into chunks
MAX_TOKENS = 1600
wikipedia_strings = []
for section in wikipedia_sections:
    wikipedia_strings.extend(split_strings_from_subsection(section, max_tokens=MAX_TOKENS))

print(f"{len(wikipedia_sections)} Wikipedia sections split into {len(wikipedia_strings)} strings.")

```

    5200 Wikipedia sections split into 6059 strings.



```python
# print example data
print(wikipedia_strings[1])

```

    Lviv bid for the 2022 Winter Olympics
    
    ==History==
    
    [[Image:Lwów - Rynek 01.JPG|thumb|right|200px|View of Rynok Square in Lviv]]
    
    On 27 May 2010, [[President of Ukraine]] [[Viktor Yanukovych]] stated during a visit to [[Lviv]] that Ukraine "will start working on the official nomination of our country as the holder of the Winter Olympic Games in [[Carpathian Mountains|Carpathians]]".
    
    In September 2012, [[government of Ukraine]] approved a document about the technical-economic substantiation of the national project "Olympic Hope 2022". This was announced by Vladyslav Kaskiv, the head of Ukraine´s Derzhinvestproekt (State investment project). The organizers announced on their website venue plans featuring Lviv as the host city and location for the "ice sport" venues, [[Volovets]] (around {{convert|185|km|mi|abbr=on}} from Lviv) as venue for the [[Alpine skiing]] competitions and [[Tysovets, Skole Raion|Tysovets]] (around {{convert|130|km|mi|abbr=on}} from Lviv) as venue for all other "snow sport" competitions. By March 2013 no other preparations than the feasibility study had been approved.
    
    On 24 October 2013, session of the Lviv City Council adopted a resolution "About submission to the International Olympic Committee for nomination of city to participate in the procedure for determining the host city of Olympic and Paralympic Winter Games in 2022".
    
    On 5 November 2013, it was confirmed that Lviv was bidding to host the [[2022 Winter Olympics]]. Lviv would host the ice sport events, while the skiing events would be held in the [[Carpathian]] mountains. This was the first bid Ukraine had ever submitted for an Olympic Games.
    
    On 30 June 2014, the International Olympic Committee announced "Lviv will turn its attention to an Olympic bid for 2026, and not continue with its application for 2022. The decision comes as a result of the present political and economic circumstances in Ukraine."
    
    Ukraine's Deputy Prime Minister Oleksandr Vilkul said that the Winter Games "will be an impetus not just for promotion of sports and tourism in Ukraine, but a very important component in the economic development of Ukraine, the attraction of the investments, the creation of new jobs, opening Ukraine to the world, returning Ukrainians working abroad to their motherland."
    
    Lviv was one of the host cities of [[UEFA Euro 2012]].


## 3. Embed document chunks

Now that we've split our library into shorter self-contained strings, we can compute embeddings for each.

(For large embedding jobs, use a script like [api_request_parallel_processor.py](api_request_parallel_processor.py) to parallelize requests while throttling to stay under rate limits.)


key: _node_content
string_value:
{"id_": "46889feb-f266-481a-8e59-9ac89daa24f8", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Embedding_Wikipedia_articles_for_search.ipynb", "title": "Embedding Wikipedia articles for search", "tokens": 4092, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "ad35521b-dea3-4465-8104-7425091d5da0", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Embedding_Wikipedia_articles_for_search.ipynb", "title": "Embedding Wikipedia articles for search", "tokens": 4092, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "092a5d71318519b610b43cf482903767e5ce1886ad354365508f58cb2cc05f13", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "6eaf1dcb-bac4-4222-93e3-86f4a4d2ff3c", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Embedding_Wikipedia_articles_for_search.ipynb", "title": "Embedding Wikipedia articles for search", "tokens": 4092, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "ab2051bd1378f8af0a56a5c098ceb2fd56b1b09411954a33524d316310bfee8b", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 15912, "end_char_idx": 17474, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
## 3. Embed document chunks

Now that we've split our library into shorter self-contained strings, we can compute embeddings for each.

(For large embedding jobs, use a script like [api_request_parallel_processor.py](api_request_parallel_processor.py) to parallelize requests while throttling to stay under rate limits.)


```python
EMBEDDING_MODEL = "text-embedding-3-small"
BATCH_SIZE = 1000  # you can submit up to 2048 embedding inputs per request

embeddings = []
for batch_start in range(0, len(wikipedia_strings), BATCH_SIZE):
    batch_end = batch_start + BATCH_SIZE
    batch = wikipedia_strings[batch_start:batch_end]
    print(f"Batch {batch_start} to {batch_end-1}")
    response = client.embeddings.create(model=EMBEDDING_MODEL, input=batch)
    for i, be in enumerate(response.data):
        assert i == be.index  # double check embeddings are in same order as input
    batch_embeddings = [e.embedding for e in response.data]
    embeddings.extend(batch_embeddings)

df = pd.DataFrame({"text": wikipedia_strings, "embedding": embeddings})

```

    Batch 0 to 999
    Batch 1000 to 1999
    Batch 2000 to 2999
    Batch 3000 to 3999
    Batch 4000 to 4999
    Batch 5000 to 5999
    Batch 6000 to 6999


## 4. Store document chunks and embeddings

Because this example only uses a few thousand strings, we'll store them in a CSV file.

(For larger datasets, use a vector database, which will be more performant.)


```python
# save document chunks and embeddings

SAVE_PATH = "data/winter_olympics_2022.csv"

df.to_csv(SAVE_PATH, index=False)

```


key: url
string_value:
https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb


key: _node_content
string_value:
{"id_": "759bd5ee-79e5-42bd-af33-222a44307615", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb", "title": "How to format inputs to ChatGPT models", "tokens": 5687, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "7f6a0328-5bf8-46ee-88a7-6980bc7f8694", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb", "title": "How to format inputs to ChatGPT models", "tokens": 5687, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "41d37741bc26fd57be6b5af37408f3347d5279f47c887c29850cc8de683ba4a0", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "7e98614a-fd32-45d7-985d-b711bfcc29b7", "node_type": "1", "metadata": {}, "hash": "4dec753a6ccb3ac73b93413931f424fd9c2cf3644c9c8f1239090da0848e7f16", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 0, "end_char_idx": 2576, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
# How to format inputs to ChatGPT models

ChatGPT is powered by `gpt-3.5-turbo` and `gpt-4`, OpenAI's most advanced models.

You can build your own applications with `gpt-3.5-turbo` or `gpt-4` using the OpenAI API.

Chat models take a series of messages as input, and return an AI-written message as output.

This guide illustrates the chat format with a few example API calls.

## 1. Import the openai library


```python
# if needed, install and/or upgrade to the latest version of the OpenAI Python library
%pip install --upgrade openai

```


```python
# import the OpenAI Python library for calling the OpenAI API
from openai import OpenAI
import os

client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY", "<your OpenAI API key if not set as env var>"))
```

## 2. An example chat completion API call

A chat completion API call parameters,
**Required**
- `model`: the name of the model you want to use (e.g., `gpt-3.5-turbo`, `gpt-4`, `gpt-3.5-turbo-16k-1106`)
- `messages`: a list of message objects, where each object has two required fields:
    - `role`: the role of the messenger (either `system`, `user`, `assistant` or `tool`)
    - `content`: the content of the message (e.g., `Write me a beautiful poem`)

Messages can also contain an optional `name` field, which give the messenger a name. E.g., `example-user`, `Alice`, `BlackbeardBot`. Names may not contain spaces.

**Optional**
- `frequency_penalty`: Penalizes tokens based on their frequency, reducing repetition.
- `logit_bias`: Modifies likelihood of specified tokens with bias values.
- `logprobs`: Returns log probabilities of output tokens if true.
- `top_logprobs`: Specifies the number of most likely tokens to return at each position.
- `max_tokens`: Sets the maximum number of generated tokens in chat completion.
- `n`: Generates a specified number of chat completion choices for each input.
- `presence_penalty`: Penalizes new tokens based on their presence in the text.
- `response_format`: Specifies the output format, e.g., JSON mode.
- `seed`: Ensures deterministic sampling with a specified seed.
- `stop`: Specifies up to 4 sequences where the API should stop generating tokens.
- `stream`: Sends partial message deltas as tokens become available.
- `temperature`: Sets the sampling temperature between 0 and 2.
- `top_p`: Uses nucleus sampling; considers tokens with top_p probability mass.
- `tools`: Lists functions the model may call.
- `tool_choice`: Controls the model's function calls (none/auto/function).
- `user`: Unique identifier for end-user monitoring and abuse detection.


key: _node_content
string_value:
{"id_": "7e98614a-fd32-45d7-985d-b711bfcc29b7", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb", "title": "How to format inputs to ChatGPT models", "tokens": 5687, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "7f6a0328-5bf8-46ee-88a7-6980bc7f8694", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb", "title": "How to format inputs to ChatGPT models", "tokens": 5687, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "41d37741bc26fd57be6b5af37408f3347d5279f47c887c29850cc8de683ba4a0", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "759bd5ee-79e5-42bd-af33-222a44307615", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb", "title": "How to format inputs to ChatGPT models", "tokens": 5687, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "b1824398be0c6eda102c91cbd7f5be6befc71924f06a0266d7cd7ad50b097a3e", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "fc3cfc1a-602d-45fd-947b-c1a88eae9591", "node_type": "1", "metadata": {}, "hash": "7bf6863c6b5a4e1149307801a62b66f77b4f1e8ea9c3f578690c69e14759df3a", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 2579, "end_char_idx": 5683, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
As of January 2024, you can also optionally submit a list of `functions` that tell GPT whether it can generate JSON to feed into a function. For details, see the [documentation](https://platform.openai.com/docs/guides/function-calling), [API reference](https://platform.openai.com/docs/api-reference/chat), or the Cookbook guide [How to call functions with chat models](How_to_call_functions_with_chat_models.ipynb).

Typically, a conversation will start with a system message that tells the assistant how to behave, followed by alternating user and assistant messages, but you are not required to follow this format.

Let's look at an example chat API calls to see how the chat format works in practice.


```python
# Example OpenAI Python library request
MODEL = "gpt-3.5-turbo"
response = client.chat.completions.create(
    model=MODEL,
    messages=[
        {"role": "system", "content": "You are a helpful assistant."},
        {"role": "user", "content": "Knock knock."},
        {"role": "assistant", "content": "Who's there?"},
        {"role": "user", "content": "Orange."},
    ],
    temperature=0,
)

```


```python
print(json.dumps(json.loads(response.model_dump_json()), indent=4))
```

    {
        "id": "chatcmpl-8dee9DuEFcg2QILtT2a6EBXZnpirM",
        "choices": [
            {
                "finish_reason": "stop",
                "index": 0,
                "logprobs": null,
                "message": {
                    "content": "Orange who?",
                    "role": "assistant",
                    "function_call": null,
                    "tool_calls": null
                }
            }
        ],
        "created": 1704461729,
        "model": "gpt-3.5-turbo-0613",
        "object": "chat.completion",
        "system_fingerprint": null,
        "usage": {
            "completion_tokens": 3,
            "prompt_tokens": 35,
            "total_tokens": 38
        }
    }


As you can see, the response object has a few fields:
- `id`: the ID of the request
- `choices`: a list of completion objects (only one, unless you set `n` greater than 1)
    - `finish_reason`: the reason the model stopped generating text (either `stop`, or `length` if `max_tokens` limit was reached)
    - `index`: The index of the choice in the list of choices.
    - `logprobs`: Log probability information for the choice.
    - `message`: the message object generated by the model
        - `content`: content of message
        - `role`: The role of the author of this message.
        - `tool_calls`: The tool calls generated by the model, such as function calls. if the tools is given
- `created`: the timestamp of the request
- `model`: the full name of the model used to generate the response
- `object`: the type of object returned (e.g., `chat.completion`)
- `system_fingerprint`: This fingerprint represents the backend configuration that the model runs with.
- `usage`: the number of tokens used to generate the replies, counting prompt, completion, and total

Extract just the reply with:


```python
response.choices[0].message.content

```




    'Orange who?'


key: _node_content
string_value:
{"id_": "fc3cfc1a-602d-45fd-947b-c1a88eae9591", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb", "title": "How to format inputs to ChatGPT models", "tokens": 5687, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "7f6a0328-5bf8-46ee-88a7-6980bc7f8694", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb", "title": "How to format inputs to ChatGPT models", "tokens": 5687, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "41d37741bc26fd57be6b5af37408f3347d5279f47c887c29850cc8de683ba4a0", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "7e98614a-fd32-45d7-985d-b711bfcc29b7", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb", "title": "How to format inputs to ChatGPT models", "tokens": 5687, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "08deb7c22eb0cddc0db6e1fccb5f079867b1b27caea5002d50a231971898fd88", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "afc4be4e-5780-4b55-bc4f-2f25bd92b27a", "node_type": "1", "metadata": {}, "hash": "be5ed72e0863a08a06d0636c7468a80fa120f48c410be10dd035d21a952e5a1c", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 4504, "end_char_idx": 5952, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
As you can see, the response object has a few fields:
- `id`: the ID of the request
- `choices`: a list of completion objects (only one, unless you set `n` greater than 1)
    - `finish_reason`: the reason the model stopped generating text (either `stop`, or `length` if `max_tokens` limit was reached)
    - `index`: The index of the choice in the list of choices.
    - `logprobs`: Log probability information for the choice.
    - `message`: the message object generated by the model
        - `content`: content of message
        - `role`: The role of the author of this message.
        - `tool_calls`: The tool calls generated by the model, such as function calls. if the tools is given
- `created`: the timestamp of the request
- `model`: the full name of the model used to generate the response
- `object`: the type of object returned (e.g., `chat.completion`)
- `system_fingerprint`: This fingerprint represents the backend configuration that the model runs with.
- `usage`: the number of tokens used to generate the replies, counting prompt, completion, and total

Extract just the reply with:


```python
response.choices[0].message.content

```




    'Orange who?'



Even non-conversation-based tasks can fit into the chat format, by placing the instruction in the first user message.

For example, to ask the model to explain asynchronous programming in the style of the pirate Blackbeard, we can structure conversation as follows:


key: _node_content
string_value:
{"id_": "afc4be4e-5780-4b55-bc4f-2f25bd92b27a", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb", "title": "How to format inputs to ChatGPT models", "tokens": 5687, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "7f6a0328-5bf8-46ee-88a7-6980bc7f8694", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb", "title": "How to format inputs to ChatGPT models", "tokens": 5687, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "41d37741bc26fd57be6b5af37408f3347d5279f47c887c29850cc8de683ba4a0", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "fc3cfc1a-602d-45fd-947b-c1a88eae9591", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb", "title": "How to format inputs to ChatGPT models", "tokens": 5687, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "ad70fad6db23a038c5933f516b5bcc65861b28593fe70c9e4e07f4a44388fbea", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "b1c33e12-bae3-46f8-8df1-a75e912362ca", "node_type": "1", "metadata": {}, "hash": "7ba9b13400d2056690a65416b53e468599581932ac4ee8d9a98089eab32ec03f", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 4504, "end_char_idx": 8544, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
As you can see, the response object has a few fields:
- `id`: the ID of the request
- `choices`: a list of completion objects (only one, unless you set `n` greater than 1)
    - `finish_reason`: the reason the model stopped generating text (either `stop`, or `length` if `max_tokens` limit was reached)
    - `index`: The index of the choice in the list of choices.
    - `logprobs`: Log probability information for the choice.
    - `message`: the message object generated by the model
        - `content`: content of message
        - `role`: The role of the author of this message.
        - `tool_calls`: The tool calls generated by the model, such as function calls. if the tools is given
- `created`: the timestamp of the request
- `model`: the full name of the model used to generate the response
- `object`: the type of object returned (e.g., `chat.completion`)
- `system_fingerprint`: This fingerprint represents the backend configuration that the model runs with.
- `usage`: the number of tokens used to generate the replies, counting prompt, completion, and total

Extract just the reply with:


```python
response.choices[0].message.content

```




    'Orange who?'



Even non-conversation-based tasks can fit into the chat format, by placing the instruction in the first user message.

For example, to ask the model to explain asynchronous programming in the style of the pirate Blackbeard, we can structure conversation as follows:


```python
# example with a system message
response = client.chat.completions.create(
    model=MODEL,
    messages=[
        {"role": "system", "content": "You are a helpful assistant."},
        {"role": "user", "content": "Explain asynchronous programming in the style of the pirate Blackbeard."},
    ],
    temperature=0,
)

print(response.choices[0].message.content)

```

    Arr, me matey! Let me tell ye a tale of asynchronous programming, in the style of the fearsome pirate Blackbeard!
    
    Picture this, me hearties. In the vast ocean of programming, there be times when ye need to perform multiple tasks at once. But fear not, for asynchronous programming be here to save the day!
    
    Ye see, in traditional programming, ye be waitin' for one task to be done before movin' on to the next. But with asynchronous programming, ye can be takin' care of multiple tasks at the same time, just like a pirate multitaskin' on the high seas!
    
    Instead of waitin' for a task to be completed, ye can be sendin' it off on its own journey, while ye move on to the next task. It be like havin' a crew of trusty sailors, each takin' care of their own duties, without waitin' for the others.
    
    Now, ye may be wonderin', how does this sorcery work? Well, me matey, it be all about callbacks and promises. When ye be sendin' off a task, ye be attachin' a callback function to it. This be like leavin' a message in a bottle, tellin' the task what to do when it be finished.
    
    While the task be sailin' on its own, ye can be movin' on to the next task, without wastin' any precious time. And when the first task be done, it be sendin' a signal back to ye, lettin' ye know it be finished. Then ye can be takin' care of the callback function, like openin' the bottle and readin' the message inside.
    
    But wait, there be more! With promises, ye can be makin' even fancier arrangements. Instead of callbacks, ye be makin' a promise that the task will be completed. It be like a contract between ye and the task, swearin' that it will be done.
    
    Ye can be attachin' multiple promises to a task, promisin' different outcomes. And when the task be finished, it be fulfillin' the promises, lettin' ye know it be done. Then ye can be handlin' the fulfillments, like collectin' the rewards of yer pirate adventures!
    
    So, me hearties, that be the tale of asynchronous programming, told in the style of the fearsome pirate Blackbeard! With callbacks and promises, ye can be takin' care of multiple tasks at once, just like a pirate conquerin' the seven seas!


key: _node_content
string_value:
{"id_": "b1c33e12-bae3-46f8-8df1-a75e912362ca", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb", "title": "How to format inputs to ChatGPT models", "tokens": 5687, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "7f6a0328-5bf8-46ee-88a7-6980bc7f8694", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb", "title": "How to format inputs to ChatGPT models", "tokens": 5687, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "41d37741bc26fd57be6b5af37408f3347d5279f47c887c29850cc8de683ba4a0", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "afc4be4e-5780-4b55-bc4f-2f25bd92b27a", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb", "title": "How to format inputs to ChatGPT models", "tokens": 5687, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "f2ffb5f2d556b2419f4b8025920bdbc991f2aaf5b08de225f38ead344b8da3b4", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "89638774-d207-43b0-bad1-b0ff17d08d3f", "node_type": "1", "metadata": {}, "hash": "45bef14035f7ebe9ef30490d89d70fc3ef99c6cc66668d446872aede9da11558", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 8548, "end_char_idx": 11484, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
# example without a system message
response = client.chat.completions.create(
    model=MODEL,
    messages=[
        {"role": "user", "content": "Explain asynchronous programming in the style of the pirate Blackbeard."},
    ],
    temperature=0,
)

print(response.choices[0].message.content)

```

    Arr, me hearties! Gather 'round and listen up, for I be tellin' ye about the mysterious art of asynchronous programming, in the style of the fearsome pirate Blackbeard!
    
    Now, ye see, in the world of programming, there be times when we need to perform tasks that take a mighty long time to complete. These tasks might involve fetchin' data from the depths of the internet, or performin' complex calculations that would make even Davy Jones scratch his head.
    
    In the olden days, we pirates used to wait patiently for each task to finish afore movin' on to the next one. But that be a waste of precious time, me hearties! We be pirates, always lookin' for ways to be more efficient and plunder more booty!
    
    That be where asynchronous programming comes in, me mateys. It be a way to tackle multiple tasks at once, without waitin' for each one to finish afore movin' on. It be like havin' a crew of scallywags workin' on different tasks simultaneously, while ye be overseein' the whole operation.
    
    Ye see, in asynchronous programming, we be breakin' down our tasks into smaller chunks called "coroutines." Each coroutine be like a separate pirate, workin' on its own task. When a coroutine be startin' its work, it don't wait for the task to finish afore movin' on to the next one. Instead, it be movin' on to the next task, lettin' the first one continue in the background.
    
    Now, ye might be wonderin', "But Blackbeard, how be we know when a task be finished if we don't wait for it?" Ah, me hearties, that be where the magic of callbacks and promises come in!
    
    When a coroutine be startin' its work, it be attachin' a callback or a promise to it. This be like leavin' a message in a bottle, tellin' the coroutine what to do when it be finished. So, while the coroutine be workin' away, the rest of the crew be movin' on to other tasks, plunderin' more booty along the way.
    
    When a coroutine be finished with its task, it be sendin' a signal to the callback or fulfillin' the promise, lettin' the rest of the crew know that it be done. Then, the crew can gather 'round and handle the results of the completed task, celebratin' their victory and countin' their plunder.
    
    So, me hearties, asynchronous programming be like havin' a crew of pirates workin' on different tasks at once, without waitin' for each one to finish afore movin' on. It be a way to be more efficient, plunder more booty, and conquer the vast seas of programming!
    
    Now, set sail, me mateys, and embrace the power of asynchronous programming like true pirates of the digital realm! Arr!


key: _node_content
string_value:
{"id_": "89638774-d207-43b0-bad1-b0ff17d08d3f", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb", "title": "How to format inputs to ChatGPT models", "tokens": 5687, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "7f6a0328-5bf8-46ee-88a7-6980bc7f8694", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb", "title": "How to format inputs to ChatGPT models", "tokens": 5687, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "41d37741bc26fd57be6b5af37408f3347d5279f47c887c29850cc8de683ba4a0", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "b1c33e12-bae3-46f8-8df1-a75e912362ca", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb", "title": "How to format inputs to ChatGPT models", "tokens": 5687, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "d5a3bffa9f7a212bb05e5c5a04989f344a07b97b2c40bdc4843fe5b583218372", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "e9f0d6a4-a76a-4493-82e0-db779ed5d7e0", "node_type": "1", "metadata": {}, "hash": "60833d3ef16ded3c94b9de0fa1c2cc6934fb80a09b955d439522768b96d9b24e", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 11487, "end_char_idx": 14789, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
## 3. Tips for instructing gpt-3.5-turbo-0301

Best practices for instructing models may change from model version to model version. The advice that follows applies to `gpt-3.5-turbo-0301` and may not apply to future models.

### System messages

The system message can be used to prime the assistant with different personalities or behaviors.

Be aware that `gpt-3.5-turbo-0301` does not generally pay as much attention to the system message as `gpt-4-0314` or `gpt-3.5-turbo-0613`. Therefore, for `gpt-3.5-turbo-0301`, we recommend placing important instructions in the user message instead. Some developers have found success in continually moving the system message near the end of the conversation to keep the model's attention from drifting away as conversations get longer.


```python
# An example of a system message that primes the assistant to explain concepts in great depth
response = client.chat.completions.create(
    model=MODEL,
    messages=[
        {"role": "system", "content": "You are a friendly and helpful teaching assistant. You explain concepts in great depth using simple terms, and you give examples to help people learn. At the end of each explanation, you ask a question to check for understanding"},
        {"role": "user", "content": "Can you explain how fractions work?"},
    ],
    temperature=0,
)

print(response.choices[0].message.content)

```

    Of course! Fractions are a way to represent parts of a whole. They are made up of two numbers: a numerator and a denominator. The numerator tells you how many parts you have, and the denominator tells you how many equal parts make up the whole.
    
    Let's take an example to understand this better. Imagine you have a pizza that is divided into 8 equal slices. If you eat 3 slices, you can represent that as the fraction 3/8. Here, the numerator is 3 because you ate 3 slices, and the denominator is 8 because the whole pizza is divided into 8 slices.
    
    Fractions can also be used to represent numbers less than 1. For example, if you eat half of a pizza, you can write it as 1/2. Here, the numerator is 1 because you ate one slice, and the denominator is 2 because the whole pizza is divided into 2 equal parts.
    
    Now, let's talk about equivalent fractions. Equivalent fractions are different fractions that represent the same amount. For example, 1/2 and 2/4 are equivalent fractions because they both represent half of something. To find equivalent fractions, you can multiply or divide both the numerator and denominator by the same number.
    
    Here's a question to check your understanding: If you have a cake divided into 12 equal slices and you eat 4 slices, what fraction of the cake did you eat?



```python
# An example of a system message that primes the assistant to give brief, to-the-point answers
response = client.chat.completions.create(
    model=MODEL,
    messages=[
        {"role": "system", "content": "You are a laconic assistant. You reply with brief, to-the-point answers with no elaboration."},
        {"role": "user", "content": "Can you explain how fractions work?"},
    ],
    temperature=0,
)

print(response.choices[0].message.content)

```

    Fractions represent parts of a whole. They have a numerator (top number) and a denominator (bottom number).


key: _node_content
string_value:
{"id_": "e9f0d6a4-a76a-4493-82e0-db779ed5d7e0", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb", "title": "How to format inputs to ChatGPT models", "tokens": 5687, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "7f6a0328-5bf8-46ee-88a7-6980bc7f8694", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb", "title": "How to format inputs to ChatGPT models", "tokens": 5687, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "41d37741bc26fd57be6b5af37408f3347d5279f47c887c29850cc8de683ba4a0", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "89638774-d207-43b0-bad1-b0ff17d08d3f", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb", "title": "How to format inputs to ChatGPT models", "tokens": 5687, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "8c19455c15e102e3d231dae22ac648ac07b280355760c7c6d2b7ff6572e0a58a", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "12a381ce-161a-4f76-9034-76548ecc0400", "node_type": "1", "metadata": {}, "hash": "06220c279e32d2014db649da58b80d9cd665c8f447c7e0a607405d607633b444", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 14209, "end_char_idx": 16513, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
# An example of a system message that primes the assistant to give brief, to-the-point answers
response = client.chat.completions.create(
    model=MODEL,
    messages=[
        {"role": "system", "content": "You are a laconic assistant. You reply with brief, to-the-point answers with no elaboration."},
        {"role": "user", "content": "Can you explain how fractions work?"},
    ],
    temperature=0,
)

print(response.choices[0].message.content)

```

    Fractions represent parts of a whole. They have a numerator (top number) and a denominator (bottom number).


### Few-shot prompting

In some cases, it's easier to show the model what you want rather than tell the model what you want.

One way to show the model what you want is with faked example messages.

For example:


```python
# An example of a faked few-shot conversation to prime the model into translating business jargon to simpler speech
response = client.chat.completions.create(
    model=MODEL,
    messages=[
        {"role": "system", "content": "You are a helpful, pattern-following assistant."},
        {"role": "user", "content": "Help me translate the following corporate jargon into plain English."},
        {"role": "assistant", "content": "Sure, I'd be happy to!"},
        {"role": "user", "content": "New synergies will help drive top-line growth."},
        {"role": "assistant", "content": "Things working well together will increase revenue."},
        {"role": "user", "content": "Let's circle back when we have more bandwidth to touch base on opportunities for increased leverage."},
        {"role": "assistant", "content": "Let's talk later when we're less busy about how to do better."},
        {"role": "user", "content": "This late pivot means we don't have time to boil the ocean for the client deliverable."},
    ],
    temperature=0,
)

print(response.choices[0].message.content)

```

    This sudden change in direction means we don't have enough time to complete the entire project for the client.


To help clarify that the example messages are not part of a real conversation, and shouldn't be referred back to by the model, you can try setting the `name` field of `system` messages to `example_user` and `example_assistant`.

Transforming the few-shot example above, we could write:


key: _node_content
string_value:
{"id_": "12a381ce-161a-4f76-9034-76548ecc0400", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb", "title": "How to format inputs to ChatGPT models", "tokens": 5687, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "7f6a0328-5bf8-46ee-88a7-6980bc7f8694", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb", "title": "How to format inputs to ChatGPT models", "tokens": 5687, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "41d37741bc26fd57be6b5af37408f3347d5279f47c887c29850cc8de683ba4a0", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "e9f0d6a4-a76a-4493-82e0-db779ed5d7e0", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb", "title": "How to format inputs to ChatGPT models", "tokens": 5687, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "642e9f06bc5b7cc940f90784ccf18696c93a82859a38ee891b14b52033bc2507", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "4bc17035-2108-4afe-9ad1-ea2bb7579309", "node_type": "1", "metadata": {}, "hash": "a0f65e9c971a3294da9c5dd81b8f9ae2d3757e66a1eb36cd849b8d1b7a5bc9ce", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 14792, "end_char_idx": 17687, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
### Few-shot prompting

In some cases, it's easier to show the model what you want rather than tell the model what you want.

One way to show the model what you want is with faked example messages.

For example:


```python
# An example of a faked few-shot conversation to prime the model into translating business jargon to simpler speech
response = client.chat.completions.create(
    model=MODEL,
    messages=[
        {"role": "system", "content": "You are a helpful, pattern-following assistant."},
        {"role": "user", "content": "Help me translate the following corporate jargon into plain English."},
        {"role": "assistant", "content": "Sure, I'd be happy to!"},
        {"role": "user", "content": "New synergies will help drive top-line growth."},
        {"role": "assistant", "content": "Things working well together will increase revenue."},
        {"role": "user", "content": "Let's circle back when we have more bandwidth to touch base on opportunities for increased leverage."},
        {"role": "assistant", "content": "Let's talk later when we're less busy about how to do better."},
        {"role": "user", "content": "This late pivot means we don't have time to boil the ocean for the client deliverable."},
    ],
    temperature=0,
)

print(response.choices[0].message.content)

```

    This sudden change in direction means we don't have enough time to complete the entire project for the client.


To help clarify that the example messages are not part of a real conversation, and shouldn't be referred back to by the model, you can try setting the `name` field of `system` messages to `example_user` and `example_assistant`.

Transforming the few-shot example above, we could write:


```python
# The business jargon translation example, but with example names for the example messages
response = client.chat.completions.create(
    model=MODEL,
    messages=[
        {"role": "system", "content": "You are a helpful, pattern-following assistant that translates corporate jargon into plain English."},
        {"role": "system", "name":"example_user", "content": "New synergies will help drive top-line growth."},
        {"role": "system", "name": "example_assistant", "content": "Things working well together will increase revenue."},
        {"role": "system", "name":"example_user", "content": "Let's circle back when we have more bandwidth to touch base on opportunities for increased leverage."},
        {"role": "system", "name": "example_assistant", "content": "Let's talk later when we're less busy about how to do better."},
        {"role": "user", "content": "This late pivot means we don't have time to boil the ocean for the client deliverable."},
    ],
    temperature=0,
)

print(response.choices[0].message.content)

```

    This sudden change in direction means we don't have enough time to complete the entire project for the client.


key: _node_content
string_value:
{"id_": "4bc17035-2108-4afe-9ad1-ea2bb7579309", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb", "title": "How to format inputs to ChatGPT models", "tokens": 5687, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "7f6a0328-5bf8-46ee-88a7-6980bc7f8694", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb", "title": "How to format inputs to ChatGPT models", "tokens": 5687, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "41d37741bc26fd57be6b5af37408f3347d5279f47c887c29850cc8de683ba4a0", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "12a381ce-161a-4f76-9034-76548ecc0400", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb", "title": "How to format inputs to ChatGPT models", "tokens": 5687, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "ff18290daebd362abb9d5978cf8ebeaee094e1ee8bbc6e9a8566bd3e7769773e", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "8986c485-b376-4bd0-a0e2-ddb48d3452ea", "node_type": "1", "metadata": {}, "hash": "754ee44774e55da89def1b74200167b08ab434651b0f63797a2c78529b2c4cc4", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 16228, "end_char_idx": 19231, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
To help clarify that the example messages are not part of a real conversation, and shouldn't be referred back to by the model, you can try setting the `name` field of `system` messages to `example_user` and `example_assistant`.

Transforming the few-shot example above, we could write:


```python
# The business jargon translation example, but with example names for the example messages
response = client.chat.completions.create(
    model=MODEL,
    messages=[
        {"role": "system", "content": "You are a helpful, pattern-following assistant that translates corporate jargon into plain English."},
        {"role": "system", "name":"example_user", "content": "New synergies will help drive top-line growth."},
        {"role": "system", "name": "example_assistant", "content": "Things working well together will increase revenue."},
        {"role": "system", "name":"example_user", "content": "Let's circle back when we have more bandwidth to touch base on opportunities for increased leverage."},
        {"role": "system", "name": "example_assistant", "content": "Let's talk later when we're less busy about how to do better."},
        {"role": "user", "content": "This late pivot means we don't have time to boil the ocean for the client deliverable."},
    ],
    temperature=0,
)

print(response.choices[0].message.content)

```

    This sudden change in direction means we don't have enough time to complete the entire project for the client.


Not every attempt at engineering conversations will succeed at first.

If your first attempts fail, don't be afraid to experiment with different ways of priming or conditioning the model.

As an example, one developer discovered an increase in accuracy when they inserted a user message that said "Great job so far, these have been perfect" to help condition the model into providing higher quality responses.

For more ideas on how to lift the reliability of the models, consider reading our guide on [techniques to increase reliability](../techniques_to_improve_reliability). It was written for non-chat models, but many of its principles still apply.

## 4. Counting tokens

When you submit your request, the API transforms the messages into a sequence of tokens.

The number of tokens used affects:
- the cost of the request
- the time it takes to generate the response
- when the reply gets cut off from hitting the maximum token limit (4,096 for `gpt-3.5-turbo` or 8,192 for `gpt-4`)

You can use the following function to count the number of tokens that a list of messages will use.

Note that the exact way that tokens are counted from messages may change from model to model. Consider the counts from the function below an estimate, not a timeless guarantee. 

In particular, requests that use the optional functions input will consume extra tokens on top of the estimates calculated below.

Read more about counting tokens in [How to count tokens with tiktoken](How_to_count_tokens_with_tiktoken.ipynb).


```python
import tiktoken


key: _node_content
string_value:
{"id_": "8986c485-b376-4bd0-a0e2-ddb48d3452ea", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb", "title": "How to format inputs to ChatGPT models", "tokens": 5687, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "7f6a0328-5bf8-46ee-88a7-6980bc7f8694", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb", "title": "How to format inputs to ChatGPT models", "tokens": 5687, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "41d37741bc26fd57be6b5af37408f3347d5279f47c887c29850cc8de683ba4a0", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "4bc17035-2108-4afe-9ad1-ea2bb7579309", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb", "title": "How to format inputs to ChatGPT models", "tokens": 5687, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "cf3e5eb87282c9b73f7bf295857318c0b489ecf57eece9d9f74cc60f9b5f87f9", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "86754901-a6eb-488a-b485-418a014d29c6", "node_type": "1", "metadata": {}, "hash": "164d8dac14ebb7ce276d08e7721b8418829202f24e76840094a511d7a7a7027d", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 17690, "end_char_idx": 20960, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
Not every attempt at engineering conversations will succeed at first.

If your first attempts fail, don't be afraid to experiment with different ways of priming or conditioning the model.

As an example, one developer discovered an increase in accuracy when they inserted a user message that said "Great job so far, these have been perfect" to help condition the model into providing higher quality responses.

For more ideas on how to lift the reliability of the models, consider reading our guide on [techniques to increase reliability](../techniques_to_improve_reliability). It was written for non-chat models, but many of its principles still apply.

## 4. Counting tokens

When you submit your request, the API transforms the messages into a sequence of tokens.

The number of tokens used affects:
- the cost of the request
- the time it takes to generate the response
- when the reply gets cut off from hitting the maximum token limit (4,096 for `gpt-3.5-turbo` or 8,192 for `gpt-4`)

You can use the following function to count the number of tokens that a list of messages will use.

Note that the exact way that tokens are counted from messages may change from model to model. Consider the counts from the function below an estimate, not a timeless guarantee. 

In particular, requests that use the optional functions input will consume extra tokens on top of the estimates calculated below.

Read more about counting tokens in [How to count tokens with tiktoken](How_to_count_tokens_with_tiktoken.ipynb).


```python
import tiktoken


def num_tokens_from_messages(messages, model="gpt-3.5-turbo-0613"):
    """Return the number of tokens used by a list of messages."""
    try:
        encoding = tiktoken.encoding_for_model(model)
    except KeyError:
        print("Warning: model not found. Using cl100k_base encoding.")
        encoding = tiktoken.get_encoding("cl100k_base")
    if model in {
        "gpt-3.5-turbo-0613",
        "gpt-3.5-turbo-16k-0613",
        "gpt-4-0314",
        "gpt-4-32k-0314",
        "gpt-4-0613",
        "gpt-4-32k-0613",
        }:
        tokens_per_message = 3
        tokens_per_name = 1
    elif model == "gpt-3.5-turbo-0301":
        tokens_per_message = 4  # every message follows <|start|>{role/name}\n{content}<|end|>\n
        tokens_per_name = -1  # if there's a name, the role is omitted
    elif "gpt-3.5-turbo" in model:
        print("Warning: gpt-3.5-turbo may update over time. Returning num tokens assuming gpt-3.5-turbo-0613.")
        return num_tokens_from_messages(messages, model="gpt-3.5-turbo-0613")
    elif "gpt-4" in model:
        print("Warning: gpt-4 may update over time. Returning num tokens assuming gpt-4-0613.")
        return num_tokens_from_messages(messages, model="gpt-4-0613")
    else:
        raise NotImplementedError(
            f"""num_tokens_from_messages() is not implemented for model {model}."""
        )
    num_tokens = 0
    for message in messages:
        num_tokens += tokens_per_message
        for key, value in message.items():
            num_tokens += len(encoding.encode(value))
            if key == "name":
                num_tokens += tokens_per_name
    num_tokens += 3  # every reply is primed with <|start|>assistant<|message|>
    return num_tokens

```


key: _node_content
string_value:
{"id_": "86754901-a6eb-488a-b485-418a014d29c6", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb", "title": "How to format inputs to ChatGPT models", "tokens": 5687, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "7f6a0328-5bf8-46ee-88a7-6980bc7f8694", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb", "title": "How to format inputs to ChatGPT models", "tokens": 5687, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "41d37741bc26fd57be6b5af37408f3347d5279f47c887c29850cc8de683ba4a0", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "8986c485-b376-4bd0-a0e2-ddb48d3452ea", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb", "title": "How to format inputs to ChatGPT models", "tokens": 5687, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "21893570a199f5e46b2ca5b4a0e2c3db87ccb7dca83b61309ab76f1b75b542bd", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 20963, "end_char_idx": 23594, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
# let's verify the function above matches the OpenAI API response
example_messages = [
    {
        "role": "system",
        "content": "You are a helpful, pattern-following assistant that translates corporate jargon into plain English.",
    },
    {
        "role": "system",
        "name": "example_user",
        "content": "New synergies will help drive top-line growth.",
    },
    {
        "role": "system",
        "name": "example_assistant",
        "content": "Things working well together will increase revenue.",
    },
    {
        "role": "system",
        "name": "example_user",
        "content": "Let's circle back when we have more bandwidth to touch base on opportunities for increased leverage.",
    },
    {
        "role": "system",
        "name": "example_assistant",
        "content": "Let's talk later when we're less busy about how to do better.",
    },
    {
        "role": "user",
        "content": "This late pivot means we don't have time to boil the ocean for the client deliverable.",
    },
]

for model in [
    # "gpt-3.5-turbo-0301",
    # "gpt-4-0314",
    # "gpt-4-0613",
    "gpt-3.5-turbo-1106",
    "gpt-3.5-turbo",
    "gpt-4",
    "gpt-4-1106-preview",
    ]:
    print(model)
    # example token count from the function defined above
    print(f"{num_tokens_from_messages(example_messages, model)} prompt tokens counted by num_tokens_from_messages().")
    # example token count from the OpenAI API
    response = client.chat.completions.create(model=model,
    messages=example_messages,
    temperature=0,
    max_tokens=1)
    token = response.usage.prompt_tokens
    print(f'{token} prompt tokens counted by the OpenAI API.')
    print()

```

    gpt-3.5-turbo-1106
    Warning: gpt-3.5-turbo may update over time. Returning num tokens assuming gpt-3.5-turbo-0613.
    129 prompt tokens counted by num_tokens_from_messages().
    129 prompt tokens counted by the OpenAI API.
    
    gpt-3.5-turbo
    Warning: gpt-3.5-turbo may update over time. Returning num tokens assuming gpt-3.5-turbo-0613.
    129 prompt tokens counted by num_tokens_from_messages().
    129 prompt tokens counted by the OpenAI API.
    
    gpt-4
    Warning: gpt-4 may update over time. Returning num tokens assuming gpt-4-0613.
    129 prompt tokens counted by num_tokens_from_messages().
    129 prompt tokens counted by the OpenAI API.
    
    gpt-4-1106-preview
    Warning: gpt-4 may update over time. Returning num tokens assuming gpt-4-0613.
    129 prompt tokens counted by num_tokens_from_messages().
    129 prompt tokens counted by the OpenAI API.
    



```python

```


key: url
string_value:
https://github.com/openai/openai-cookbook/blob/main/examples/How_to_call_functions_with_chat_models.ipynb


key: _node_content
string_value:
{"id_": "febddef8-d4aa-48a2-a2a3-b2395e900df3", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_call_functions_with_chat_models.ipynb", "title": "How to call functions with chat models", "tokens": 4334, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "4ff1d23d-3d6a-4a22-9968-721381f66e1e", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_call_functions_with_chat_models.ipynb", "title": "How to call functions with chat models", "tokens": 4334, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "08b8b6fa48b1e1e14179f154ae2248c8cb28ce70546f10038658b577ad2e0150", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "5f22055c-e443-4698-ac49-afde8a5d4c4a", "node_type": "1", "metadata": {}, "hash": "38402decf5664123218753b91568d62c3e7c4d7de554935f21d6d68fadf56395", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 0, "end_char_idx": 2582, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
# How to call functions with chat models

This notebook covers how to use the Chat Completions API in combination with external functions to extend the capabilities of GPT models.

`tools` is an optional parameter in the Chat Completion API which can be used to provide function specifications. The purpose of this is to enable models to generate function arguments which adhere to the provided specifications. Note that the API will not actually execute any function calls. It is up to developers to execute function calls using model outputs.

Within the `tools` parameter, if the `functions` parameter is provided then by default the model will decide when it is appropriate to use one of the functions. The API can be forced to use a specific function by setting the `tool_choice` parameter to `{"type": "function", "function": {"name": "my_function"}}`. The API can also be forced to not use any function by setting the `tool_choice` parameter to `"none"`. If a function is used, the output will contain `"finish_reason": "tool_calls"` in the response, as well as a `tool_calls` object that has the name of the function and the generated function arguments.

### Overview

This notebook contains the following 2 sections:

- **How to generate function arguments:** Specify a set of functions and use the API to generate function arguments.
- **How to call functions with model generated arguments:** Close the loop by actually executing functions with model generated arguments.

## How to generate function arguments


```python
!pip install scipy --quiet
!pip install tenacity --quiet
!pip install tiktoken --quiet
!pip install termcolor --quiet
!pip install openai --quiet
```


```python
import json
from openai import OpenAI
from tenacity import retry, wait_random_exponential, stop_after_attempt
from termcolor import colored  

GPT_MODEL = "gpt-4o"
client = OpenAI()
```

### Utilities

First let's define a few utilities for making calls to the Chat Completions API and for maintaining and keeping track of the conversation state.


```python
@retry(wait=wait_random_exponential(multiplier=1, max=40), stop=stop_after_attempt(3))
def chat_completion_request(messages, tools=None, tool_choice=None, model=GPT_MODEL):
    try:
        response = client.chat.completions.create(
            model=model,
            messages=messages,
            tools=tools,
            tool_choice=tool_choice,
        )
        return response
    except Exception as e:
        print("Unable to generate ChatCompletion response")
        print(f"Exception: {e}")
        return e

```


key: _node_content
string_value:
{"id_": "5f22055c-e443-4698-ac49-afde8a5d4c4a", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_call_functions_with_chat_models.ipynb", "title": "How to call functions with chat models", "tokens": 4334, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "4ff1d23d-3d6a-4a22-9968-721381f66e1e", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_call_functions_with_chat_models.ipynb", "title": "How to call functions with chat models", "tokens": 4334, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "08b8b6fa48b1e1e14179f154ae2248c8cb28ce70546f10038658b577ad2e0150", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "febddef8-d4aa-48a2-a2a3-b2395e900df3", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_call_functions_with_chat_models.ipynb", "title": "How to call functions with chat models", "tokens": 4334, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "692ee8c24e751e153c43ea6c15f7e4c4d5c05158af9ea4a132e65218648c2ce9", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "7df4ce6b-86ac-49ab-a740-b03126aae538", "node_type": "1", "metadata": {}, "hash": "c25252aca9500956ac40183d2226d83c21cdbebf8d8bfb6719d5760c9ac56e3e", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 1525, "end_char_idx": 3849, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
!pip install scipy --quiet
!pip install tenacity --quiet
!pip install tiktoken --quiet
!pip install termcolor --quiet
!pip install openai --quiet
```


```python
import json
from openai import OpenAI
from tenacity import retry, wait_random_exponential, stop_after_attempt
from termcolor import colored  

GPT_MODEL = "gpt-4o"
client = OpenAI()
```

### Utilities

First let's define a few utilities for making calls to the Chat Completions API and for maintaining and keeping track of the conversation state.


```python
@retry(wait=wait_random_exponential(multiplier=1, max=40), stop=stop_after_attempt(3))
def chat_completion_request(messages, tools=None, tool_choice=None, model=GPT_MODEL):
    try:
        response = client.chat.completions.create(
            model=model,
            messages=messages,
            tools=tools,
            tool_choice=tool_choice,
        )
        return response
    except Exception as e:
        print("Unable to generate ChatCompletion response")
        print(f"Exception: {e}")
        return e

```


```python
def pretty_print_conversation(messages):
    role_to_color = {
        "system": "red",
        "user": "green",
        "assistant": "blue",
        "function": "magenta",
    }
    
    for message in messages:
        if message["role"] == "system":
            print(colored(f"system: {message['content']}\n", role_to_color[message["role"]]))
        elif message["role"] == "user":
            print(colored(f"user: {message['content']}\n", role_to_color[message["role"]]))
        elif message["role"] == "assistant" and message.get("function_call"):
            print(colored(f"assistant: {message['function_call']}\n", role_to_color[message["role"]]))
        elif message["role"] == "assistant" and not message.get("function_call"):
            print(colored(f"assistant: {message['content']}\n", role_to_color[message["role"]]))
        elif message["role"] == "function":
            print(colored(f"function ({message['name']}): {message['content']}\n", role_to_color[message["role"]]))

```

### Basic concepts

Let's create some function specifications to interface with a hypothetical weather API. We'll pass these function specification to the Chat Completions API in order to generate function arguments that adhere to the specification.


key: _node_content
string_value:
{"id_": "7df4ce6b-86ac-49ab-a740-b03126aae538", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_call_functions_with_chat_models.ipynb", "title": "How to call functions with chat models", "tokens": 4334, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "4ff1d23d-3d6a-4a22-9968-721381f66e1e", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_call_functions_with_chat_models.ipynb", "title": "How to call functions with chat models", "tokens": 4334, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "08b8b6fa48b1e1e14179f154ae2248c8cb28ce70546f10038658b577ad2e0150", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "5f22055c-e443-4698-ac49-afde8a5d4c4a", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_call_functions_with_chat_models.ipynb", "title": "How to call functions with chat models", "tokens": 4334, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "f8ed1fa8527613ef3133d85c7cc6bded6a198af93b2b58064e0ee48fb2994f5f", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "d40a24ce-41d9-42e7-a704-fa9a73a24f3b", "node_type": "1", "metadata": {}, "hash": "022db31fedbfbff67071453c29d490ced17f7f6fa8b6be2e235c1103efc36968", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 2585, "end_char_idx": 6219, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
def pretty_print_conversation(messages):
    role_to_color = {
        "system": "red",
        "user": "green",
        "assistant": "blue",
        "function": "magenta",
    }
    
    for message in messages:
        if message["role"] == "system":
            print(colored(f"system: {message['content']}\n", role_to_color[message["role"]]))
        elif message["role"] == "user":
            print(colored(f"user: {message['content']}\n", role_to_color[message["role"]]))
        elif message["role"] == "assistant" and message.get("function_call"):
            print(colored(f"assistant: {message['function_call']}\n", role_to_color[message["role"]]))
        elif message["role"] == "assistant" and not message.get("function_call"):
            print(colored(f"assistant: {message['content']}\n", role_to_color[message["role"]]))
        elif message["role"] == "function":
            print(colored(f"function ({message['name']}): {message['content']}\n", role_to_color[message["role"]]))

```

### Basic concepts

Let's create some function specifications to interface with a hypothetical weather API. We'll pass these function specification to the Chat Completions API in order to generate function arguments that adhere to the specification.


```python
tools = [
    {
        "type": "function",
        "function": {
            "name": "get_current_weather",
            "description": "Get the current weather",
            "parameters": {
                "type": "object",
                "properties": {
                    "location": {
                        "type": "string",
                        "description": "The city and state, e.g. San Francisco, CA",
                    },
                    "format": {
                        "type": "string",
                        "enum": ["celsius", "fahrenheit"],
                        "description": "The temperature unit to use. Infer this from the users location.",
                    },
                },
                "required": ["location", "format"],
            },
        }
    },
    {
        "type": "function",
        "function": {
            "name": "get_n_day_weather_forecast",
            "description": "Get an N-day weather forecast",
            "parameters": {
                "type": "object",
                "properties": {
                    "location": {
                        "type": "string",
                        "description": "The city and state, e.g. San Francisco, CA",
                    },
                    "format": {
                        "type": "string",
                        "enum": ["celsius", "fahrenheit"],
                        "description": "The temperature unit to use. Infer this from the users location.",
                    },
                    "num_days": {
                        "type": "integer",
                        "description": "The number of days to forecast",
                    }
                },
                "required": ["location", "format", "num_days"]
            },
        }
    },
]
```

If we prompt the model about the current weather, it will respond with some clarifying questions.


```python
messages = []
messages.append({"role": "system", "content": "Don't make assumptions about what values to plug into functions. Ask for clarification if a user request is ambiguous."})
messages.append({"role": "user", "content": "What's the weather like today"})
chat_response = chat_completion_request(
    messages, tools=tools
)
assistant_message = chat_response.choices[0].message
messages.append(assistant_message)
assistant_message

```


key: _node_content
string_value:
{"id_": "d40a24ce-41d9-42e7-a704-fa9a73a24f3b", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_call_functions_with_chat_models.ipynb", "title": "How to call functions with chat models", "tokens": 4334, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "4ff1d23d-3d6a-4a22-9968-721381f66e1e", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_call_functions_with_chat_models.ipynb", "title": "How to call functions with chat models", "tokens": 4334, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "08b8b6fa48b1e1e14179f154ae2248c8cb28ce70546f10038658b577ad2e0150", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "7df4ce6b-86ac-49ab-a740-b03126aae538", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_call_functions_with_chat_models.ipynb", "title": "How to call functions with chat models", "tokens": 4334, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "aaa43881faa9f440d16889e3207b4f3303d8a94a9d1e92fdf3e20c868a159cde", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "188d1dbc-e8e1-4e40-8124-cabb7335799f", "node_type": "1", "metadata": {}, "hash": "172f20db166ba3a65f270dc2f6e81aabb188299f42e8761f2b1f28d440d12a53", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 5769, "end_char_idx": 8918, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
messages = []
messages.append({"role": "system", "content": "Don't make assumptions about what values to plug into functions. Ask for clarification if a user request is ambiguous."})
messages.append({"role": "user", "content": "What's the weather like today"})
chat_response = chat_completion_request(
    messages, tools=tools
)
assistant_message = chat_response.choices[0].message
messages.append(assistant_message)
assistant_message

```




    ChatCompletionMessage(content='Sure, can you please provide me with the name of your city and state?', role='assistant', function_call=None, tool_calls=None)



Once we provide the missing information, it will generate the appropriate function arguments for us.


```python
messages.append({"role": "user", "content": "I'm in Glasgow, Scotland."})
chat_response = chat_completion_request(
    messages, tools=tools
)
assistant_message = chat_response.choices[0].message
messages.append(assistant_message)
assistant_message

```




    ChatCompletionMessage(content=None, role='assistant', function_call=None, tool_calls=[ChatCompletionMessageToolCall(id='call_xb7QwwNnx90LkmhtlW0YrgP2', function=Function(arguments='{"location":"Glasgow, Scotland","format":"celsius"}', name='get_current_weather'), type='function')])



By prompting it differently, we can get it to target the other function we've told it about.


```python
messages = []
messages.append({"role": "system", "content": "Don't make assumptions about what values to plug into functions. Ask for clarification if a user request is ambiguous."})
messages.append({"role": "user", "content": "what is the weather going to be like in Glasgow, Scotland over the next x days"})
chat_response = chat_completion_request(
    messages, tools=tools
)
assistant_message = chat_response.choices[0].message
messages.append(assistant_message)
assistant_message

```




    ChatCompletionMessage(content='To provide you with the weather forecast for Glasgow, Scotland, could you please specify the number of days you would like the forecast for?', role='assistant', function_call=None, tool_calls=None)



Once again, the model is asking us for clarification because it doesn't have enough information yet. In this case it already knows the location for the forecast, but it needs to know how many days are required in the forecast.


```python
messages.append({"role": "user", "content": "5 days"})
chat_response = chat_completion_request(
    messages, tools=tools
)
chat_response.choices[0]

```




    Choice(finish_reason='tool_calls', index=0, logprobs=None, message=ChatCompletionMessage(content=None, role='assistant', function_call=None, tool_calls=[ChatCompletionMessageToolCall(id='call_34PBraFdNN6KR95uD5rHF8Aw', function=Function(arguments='{"location":"Glasgow, Scotland","format":"celsius","num_days":5}', name='get_n_day_weather_forecast'), type='function')]))



#### Forcing the use of specific functions or no function

We can force the model to use a specific function, for example get_n_day_weather_forecast by using the function_call argument. By doing so, we force the model to make assumptions about how to use it.


key: _node_content
string_value:
{"id_": "188d1dbc-e8e1-4e40-8124-cabb7335799f", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_call_functions_with_chat_models.ipynb", "title": "How to call functions with chat models", "tokens": 4334, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "4ff1d23d-3d6a-4a22-9968-721381f66e1e", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_call_functions_with_chat_models.ipynb", "title": "How to call functions with chat models", "tokens": 4334, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "08b8b6fa48b1e1e14179f154ae2248c8cb28ce70546f10038658b577ad2e0150", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "d40a24ce-41d9-42e7-a704-fa9a73a24f3b", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_call_functions_with_chat_models.ipynb", "title": "How to call functions with chat models", "tokens": 4334, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "deb02c6fb26802ead49bdee85693e10741f3edc9cd394f5deaf90a8a0eda26d5", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "fa889866-992d-4429-ac83-fd0834805e33", "node_type": "1", "metadata": {}, "hash": "c406b2b49109cc4139f8e07a224881a1a1f24da3eed54ca64fa76e89cef6838d", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 7145, "end_char_idx": 10248, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
messages = []
messages.append({"role": "system", "content": "Don't make assumptions about what values to plug into functions. Ask for clarification if a user request is ambiguous."})
messages.append({"role": "user", "content": "what is the weather going to be like in Glasgow, Scotland over the next x days"})
chat_response = chat_completion_request(
    messages, tools=tools
)
assistant_message = chat_response.choices[0].message
messages.append(assistant_message)
assistant_message

```




    ChatCompletionMessage(content='To provide you with the weather forecast for Glasgow, Scotland, could you please specify the number of days you would like the forecast for?', role='assistant', function_call=None, tool_calls=None)



Once again, the model is asking us for clarification because it doesn't have enough information yet. In this case it already knows the location for the forecast, but it needs to know how many days are required in the forecast.


```python
messages.append({"role": "user", "content": "5 days"})
chat_response = chat_completion_request(
    messages, tools=tools
)
chat_response.choices[0]

```




    Choice(finish_reason='tool_calls', index=0, logprobs=None, message=ChatCompletionMessage(content=None, role='assistant', function_call=None, tool_calls=[ChatCompletionMessageToolCall(id='call_34PBraFdNN6KR95uD5rHF8Aw', function=Function(arguments='{"location":"Glasgow, Scotland","format":"celsius","num_days":5}', name='get_n_day_weather_forecast'), type='function')]))



#### Forcing the use of specific functions or no function

We can force the model to use a specific function, for example get_n_day_weather_forecast by using the function_call argument. By doing so, we force the model to make assumptions about how to use it.


```python
# in this cell we force the model to use get_n_day_weather_forecast
messages = []
messages.append({"role": "system", "content": "Don't make assumptions about what values to plug into functions. Ask for clarification if a user request is ambiguous."})
messages.append({"role": "user", "content": "Give me a weather report for Toronto, Canada."})
chat_response = chat_completion_request(
    messages, tools=tools, tool_choice={"type": "function", "function": {"name": "get_n_day_weather_forecast"}}
)
chat_response.choices[0].message
```




    ChatCompletionMessage(content=None, role='assistant', function_call=None, tool_calls=[ChatCompletionMessageToolCall(id='call_FImGxrLowOAOszCaaQqQWmEN', function=Function(arguments='{"location":"Toronto, Canada","format":"celsius","num_days":7}', name='get_n_day_weather_forecast'), type='function')])




```python
# if we don't force the model to use get_n_day_weather_forecast it may not
messages = []
messages.append({"role": "system", "content": "Don't make assumptions about what values to plug into functions. Ask for clarification if a user request is ambiguous."})
messages.append({"role": "user", "content": "Give me a weather report for Toronto, Canada."})
chat_response = chat_completion_request(
    messages, tools=tools
)
chat_response.choices[0].message
```


key: _node_content
string_value:
{"id_": "fa889866-992d-4429-ac83-fd0834805e33", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_call_functions_with_chat_models.ipynb", "title": "How to call functions with chat models", "tokens": 4334, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "4ff1d23d-3d6a-4a22-9968-721381f66e1e", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_call_functions_with_chat_models.ipynb", "title": "How to call functions with chat models", "tokens": 4334, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "08b8b6fa48b1e1e14179f154ae2248c8cb28ce70546f10038658b577ad2e0150", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "188d1dbc-e8e1-4e40-8124-cabb7335799f", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_call_functions_with_chat_models.ipynb", "title": "How to call functions with chat models", "tokens": 4334, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "228ea967d13f86601eba8c506d7b7a0b6dd4d77349f2e5e701535adcea4fdc8a", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "2668ea51-9188-4bb8-af1e-449eb1628403", "node_type": "1", "metadata": {}, "hash": "df9beb088672f6f75aafd9a9df37da403bae8eb37fb12158310eeb057b99eb07", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 8660, "end_char_idx": 11601, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
#### Forcing the use of specific functions or no function

We can force the model to use a specific function, for example get_n_day_weather_forecast by using the function_call argument. By doing so, we force the model to make assumptions about how to use it.


```python
# in this cell we force the model to use get_n_day_weather_forecast
messages = []
messages.append({"role": "system", "content": "Don't make assumptions about what values to plug into functions. Ask for clarification if a user request is ambiguous."})
messages.append({"role": "user", "content": "Give me a weather report for Toronto, Canada."})
chat_response = chat_completion_request(
    messages, tools=tools, tool_choice={"type": "function", "function": {"name": "get_n_day_weather_forecast"}}
)
chat_response.choices[0].message
```




    ChatCompletionMessage(content=None, role='assistant', function_call=None, tool_calls=[ChatCompletionMessageToolCall(id='call_FImGxrLowOAOszCaaQqQWmEN', function=Function(arguments='{"location":"Toronto, Canada","format":"celsius","num_days":7}', name='get_n_day_weather_forecast'), type='function')])




```python
# if we don't force the model to use get_n_day_weather_forecast it may not
messages = []
messages.append({"role": "system", "content": "Don't make assumptions about what values to plug into functions. Ask for clarification if a user request is ambiguous."})
messages.append({"role": "user", "content": "Give me a weather report for Toronto, Canada."})
chat_response = chat_completion_request(
    messages, tools=tools
)
chat_response.choices[0].message
```




    ChatCompletionMessage(content=None, role='assistant', function_call=None, tool_calls=[ChatCompletionMessageToolCall(id='call_n84kYFqjNFDPNGDEnjnrd2KC', function=Function(arguments='{"location": "Toronto, Canada", "format": "celsius"}', name='get_current_weather'), type='function'), ChatCompletionMessageToolCall(id='call_AEs3AFhJc9pn42hWSbHTaIDh', function=Function(arguments='{"location": "Toronto, Canada", "format": "celsius", "num_days": 3}', name='get_n_day_weather_forecast'), type='function')])



We can also force the model to not use a function at all. By doing so we prevent it from producing a proper function call.


```python
messages = []
messages.append({"role": "system", "content": "Don't make assumptions about what values to plug into functions. Ask for clarification if a user request is ambiguous."})
messages.append({"role": "user", "content": "Give me the current weather (use Celcius) for Toronto, Canada."})
chat_response = chat_completion_request(
    messages, tools=tools, tool_choice="none"
)
chat_response.choices[0].message

```




    ChatCompletionMessage(content="Sure, I'll get the current weather for Toronto, Canada in Celsius.", role='assistant', function_call=None, tool_calls=None)



### Parallel Function Calling

Newer models such as gpt-4o or gpt-3.5-turbo can call multiple functions in one turn.


key: _node_content
string_value:
{"id_": "2668ea51-9188-4bb8-af1e-449eb1628403", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_call_functions_with_chat_models.ipynb", "title": "How to call functions with chat models", "tokens": 4334, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "4ff1d23d-3d6a-4a22-9968-721381f66e1e", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_call_functions_with_chat_models.ipynb", "title": "How to call functions with chat models", "tokens": 4334, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "08b8b6fa48b1e1e14179f154ae2248c8cb28ce70546f10038658b577ad2e0150", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "fa889866-992d-4429-ac83-fd0834805e33", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_call_functions_with_chat_models.ipynb", "title": "How to call functions with chat models", "tokens": 4334, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "34bdd89d21d9e65d6cd21c2e233a12d5ab0bf0c6cc573747f7398de2359c929f", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "b766b699-8224-4446-ade8-1b483b640b39", "node_type": "1", "metadata": {}, "hash": "8a11f50154f079af3ebee974f2265a1e7cb82a75d310223863c515701f312b9a", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 10257, "end_char_idx": 13337, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
ChatCompletionMessage(content=None, role='assistant', function_call=None, tool_calls=[ChatCompletionMessageToolCall(id='call_n84kYFqjNFDPNGDEnjnrd2KC', function=Function(arguments='{"location": "Toronto, Canada", "format": "celsius"}', name='get_current_weather'), type='function'), ChatCompletionMessageToolCall(id='call_AEs3AFhJc9pn42hWSbHTaIDh', function=Function(arguments='{"location": "Toronto, Canada", "format": "celsius", "num_days": 3}', name='get_n_day_weather_forecast'), type='function')])



We can also force the model to not use a function at all. By doing so we prevent it from producing a proper function call.


```python
messages = []
messages.append({"role": "system", "content": "Don't make assumptions about what values to plug into functions. Ask for clarification if a user request is ambiguous."})
messages.append({"role": "user", "content": "Give me the current weather (use Celcius) for Toronto, Canada."})
chat_response = chat_completion_request(
    messages, tools=tools, tool_choice="none"
)
chat_response.choices[0].message

```




    ChatCompletionMessage(content="Sure, I'll get the current weather for Toronto, Canada in Celsius.", role='assistant', function_call=None, tool_calls=None)



### Parallel Function Calling

Newer models such as gpt-4o or gpt-3.5-turbo can call multiple functions in one turn.


```python
messages = []
messages.append({"role": "system", "content": "Don't make assumptions about what values to plug into functions. Ask for clarification if a user request is ambiguous."})
messages.append({"role": "user", "content": "what is the weather going to be like in San Francisco and Glasgow over the next 4 days"})
chat_response = chat_completion_request(
    messages, tools=tools, model=GPT_MODEL
)

assistant_message = chat_response.choices[0].message.tool_calls
assistant_message
```




    [ChatCompletionMessageToolCall(id='call_ObhLiJwaHwc3U1KyB4Pdpx8y', function=Function(arguments='{"location": "San Francisco, CA", "format": "fahrenheit", "num_days": 4}', name='get_n_day_weather_forecast'), type='function'),
     ChatCompletionMessageToolCall(id='call_5YRgeZ0MGBMFKE3hZiLouwg7', function=Function(arguments='{"location": "Glasgow, SCT", "format": "celsius", "num_days": 4}', name='get_n_day_weather_forecast'), type='function')]



## How to call functions with model generated arguments

In our next example, we'll demonstrate how to execute functions whose inputs are model-generated, and use this to implement an agent that can answer questions for us about a database. For simplicity we'll use the [Chinook sample database](https://www.sqlitetutorial.net/sqlite-sample-database/).

*Note:* SQL generation can be high-risk in a production environment since models are not perfectly reliable at generating correct SQL.

### Specifying a function to execute SQL queries

First let's define some helpful utility functions to extract data from a SQLite database.


```python
import sqlite3

conn = sqlite3.connect("data/Chinook.db")
print("Opened database successfully")
```

    Opened database successfully


key: _node_content
string_value:
{"id_": "b766b699-8224-4446-ade8-1b483b640b39", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_call_functions_with_chat_models.ipynb", "title": "How to call functions with chat models", "tokens": 4334, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "4ff1d23d-3d6a-4a22-9968-721381f66e1e", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_call_functions_with_chat_models.ipynb", "title": "How to call functions with chat models", "tokens": 4334, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "08b8b6fa48b1e1e14179f154ae2248c8cb28ce70546f10038658b577ad2e0150", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "2668ea51-9188-4bb8-af1e-449eb1628403", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_call_functions_with_chat_models.ipynb", "title": "How to call functions with chat models", "tokens": 4334, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "763c025389dba171a5eae7c26b9fb99139290a5d8bb1c9d485dad8dd22164b3f", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "73bb2ff1-dcc1-4ef6-b4b2-ef4b3886accc", "node_type": "1", "metadata": {}, "hash": "40eea6c7cf635a5045b7cc0b459c1320f894f6f7bff0de859bccc766e22131ab", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 12113, "end_char_idx": 14828, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
[ChatCompletionMessageToolCall(id='call_ObhLiJwaHwc3U1KyB4Pdpx8y', function=Function(arguments='{"location": "San Francisco, CA", "format": "fahrenheit", "num_days": 4}', name='get_n_day_weather_forecast'), type='function'),
     ChatCompletionMessageToolCall(id='call_5YRgeZ0MGBMFKE3hZiLouwg7', function=Function(arguments='{"location": "Glasgow, SCT", "format": "celsius", "num_days": 4}', name='get_n_day_weather_forecast'), type='function')]



## How to call functions with model generated arguments

In our next example, we'll demonstrate how to execute functions whose inputs are model-generated, and use this to implement an agent that can answer questions for us about a database. For simplicity we'll use the [Chinook sample database](https://www.sqlitetutorial.net/sqlite-sample-database/).

*Note:* SQL generation can be high-risk in a production environment since models are not perfectly reliable at generating correct SQL.

### Specifying a function to execute SQL queries

First let's define some helpful utility functions to extract data from a SQLite database.


```python
import sqlite3

conn = sqlite3.connect("data/Chinook.db")
print("Opened database successfully")
```

    Opened database successfully



```python
def get_table_names(conn):
    """Return a list of table names."""
    table_names = []
    tables = conn.execute("SELECT name FROM sqlite_master WHERE type='table';")
    for table in tables.fetchall():
        table_names.append(table[0])
    return table_names


def get_column_names(conn, table_name):
    """Return a list of column names."""
    column_names = []
    columns = conn.execute(f"PRAGMA table_info('{table_name}');").fetchall()
    for col in columns:
        column_names.append(col[1])
    return column_names


def get_database_info(conn):
    """Return a list of dicts containing the table name and columns for each table in the database."""
    table_dicts = []
    for table_name in get_table_names(conn):
        columns_names = get_column_names(conn, table_name)
        table_dicts.append({"table_name": table_name, "column_names": columns_names})
    return table_dicts

```

Now can use these utility functions to extract a representation of the database schema.


```python
database_schema_dict = get_database_info(conn)
database_schema_string = "\n".join(
    [
        f"Table: {table['table_name']}\nColumns: {', '.join(table['column_names'])}"
        for table in database_schema_dict
    ]
)
```

As before, we'll define a function specification for the function we'd like the API to generate arguments for. Notice that we are inserting the database schema into the function specification. This will be important for the model to know about.


key: _node_content
string_value:
{"id_": "73bb2ff1-dcc1-4ef6-b4b2-ef4b3886accc", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_call_functions_with_chat_models.ipynb", "title": "How to call functions with chat models", "tokens": 4334, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "4ff1d23d-3d6a-4a22-9968-721381f66e1e", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_call_functions_with_chat_models.ipynb", "title": "How to call functions with chat models", "tokens": 4334, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "08b8b6fa48b1e1e14179f154ae2248c8cb28ce70546f10038658b577ad2e0150", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "b766b699-8224-4446-ade8-1b483b640b39", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_call_functions_with_chat_models.ipynb", "title": "How to call functions with chat models", "tokens": 4334, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "42e72e23a328a2d8f393cbeb9b941452c339b5299f50de5529ebbd96cabd70b1", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "ad354a62-49f5-4633-8168-266c4d2b687e", "node_type": "1", "metadata": {}, "hash": "713b5fbb7f0a80047521709fbd7b998d3c14e9ce32db67f0ad62d3fb004f4749", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 13194, "end_char_idx": 16862, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
import sqlite3

conn = sqlite3.connect("data/Chinook.db")
print("Opened database successfully")
```

    Opened database successfully



```python
def get_table_names(conn):
    """Return a list of table names."""
    table_names = []
    tables = conn.execute("SELECT name FROM sqlite_master WHERE type='table';")
    for table in tables.fetchall():
        table_names.append(table[0])
    return table_names


def get_column_names(conn, table_name):
    """Return a list of column names."""
    column_names = []
    columns = conn.execute(f"PRAGMA table_info('{table_name}');").fetchall()
    for col in columns:
        column_names.append(col[1])
    return column_names


def get_database_info(conn):
    """Return a list of dicts containing the table name and columns for each table in the database."""
    table_dicts = []
    for table_name in get_table_names(conn):
        columns_names = get_column_names(conn, table_name)
        table_dicts.append({"table_name": table_name, "column_names": columns_names})
    return table_dicts

```

Now can use these utility functions to extract a representation of the database schema.


```python
database_schema_dict = get_database_info(conn)
database_schema_string = "\n".join(
    [
        f"Table: {table['table_name']}\nColumns: {', '.join(table['column_names'])}"
        for table in database_schema_dict
    ]
)
```

As before, we'll define a function specification for the function we'd like the API to generate arguments for. Notice that we are inserting the database schema into the function specification. This will be important for the model to know about.


```python
tools = [
    {
        "type": "function",
        "function": {
            "name": "ask_database",
            "description": "Use this function to answer user questions about music. Input should be a fully formed SQL query.",
            "parameters": {
                "type": "object",
                "properties": {
                    "query": {
                        "type": "string",
                        "description": f"""
                                SQL query extracting info to answer the user's question.
                                SQL should be written using this database schema:
                                {database_schema_string}
                                The query should be returned in plain text, not in JSON.
                                """,
                    }
                },
                "required": ["query"],
            },
        }
    }
]
```

### Executing SQL queries

Now let's implement the function that will actually excute queries against the database.


```python
def ask_database(conn, query):
    """Function to query SQLite database with a provided SQL query."""
    try:
        results = str(conn.execute(query).fetchall())
    except Exception as e:
        results = f"query failed with error: {e}"
    return results
```

##### Steps to invoke a function call using Chat Completions API: 

**Step 1**: Prompt the model with content that may result in model selecting a tool to use. The description of the tools such as a function names and signature is defined in the 'Tools' list and passed to the model in API call. If selected, the function name and parameters are included in the response.<br>
  
**Step 2**: Check programmatically if model wanted to call a function. If true, proceed to step 3. <br>  
**Step 3**: Extract the function name and parameters from response, call the function with parameters. Append the result to messages. <br>    
**Step 4**: Invoke the chat completions API with the message list to get the response.


key: _node_content
string_value:
{"id_": "ad354a62-49f5-4633-8168-266c4d2b687e", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_call_functions_with_chat_models.ipynb", "title": "How to call functions with chat models", "tokens": 4334, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "4ff1d23d-3d6a-4a22-9968-721381f66e1e", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_call_functions_with_chat_models.ipynb", "title": "How to call functions with chat models", "tokens": 4334, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "08b8b6fa48b1e1e14179f154ae2248c8cb28ce70546f10038658b577ad2e0150", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "73bb2ff1-dcc1-4ef6-b4b2-ef4b3886accc", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_call_functions_with_chat_models.ipynb", "title": "How to call functions with chat models", "tokens": 4334, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "31c1386700b17f9f4996aad2342d37b4ac0aea9c1a9d0b5ddceb1a6c95633fa2", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "685bc394-689b-42aa-87db-111a7fbc902b", "node_type": "1", "metadata": {}, "hash": "07084e56be7dfbd48cf8bbd2718c15bf668cf6b93146d59675d4b4f241455355", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 14831, "end_char_idx": 17906, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
tools = [
    {
        "type": "function",
        "function": {
            "name": "ask_database",
            "description": "Use this function to answer user questions about music. Input should be a fully formed SQL query.",
            "parameters": {
                "type": "object",
                "properties": {
                    "query": {
                        "type": "string",
                        "description": f"""
                                SQL query extracting info to answer the user's question.
                                SQL should be written using this database schema:
                                {database_schema_string}
                                The query should be returned in plain text, not in JSON.
                                """,
                    }
                },
                "required": ["query"],
            },
        }
    }
]
```

### Executing SQL queries

Now let's implement the function that will actually excute queries against the database.


```python
def ask_database(conn, query):
    """Function to query SQLite database with a provided SQL query."""
    try:
        results = str(conn.execute(query).fetchall())
    except Exception as e:
        results = f"query failed with error: {e}"
    return results
```

##### Steps to invoke a function call using Chat Completions API: 

**Step 1**: Prompt the model with content that may result in model selecting a tool to use. The description of the tools such as a function names and signature is defined in the 'Tools' list and passed to the model in API call. If selected, the function name and parameters are included in the response.<br>
  
**Step 2**: Check programmatically if model wanted to call a function. If true, proceed to step 3. <br>  
**Step 3**: Extract the function name and parameters from response, call the function with parameters. Append the result to messages. <br>    
**Step 4**: Invoke the chat completions API with the message list to get the response. 


```python
# Step #1: Prompt with content that may result in function call. In this case the model can identify the information requested by the user is potentially available in the database schema passed to the model in Tools description. 
messages = [{
    "role":"user", 
    "content": "What is the name of the album with the most tracks?"
}]

response = client.chat.completions.create(
    model='gpt-4o', 
    messages=messages, 
    tools= tools, 
    tool_choice="auto"
)

# Append the message to messages list
response_message = response.choices[0].message 
messages.append(response_message)

print(response_message)
```

    ChatCompletionMessage(content=None, role='assistant', function_call=None, tool_calls=[ChatCompletionMessageToolCall(id='call_wDN8uLjq2ofuU6rVx1k8Gw0e', function=Function(arguments='{"query":"SELECT Album.Title, COUNT(Track.TrackId) AS TrackCount FROM Album INNER JOIN Track ON Album.AlbumId = Track.AlbumId GROUP BY Album.Title ORDER BY TrackCount DESC LIMIT 1;"}', name='ask_database'), type='function')])


key: _node_content
string_value:
{"id_": "685bc394-689b-42aa-87db-111a7fbc902b", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_call_functions_with_chat_models.ipynb", "title": "How to call functions with chat models", "tokens": 4334, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "4ff1d23d-3d6a-4a22-9968-721381f66e1e", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_call_functions_with_chat_models.ipynb", "title": "How to call functions with chat models", "tokens": 4334, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "08b8b6fa48b1e1e14179f154ae2248c8cb28ce70546f10038658b577ad2e0150", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "ad354a62-49f5-4633-8168-266c4d2b687e", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_call_functions_with_chat_models.ipynb", "title": "How to call functions with chat models", "tokens": 4334, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "eef38f8bb036013445fd46e0ed7a7d2baf2ea13cf89a3b52d8feb2309121246a", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 16866, "end_char_idx": 19767, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
# Step #1: Prompt with content that may result in function call. In this case the model can identify the information requested by the user is potentially available in the database schema passed to the model in Tools description. 
messages = [{
    "role":"user", 
    "content": "What is the name of the album with the most tracks?"
}]

response = client.chat.completions.create(
    model='gpt-4o', 
    messages=messages, 
    tools= tools, 
    tool_choice="auto"
)

# Append the message to messages list
response_message = response.choices[0].message 
messages.append(response_message)

print(response_message)
```

    ChatCompletionMessage(content=None, role='assistant', function_call=None, tool_calls=[ChatCompletionMessageToolCall(id='call_wDN8uLjq2ofuU6rVx1k8Gw0e', function=Function(arguments='{"query":"SELECT Album.Title, COUNT(Track.TrackId) AS TrackCount FROM Album INNER JOIN Track ON Album.AlbumId = Track.AlbumId GROUP BY Album.Title ORDER BY TrackCount DESC LIMIT 1;"}', name='ask_database'), type='function')])



```python
# Step 2: determine if the response from the model includes a tool call.   
tool_calls = response_message.tool_calls
if tool_calls:
    # If true the model will return the name of the tool / function to call and the argument(s)  
    tool_call_id = tool_calls[0].id
    tool_function_name = tool_calls[0].function.name
    tool_query_string = json.loads(tool_calls[0].function.arguments)['query']

    # Step 3: Call the function and retrieve results. Append the results to the messages list.      
    if tool_function_name == 'ask_database':
        results = ask_database(conn, tool_query_string)
        
        messages.append({
            "role":"tool", 
            "tool_call_id":tool_call_id, 
            "name": tool_function_name, 
            "content":results
        })
        
        # Step 4: Invoke the chat completions API with the function response appended to the messages list
        # Note that messages with role 'tool' must be a response to a preceding message with 'tool_calls'
        model_response_with_function_call = client.chat.completions.create(
            model="gpt-4o",
            messages=messages,
        )  # get a new response from the model where it can see the function response
        print(model_response_with_function_call.choices[0].message.content)
    else: 
        print(f"Error: function {tool_function_name} does not exist")
else: 
    # Model did not identify a function to call, result can be returned to the user 
    print(response_message.content) 
```

    The album with the most tracks is titled "Greatest Hits," which contains 57 tracks.


## Next Steps

See our other [notebook](How_to_call_functions_for_knowledge_retrieval.ipynb) that demonstrates how to use the Chat Completions API and functions for knowledge retrieval to interact conversationally with a knowledge base.


key: url
string_value:
https://github.com/openai/openai-cookbook/blob/main/examples/How_to_build_an_agent_with_the_node_sdk.ipynb


key: _node_content
string_value:
{"id_": "1cc446bc-dad5-4079-b913-85e220631d4d", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_build_an_agent_with_the_node_sdk.ipynb", "title": "How to build an agent with the Node.js SDK", "tokens": 3947, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "4ad93f91-d133-44e9-937b-de619be42a4e", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_build_an_agent_with_the_node_sdk.ipynb", "title": "How to build an agent with the Node.js SDK", "tokens": 3947, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "cabec513f13e9d8fd1f6b045c22735847796e4c9e04e9f7c7763932590bb6b4d", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "47339443-587f-4ed5-bfb4-f9aa51135dc7", "node_type": "1", "metadata": {}, "hash": "ca1abd482282c8464f2b2e19ea362028f7b844b8c6a27ddeabe6f63a2ddaa246", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 0, "end_char_idx": 3146, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
# How to build an agent with the Node.js SDK

OpenAI functions enable your app to take action based on user inputs. This means that it can, e.g., search the web, send emails, or book tickets on behalf of your users, making it more powerful than a regular chatbot.

In this tutorial, you will build an app that uses OpenAI functions along with the latest version of the Node.js SDK. The app runs in the browser, so you only need a code editor and, e.g., VS Code Live Server to follow along locally. Alternatively, write your code directly in the browser via [this code playground at Scrimba.](https://scrimba.com/scrim/c6r3LkU9)

## What you will build

Our app is a simple agent that helps you find activities in your area.
It has access to two functions, `getLocation()` and `getCurrentWeather()`,
which means it can figure out where you’re located and what the weather
is at the moment.

At this point, it's important to understand that
OpenAI doesn't execute any code for you. It just tells your app which
functions it should use in a given scenario, and then leaves it up to
your app to invoke them.

Once our agent knows your location and the weather, it'll use GPT’s
internal knowledge to suggest suitable local activities for you.

## Importing the SDK and authenticating with OpenAI

We start by importing the OpenAI SDK at the top of our JavaScript file and authenticate with our API key, which we have stored as an environment variable.

```js
import OpenAI from "openai";

const openai = new OpenAI({
  apiKey: process.env.OPENAI_API_KEY,
  dangerouslyAllowBrowser: true,
});
```

Since we're running our code in a browser environment at Scrimba, we also need to set `dangerouslyAllowBrowser: true` to confirm we understand the risks involved with client-side API requests. Please note that you should move these requests over to a Node server in a production app.

## Creating our two functions

Next, we'll create the two functions. The first one - `getLocation` -
uses the [IP API](https://ipapi.co/) to get the location of the
user.

```js
async function getLocation() {
  const response = await fetch("https://ipapi.co/json/");
  const locationData = await response.json();
  return locationData;
}
```

The IP API returns a bunch of data about your location, including your
latitude and longitude, which we’ll use as arguments in the second
function `getCurrentWeather`. It uses the [Open Meteo
API](https://open-meteo.com/) to get the current weather data, like
this:

```js
async function getCurrentWeather(latitude, longitude) {
  const url = `https://api.open-meteo.com/v1/forecast?latitude=${latitude}&longitude=${longitude}&hourly=apparent_temperature`;
  const response = await fetch(url);
  const weatherData = await response.json();
  return weatherData;
}
```

## Describing our functions for OpenAI

For OpenAI to understand the purpose of these functions, we need to
describe them using a specific schema. We'll create an array called
`tools` that contains one object per function. Each object
will have two keys: `type`, `function`, and the `function` key has 
three subkeys: `name`, `description`, and `parameters`.


key: _node_content
string_value:
{"id_": "47339443-587f-4ed5-bfb4-f9aa51135dc7", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_build_an_agent_with_the_node_sdk.ipynb", "title": "How to build an agent with the Node.js SDK", "tokens": 3947, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "4ad93f91-d133-44e9-937b-de619be42a4e", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_build_an_agent_with_the_node_sdk.ipynb", "title": "How to build an agent with the Node.js SDK", "tokens": 3947, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "cabec513f13e9d8fd1f6b045c22735847796e4c9e04e9f7c7763932590bb6b4d", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "1cc446bc-dad5-4079-b913-85e220631d4d", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_build_an_agent_with_the_node_sdk.ipynb", "title": "How to build an agent with the Node.js SDK", "tokens": 3947, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "20cbdefb789702661907f51429a6340a93e466cea0b2d45b337588100b171136", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "1e3c3053-6dd4-407a-a0d8-32e8c502aca6", "node_type": "1", "metadata": {}, "hash": "ca2e5d26e7b9ee64c4d5aafca21d6281e27eabbde2460e570eb871d0b9c40752", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 1785, "end_char_idx": 4631, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
Please note that you should move these requests over to a Node server in a production app.

## Creating our two functions

Next, we'll create the two functions. The first one - `getLocation` -
uses the [IP API](https://ipapi.co/) to get the location of the
user.

```js
async function getLocation() {
  const response = await fetch("https://ipapi.co/json/");
  const locationData = await response.json();
  return locationData;
}
```

The IP API returns a bunch of data about your location, including your
latitude and longitude, which we’ll use as arguments in the second
function `getCurrentWeather`. It uses the [Open Meteo
API](https://open-meteo.com/) to get the current weather data, like
this:

```js
async function getCurrentWeather(latitude, longitude) {
  const url = `https://api.open-meteo.com/v1/forecast?latitude=${latitude}&longitude=${longitude}&hourly=apparent_temperature`;
  const response = await fetch(url);
  const weatherData = await response.json();
  return weatherData;
}
```

## Describing our functions for OpenAI

For OpenAI to understand the purpose of these functions, we need to
describe them using a specific schema. We'll create an array called
`tools` that contains one object per function. Each object
will have two keys: `type`, `function`, and the `function` key has 
three subkeys: `name`, `description`, and `parameters`.

```js
const tools = [
  {
    type: "function",
    function: {
      name: "getCurrentWeather",
      description: "Get the current weather in a given location",
      parameters: {
        type: "object",
        properties: {
          latitude: {
            type: "string",
          },
          longitude: {
            type: "string",
          },
        },
        required: ["longitude", "latitude"],
      },
    }
  },
  {
    type: "function",
    function: {
      name: "getLocation",
      description: "Get the user's location based on their IP address",
      parameters: {
        type: "object",
        properties: {},
      },
    }
  },
];
```

## Setting up the messages array

We also need to define a `messages` array. This will keep track of all of the messages back and forth between our app and OpenAI.

The first object in the array should always have the `role` property set to `"system"`, which tells OpenAI that this is how we want it to behave.

```js
const messages = [
  {
    role: "system",
    content:
      "You are a helpful assistant. Only use the functions you have been provided with.",
  },
];
```

## Creating the agent function

We are now ready to build the logic of our app, which lives in the
`agent` function. It is asynchronous and takes one argument: the
`userInput`.

We start by pushing the `userInput` to the messages array. This time, we set the `role` to `"user"`, so that OpenAI knows that this is the input from the user.


key: _node_content
string_value:
{"id_": "1e3c3053-6dd4-407a-a0d8-32e8c502aca6", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_build_an_agent_with_the_node_sdk.ipynb", "title": "How to build an agent with the Node.js SDK", "tokens": 3947, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "4ad93f91-d133-44e9-937b-de619be42a4e", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_build_an_agent_with_the_node_sdk.ipynb", "title": "How to build an agent with the Node.js SDK", "tokens": 3947, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "cabec513f13e9d8fd1f6b045c22735847796e4c9e04e9f7c7763932590bb6b4d", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "47339443-587f-4ed5-bfb4-f9aa51135dc7", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_build_an_agent_with_the_node_sdk.ipynb", "title": "How to build an agent with the Node.js SDK", "tokens": 3947, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "db359b9f5f57518f2cab194e5e6a6a29ea41753850abcdd8e95fe6a2004d90db", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "88ae7f6b-e5dc-478a-8767-12e5c5802273", "node_type": "1", "metadata": {}, "hash": "a5196a1bcd36d11233917bb7cf99623661cf52416c626958c07e77776b086e64", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 3011, "end_char_idx": 5625, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
Each object
will have two keys: `type`, `function`, and the `function` key has 
three subkeys: `name`, `description`, and `parameters`.

```js
const tools = [
  {
    type: "function",
    function: {
      name: "getCurrentWeather",
      description: "Get the current weather in a given location",
      parameters: {
        type: "object",
        properties: {
          latitude: {
            type: "string",
          },
          longitude: {
            type: "string",
          },
        },
        required: ["longitude", "latitude"],
      },
    }
  },
  {
    type: "function",
    function: {
      name: "getLocation",
      description: "Get the user's location based on their IP address",
      parameters: {
        type: "object",
        properties: {},
      },
    }
  },
];
```

## Setting up the messages array

We also need to define a `messages` array. This will keep track of all of the messages back and forth between our app and OpenAI.

The first object in the array should always have the `role` property set to `"system"`, which tells OpenAI that this is how we want it to behave.

```js
const messages = [
  {
    role: "system",
    content:
      "You are a helpful assistant. Only use the functions you have been provided with.",
  },
];
```

## Creating the agent function

We are now ready to build the logic of our app, which lives in the
`agent` function. It is asynchronous and takes one argument: the
`userInput`.

We start by pushing the `userInput` to the messages array. This time, we set the `role` to `"user"`, so that OpenAI knows that this is the input from the user.

```js
async function agent(userInput) {
  messages.push({
    role: "user",
    content: userInput,
  });
  const response = await openai.chat.completions.create({
    model: "gpt-4",
    messages: messages,
    tools: tools,
  });
  console.log(response);
}
```

Next, we'll send a request to the Chat completions endpoint via the
`chat.completions.create()` method in the Node SDK. This method takes a
configuration object as an argument. In it, we'll specify three
properties:

- `model` - Decides which AI model we want to use (in our case,
  GPT-4).
- `messages` - The entire history of messages between the user and the
  AI up until this point.
- `tools` - A list of tools the model may call. Currently, only 
  functions are supported as a tool., we'll we use the `tools` array we 
  created earlier.

## Running our app with a simple input

Let's try to run the `agent` with an input that requires a function call to give a suitable reply.

```js
agent("Where am I located right now?


key: _node_content
string_value:
{"id_": "88ae7f6b-e5dc-478a-8767-12e5c5802273", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_build_an_agent_with_the_node_sdk.ipynb", "title": "How to build an agent with the Node.js SDK", "tokens": 3947, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "4ad93f91-d133-44e9-937b-de619be42a4e", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_build_an_agent_with_the_node_sdk.ipynb", "title": "How to build an agent with the Node.js SDK", "tokens": 3947, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "cabec513f13e9d8fd1f6b045c22735847796e4c9e04e9f7c7763932590bb6b4d", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "1e3c3053-6dd4-407a-a0d8-32e8c502aca6", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_build_an_agent_with_the_node_sdk.ipynb", "title": "How to build an agent with the Node.js SDK", "tokens": 3947, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "b12caabc68470e7dd969e4fb949171bb4e6bade858f5ae18ad286505262b95e7", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "ce8344fc-7707-488b-9270-306cee3474c4", "node_type": "1", "metadata": {}, "hash": "156ab0952c55e053cf558d83723ada32eaeede13fd629274da1d9f9cb20b9b8a", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 4129, "end_char_idx": 6904, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```js
const messages = [
  {
    role: "system",
    content:
      "You are a helpful assistant. Only use the functions you have been provided with.",
  },
];
```

## Creating the agent function

We are now ready to build the logic of our app, which lives in the
`agent` function. It is asynchronous and takes one argument: the
`userInput`.

We start by pushing the `userInput` to the messages array. This time, we set the `role` to `"user"`, so that OpenAI knows that this is the input from the user.

```js
async function agent(userInput) {
  messages.push({
    role: "user",
    content: userInput,
  });
  const response = await openai.chat.completions.create({
    model: "gpt-4",
    messages: messages,
    tools: tools,
  });
  console.log(response);
}
```

Next, we'll send a request to the Chat completions endpoint via the
`chat.completions.create()` method in the Node SDK. This method takes a
configuration object as an argument. In it, we'll specify three
properties:

- `model` - Decides which AI model we want to use (in our case,
  GPT-4).
- `messages` - The entire history of messages between the user and the
  AI up until this point.
- `tools` - A list of tools the model may call. Currently, only 
  functions are supported as a tool., we'll we use the `tools` array we 
  created earlier.

## Running our app with a simple input

Let's try to run the `agent` with an input that requires a function call to give a suitable reply.

```js
agent("Where am I located right now?");
```

When we run the code above, we see the response from OpenAI logged out
to the console like this:

```js
{
    id: "chatcmpl-84ojoEJtyGnR6jRHK2Dl4zTtwsa7O",
    object: "chat.completion",
    created: 1696159040,
    model: "gpt-4-0613",
    choices: [{
        index: 0,
        message: {
            role: "assistant",
            content: null,
            tool_calls: [
              id: "call_CBwbo9qoXUn1kTR5pPuv6vR1",
              type: "function",
              function: {
                name: "getLocation",
                arguments: "{}"
              }
            ]
        },
        logprobs: null,
        finish_reason: "tool_calls" // OpenAI wants us to call a function
    }],
    usage: {
        prompt_tokens: 134,
        completion_tokens: 6,
        total_tokens: 140
    }
     system_fingerprint: null
}
```

This response tells us that we should call one of our functions, as it contains the following key: `finish_reason: "tool_calls"`.

The name of the function can be found in the
`response.choices[0].message.tool_calls[0].function.name` key, which is set to
`"getLocation"`.

## Turning the OpenAI response into a function call

Now that we have the name of the function as a string, we'll need to
translate that into a function call.


key: _node_content
string_value:
{"id_": "ce8344fc-7707-488b-9270-306cee3474c4", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_build_an_agent_with_the_node_sdk.ipynb", "title": "How to build an agent with the Node.js SDK", "tokens": 3947, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "4ad93f91-d133-44e9-937b-de619be42a4e", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_build_an_agent_with_the_node_sdk.ipynb", "title": "How to build an agent with the Node.js SDK", "tokens": 3947, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "cabec513f13e9d8fd1f6b045c22735847796e4c9e04e9f7c7763932590bb6b4d", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "88ae7f6b-e5dc-478a-8767-12e5c5802273", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_build_an_agent_with_the_node_sdk.ipynb", "title": "How to build an agent with the Node.js SDK", "tokens": 3947, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "cacddd8f489fa1ea7ff83254a63aa6b3b580368bb1e27775f6a503e4cca11614", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "beee133c-b656-4baa-9c68-4618ec5bbd36", "node_type": "1", "metadata": {}, "hash": "5bac127fd4eb5b14b3f065fd46d55d20acac5b808a38b458e7e9f841674a8fbf", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 5443, "end_char_idx": 8167, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
## Running our app with a simple input

Let's try to run the `agent` with an input that requires a function call to give a suitable reply.

```js
agent("Where am I located right now?");
```

When we run the code above, we see the response from OpenAI logged out
to the console like this:

```js
{
    id: "chatcmpl-84ojoEJtyGnR6jRHK2Dl4zTtwsa7O",
    object: "chat.completion",
    created: 1696159040,
    model: "gpt-4-0613",
    choices: [{
        index: 0,
        message: {
            role: "assistant",
            content: null,
            tool_calls: [
              id: "call_CBwbo9qoXUn1kTR5pPuv6vR1",
              type: "function",
              function: {
                name: "getLocation",
                arguments: "{}"
              }
            ]
        },
        logprobs: null,
        finish_reason: "tool_calls" // OpenAI wants us to call a function
    }],
    usage: {
        prompt_tokens: 134,
        completion_tokens: 6,
        total_tokens: 140
    }
     system_fingerprint: null
}
```

This response tells us that we should call one of our functions, as it contains the following key: `finish_reason: "tool_calls"`.

The name of the function can be found in the
`response.choices[0].message.tool_calls[0].function.name` key, which is set to
`"getLocation"`.

## Turning the OpenAI response into a function call

Now that we have the name of the function as a string, we'll need to
translate that into a function call. To help us with that, we'll gather
both of our functions in an object called `availableTools`:

```js
const availableTools = {
  getCurrentWeather,
  getLocation,
};
```

This is handy because we'll be able to access the `getLocation` function
via bracket notation and the string we got back from OpenAI, like this:
`availableTools["getLocation"]`.

```js
const { finish_reason, message } = response.choices[0];

if (finish_reason === "tool_calls" && message.tool_calls) {
  const functionName = message.tool_calls[0].function.name;
  const functionToCall = availableTools[functionName];
  const functionArgs = JSON.parse(message.tool_calls[0].function.arguments);
  const functionArgsArr = Object.values(functionArgs);
  const functionResponse = await functionToCall.apply(null, functionArgsArr);
  console.log(functionResponse);
}
```

We're also grabbing ahold of any arguments OpenAI wants us to pass into
the function: `message.tool_calls[0].function.arguments`.
However, we won't need any arguments for this first function call.

If we run the code again with the same input
(`"Where am I located right now?"`), we'll see that `functionResponse`
is an object filled with location about where the user is located right
now. In my case, that is Oslo, Norway.


key: _node_content
string_value:
{"id_": "beee133c-b656-4baa-9c68-4618ec5bbd36", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_build_an_agent_with_the_node_sdk.ipynb", "title": "How to build an agent with the Node.js SDK", "tokens": 3947, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "4ad93f91-d133-44e9-937b-de619be42a4e", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_build_an_agent_with_the_node_sdk.ipynb", "title": "How to build an agent with the Node.js SDK", "tokens": 3947, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "cabec513f13e9d8fd1f6b045c22735847796e4c9e04e9f7c7763932590bb6b4d", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "ce8344fc-7707-488b-9270-306cee3474c4", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_build_an_agent_with_the_node_sdk.ipynb", "title": "How to build an agent with the Node.js SDK", "tokens": 3947, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "0cfb7461a4eae1e537a3ab64b27a99e113f280a2829f757057a278ec8eb81c5b", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "a16370de-617b-4ab5-bb89-e2b725afe49f", "node_type": "1", "metadata": {}, "hash": "4acfcc5b2f43f077fca35e309599e288082196449b9519ce1e67e1c153393778", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 6604, "end_char_idx": 9553, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
The name of the function can be found in the
`response.choices[0].message.tool_calls[0].function.name` key, which is set to
`"getLocation"`.

## Turning the OpenAI response into a function call

Now that we have the name of the function as a string, we'll need to
translate that into a function call. To help us with that, we'll gather
both of our functions in an object called `availableTools`:

```js
const availableTools = {
  getCurrentWeather,
  getLocation,
};
```

This is handy because we'll be able to access the `getLocation` function
via bracket notation and the string we got back from OpenAI, like this:
`availableTools["getLocation"]`.

```js
const { finish_reason, message } = response.choices[0];

if (finish_reason === "tool_calls" && message.tool_calls) {
  const functionName = message.tool_calls[0].function.name;
  const functionToCall = availableTools[functionName];
  const functionArgs = JSON.parse(message.tool_calls[0].function.arguments);
  const functionArgsArr = Object.values(functionArgs);
  const functionResponse = await functionToCall.apply(null, functionArgsArr);
  console.log(functionResponse);
}
```

We're also grabbing ahold of any arguments OpenAI wants us to pass into
the function: `message.tool_calls[0].function.arguments`.
However, we won't need any arguments for this first function call.

If we run the code again with the same input
(`"Where am I located right now?"`), we'll see that `functionResponse`
is an object filled with location about where the user is located right
now. In my case, that is Oslo, Norway.

```js
{ip: "193.212.60.170", network: "193.212.60.0/23", version: "IPv4", city: "Oslo", region: "Oslo County", region_code: "03", country: "NO", country_name: "Norway", country_code: "NO", country_code_iso3: "NOR", country_capital: "Oslo", country_tld: ".no", continent_code: "EU", in_eu: false, postal: "0026", latitude: 59.955, longitude: 10.859, timezone: "Europe/Oslo", utc_offset: "+0200", country_calling_code: "+47", currency: "NOK", currency_name: "Krone", languages: "no,nb,nn,se,fi", country_area: 324220, country_population: 5314336, asn: "AS2119", org: "Telenor Norge AS"}
```

We'll add this data to a new item in the `messages` array, where we also
specify the name of the function we called.

```js
messages.push({
  role: "function",
  name: functionName,
  content: `The result of the last function was this: ${JSON.stringify(
    functionResponse
  )}
  `,
});
```

Notice that the `role` is set to `"function"`. This tells OpenAI
that the `content` parameter contains the result of the function call
and not the input from the user.

At this point, we need to send a new request to OpenAI with this updated
`messages` array. However, we don’t want to hard code a new function
call, as our agent might need to go back and forth between itself and
GPT several times until it has found the final answer for the user.

This can be solved in several different ways, e.g.


key: _node_content
string_value:
{"id_": "a16370de-617b-4ab5-bb89-e2b725afe49f", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_build_an_agent_with_the_node_sdk.ipynb", "title": "How to build an agent with the Node.js SDK", "tokens": 3947, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "4ad93f91-d133-44e9-937b-de619be42a4e", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_build_an_agent_with_the_node_sdk.ipynb", "title": "How to build an agent with the Node.js SDK", "tokens": 3947, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "cabec513f13e9d8fd1f6b045c22735847796e4c9e04e9f7c7763932590bb6b4d", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "beee133c-b656-4baa-9c68-4618ec5bbd36", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_build_an_agent_with_the_node_sdk.ipynb", "title": "How to build an agent with the Node.js SDK", "tokens": 3947, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "c215cd1adffde1e5d9b954eff1be41a819c6dc9ca9b7e7a238c4f3f2f85a4ad9", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "c941ad70-bc33-4c59-b1e4-411009c3eb69", "node_type": "1", "metadata": {}, "hash": "b1885671f8b1758a1018aaa2f5bff352cb9ead17165d574b27a5892209cfc6f6", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 8877, "end_char_idx": 12087, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```js
messages.push({
  role: "function",
  name: functionName,
  content: `The result of the last function was this: ${JSON.stringify(
    functionResponse
  )}
  `,
});
```

Notice that the `role` is set to `"function"`. This tells OpenAI
that the `content` parameter contains the result of the function call
and not the input from the user.

At this point, we need to send a new request to OpenAI with this updated
`messages` array. However, we don’t want to hard code a new function
call, as our agent might need to go back and forth between itself and
GPT several times until it has found the final answer for the user.

This can be solved in several different ways, e.g. recursion, a
while-loop, or a for-loop. We'll use a good old for-loop for the sake of
simplicity.

## Creating the loop

At the top of the `agent` function, we'll create a loop that lets us run
the entire procedure up to five times.

If we get back `finish_reason: "tool_calls"` from GPT, we'll just
push the result of the function call to the `messages` array and jump to
the next iteration of the loop, triggering a new request.

If we get `finish_reason: "stop"` back, then GPT has found a suitable
answer, so we'll return the function and cancel the loop.

```js
for (let i = 0; i < 5; i++) {
  const response = await openai.chat.completions.create({
    model: "gpt-4",
    messages: messages,
    tools: tools,
  });
  const { finish_reason, message } = response.choices[0];

  if (finish_reason === "tool_calls" && message.tool_calls) {
    const functionName = message.tool_calls[0].function.name;
    const functionToCall = availableTools[functionName];
    const functionArgs = JSON.parse(message.tool_calls[0].function.arguments);
    const functionArgsArr = Object.values(functionArgs);
    const functionResponse = await functionToCall.apply(null, functionArgsArr);

    messages.push({
      role: "function",
      name: functionName,
      content: `
          The result of the last function was this: ${JSON.stringify(
            functionResponse
          )}
          `,
    });
  } else if (finish_reason === "stop") {
    messages.push(message);
    return message.content;
  }
}
return "The maximum number of iterations has been met without a suitable answer. Please try again with a more specific input.";
```

If we don't see a `finish_reason: "stop"` within our five iterations,
we'll return a message saying we couldn’t find a suitable answer.

## Running the final app

At this point, we are ready to try our app! I'll ask the agent to
suggest some activities based on my location and the current weather.

```js
const response = await agent(
  "Please suggest some activities based on my location and the current weather."
);
console.log(response);
```

Here's what we see in the console (formatted to make it easier to read):

```js
Based on your current location in Oslo, Norway and the weather (15°C and snowy),
here are some activity suggestions:

1. A visit to the Oslo Winter Park for skiing or snowboarding.
2. Enjoy a cosy day at a local café or restaurant.
3. Visit one of Oslo's many museums. The Fram Museum or Viking Ship Museum offer interesting insights into Norway’s seafaring history.
4.


key: _node_content
string_value:
{"id_": "c941ad70-bc33-4c59-b1e4-411009c3eb69", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_build_an_agent_with_the_node_sdk.ipynb", "title": "How to build an agent with the Node.js SDK", "tokens": 3947, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "4ad93f91-d133-44e9-937b-de619be42a4e", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_build_an_agent_with_the_node_sdk.ipynb", "title": "How to build an agent with the Node.js SDK", "tokens": 3947, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "cabec513f13e9d8fd1f6b045c22735847796e4c9e04e9f7c7763932590bb6b4d", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "a16370de-617b-4ab5-bb89-e2b725afe49f", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_build_an_agent_with_the_node_sdk.ipynb", "title": "How to build an agent with the Node.js SDK", "tokens": 3947, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "7ded74d92f10a5c650e347abcccfa8aff1eea50e3e4992c4625c04c39e8a5651", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "aebb62e8-9d73-4722-b655-7aa3f6486bfd", "node_type": "1", "metadata": {}, "hash": "878dc08538da8f65fe66c6a3dee4de13f8bb480fe45b4cf715ad1ce893704a08", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 11138, "end_char_idx": 13479, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
Please try again with a more specific input.";
```

If we don't see a `finish_reason: "stop"` within our five iterations,
we'll return a message saying we couldn’t find a suitable answer.

## Running the final app

At this point, we are ready to try our app! I'll ask the agent to
suggest some activities based on my location and the current weather.

```js
const response = await agent(
  "Please suggest some activities based on my location and the current weather."
);
console.log(response);
```

Here's what we see in the console (formatted to make it easier to read):

```js
Based on your current location in Oslo, Norway and the weather (15°C and snowy),
here are some activity suggestions:

1. A visit to the Oslo Winter Park for skiing or snowboarding.
2. Enjoy a cosy day at a local café or restaurant.
3. Visit one of Oslo's many museums. The Fram Museum or Viking Ship Museum offer interesting insights into Norway’s seafaring history.
4. Take a stroll in the snowy streets and enjoy the beautiful winter landscape.
5. Enjoy a nice book by the fireplace in a local library.
6. Take a fjord sightseeing cruise to enjoy the snowy landscapes.

Always remember to bundle up and stay warm. Enjoy your day!
```

If we peak under the hood, and log out `response.choices[0].message` in
each iteration of the loop, we'll see that GPT has instructed us to use
both our functions before coming up with an answer.

First, it tells us to call the `getLocation` function. Then it tells us
to call the `getCurrentWeather` function with
`"longitude": "10.859", "latitude": "59.955"` passed in as the
arguments. This is data it got back from the first function call we did.

```js
{"role":"assistant","content":null,"tool_calls":[{"id":"call_Cn1KH8mtHQ2AMbyNwNJTweEP","type":"function","function":{"name":"getLocation","arguments":"{}"}}]}
{"role":"assistant","content":null,"tool_calls":[{"id":"call_uc1oozJfGTvYEfIzzcsfXfOl","type":"function","function":{"name":"getCurrentWeather","arguments":"{\n\"latitude\": \"10.859\",\n\"longitude\": \"59.955\"\n}"}}]}
```

You've now built an AI agent using OpenAI functions and the Node.js SDK! If you're looking for an extra challenge, consider enhancing this app. For example, you could add a function that fetches up-to-date information on events and activities in the user's location.

Happy coding!


key: _node_content
string_value:
{"id_": "aebb62e8-9d73-4722-b655-7aa3f6486bfd", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_build_an_agent_with_the_node_sdk.ipynb", "title": "How to build an agent with the Node.js SDK", "tokens": 3947, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "4ad93f91-d133-44e9-937b-de619be42a4e", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_build_an_agent_with_the_node_sdk.ipynb", "title": "How to build an agent with the Node.js SDK", "tokens": 3947, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "cabec513f13e9d8fd1f6b045c22735847796e4c9e04e9f7c7763932590bb6b4d", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "c941ad70-bc33-4c59-b1e4-411009c3eb69", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_build_an_agent_with_the_node_sdk.ipynb", "title": "How to build an agent with the Node.js SDK", "tokens": 3947, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "9a7f6422814ceed039b2aa33052b7f96c492e5bb2c09f3fb3b148c922df137da", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "0e4cfe2f-3871-4ac6-af2e-7f5e110cfc3f", "node_type": "1", "metadata": {}, "hash": "f7865ffd704d0cba63385c58a1801b60b329c1eb19d631c990a51a26b8af936f", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 11950, "end_char_idx": 14926, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
3. Visit one of Oslo's many museums. The Fram Museum or Viking Ship Museum offer interesting insights into Norway’s seafaring history.
4. Take a stroll in the snowy streets and enjoy the beautiful winter landscape.
5. Enjoy a nice book by the fireplace in a local library.
6. Take a fjord sightseeing cruise to enjoy the snowy landscapes.

Always remember to bundle up and stay warm. Enjoy your day!
```

If we peak under the hood, and log out `response.choices[0].message` in
each iteration of the loop, we'll see that GPT has instructed us to use
both our functions before coming up with an answer.

First, it tells us to call the `getLocation` function. Then it tells us
to call the `getCurrentWeather` function with
`"longitude": "10.859", "latitude": "59.955"` passed in as the
arguments. This is data it got back from the first function call we did.

```js
{"role":"assistant","content":null,"tool_calls":[{"id":"call_Cn1KH8mtHQ2AMbyNwNJTweEP","type":"function","function":{"name":"getLocation","arguments":"{}"}}]}
{"role":"assistant","content":null,"tool_calls":[{"id":"call_uc1oozJfGTvYEfIzzcsfXfOl","type":"function","function":{"name":"getCurrentWeather","arguments":"{\n\"latitude\": \"10.859\",\n\"longitude\": \"59.955\"\n}"}}]}
```

You've now built an AI agent using OpenAI functions and the Node.js SDK! If you're looking for an extra challenge, consider enhancing this app. For example, you could add a function that fetches up-to-date information on events and activities in the user's location.

Happy coding!

<details>
<summary>Complete code</summary>

```js
import OpenAI from "openai";

const openai = new OpenAI({
  apiKey: process.env.OPENAI_API_KEY,
  dangerouslyAllowBrowser: true,
});

async function getLocation() {
  const response = await fetch("https://ipapi.co/json/");
  const locationData = await response.json();
  return locationData;
}

async function getCurrentWeather(latitude, longitude) {
  const url = `https://api.open-meteo.com/v1/forecast?latitude=${latitude}&longitude=${longitude}&hourly=apparent_temperature`;
  const response = await fetch(url);
  const weatherData = await response.json();
  return weatherData;
}

const tools = [
  {
    type: "function",
    function: {
      name: "getCurrentWeather",
      description: "Get the current weather in a given location",
      parameters: {
        type: "object",
        properties: {
          latitude: {
            type: "string",
          },
          longitude: {
            type: "string",
          },
        },
        required: ["longitude", "latitude"],
      },
    }
  },
  {
    type: "function",
    function: {
      name: "getLocation",
      description: "Get the user's location based on their IP address",
      parameters: {
        type: "object",
        properties: {},
      },
    }
  },
];

const availableTools = {
  getCurrentWeather,
  getLocation,
};

const messages = [
  {
    role: "system",
    content: `You are a helpful assistant.


key: _node_content
string_value:
{"id_": "0e4cfe2f-3871-4ac6-af2e-7f5e110cfc3f", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_build_an_agent_with_the_node_sdk.ipynb", "title": "How to build an agent with the Node.js SDK", "tokens": 3947, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "4ad93f91-d133-44e9-937b-de619be42a4e", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_build_an_agent_with_the_node_sdk.ipynb", "title": "How to build an agent with the Node.js SDK", "tokens": 3947, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "cabec513f13e9d8fd1f6b045c22735847796e4c9e04e9f7c7763932590bb6b4d", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "aebb62e8-9d73-4722-b655-7aa3f6486bfd", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/How_to_build_an_agent_with_the_node_sdk.ipynb", "title": "How to build an agent with the Node.js SDK", "tokens": 3947, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "94e167d95fce4fab12cc1c004b036bed96d469db8284e3e54cb2a9f5d6ee202f", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 13271, "end_char_idx": 16413, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
If you're looking for an extra challenge, consider enhancing this app. For example, you could add a function that fetches up-to-date information on events and activities in the user's location.

Happy coding!

<details>
<summary>Complete code</summary>

```js
import OpenAI from "openai";

const openai = new OpenAI({
  apiKey: process.env.OPENAI_API_KEY,
  dangerouslyAllowBrowser: true,
});

async function getLocation() {
  const response = await fetch("https://ipapi.co/json/");
  const locationData = await response.json();
  return locationData;
}

async function getCurrentWeather(latitude, longitude) {
  const url = `https://api.open-meteo.com/v1/forecast?latitude=${latitude}&longitude=${longitude}&hourly=apparent_temperature`;
  const response = await fetch(url);
  const weatherData = await response.json();
  return weatherData;
}

const tools = [
  {
    type: "function",
    function: {
      name: "getCurrentWeather",
      description: "Get the current weather in a given location",
      parameters: {
        type: "object",
        properties: {
          latitude: {
            type: "string",
          },
          longitude: {
            type: "string",
          },
        },
        required: ["longitude", "latitude"],
      },
    }
  },
  {
    type: "function",
    function: {
      name: "getLocation",
      description: "Get the user's location based on their IP address",
      parameters: {
        type: "object",
        properties: {},
      },
    }
  },
];

const availableTools = {
  getCurrentWeather,
  getLocation,
};

const messages = [
  {
    role: "system",
    content: `You are a helpful assistant. Only use the functions you have been provided with.`,
  },
];

async function agent(userInput) {
  messages.push({
    role: "user",
    content: userInput,
  });

  for (let i = 0; i < 5; i++) {
    const response = await openai.chat.completions.create({
      model: "gpt-4",
      messages: messages,
      tools: tools,
    });

    const { finish_reason, message } = response.choices[0];

    if (finish_reason === "tool_calls" && message.tool_calls) {
      const functionName = message.tool_calls[0].function.name;
      const functionToCall = availableTools[functionName];
      const functionArgs = JSON.parse(message.tool_calls[0].function.arguments);
      const functionArgsArr = Object.values(functionArgs);
      const functionResponse = await functionToCall.apply(
        null,
        functionArgsArr
      );

      messages.push({
        role: "function",
        name: functionName,
        content: `
                The result of the last function was this: ${JSON.stringify(
                  functionResponse
                )}
                `,
      });
    } else if (finish_reason === "stop") {
      messages.push(message);
      return message.content;
    }
  }
  return "The maximum number of iterations has been met without a suitable answer. Please try again with a more specific input.";
}

const response = await agent(
  "Please suggest some activities based on my location and the weather."
);

console.log("response:", response);
```

</details>


key: _node_content
string_value:
{"id_": "8632a754-7f5c-4e80-8d33-2607e6d67b66", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_processing_guide.ipynb", "title": "Enhancing Whisper transcriptions: pre- & post-processing techniques", "tokens": 4316, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "99649ca4-0175-4611-aa0f-7e9d989b22dc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_processing_guide.ipynb", "title": "Enhancing Whisper transcriptions: pre- & post-processing techniques", "tokens": 4316, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "00d33af3a6ebd0a2f548e2680375838359dc6c733bf1cfcd8d5af64d43cf3e38", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "d2801f78-b85a-408e-91c3-ea93bbac4763", "node_type": "1", "metadata": {}, "hash": "ce2a580a5afd89fd71b66665fe1b4b24f3e9d9034b478faeaf45c0e25150c635", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 0, "end_char_idx": 3155, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
# Enhancing Whisper transcriptions: pre- & post-processing techniques

This notebook offers a guide to improve the Whisper's transcriptions. We'll streamline your audio data via trimming and segmentation, enhancing Whisper's transcription quality. After transcriptions, we'll refine the output by adding punctuation, adjusting product terminology (e.g., 'five two nine' to '529'), and mitigating Unicode issues. These strategies will help improve the clarity of your transcriptions, but remember, customization based on your unique use-case may be beneficial.



## Setup

To get started let's import a few different libraries:

- [PyDub](http://pydub.com/) is a simple and easy-to-use Python library for audio processing tasks such as slicing, concatenating, and exporting audio files.

- The `Audio` class from the `IPython.display` module allows you to create an audio control that can play sound in Jupyter notebooks, providing a straightforward way to play audio data directly in your notebook.

- For our audio file, we'll use a fictional earnings call written by ChatGPT and read aloud by the author.This audio file is relatively short, but hopefully provides you with an illustrative idea of how these pre and post processing steps can be applied to any audio file. 


```python
from openai import OpenAI
import os
import urllib
from IPython.display import Audio
from pathlib import Path
from pydub import AudioSegment
import ssl
```


```python
client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY", "<your OpenAI API key if not set as env var>"))
```


```python
# set download paths
earnings_call_remote_filepath = "https://cdn.openai.com/API/examples/data/EarningsCall.wav"

# set local save locations
earnings_call_filepath = "data/EarningsCall.wav"

# download example audio files and save locally
ssl._create_default_https_context = ssl._create_unverified_context
urllib.request.urlretrieve(earnings_call_remote_filepath, earnings_call_filepath)
```




    ('data/EarningsCall.wav', <http.client.HTTPMessage at 0x11be41f50>)



At times, files with long silences at the beginning can cause Whisper to transcribe the audio incorrectly. We'll use Pydub to detect and trim the silence. 

Here, we've set the decibel threshold of 20. You can change this if you would like.


```python
# Function to detect leading silence
# Returns the number of milliseconds until the first sound (chunk averaging more than X decibels)
def milliseconds_until_sound(sound, silence_threshold_in_decibels=-20.0, chunk_size=10):
    trim_ms = 0  # ms

    assert chunk_size > 0  # to avoid infinite loop
    while sound[trim_ms:trim_ms+chunk_size].dBFS < silence_threshold_in_decibels and trim_ms < len(sound):
        trim_ms += chunk_size

    return trim_ms

```


```python
def trim_start(filepath):
    path = Path(filepath)
    directory = path.parent
    filename = path.name
    audio = AudioSegment.from_file(filepath, format="wav")
    start_trim = milliseconds_until_sound(audio)
    trimmed = audio[start_trim:]
    new_filename = directory / f"trimmed_{filename}"
    trimmed.export(new_filename, format="wav")
    return trimmed, new_filename

```


key: _node_content
string_value:
{"id_": "d2801f78-b85a-408e-91c3-ea93bbac4763", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_processing_guide.ipynb", "title": "Enhancing Whisper transcriptions: pre- & post-processing techniques", "tokens": 4316, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "99649ca4-0175-4611-aa0f-7e9d989b22dc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_processing_guide.ipynb", "title": "Enhancing Whisper transcriptions: pre- & post-processing techniques", "tokens": 4316, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "00d33af3a6ebd0a2f548e2680375838359dc6c733bf1cfcd8d5af64d43cf3e38", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "8632a754-7f5c-4e80-8d33-2607e6d67b66", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_processing_guide.ipynb", "title": "Enhancing Whisper transcriptions: pre- & post-processing techniques", "tokens": 4316, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "b7651bc69de64d6fe8c9ee5c72db26ab88727a028cc06721cb4a458e7c6768d7", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "431d2f3e-8d11-430a-b7ac-17a9679e30fd", "node_type": "1", "metadata": {}, "hash": "250343fa2b7446d5d05e80aa64a3da5a50a7ea2b7842df0bcafe033caa211721", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 1565, "end_char_idx": 5033, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
# set download paths
earnings_call_remote_filepath = "https://cdn.openai.com/API/examples/data/EarningsCall.wav"

# set local save locations
earnings_call_filepath = "data/EarningsCall.wav"

# download example audio files and save locally
ssl._create_default_https_context = ssl._create_unverified_context
urllib.request.urlretrieve(earnings_call_remote_filepath, earnings_call_filepath)
```




    ('data/EarningsCall.wav', <http.client.HTTPMessage at 0x11be41f50>)



At times, files with long silences at the beginning can cause Whisper to transcribe the audio incorrectly. We'll use Pydub to detect and trim the silence. 

Here, we've set the decibel threshold of 20. You can change this if you would like.


```python
# Function to detect leading silence
# Returns the number of milliseconds until the first sound (chunk averaging more than X decibels)
def milliseconds_until_sound(sound, silence_threshold_in_decibels=-20.0, chunk_size=10):
    trim_ms = 0  # ms

    assert chunk_size > 0  # to avoid infinite loop
    while sound[trim_ms:trim_ms+chunk_size].dBFS < silence_threshold_in_decibels and trim_ms < len(sound):
        trim_ms += chunk_size

    return trim_ms

```


```python
def trim_start(filepath):
    path = Path(filepath)
    directory = path.parent
    filename = path.name
    audio = AudioSegment.from_file(filepath, format="wav")
    start_trim = milliseconds_until_sound(audio)
    trimmed = audio[start_trim:]
    new_filename = directory / f"trimmed_{filename}"
    trimmed.export(new_filename, format="wav")
    return trimmed, new_filename

```


```python
def transcribe_audio(file,output_dir):
    audio_path = os.path.join(output_dir, file)
    with open(audio_path, 'rb') as audio_data:
        transcription = client.audio.transcriptions.create(
            model="whisper-1", file=audio_data)
        return transcription.text
```

At times, we've seen unicode character injection in transcripts, removing any non-ASCII characters should help mitigate this issue.

Keep in mind you should not use this function if you are transcribing in Greek, Cyrillic, Arabic, Chinese, etc


```python
# Define function to remove non-ascii characters
def remove_non_ascii(text):
    return ''.join(i for i in text if ord(i)<128)

```

This function will add formatting and punctuation to our transcript. Whisper generates a transcript with punctuation but without formatting.


```python
# Define function to add punctuation
def punctuation_assistant(ascii_transcript):

    system_prompt = """You are a helpful assistant that adds punctuation to text.
      Preserve the original words and only insert necessary punctuation such as periods,
     commas, capialization, symbols like dollar sings or percentage signs, and formatting.
     Use only the context provided. If there is no context provided say, 'No context provided'\n"""
    response = client.chat.completions.create(
        model="gpt-3.5-turbo",
        temperature=0,
        messages=[
            {
                "role": "system",
                "content": system_prompt
            },
            {
                "role": "user",
                "content": ascii_transcript
            }
        ]
    )
    return response

```

Our audio file is a recording from a fake earnings call that includes a lot of financial products. This function can help ensure that if Whisper transcribes these financial product names incorrectly, that they can be corrected.


key: _node_content
string_value:
{"id_": "431d2f3e-8d11-430a-b7ac-17a9679e30fd", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_processing_guide.ipynb", "title": "Enhancing Whisper transcriptions: pre- & post-processing techniques", "tokens": 4316, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "99649ca4-0175-4611-aa0f-7e9d989b22dc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_processing_guide.ipynb", "title": "Enhancing Whisper transcriptions: pre- & post-processing techniques", "tokens": 4316, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "00d33af3a6ebd0a2f548e2680375838359dc6c733bf1cfcd8d5af64d43cf3e38", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "d2801f78-b85a-408e-91c3-ea93bbac4763", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_processing_guide.ipynb", "title": "Enhancing Whisper transcriptions: pre- & post-processing techniques", "tokens": 4316, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "49ccad75f62a6bd49d801c93e18df1d5c73e31299188f0771c64529518f6a371", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "308df467-d6a3-4f7c-9ab8-77a1daa33d6e", "node_type": "1", "metadata": {}, "hash": "4e9490c34d3fe134717faef8ddbacf62f2eefefbeea076dcecf05bc590620988", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 3158, "end_char_idx": 7246, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
def transcribe_audio(file,output_dir):
    audio_path = os.path.join(output_dir, file)
    with open(audio_path, 'rb') as audio_data:
        transcription = client.audio.transcriptions.create(
            model="whisper-1", file=audio_data)
        return transcription.text
```

At times, we've seen unicode character injection in transcripts, removing any non-ASCII characters should help mitigate this issue.

Keep in mind you should not use this function if you are transcribing in Greek, Cyrillic, Arabic, Chinese, etc


```python
# Define function to remove non-ascii characters
def remove_non_ascii(text):
    return ''.join(i for i in text if ord(i)<128)

```

This function will add formatting and punctuation to our transcript. Whisper generates a transcript with punctuation but without formatting.


```python
# Define function to add punctuation
def punctuation_assistant(ascii_transcript):

    system_prompt = """You are a helpful assistant that adds punctuation to text.
      Preserve the original words and only insert necessary punctuation such as periods,
     commas, capialization, symbols like dollar sings or percentage signs, and formatting.
     Use only the context provided. If there is no context provided say, 'No context provided'\n"""
    response = client.chat.completions.create(
        model="gpt-3.5-turbo",
        temperature=0,
        messages=[
            {
                "role": "system",
                "content": system_prompt
            },
            {
                "role": "user",
                "content": ascii_transcript
            }
        ]
    )
    return response

```

Our audio file is a recording from a fake earnings call that includes a lot of financial products. This function can help ensure that if Whisper transcribes these financial product names incorrectly, that they can be corrected. 


```python
# Define function to fix product mispellings
def product_assistant(ascii_transcript):
    system_prompt = """You are an intelligent assistant specializing in financial products;
    your task is to process transcripts of earnings calls, ensuring that all references to
     financial products and common financial terms are in the correct format. For each
     financial product or common term that is typically abbreviated as an acronym, the full term 
    should be spelled out followed by the acronym in parentheses. For example, '401k' should be
     transformed to '401(k) retirement savings plan', 'HSA' should be transformed to 'Health Savings Account (HSA)'
    , 'ROA' should be transformed to 'Return on Assets (ROA)', 'VaR' should be transformed to 'Value at Risk (VaR)'
, and 'PB' should be transformed to 'Price to Book (PB) ratio'. Similarly, transform spoken numbers representing 
financial products into their numeric representations, followed by the full name of the product in parentheses. 
For instance, 'five two nine' to '529 (Education Savings Plan)' and 'four zero one k' to '401(k) (Retirement Savings Plan)'.
 However, be aware that some acronyms can have different meanings based on the context (e.g., 'LTV' can stand for 
'Loan to Value' or 'Lifetime Value'). You will need to discern from the context which term is being referred to 
and apply the appropriate transformation. In cases where numerical figures or metrics are spelled out but do not 
represent specific financial products (like 'twenty three percent'), these should be left as is. Your role is to
 analyze and adjust financial product terminology in the text. Once you've done that, produce the adjusted 
 transcript and a list of the words you've changed"""
    response = client.chat.completions.create(
        model="gpt-4",
        temperature=0,
        messages=[
            {
                "role": "system",
                "content": system_prompt
            },
            {
                "role": "user",
                "content": ascii_transcript
            }
        ]
    )
    return response

```

This function will create a new file with 'trimmed' appended to the original file name


key: _node_content
string_value:
{"id_": "308df467-d6a3-4f7c-9ab8-77a1daa33d6e", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_processing_guide.ipynb", "title": "Enhancing Whisper transcriptions: pre- & post-processing techniques", "tokens": 4316, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "99649ca4-0175-4611-aa0f-7e9d989b22dc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_processing_guide.ipynb", "title": "Enhancing Whisper transcriptions: pre- & post-processing techniques", "tokens": 4316, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "00d33af3a6ebd0a2f548e2680375838359dc6c733bf1cfcd8d5af64d43cf3e38", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "431d2f3e-8d11-430a-b7ac-17a9679e30fd", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_processing_guide.ipynb", "title": "Enhancing Whisper transcriptions: pre- & post-processing techniques", "tokens": 4316, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "357779f326a151f346110788130a62a8f03004c2ebb0252283cfca70bf0a2215", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "4b18c247-de9c-46e4-a0df-a63370b3c500", "node_type": "1", "metadata": {}, "hash": "89ee532b7d56a10764bc114f8dd5f5ed9b8e09da8c99c87e082d0e39eca7e8b8", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 7249, "end_char_idx": 9037, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
# Trim the start of the original audio file
trimmed_audio = trim_start(earnings_call_filepath)

```


```python
trimmed_audio, trimmed_filename = trim_start(earnings_call_filepath)

```

Our fake earnings report audio file is fairly short in length, so we'll adjust the segments accordingly. Keep in mind you can adjust the segment length as you need.


```python
# Segment audio
trimmed_audio = AudioSegment.from_wav(trimmed_filename)  # Load the trimmed audio file

one_minute = 1 * 60 * 1000  # Duration for each segment (in milliseconds)

start_time = 0  # Start time for the first segment

i = 0  # Index for naming the segmented files

output_dir_trimmed = "trimmed_earnings_directory"  # Output directory for the segmented files

if not os.path.isdir(output_dir_trimmed):  # Create the output directory if it does not exist
    os.makedirs(output_dir_trimmed)

while start_time < len(trimmed_audio):  # Loop over the trimmed audio file
    segment = trimmed_audio[start_time:start_time + one_minute]  # Extract a segment
    segment.export(os.path.join(output_dir_trimmed, f"trimmed_{i:02d}.wav"), format="wav")  # Save the segment
    start_time += one_minute  # Update the start time for the next segment
    i += 1  # Increment the index for naming the next file

```


```python
# Get list of trimmed and segmented audio files and sort them numerically
audio_files = sorted(
    (f for f in os.listdir(output_dir_trimmed) if f.endswith(".wav")),
    key=lambda f: int(''.join(filter(str.isdigit, f)))
)

```


```python
# Use a loop to apply the transcribe function to all audio files
transcriptions = [transcribe_audio(file, output_dir_trimmed) for file in audio_files]

```


```python
# Concatenate the transcriptions
full_transcript = ' '.join(transcriptions)
```


key: _node_content
string_value:
{"id_": "4b18c247-de9c-46e4-a0df-a63370b3c500", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_processing_guide.ipynb", "title": "Enhancing Whisper transcriptions: pre- & post-processing techniques", "tokens": 4316, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "99649ca4-0175-4611-aa0f-7e9d989b22dc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_processing_guide.ipynb", "title": "Enhancing Whisper transcriptions: pre- & post-processing techniques", "tokens": 4316, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "00d33af3a6ebd0a2f548e2680375838359dc6c733bf1cfcd8d5af64d43cf3e38", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "308df467-d6a3-4f7c-9ab8-77a1daa33d6e", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_processing_guide.ipynb", "title": "Enhancing Whisper transcriptions: pre- & post-processing techniques", "tokens": 4316, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "6f09397fb6a831915488e7a6c399aecefddce672c8539c1d2d5ea5c844268d17", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "a27d6f50-5aa0-4818-8f18-206c89742da6", "node_type": "1", "metadata": {}, "hash": "72c5e5e2d75b959ff6b4aca1db084e20fe4e3fba4b65ec1826257384dc8a8d30", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 7613, "end_char_idx": 11303, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
# Segment audio
trimmed_audio = AudioSegment.from_wav(trimmed_filename)  # Load the trimmed audio file

one_minute = 1 * 60 * 1000  # Duration for each segment (in milliseconds)

start_time = 0  # Start time for the first segment

i = 0  # Index for naming the segmented files

output_dir_trimmed = "trimmed_earnings_directory"  # Output directory for the segmented files

if not os.path.isdir(output_dir_trimmed):  # Create the output directory if it does not exist
    os.makedirs(output_dir_trimmed)

while start_time < len(trimmed_audio):  # Loop over the trimmed audio file
    segment = trimmed_audio[start_time:start_time + one_minute]  # Extract a segment
    segment.export(os.path.join(output_dir_trimmed, f"trimmed_{i:02d}.wav"), format="wav")  # Save the segment
    start_time += one_minute  # Update the start time for the next segment
    i += 1  # Increment the index for naming the next file

```


```python
# Get list of trimmed and segmented audio files and sort them numerically
audio_files = sorted(
    (f for f in os.listdir(output_dir_trimmed) if f.endswith(".wav")),
    key=lambda f: int(''.join(filter(str.isdigit, f)))
)

```


```python
# Use a loop to apply the transcribe function to all audio files
transcriptions = [transcribe_audio(file, output_dir_trimmed) for file in audio_files]

```


```python
# Concatenate the transcriptions
full_transcript = ' '.join(transcriptions)
```


```python
print(full_transcript)
```

    Good afternoon, everyone. And welcome to FinTech Plus Sync's second quarter 2023 earnings call. I'm John Doe, CEO of FinTech Plus. We've had a stellar Q2 with a revenue of 125 million, a 25% increase year over year. Our gross profit margin stands at a solid 58%, due in part to cost efficiencies gained from our scalable business model. Our EBITDA has surged to 37.5 million, translating to a remarkable 30% EBITDA margin. Our net income for the quarter rose to 16 million, which is a noteworthy increase from 10 million in Q2 2022. Our total addressable market has grown substantially thanks to the expansion of our high yield savings product line and the new RoboAdvisor platform. We've been diversifying our asset-backed securities portfolio, investing heavily in collateralized. debt obligations, and residential mortgage-backed securities. We've also invested $25 million in AAA rated corporate bonds, enhancing our risk adjusted returns. As for our balance sheet, total assets reached $1.5 billion with total liabilities at $900 million, leaving us with a solid equity base of $600 million. Our debt-to-equity ratio stands at 1.5, a healthy figure considering our expansionary phase. We continue to see substantial organic user growth, with customer acquisition cost dropping by 15% and lifetime value growing by 25%. Our LTVCAC ratio is at an impressive 3.5%. In terms of risk management, we have a value-at-risk model in place with a 99%... confidence level indicating that our maximum loss will not exceed 5 million in the next trading day. We've adopted a conservative approach to managing our leverage and have a healthy tier one capital ratio of 12.5%. Our forecast for the coming quarter is positive. We expect revenue to be around 135 million and 8% quarter over quarter growth driven primarily by our cutting edge blockchain solutions and AI driven predictive analytics. We're also excited about the upcoming IPO of our FinTech subsidiary Pay Plus, which we expect to raise 200 million, significantly bolstering our liquidity and paving the way for aggressive growth strategies. We thank our shareholders for their continued faith in us and we look forward to an even more successful Q3. Thank you so much.


key: _node_content
string_value:
{"id_": "a27d6f50-5aa0-4818-8f18-206c89742da6", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_processing_guide.ipynb", "title": "Enhancing Whisper transcriptions: pre- & post-processing techniques", "tokens": 4316, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "99649ca4-0175-4611-aa0f-7e9d989b22dc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_processing_guide.ipynb", "title": "Enhancing Whisper transcriptions: pre- & post-processing techniques", "tokens": 4316, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "00d33af3a6ebd0a2f548e2680375838359dc6c733bf1cfcd8d5af64d43cf3e38", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "4b18c247-de9c-46e4-a0df-a63370b3c500", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_processing_guide.ipynb", "title": "Enhancing Whisper transcriptions: pre- & post-processing techniques", "tokens": 4316, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "2fbe822e0a51bc2f50a17fb982f43a8771ec66786efe940e8cb23eb539c42705", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "5c7a825c-6793-48a1-9fbf-b0c5d991b7f7", "node_type": "1", "metadata": {}, "hash": "a09ab78b36bb4244b7e67208f56c917b7256e350437cf29fc68c3ac79326f7b7", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 11307, "end_char_idx": 13934, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
# Remove non-ascii characters from the transcript
ascii_transcript = remove_non_ascii(full_transcript)
```


```python
print(ascii_transcript)
```

    Good afternoon, everyone. And welcome to FinTech Plus Sync's second quarter 2023 earnings call. I'm John Doe, CEO of FinTech Plus. We've had a stellar Q2 with a revenue of 125 million, a 25% increase year over year. Our gross profit margin stands at a solid 58%, due in part to cost efficiencies gained from our scalable business model. Our EBITDA has surged to 37.5 million, translating to a remarkable 30% EBITDA margin. Our net income for the quarter rose to 16 million, which is a noteworthy increase from 10 million in Q2 2022. Our total addressable market has grown substantially thanks to the expansion of our high yield savings product line and the new RoboAdvisor platform. We've been diversifying our asset-backed securities portfolio, investing heavily in collateralized. debt obligations, and residential mortgage-backed securities. We've also invested $25 million in AAA rated corporate bonds, enhancing our risk adjusted returns. As for our balance sheet, total assets reached $1.5 billion with total liabilities at $900 million, leaving us with a solid equity base of $600 million. Our debt-to-equity ratio stands at 1.5, a healthy figure considering our expansionary phase. We continue to see substantial organic user growth, with customer acquisition cost dropping by 15% and lifetime value growing by 25%. Our LTVCAC ratio is at an impressive 3.5%. In terms of risk management, we have a value-at-risk model in place with a 99%... confidence level indicating that our maximum loss will not exceed 5 million in the next trading day. We've adopted a conservative approach to managing our leverage and have a healthy tier one capital ratio of 12.5%. Our forecast for the coming quarter is positive. We expect revenue to be around 135 million and 8% quarter over quarter growth driven primarily by our cutting edge blockchain solutions and AI driven predictive analytics. We're also excited about the upcoming IPO of our FinTech subsidiary Pay Plus, which we expect to raise 200 million, significantly bolstering our liquidity and paving the way for aggressive growth strategies. We thank our shareholders for their continued faith in us and we look forward to an even more successful Q3. Thank you so much.



```python
# Use punctuation assistant function
response = punctuation_assistant(ascii_transcript)
```


```python
# Extract the punctuated transcript from the model's response
punctuated_transcript = response.choices[0].message.content

```


key: _node_content
string_value:
{"id_": "5c7a825c-6793-48a1-9fbf-b0c5d991b7f7", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_processing_guide.ipynb", "title": "Enhancing Whisper transcriptions: pre- & post-processing techniques", "tokens": 4316, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "99649ca4-0175-4611-aa0f-7e9d989b22dc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_processing_guide.ipynb", "title": "Enhancing Whisper transcriptions: pre- & post-processing techniques", "tokens": 4316, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "00d33af3a6ebd0a2f548e2680375838359dc6c733bf1cfcd8d5af64d43cf3e38", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "a27d6f50-5aa0-4818-8f18-206c89742da6", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_processing_guide.ipynb", "title": "Enhancing Whisper transcriptions: pre- & post-processing techniques", "tokens": 4316, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "97c25333f44d3c8728d39c9b09730c39b08446c785a9a9f55e991beaa4e4cc39", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "482495fd-b8d0-40a0-bc18-8853d144969d", "node_type": "1", "metadata": {}, "hash": "7a40d3431e7ccc2e456c6789aa456a094be82379c6df6fd6e6fc3e28e4597763", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 13694, "end_char_idx": 16440, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
# Use punctuation assistant function
response = punctuation_assistant(ascii_transcript)
```


```python
# Extract the punctuated transcript from the model's response
punctuated_transcript = response.choices[0].message.content

```


```python
print(punctuated_transcript)
```

    Good afternoon, everyone. And welcome to FinTech Plus Sync's second quarter 2023 earnings call. I'm John Doe, CEO of FinTech Plus. We've had a stellar Q2 with a revenue of $125 million, a 25% increase year over year. Our gross profit margin stands at a solid 58%, due in part to cost efficiencies gained from our scalable business model. Our EBITDA has surged to $37.5 million, translating to a remarkable 30% EBITDA margin. Our net income for the quarter rose to $16 million, which is a noteworthy increase from $10 million in Q2 2022. Our total addressable market has grown substantially thanks to the expansion of our high yield savings product line and the new RoboAdvisor platform. We've been diversifying our asset-backed securities portfolio, investing heavily in collateralized debt obligations, and residential mortgage-backed securities. We've also invested $25 million in AAA rated corporate bonds, enhancing our risk-adjusted returns. As for our balance sheet, total assets reached $1.5 billion with total liabilities at $900 million, leaving us with a solid equity base of $600 million. Our debt-to-equity ratio stands at 1.5, a healthy figure considering our expansionary phase. We continue to see substantial organic user growth, with customer acquisition cost dropping by 15% and lifetime value growing by 25%. Our LTVCAC ratio is at an impressive 3.5%. In terms of risk management, we have a value-at-risk model in place with a 99% confidence level indicating that our maximum loss will not exceed $5 million in the next trading day. We've adopted a conservative approach to managing our leverage and have a healthy tier one capital ratio of 12.5%. Our forecast for the coming quarter is positive. We expect revenue to be around $135 million and 8% quarter over quarter growth driven primarily by our cutting-edge blockchain solutions and AI-driven predictive analytics. We're also excited about the upcoming IPO of our FinTech subsidiary Pay Plus, which we expect to raise $200 million, significantly bolstering our liquidity and paving the way for aggressive growth strategies. We thank our shareholders for their continued faith in us and we look forward to an even more successful Q3. Thank you so much.



```python
# Use product assistant function
response = product_assistant(punctuated_transcript)

```


```python
# Extract the final transcript from the model's response
final_transcript = response.choices[0].message.content
```


key: _node_content
string_value:
{"id_": "482495fd-b8d0-40a0-bc18-8853d144969d", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_processing_guide.ipynb", "title": "Enhancing Whisper transcriptions: pre- & post-processing techniques", "tokens": 4316, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "99649ca4-0175-4611-aa0f-7e9d989b22dc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_processing_guide.ipynb", "title": "Enhancing Whisper transcriptions: pre- & post-processing techniques", "tokens": 4316, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "00d33af3a6ebd0a2f548e2680375838359dc6c733bf1cfcd8d5af64d43cf3e38", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "5c7a825c-6793-48a1-9fbf-b0c5d991b7f7", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Whisper_processing_guide.ipynb", "title": "Enhancing Whisper transcriptions: pre- & post-processing techniques", "tokens": 4316, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "9e665a2e34416e208af214fe9d33bbe613316b793a4315ab49b946bb56715b03", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 16213, "end_char_idx": 19472, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
# Use product assistant function
response = product_assistant(punctuated_transcript)

```


```python
# Extract the final transcript from the model's response
final_transcript = response.choices[0].message.content
```


```python
print(final_transcript)
```

    Good afternoon, everyone. And welcome to FinTech Plus Sync's second quarter 2023 earnings call. I'm John Doe, CEO of FinTech Plus. We've had a stellar second quarter (Q2) with a revenue of $125 million, a 25% increase year over year. Our gross profit margin stands at a solid 58%, due in part to cost efficiencies gained from our scalable business model. Our Earnings Before Interest, Taxes, Depreciation, and Amortization (EBITDA) has surged to $37.5 million, translating to a remarkable 30% EBITDA margin. Our net income for the quarter rose to $16 million, which is a noteworthy increase from $10 million in second quarter (Q2) 2022. Our total addressable market has grown substantially thanks to the expansion of our high yield savings product line and the new RoboAdvisor platform. We've been diversifying our asset-backed securities portfolio, investing heavily in Collateralized Debt Obligations (CDOs), and Residential Mortgage-Backed Securities (RMBS). We've also invested $25 million in AAA rated corporate bonds, enhancing our risk-adjusted returns. As for our balance sheet, total assets reached $1.5 billion with total liabilities at $900 million, leaving us with a solid equity base of $600 million. Our Debt-to-Equity (D/E) ratio stands at 1.5, a healthy figure considering our expansionary phase. We continue to see substantial organic user growth, with Customer Acquisition Cost (CAC) dropping by 15% and Lifetime Value (LTV) growing by 25%. Our LTV to CAC (LTVCAC) ratio is at an impressive 3.5%. In terms of risk management, we have a Value at Risk (VaR) model in place with a 99% confidence level indicating that our maximum loss will not exceed $5 million in the next trading day. We've adopted a conservative approach to managing our leverage and have a healthy Tier 1 Capital ratio of 12.5%. Our forecast for the coming quarter is positive. We expect revenue to be around $135 million and 8% quarter over quarter growth driven primarily by our cutting-edge blockchain solutions and AI-driven predictive analytics. We're also excited about the upcoming Initial Public Offering (IPO) of our FinTech subsidiary Pay Plus, which we expect to raise $200 million, significantly bolstering our liquidity and paving the way for aggressive growth strategies. We thank our shareholders for their continued faith in us and we look forward to an even more successful third quarter (Q3). Thank you so much.
    
    Words Changed:
    1. Q2 -> second quarter (Q2)
    2. EBITDA -> Earnings Before Interest, Taxes, Depreciation, and Amortization (EBITDA)
    3. Q2 2022 -> second quarter (Q2) 2022
    4. CDOs -> Collateralized Debt Obligations (CDOs)
    5. RMBS -> Residential Mortgage-Backed Securities (RMBS)
    6. D/E -> Debt-to-Equity (D/E)
    7. CAC -> Customer Acquisition Cost (CAC)
    8. LTV -> Lifetime Value (LTV)
    9. LTVCAC -> LTV to CAC (LTVCAC)
    10. VaR -> Value at Risk (VaR)
    11. IPO -> Initial Public Offering (IPO)
    12. Q3 -> third quarter (Q3)


key: url
string_value:
https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_a_search_API.ipynb


key: _node_content
string_value:
{"id_": "024b85c5-cea2-456c-872c-f8e7370d7bd8", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_a_search_API.ipynb", "title": "Question answering using a search API and re-ranking", "tokens": 4009, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "1f39c952-9294-45e5-a9b4-ccb3c1e0dfdc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_a_search_API.ipynb", "title": "Question answering using a search API and re-ranking", "tokens": 4009, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "0b5f2f8ff663acd44e24d8fe522fec60044e4f369875d21f6b9074e4c97a4414", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "165d94e4-87e0-46b4-bc47-bd340b1bf883", "node_type": "1", "metadata": {}, "hash": "c4254d8c62443e86b16503c909db9f2a9579fa2e50c7ccc6501a14d591220afa", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 0, "end_char_idx": 2849, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
# Question answering using a search API and re-ranking

Searching for relevant information can sometimes feel like looking for a needle in a haystack, but don’t despair, GPTs can actually do a lot of this work for us. In this guide we explore a way to augment existing search systems with various AI techniques, helping us sift through the noise.

Two ways of retrieving information for GPT are:

1. **Mimicking Human Browsing:** [GPT triggers a search](https://openai.com/blog/chatgpt-plugins#browsing), evaluates the results, and modifies the search query if necessary. It can also follow up on specific search results to form a chain of thought, much like a human user would do.
2. **Retrieval with Embeddings:** Calculate [embeddings](https://platform.openai.com/docs/guides/embeddings) for your content and a user query, and then [retrieve the content](Question_answering_using_embeddings.ipynb) most related as measured by cosine similarity. This technique is [used heavily](https://blog.google/products/search/search-language-understanding-bert/) by search engines like Google.

These approaches are both promising, but each has their shortcomings: the first one can be slow due to its iterative nature and the second one requires embedding your entire knowledge base in advance, continuously embedding new content and maintaining a vector database.

By combining these approaches, and drawing inspiration from [re-ranking](https://www.sbert.net/examples/applications/retrieve_rerank/README.html) methods, we identify an approach that sits in the middle. **This approach can be implemented on top of any existing search system, like the Slack search API, or an internal ElasticSearch instance with private data**. Here’s how it works:

![search_augmented_by_query_generation_and_embeddings_reranking.png](../images/search_rerank_answer.png)

**Step 1: Search**

1.  User asks a question.
2.  GPT generates a list of potential queries.
3.  Search queries are executed in parallel.

**Step 2: Re-rank**

1.  Embeddings for each result are used to calculate semantic similarity to a generated hypothetical ideal answer to the user question.
2.  Results are ranked and filtered based on this similarity metric.

**Step 3: Answer**

1.  Given the top search results, the model generates an answer to the user’s question, including references and links.

This hybrid approach offers relatively low latency and can be integrated into any existing search endpoint, without requiring the upkeep of a vector database. Let's dive into it! We will use the [News API](https://newsapi.org/) as an example domain to search over.

## Setup

In addition to your `OPENAI_API_KEY`, you'll have to include a `NEWS_API_KEY` in your environment. You can get an API key [here](https://newsapi.org/).



```python
%%capture
%env NEWS_API_KEY = YOUR_NEWS_API_KEY

```


key: _node_content
string_value:
{"id_": "165d94e4-87e0-46b4-bc47-bd340b1bf883", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_a_search_API.ipynb", "title": "Question answering using a search API and re-ranking", "tokens": 4009, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "1f39c952-9294-45e5-a9b4-ccb3c1e0dfdc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_a_search_API.ipynb", "title": "Question answering using a search API and re-ranking", "tokens": 4009, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "0b5f2f8ff663acd44e24d8fe522fec60044e4f369875d21f6b9074e4c97a4414", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "024b85c5-cea2-456c-872c-f8e7370d7bd8", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_a_search_API.ipynb", "title": "Question answering using a search API and re-ranking", "tokens": 4009, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "f9efdd4af98838185fdfa9d6cfe18acdcb7e0710662f83d0478429bb43ec299d", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "e9d5395c-50fc-4808-8cd0-69061f96418d", "node_type": "1", "metadata": {}, "hash": "ed9d4b732ef247f18cba6b0c12366e7e1a84df9d7dc3bb4c031147fceb797ccf", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 2787, "end_char_idx": 5980, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
%%capture
%env NEWS_API_KEY = YOUR_NEWS_API_KEY

```


```python
# Dependencies
from datetime import date, timedelta  # date handling for fetching recent news
from IPython import display  # for pretty printing
import json  # for parsing the JSON api responses and model outputs
from numpy import dot  # for cosine similarity
from openai import OpenAI
import os  # for loading environment variables
import requests  # for making the API requests
from tqdm.notebook import tqdm  # for printing progress bars

client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY", "<your OpenAI API key if not set as env var>"))

# Load environment variables
news_api_key = os.getenv("NEWS_API_KEY")

GPT_MODEL = "gpt-3.5-turbo"


# Helper functions
def json_gpt(input: str):
    completion = client.chat.completions.create(model=GPT_MODEL,
    messages=[
        {"role": "system", "content": "Output only valid JSON"},
        {"role": "user", "content": input},
    ],
    temperature=0.5)

    text = completion.choices[0].message.content
    parsed = json.loads(text)

    return parsed


def embeddings(input: list[str]) -> list[list[str]]:
    response = client.embeddings.create(model="text-embedding-3-small", input=input)
    return [data.embedding for data in response.data]
```

## 1. Search

It all starts with a user question.



```python
# User asks a question
USER_QUESTION = "Who won the NBA championship? And who was the MVP? Tell me a bit about the last game."
```

Now, in order to be as exhaustive as possible, we use the model to generate a list of diverse queries based on this question.



```python
QUERIES_INPUT = f"""
You have access to a search API that returns recent news articles.
Generate an array of search queries that are relevant to this question.
Use a variation of related keywords for the queries, trying to be as general as possible.
Include as many queries as you can think of, including and excluding terms.
For example, include queries like ['keyword_1 keyword_2', 'keyword_1', 'keyword_2'].
Be creative. The more queries you include, the more likely you are to find relevant results.

User question: {USER_QUESTION}

Format: {{"queries": ["query_1", "query_2", "query_3"]}}
"""

queries = json_gpt(QUERIES_INPUT)["queries"]

# Let's include the original question as well for good measure
queries.append(USER_QUESTION)

queries
```




    ['NBA championship winner',
     'MVP of NBA championship',
     'Last game of NBA championship',
     'NBA finals winner',
     'Most valuable player of NBA championship',
     'Finals game of NBA',
     'Who won the NBA finals',
     'NBA championship game summary',
     'NBA finals MVP',
     'Champion of NBA playoffs',
     'NBA finals last game highlights',
     'NBA championship series result',
     'NBA finals game score',
     'NBA finals game recap',
     'NBA champion team and player',
     'NBA finals statistics',
     'NBA championship final score',
     'NBA finals best player',
     'NBA playoffs champion and MVP',
     'NBA finals game analysis',
     'Who won the NBA championship? And who was the MVP? Tell me a bit about the last game.']



The queries look good, so let's run the searches.


key: _node_content
string_value:
{"id_": "e9d5395c-50fc-4808-8cd0-69061f96418d", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_a_search_API.ipynb", "title": "Question answering using a search API and re-ranking", "tokens": 4009, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "1f39c952-9294-45e5-a9b4-ccb3c1e0dfdc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_a_search_API.ipynb", "title": "Question answering using a search API and re-ranking", "tokens": 4009, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "0b5f2f8ff663acd44e24d8fe522fec60044e4f369875d21f6b9074e4c97a4414", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "165d94e4-87e0-46b4-bc47-bd340b1bf883", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_a_search_API.ipynb", "title": "Question answering using a search API and re-ranking", "tokens": 4009, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "13439823af75edc32c4b542741e29f51770ba0f981f66fe7d15e272f97cd3100", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "14b182b5-550c-4f83-b242-dec04fe08c01", "node_type": "1", "metadata": {}, "hash": "aeb0caee2ade7eac44215e7ed07742a4daf74ee655d796dee1294b507b632be2", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 4395, "end_char_idx": 7202, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
QUERIES_INPUT = f"""
You have access to a search API that returns recent news articles.
Generate an array of search queries that are relevant to this question.
Use a variation of related keywords for the queries, trying to be as general as possible.
Include as many queries as you can think of, including and excluding terms.
For example, include queries like ['keyword_1 keyword_2', 'keyword_1', 'keyword_2'].
Be creative. The more queries you include, the more likely you are to find relevant results.

User question: {USER_QUESTION}

Format: {{"queries": ["query_1", "query_2", "query_3"]}}
"""

queries = json_gpt(QUERIES_INPUT)["queries"]

# Let's include the original question as well for good measure
queries.append(USER_QUESTION)

queries
```




    ['NBA championship winner',
     'MVP of NBA championship',
     'Last game of NBA championship',
     'NBA finals winner',
     'Most valuable player of NBA championship',
     'Finals game of NBA',
     'Who won the NBA finals',
     'NBA championship game summary',
     'NBA finals MVP',
     'Champion of NBA playoffs',
     'NBA finals last game highlights',
     'NBA championship series result',
     'NBA finals game score',
     'NBA finals game recap',
     'NBA champion team and player',
     'NBA finals statistics',
     'NBA championship final score',
     'NBA finals best player',
     'NBA playoffs champion and MVP',
     'NBA finals game analysis',
     'Who won the NBA championship? And who was the MVP? Tell me a bit about the last game.']



The queries look good, so let's run the searches.



```python
def search_news(
    query: str,
    news_api_key: str = news_api_key,
    num_articles: int = 50,
    from_datetime: str = "2023-06-01",  # the 2023 NBA finals were played in June 2023
    to_datetime: str = "2023-06-30",
) -> dict:
    response = requests.get(
        "https://newsapi.org/v2/everything",
        params={
            "q": query,
            "apiKey": news_api_key,
            "pageSize": num_articles,
            "sortBy": "relevancy",
            "from": from_datetime,
            "to": to_datetime,
        },
    )

    return response.json()


articles = []

for query in tqdm(queries):
    result = search_news(query)
    if result["status"] == "ok":
        articles = articles + result["articles"]
    else:
        raise Exception(result["message"])

# remove duplicates
articles = list({article["url"]: article for article in articles}.values())

print("Total number of articles:", len(articles))
print("Top 5 articles of query 1:", "\n")

for article in articles[0:5]:
    print("Title:", article["title"])
    print("Description:", article["description"])
    print("Content:", article["content"][0:100] + "...")
    print()

```


      0%|          | 0/21 [00:00<?, ?it/s]


key: _node_content
string_value:
{"id_": "14b182b5-550c-4f83-b242-dec04fe08c01", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_a_search_API.ipynb", "title": "Question answering using a search API and re-ranking", "tokens": 4009, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "1f39c952-9294-45e5-a9b4-ccb3c1e0dfdc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_a_search_API.ipynb", "title": "Question answering using a search API and re-ranking", "tokens": 4009, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "0b5f2f8ff663acd44e24d8fe522fec60044e4f369875d21f6b9074e4c97a4414", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "e9d5395c-50fc-4808-8cd0-69061f96418d", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_a_search_API.ipynb", "title": "Question answering using a search API and re-ranking", "tokens": 4009, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "21145a569215dd4ee4d6d59454adcfefd82c350811fb50a5c27a9b64041a3ee9", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "a39241b0-d3c5-4e9e-a9f7-43b571f59c67", "node_type": "1", "metadata": {}, "hash": "4925777faecc5576431ca759e371dfac1edc1e951f5c6a42f16d812accbc3b01", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 5931, "end_char_idx": 9828, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
The queries look good, so let's run the searches.



```python
def search_news(
    query: str,
    news_api_key: str = news_api_key,
    num_articles: int = 50,
    from_datetime: str = "2023-06-01",  # the 2023 NBA finals were played in June 2023
    to_datetime: str = "2023-06-30",
) -> dict:
    response = requests.get(
        "https://newsapi.org/v2/everything",
        params={
            "q": query,
            "apiKey": news_api_key,
            "pageSize": num_articles,
            "sortBy": "relevancy",
            "from": from_datetime,
            "to": to_datetime,
        },
    )

    return response.json()


articles = []

for query in tqdm(queries):
    result = search_news(query)
    if result["status"] == "ok":
        articles = articles + result["articles"]
    else:
        raise Exception(result["message"])

# remove duplicates
articles = list({article["url"]: article for article in articles}.values())

print("Total number of articles:", len(articles))
print("Top 5 articles of query 1:", "\n")

for article in articles[0:5]:
    print("Title:", article["title"])
    print("Description:", article["description"])
    print("Content:", article["content"][0:100] + "...")
    print()

```


      0%|          | 0/21 [00:00<?, ?it/s]


    Total number of articles: 554
    Top 5 articles of query 1: 
    
    Title: Nascar takes on Le Mans as LeBron James gets centenary race under way
    Description: <ul><li>Nascar has presence at iconic race for first time since 1976</li><li>NBA superstar LeBron James waves flag as honorary starter</li></ul>The crowd chanted “U-S-A! U-S-A!” as Nascar driver lineup for the 24 Hours of Le Mans passed through the city cente…
    Content: The crowd chanted U-S-A! U-S-A! as Nascar driver lineup for the 24 Hours of Le Mans passed through t...
    
    Title: NBA finals predictions: Nuggets or Heat? Our writers share their picks
    Description: Denver or Miami? Our contributors pick the winner, key players and dark horses before the NBA’s grand finale tips offA lot has been made of the importance of a balanced roster with continuity, but, somehow, still not enough. The Nuggets are the prime example …
    Content: The Nuggets are here because 
    A lot has been made of the importance of a balanced roster with conti...
    
    Title: Unboxing: Michelob ULTRA and Artist Futura Enshrine the NBA Championship In Custom Hand-Painted Bottles
    Description: As the 2022-2023 NBA Championship nears the end, Michelob ULTRA brings joy to sports fans who will gather to watch the showdown between the Denver Nuggets and Miami Heat. The beermaker teamed up with artist Futura to remix its newly-designed 2023 Champ Bottle…
    Content: As the 2022-2023 NBA Championship nears the end, Michelob ULTRA brings joy to sports fans who will g...
    
    Title: Futura and Michelob ULTRA Toast to the NBA Finals With Abstract Artwork Crafted From the Brand’s 2023 Limited-Edition Championship Bottles
    Description: The sun is out to play, and so is Michelob ULTRA. With the 2022-2023 NBA Finals underway, the beermaker is back with its celebratory NBA Champ Bottles. This year, the self-proclaimed MVP of joy is dropping a limited-edition bottle made in collaboration with a…
    Content: The sun is out to play, and so is Michelob ULTRA. With the 2022-2023 NBA Finals underway, the beerma...
    
    Title: Signed and Delivered, Futura and Michelob ULTRA Will Gift Hand-Painted Bottles to This Year’s NBA Championship Team
    Description: Michelob ULTRA, the MVP of joy and official beer sponsor of the NBA is back to celebrate with basketball lovers and sports fans around the globe as the NBA 2022-2023 season comes to a nail-biting close. In collaboration with artist Futura, Michelob ULTRA will…
    Content: Michelob ULTRA, the MVP of joy and official beer sponsor of the NBA is back to celebrate with basket...


key: _node_content
string_value:
{"id_": "a39241b0-d3c5-4e9e-a9f7-43b571f59c67", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_a_search_API.ipynb", "title": "Question answering using a search API and re-ranking", "tokens": 4009, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "1f39c952-9294-45e5-a9b4-ccb3c1e0dfdc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_a_search_API.ipynb", "title": "Question answering using a search API and re-ranking", "tokens": 4009, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "0b5f2f8ff663acd44e24d8fe522fec60044e4f369875d21f6b9074e4c97a4414", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "14b182b5-550c-4f83-b242-dec04fe08c01", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_a_search_API.ipynb", "title": "Question answering using a search API and re-ranking", "tokens": 4009, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "b74768946f75a08e7af0307da49c4d962e899d34e79e8a8fe0f199eadc0c9ee3", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "f09576f0-e057-40ed-acea-0039f4e8ac0a", "node_type": "1", "metadata": {}, "hash": "943746c439027f798ce77ace1d780f4e946f3f4546acfa84311cbaf7d3eddc71", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 9836, "end_char_idx": 12345, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
As we can see, oftentimes, the search queries will return a large number of results, many of which are not relevant to the original question asked by the user. In order to improve the quality of the final answer, we use embeddings to re-rank and filter the results.

## 2. Re-rank

Drawing inspiration from [HyDE (Gao et al.)](https://arxiv.org/abs/2212.10496), we first generate a hypothetical ideal answer to rerank our compare our results against. This helps prioritize results that look like good answers, rather than those similar to our question. Here’s the prompt we use to generate our hypothetical answer.



```python
HA_INPUT = f"""
Generate a hypothetical answer to the user's question. This answer will be used to rank search results. 
Pretend you have all the information you need to answer, but don't use any actual facts. Instead, use placeholders
like NAME did something, or NAME said something at PLACE. 

User question: {USER_QUESTION}

Format: {{"hypotheticalAnswer": "hypothetical answer text"}}
"""

hypothetical_answer = json_gpt(HA_INPUT)["hypotheticalAnswer"]

hypothetical_answer

```




    'The NBA championship was won by TEAM NAME. The MVP was awarded to PLAYER NAME. The last game was held at STADIUM NAME, where both teams played with great energy and enthusiasm. It was a close game, but in the end, TEAM NAME emerged victorious.'



Now, let's generate embeddings for the search results and the hypothetical answer. We then calculate the cosine distance between these embeddings, giving us a semantic similarity metric. Note that we can simply calculate the dot product in lieu of doing a full cosine similarity calculation since the OpenAI embeddings are returned normalized in our API.



```python
hypothetical_answer_embedding = embeddings(hypothetical_answer)[0]
article_embeddings = embeddings(
    [
        f"{article['title']} {article['description']} {article['content'][0:100]}"
        for article in articles
    ]
)

# Calculate cosine similarity
cosine_similarities = []
for article_embedding in article_embeddings:
    cosine_similarities.append(dot(hypothetical_answer_embedding, article_embedding))

cosine_similarities[0:10]

```




    [0.7854456526852069,
     0.8086023500072106,
     0.8002998147018501,
     0.7961229569526956,
     0.798354506673743,
     0.758216458795653,
     0.7753754083127359,
     0.7494958338411927,
     0.804733946801739,
     0.8405965885235218]



Finally, we use these similarity scores to sort and filter the results.


key: _node_content
string_value:
{"id_": "f09576f0-e057-40ed-acea-0039f4e8ac0a", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_a_search_API.ipynb", "title": "Question answering using a search API and re-ranking", "tokens": 4009, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "1f39c952-9294-45e5-a9b4-ccb3c1e0dfdc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_a_search_API.ipynb", "title": "Question answering using a search API and re-ranking", "tokens": 4009, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "0b5f2f8ff663acd44e24d8fe522fec60044e4f369875d21f6b9074e4c97a4414", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "a39241b0-d3c5-4e9e-a9f7-43b571f59c67", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_a_search_API.ipynb", "title": "Question answering using a search API and re-ranking", "tokens": 4009, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "7937ba5c70bc9a196b798d638f378d8f6229809e034789ce7b484a217f62abbe", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "0496be52-3b71-44db-8906-70dfe15e2a0d", "node_type": "1", "metadata": {}, "hash": "8928b7b99b2e7d118441c4164cf190081aa3eabe0c5e0f0478aa153ed8c50f87", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 10955, "end_char_idx": 14906, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
'The NBA championship was won by TEAM NAME. The MVP was awarded to PLAYER NAME. The last game was held at STADIUM NAME, where both teams played with great energy and enthusiasm. It was a close game, but in the end, TEAM NAME emerged victorious.'



Now, let's generate embeddings for the search results and the hypothetical answer. We then calculate the cosine distance between these embeddings, giving us a semantic similarity metric. Note that we can simply calculate the dot product in lieu of doing a full cosine similarity calculation since the OpenAI embeddings are returned normalized in our API.



```python
hypothetical_answer_embedding = embeddings(hypothetical_answer)[0]
article_embeddings = embeddings(
    [
        f"{article['title']} {article['description']} {article['content'][0:100]}"
        for article in articles
    ]
)

# Calculate cosine similarity
cosine_similarities = []
for article_embedding in article_embeddings:
    cosine_similarities.append(dot(hypothetical_answer_embedding, article_embedding))

cosine_similarities[0:10]

```




    [0.7854456526852069,
     0.8086023500072106,
     0.8002998147018501,
     0.7961229569526956,
     0.798354506673743,
     0.758216458795653,
     0.7753754083127359,
     0.7494958338411927,
     0.804733946801739,
     0.8405965885235218]



Finally, we use these similarity scores to sort and filter the results.



```python
scored_articles = zip(articles, cosine_similarities)

# Sort articles by cosine similarity
sorted_articles = sorted(scored_articles, key=lambda x: x[1], reverse=True)

# Print top 5 articles
print("Top 5 articles:", "\n")

for article, score in sorted_articles[0:5]:
    print("Title:", article["title"])
    print("Description:", article["description"])
    print("Content:", article["content"][0:100] + "...")
    print("Score:", score)
    print()

```

    Top 5 articles: 
    
    Title: NBA Finals: Denver Nuggets beat Miami Hea, lift thier first-ever NBA title
    Description: Denver Nuggets won their maiden NBA Championship trophy defeating Miami Heat 94-89 in Game 5 of the NBA Final held on Tuesday at the Ball Arena in Denver
    Content: Denver Nuggets won their maiden NBA Championship trophy defeating Miami Heat 94-89 in Game 5 of the ...
    Score: 0.8445817523602124
    
    Title: Photos: Denver Nuggets celebrate their first NBA title
    Description: The Nuggets capped off an impressive postseason by beating the Miami Heat in the NBA Finals.
    Content: Thousands of supporters watched along the streets of Denver, Colorado as the US National Basketball ...
    Score: 0.842070667753606
    
    Title: Denver Nuggets win first NBA championship title in Game 5 victory over Miami Heat
    Description: The Denver Nuggets won their first NBA championship Monday night, downing the Miami Heat 94-89 at Ball Arena in Denver to take Game 5 of the NBA Finals.
    Content: The Denver Nuggets won their first NBA championship Monday night, downing the Miami Heat 94-89 at Ba...
    Score: 0.8409346078172385
    
    Title: Denver Nuggets Capture Their First NBA Championship Behind Unbreakable Chemistry
    Description: After 47 years of waiting, the Denver Nuggets are NBA champions. Led by Nikola Jokic and Jamal Murray, they reached the mountain top by staying true to themselves.
    Content: DENVER, CO - JUNE 12: Jamal Murray (27) of the Denver Nuggets celebrates as he leaves the court ... ...
    Score: 0.8405965885235218
    
    Title: NBA Finals: Nikola Jokic, Denver Nuggets survive Miami Heat to secure franchise's first NBA championship
    Description: In a rock-fight of a Game 5, the Denver Nuggets reached the NBA mountaintop from the foothills of the Rockies, winning their first-ever championship and setting Nikola Jokic's legacy as an all-timer in stone.
    Content: DENVER, COLORADO - JUNE 12: Jamal Murray #27 of the Denver Nuggets reacts during the fourth quarter ...
    Score: 0.8389716330890262


key: _node_content
string_value:
{"id_": "0496be52-3b71-44db-8906-70dfe15e2a0d", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_a_search_API.ipynb", "title": "Question answering using a search API and re-ranking", "tokens": 4009, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "1f39c952-9294-45e5-a9b4-ccb3c1e0dfdc", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_a_search_API.ipynb", "title": "Question answering using a search API and re-ranking", "tokens": 4009, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "0b5f2f8ff663acd44e24d8fe522fec60044e4f369875d21f6b9074e4c97a4414", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "f09576f0-e057-40ed-acea-0039f4e8ac0a", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Question_answering_using_a_search_API.ipynb", "title": "Question answering using a search API and re-ranking", "tokens": 4009, "retrieve_doc": true, "source": "OpenAI_Cookbooks"}, "hash": "562e9db097ed6980676cf72d2e9fbd3101b5c31540e1fbb471950cbb743c48b4", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 14914, "end_char_idx": 17018, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
Awesome! These results look a lot more relevant to our original query. Now, let's use the top 5 results to generate a final answer.

## 3. Answer



```python
formatted_top_results = [
    {
        "title": article["title"],
        "description": article["description"],
        "url": article["url"],
    }
    for article, _score in sorted_articles[0:5]
]

ANSWER_INPUT = f"""
Generate an answer to the user's question based on the given search results. 
TOP_RESULTS: {formatted_top_results}
USER_QUESTION: {USER_QUESTION}

Include as much information as possible in the answer. Reference the relevant search result urls as markdown links.
"""

completion = client.chat.completions.create(
    model=GPT_MODEL,
    messages=[{"role": "user", "content": ANSWER_INPUT}],
    temperature=0.5,
    stream=True,
)

text = ""
for chunk in completion:
    text += chunk.choices[0].delta.content
    display.clear_output(wait=True)
    display.display(display.Markdown(text))
```


The Denver Nuggets won their first-ever NBA championship by defeating the Miami Heat 94-89 in Game 5 of the NBA Finals held on Tuesday at the Ball Arena in Denver, according to this [Business Standard article](https://www.business-standard.com/sports/other-sports-news/nba-finals-denver-nuggets-beat-miami-hea-lift-thier-first-ever-nba-title-123061300285_1.html). Nikola Jokic, the Nuggets' center, was named the NBA Finals MVP. In a rock-fight of a Game 5, the Nuggets reached the NBA mountaintop, securing their franchise's first NBA championship and setting Nikola Jokic's legacy as an all-timer in stone, according to this [Yahoo Sports article](https://sports.yahoo.com/nba-finals-nikola-jokic-denver-nuggets-survive-miami-heat-to-secure-franchises-first-nba-championship-030321214.html). For more information and photos of the Nuggets' celebration, check out this [Al Jazeera article](https://www.aljazeera.com/gallery/2023/6/15/photos-denver-nuggets-celebrate-their-first-nba-title) and this [CNN article](https://www.cnn.com/2023/06/12/sport/denver-nuggets-nba-championship-spt-intl?cid=external-feeds_iluminar_yahoo).


key: url
string_value:
https://github.com/openai/openai-cookbook/blob/main/examples/Search_reranking_with_cross-encoders.ipynb


key: _node_content
string_value:
{"id_": "beb420ee-ba57-4e58-8de5-7b6855f6003b", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Search_reranking_with_cross-encoders.ipynb", "title": "Search reranking with cross-encoders", "tokens": 8550, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "64f6fb54-183f-4a0b-a1e1-9040a6717110", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Search_reranking_with_cross-encoders.ipynb", "title": "Search reranking with cross-encoders", "tokens": 8550, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "hash": "e8e0acaf40970dd551e968e92897d3415bd23a6e4bc77c0be91f1270b664426c", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "72ede367-4569-42b8-b282-d1b3ad8af6b3", "node_type": "1", "metadata": {}, "hash": "16a59da98549bc0110606c7cd53af1102ff09372c7b472dfbb3e335e33c89672", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 0, "end_char_idx": 3275, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
# Search reranking with cross-encoders

This notebook takes you through examples of using a cross-encoder to re-rank search results.

This is a common use case with our customers, where you've implemented semantic search using embeddings (produced using a [bi-encoder](https://www.sbert.net/examples/applications/retrieve_rerank/README.html#retrieval-bi-encoder)) but the results are not as accurate as your use case requires. A possible cause is that there is some business rule you can use to rerank the documents such as how recent or how popular a document is. 

However, often there are subtle domain-specific rules that help determine relevancy, and this is where a cross-encoder can be useful. Cross-encoders are more accurate than bi-encoders but they don't scale well, so using them to re-order a shortened list returned by semantic search is the ideal use case.

### Example

Consider a search task with D documents and Q queries.

The brute force approach of computing every pairwise relevance is expensive; its cost scales as ```D * Q```. This is known as **cross-encoding**.

A faster approach is **embeddings-based search**, in which an embedding is computed once for each document and query, and then re-used multiple times to cheaply compute pairwise relevance. Because embeddings are only computed once, its cost scales as ```D + Q```. This is known as **bi-encoding**.

Although embeddings-based search is faster, the quality can be worse. To get the best of both, one common approach is to use embeddings (or another bi-encoder) to cheaply identify top candidates, and then use GPT (or another cross-encoder) to expensively re-rank those top candidates. The cost of this hybrid approach scales as ```(D + Q) * cost of embedding + (N * Q) * cost of re-ranking```, where ```N``` is the number of candidates re-ranked.

### Walkthrough

To illustrate this approach we'll use ```text-davinci-003``` with ```logprobs``` enabled to build a GPT-powered cross-encoder. Our GPT models have strong general language understanding, which when tuned with some few-shot examples can provide a simple and effective cross-encoding option.

This notebook drew on this great [article](https://weaviate.io/blog/cross-encoders-as-reranker) by Weaviate, and this [excellent explanation](https://www.sbert.net/examples/applications/cross-encoder/README.html) of bi-encoders vs. cross-encoders from Sentence Transformers.


```python
!pip install openai
!pip install arxiv
!pip install tenacity
!pip install pandas
!pip install tiktoken
```


```python
import arxiv
from math import exp
import openai
import os
import pandas as pd
from tenacity import retry, wait_random_exponential, stop_after_attempt
import tiktoken

client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY", "<your OpenAI API key if not set as env var>"))

OPENAI_MODEL = "gpt-4"
```

## Search

We'll use the arXiv search service for this example, but this step could be performed by any search service you have. The key item to consider is over-fetching slightly to capture all the potentially relevant documents, before re-sorting them.



```python
query = "how do bi-encoders work for sentence embeddings"
search = arxiv.Search(
    query=query, max_results=20, sort_by=arxiv.SortCriterion.Relevance
)
```


key: _node_content
string_value:
{"id_": "72ede367-4569-42b8-b282-d1b3ad8af6b3", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Search_reranking_with_cross-encoders.ipynb", "title": "Search reranking with cross-encoders", "tokens": 8550, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "64f6fb54-183f-4a0b-a1e1-9040a6717110", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Search_reranking_with_cross-encoders.ipynb", "title": "Search reranking with cross-encoders", "tokens": 8550, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "hash": "e8e0acaf40970dd551e968e92897d3415bd23a6e4bc77c0be91f1270b664426c", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "beb420ee-ba57-4e58-8de5-7b6855f6003b", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Search_reranking_with_cross-encoders.ipynb", "title": "Search reranking with cross-encoders", "tokens": 8550, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "hash": "dcaa9514d8479d9a4512b09b03abc3067d2e5b40a26c50b13a48fa69684220fc", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "fc95a9dc-78f4-47b3-8183-e08afa5d87aa", "node_type": "1", "metadata": {}, "hash": "3fdb09420e94baba44c98dfab1fd368e4cd8dce2bf4c9e74fce31e2f27fcbb49", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 2420, "end_char_idx": 5391, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


key: chroma:document
string_value:
```python
!pip install openai
!pip install arxiv
!pip install tenacity
!pip install pandas
!pip install tiktoken
```


```python
import arxiv
from math import exp
import openai
import os
import pandas as pd
from tenacity import retry, wait_random_exponential, stop_after_attempt
import tiktoken

client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY", "<your OpenAI API key if not set as env var>"))

OPENAI_MODEL = "gpt-4"
```

## Search

We'll use the arXiv search service for this example, but this step could be performed by any search service you have. The key item to consider is over-fetching slightly to capture all the potentially relevant documents, before re-sorting them.



```python
query = "how do bi-encoders work for sentence embeddings"
search = arxiv.Search(
    query=query, max_results=20, sort_by=arxiv.SortCriterion.Relevance
)
```


```python
result_list = []

for result in search.results():
    result_dict = {}

    result_dict.update({"title": result.title})
    result_dict.update({"summary": result.summary})

    # Taking the first url provided
    result_dict.update({"article_url": [x.href for x in result.links][0]})
    result_dict.update({"pdf_url": [x.href for x in result.links][1]})
    result_list.append(result_dict)
```


```python
result_list[0]
```




    {'title': 'SBERT studies Meaning Representations: Decomposing Sentence Embeddings into Explainable Semantic Features',
     'summary': 'Models based on large-pretrained language models, such as S(entence)BERT,\nprovide effective and efficient sentence embeddings that show high correlation\nto human similarity ratings, but lack interpretability. On the other hand,\ngraph metrics for graph-based meaning representations (e.g., Abstract Meaning\nRepresentation, AMR) can make explicit the semantic aspects in which two\nsentences are similar. However, such metrics tend to be slow, rely on parsers,\nand do not reach state-of-the-art performance when rating sentence similarity.\n  In this work, we aim at the best of both worlds, by learning to induce\n$S$emantically $S$tructured $S$entence BERT embeddings (S$^3$BERT). Our\nS$^3$BERT embeddings are composed of explainable sub-embeddings that emphasize\nvarious semantic sentence features (e.g., semantic roles, negation, or\nquantification). We show how to i) learn a decomposition of the sentence\nembeddings into semantic features, through approximation of a suite of\ninterpretable AMR graph metrics, and how to ii) preserve the overall power of\nthe neural embeddings by controlling the decomposition learning process with a\nsecond objective that enforces consistency with the similarity ratings of an\nSBERT teacher model. In our experimental studies, we show that our approach\noffers interpretability -- while fully preserving the effectiveness and\nefficiency of the neural sentence embeddings.',
     'article_url': 'http://arxiv.org/abs/2206.07023v2',
     'pdf_url': 'http://arxiv.org/pdf/2206.07023v2'}


key: _node_content
string_value:
{"id_": "fc95a9dc-78f4-47b3-8183-e08afa5d87aa", "embedding": null, "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Search_reranking_with_cross-encoders.ipynb", "title": "Search reranking with cross-encoders", "tokens": 8550, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "excluded_embed_metadata_keys": ["url", "tokens", "retrieve_doc", "source"], "excluded_llm_metadata_keys": ["title", "tokens", "retrieve_doc", "source"], "relationships": {"1": {"node_id": "64f6fb54-183f-4a0b-a1e1-9040a6717110", "node_type": "4", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Search_reranking_with_cross-encoders.ipynb", "title": "Search reranking with cross-encoders", "tokens": 8550, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "hash": "e8e0acaf40970dd551e968e92897d3415bd23a6e4bc77c0be91f1270b664426c", "class_name": "RelatedNodeInfo"}, "2": {"node_id": "72ede367-4569-42b8-b282-d1b3ad8af6b3", "node_type": "1", "metadata": {"url": "https://github.com/openai/openai-cookbook/blob/main/examples/Search_reranking_with_cross-encoders.ipynb", "title": "Search reranking with cross-encoders", "tokens": 8550, "retrieve_doc": false, "source": "OpenAI_Cookbooks"}, "hash": "9b67c42a2a07c57938fa519a73b2d5c76c7d5d37eb8ee83d4b4ad53cd8dff522", "class_name": "RelatedNodeInfo"}, "3": {"node_id": "3cf46980-4bab-4596-ba55-eae07d6eb399", "node_type": "1", "metadata": {}, "hash": "c69f519d93f62a0553c1392de1d6e130e8779161aea688eb237db3e36ead483f", "class_name": "RelatedNodeInfo"}}, "text": "", "mimetype": "text/plain", "start_char_idx": 3685, "end_char_idx": 7209, "text_template": "{metadata_str}\n\n{content}", "metadata_template": "{key}: {value}", "metadata_seperator": "\n", "class_name": "TextNode"}


