diff --git "a/data/general_plots.csv" "b/data/general_plots.csv"
--- "a/data/general_plots.csv"
+++ "b/data/general_plots.csv"
@@ -1,659 +1,617 @@
-path,plot_object,header,description,plot_json,arguments,df
-./html/plots/response_times.html,"Figure({
+plot_name,path,plot_object,header,description,plot_json,arguments,df
+execution_costs,./html/plots/execution_costs.html,"Figure({
'data': [{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': '01-ai Yi Chat (34B)',
+ 'hovertemplate': 'Model=%{y}
Execution cost ($)=%{x}',
+ 'legendgroup': 'gpt-4',
'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
- 'name': '01-ai Yi Chat (34B)',
- 'offsetgroup': '01-ai Yi Chat (34B)',
- 'orientation': 'v',
+ 'name': 'gpt-4',
+ 'offsetgroup': 'gpt-4',
+ 'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['01-ai Yi Chat (34B)'], dtype=object),
+ 'x': array([0.0249]),
'xaxis': 'x',
- 'y': array([nan]),
+ 'y': array(['gpt-4'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'Chronos Hermes (13B)',
+ 'hovertemplate': 'Model=%{y}
Execution cost ($)=%{x}',
+ 'legendgroup': 'gpt-4-turbo',
'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
- 'name': 'Chronos Hermes (13B)',
- 'offsetgroup': 'Chronos Hermes (13B)',
- 'orientation': 'v',
+ 'name': 'gpt-4-turbo',
+ 'offsetgroup': 'gpt-4-turbo',
+ 'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['Chronos Hermes (13B)'], dtype=object),
+ 'x': array([0.0229]),
'xaxis': 'x',
- 'y': array([4.16887414]),
+ 'y': array(['gpt-4-turbo'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'Falcon Instruct (7B)',
+ 'hovertemplate': 'Model=%{y}
Execution cost ($)=%{x}',
+ 'legendgroup': 'gpt-3.5-turbo',
'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
- 'name': 'Falcon Instruct (7B)',
- 'offsetgroup': 'Falcon Instruct (7B)',
- 'orientation': 'v',
+ 'name': 'gpt-3.5-turbo',
+ 'offsetgroup': 'gpt-3.5-turbo',
+ 'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['Falcon Instruct (7B)'], dtype=object),
+ 'x': array([0.00083]),
'xaxis': 'x',
- 'y': array([1.07813787]),
+ 'y': array(['gpt-3.5-turbo'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'LLaMA-2 Chat (7B)',
+ 'hovertemplate': 'Model=%{y}
Execution cost ($)=%{x}',
+ 'legendgroup': 'llama-2-70b-chat',
'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
- 'name': 'LLaMA-2 Chat (7B)',
- 'offsetgroup': 'LLaMA-2 Chat (7B)',
- 'orientation': 'v',
+ 'name': 'llama-2-70b-chat',
+ 'offsetgroup': 'llama-2-70b-chat',
+ 'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['LLaMA-2 Chat (7B)'], dtype=object),
+ 'x': array([0.000711]),
'xaxis': 'x',
- 'y': array([1.68668771]),
+ 'y': array(['llama-2-70b-chat'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'Mistral (7B) Instruct v0.2 (Together AI)',
+ 'hovertemplate': 'Model=%{y}
Execution cost ($)=%{x}',
+ 'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
- 'name': 'Mistral (7B) Instruct v0.2 (Together AI)',
- 'offsetgroup': 'Mistral (7B) Instruct v0.2 (Together AI)',
- 'orientation': 'v',
+ 'name': 'Mixtral-8x7B-Instruct-v0.1',
+ 'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
+ 'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['Mistral (7B) Instruct v0.2 (Together AI)'], dtype=object),
+ 'x': array([0.000711]),
'xaxis': 'x',
- 'y': array([1.77096045]),
+ 'y': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'Mistral-7B-Instruct-v0.2',
+ 'hovertemplate': 'Model=%{y}
Execution cost ($)=%{x}',
+ 'legendgroup': 'gemini-pro',
'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
- 'name': 'Mistral-7B-Instruct-v0.2',
- 'offsetgroup': 'Mistral-7B-Instruct-v0.2',
- 'orientation': 'v',
+ 'name': 'gemini-pro',
+ 'offsetgroup': 'gemini-pro',
+ 'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['Mistral-7B-Instruct-v0.2'], dtype=object),
+ 'x': array([0.0001975]),
'xaxis': 'x',
- 'y': array([3.27102681]),
+ 'y': array(['gemini-pro'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
+ 'hovertemplate': 'Model=%{y}
Execution cost ($)=%{x}',
+ 'legendgroup': 'chat-bison (PaLM 2)',
'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
- 'name': 'Mixtral-8x7B-Instruct-v0.1',
- 'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
- 'orientation': 'v',
+ 'name': 'chat-bison (PaLM 2)',
+ 'offsetgroup': 'chat-bison (PaLM 2)',
+ 'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
+ 'x': array([0.0001975]),
'xaxis': 'x',
- 'y': array([4.83500647]),
+ 'y': array(['chat-bison (PaLM 2)'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'OpenHermes-2.5-Mistral (7B)',
+ 'hovertemplate': 'Model=%{y}
Execution cost ($)=%{x}',
+ 'legendgroup': 'chat-bison-32k (PaLM 2 32K)',
'marker': {'color': '#B6E880', 'pattern': {'shape': ''}},
- 'name': 'OpenHermes-2.5-Mistral (7B)',
- 'offsetgroup': 'OpenHermes-2.5-Mistral (7B)',
- 'orientation': 'v',
- 'showlegend': True,
- 'textposition': 'auto',
- 'type': 'bar',
- 'x': array(['OpenHermes-2.5-Mistral (7B)'], dtype=object),
- 'xaxis': 'x',
- 'y': array([3.33096337]),
- 'yaxis': 'y'},
- {'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'Qwen 1.5 Chat (7B)',
- 'marker': {'color': '#FF97FF', 'pattern': {'shape': ''}},
- 'name': 'Qwen 1.5 Chat (7B)',
- 'offsetgroup': 'Qwen 1.5 Chat (7B)',
- 'orientation': 'v',
- 'showlegend': True,
- 'textposition': 'auto',
- 'type': 'bar',
- 'x': array(['Qwen 1.5 Chat (7B)'], dtype=object),
- 'xaxis': 'x',
- 'y': array([nan]),
- 'yaxis': 'y'},
- {'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'RedPajama-INCITE Chat (7B)',
- 'marker': {'color': '#FECB52', 'pattern': {'shape': ''}},
- 'name': 'RedPajama-INCITE Chat (7B)',
- 'offsetgroup': 'RedPajama-INCITE Chat (7B)',
- 'orientation': 'v',
+ 'name': 'chat-bison-32k (PaLM 2 32K)',
+ 'offsetgroup': 'chat-bison-32k (PaLM 2 32K)',
+ 'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['RedPajama-INCITE Chat (7B)'], dtype=object),
+ 'x': array([0.0001975]),
'xaxis': 'x',
- 'y': array([1.34753919]),
- 'yaxis': 'y'},
- {'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'Snorkel Mistral PairRM DPO (7B)',
+ 'y': array(['chat-bison-32k (PaLM 2 32K)'], dtype=object),
+ 'yaxis': 'y'}],
+ 'layout': {'barmode': 'relative',
+ 'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
+ 'template': '...',
+ 'title': {'text': 'Costs of execution of 20 test queries per model'},
+ 'xaxis': {'anchor': 'y', 'domain': [0.0, 1.0], 'title': {'text': 'Execution cost ($)'}},
+ 'yaxis': {'anchor': 'x',
+ 'categoryarray': [chat-bison-32k (PaLM 2 32K), chat-bison
+ (PaLM 2), gemini-pro,
+ Mixtral-8x7B-Instruct-v0.1,
+ llama-2-70b-chat, gpt-3.5-turbo,
+ gpt-4-turbo, gpt-4],
+ 'categoryorder': 'array',
+ 'domain': [0.0, 1.0],
+ 'title': {'text': 'Model'}}}
+})",Costs of execution of 20 test queries per model,,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.0249],""xaxis"":""x"",""y"":[""gpt-4""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.0229],""xaxis"":""x"",""y"":[""gpt-4-turbo""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.00083],""xaxis"":""x"",""y"":[""gpt-3.5-turbo""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.000711],""xaxis"":""x"",""y"":[""llama-2-70b-chat""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.000711],""xaxis"":""x"",""y"":[""Mixtral-8x7B-Instruct-v0.1""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gemini-pro"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""gemini-pro"",""offsetgroup"":""gemini-pro"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.0001975],""xaxis"":""x"",""y"":[""gemini-pro""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison (PaLM 2)"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""chat-bison (PaLM 2)"",""offsetgroup"":""chat-bison (PaLM 2)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.0001975],""xaxis"":""x"",""y"":[""chat-bison (PaLM 2)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k (PaLM 2 32K)"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k (PaLM 2 32K)"",""offsetgroup"":""chat-bison-32k (PaLM 2 32K)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.0001975],""xaxis"":""x"",""y"":[""chat-bison-32k (PaLM 2 32K)""],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Execution cost ($)""}},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Model""},""categoryorder"":""array"",""categoryarray"":[""chat-bison-32k (PaLM 2 32K)"",""chat-bison (PaLM 2)"",""gemini-pro"",""Mixtral-8x7B-Instruct-v0.1"",""llama-2-70b-chat"",""gpt-3.5-turbo"",""gpt-4-turbo"",""gpt-4""]},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Costs of execution of 20 test queries per model""},""barmode"":""relative""}}","{""y"": ""model"", ""x"": ""model_query_costs"", ""color"": ""model"", ""orientation"": ""h"", ""title"": ""Costs of execution of 20 test queries per model"", ""labels"": {""model"": ""Model"", ""model_query_costs"": ""Execution cost ($)""}}",",model_query_costs,model
+2,0.0249,gpt-4
+1,0.0229,gpt-4-turbo
+0,0.00083,gpt-3.5-turbo
+3,0.000711,llama-2-70b-chat
+4,0.000711,Mixtral-8x7B-Instruct-v0.1
+8,0.0001975,gemini-pro
+9,0.0001975,chat-bison (PaLM 2)
+10,0.0001975,chat-bison-32k (PaLM 2 32K)
+"
+model_sizes,./html/plots/model_sizes.html,"Figure({
+ 'data': [{'alignmentgroup': 'True',
+ 'hovertemplate': 'Model=%{y}
Model size (B params)=%{x}',
+ 'legendgroup': 'llama-2-70b-chat',
'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
- 'name': 'Snorkel Mistral PairRM DPO (7B)',
- 'offsetgroup': 'Snorkel Mistral PairRM DPO (7B)',
- 'orientation': 'v',
+ 'name': 'llama-2-70b-chat',
+ 'offsetgroup': 'llama-2-70b-chat',
+ 'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['Snorkel Mistral PairRM DPO (7B)'], dtype=object),
+ 'x': array([70.]),
'xaxis': 'x',
- 'y': array([4.39964783]),
+ 'y': array(['llama-2-70b-chat'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+ 'hovertemplate': 'Model=%{y}
Model size (B params)=%{x}',
+ 'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
- 'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
- 'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
- 'orientation': 'v',
+ 'name': 'Mixtral-8x7B-Instruct-v0.1',
+ 'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
+ 'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['TinyLlama/TinyLlama-1.1B-Chat-v1.0'], dtype=object),
+ 'x': array([56.]),
'xaxis': 'x',
- 'y': array([1.79273835]),
+ 'y': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'Upstage SOLAR Instruct v1 (11B)',
+ 'hovertemplate': 'Model=%{y}
Model size (B params)=%{x}',
+ 'legendgroup': 'WizardLM v1.2 (13B)',
'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
- 'name': 'Upstage SOLAR Instruct v1 (11B)',
- 'offsetgroup': 'Upstage SOLAR Instruct v1 (11B)',
- 'orientation': 'v',
+ 'name': 'WizardLM v1.2 (13B)',
+ 'offsetgroup': 'WizardLM v1.2 (13B)',
+ 'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['Upstage SOLAR Instruct v1 (11B)'], dtype=object),
+ 'x': array([13.]),
'xaxis': 'x',
- 'y': array([nan]),
+ 'y': array(['WizardLM v1.2 (13B)'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'Vicuna v1.5 (7B)',
+ 'hovertemplate': 'Model=%{y}
Model size (B params)=%{x}',
+ 'legendgroup': 'Upstage SOLAR Instruct v1 (11B)',
'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
- 'name': 'Vicuna v1.5 (7B)',
- 'offsetgroup': 'Vicuna v1.5 (7B)',
- 'orientation': 'v',
+ 'name': 'Upstage SOLAR Instruct v1 (11B)',
+ 'offsetgroup': 'Upstage SOLAR Instruct v1 (11B)',
+ 'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['Vicuna v1.5 (7B)'], dtype=object),
+ 'x': array([11.]),
'xaxis': 'x',
- 'y': array([nan]),
+ 'y': array(['Upstage SOLAR Instruct v1 (11B)'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'WizardLM v1.2 (13B)',
+ 'hovertemplate': 'Model=%{y}
Model size (B params)=%{x}',
+ 'legendgroup': 'zephyr-7b-beta',
'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
- 'name': 'WizardLM v1.2 (13B)',
- 'offsetgroup': 'WizardLM v1.2 (13B)',
- 'orientation': 'v',
+ 'name': 'zephyr-7b-beta',
+ 'offsetgroup': 'zephyr-7b-beta',
+ 'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['WizardLM v1.2 (13B)'], dtype=object),
+ 'x': array([7.]),
'xaxis': 'x',
- 'y': array([3.35003853]),
+ 'y': array(['zephyr-7b-beta'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'chat-bison',
+ 'hovertemplate': 'Model=%{y}
Model size (B params)=%{x}',
+ 'legendgroup': 'Mistral-7B-Instruct-v0.2',
'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
- 'name': 'chat-bison',
- 'offsetgroup': 'chat-bison',
- 'orientation': 'v',
+ 'name': 'Mistral-7B-Instruct-v0.2',
+ 'offsetgroup': 'Mistral-7B-Instruct-v0.2',
+ 'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['chat-bison'], dtype=object),
+ 'x': array([7.]),
'xaxis': 'x',
- 'y': array([3.89024363]),
+ 'y': array(['Mistral-7B-Instruct-v0.2'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'chat-bison (PaLM 2)',
+ 'hovertemplate': 'Model=%{y}
Model size (B params)=%{x}',
+ 'legendgroup': 'Vicuna v1.5 (7B)',
'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
- 'name': 'chat-bison (PaLM 2)',
- 'offsetgroup': 'chat-bison (PaLM 2)',
- 'orientation': 'v',
+ 'name': 'Vicuna v1.5 (7B)',
+ 'offsetgroup': 'Vicuna v1.5 (7B)',
+ 'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['chat-bison (PaLM 2)'], dtype=object),
+ 'x': array([7.]),
'xaxis': 'x',
- 'y': array([2.66830386]),
+ 'y': array(['Vicuna v1.5 (7B)'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'chat-bison-32k',
+ 'hovertemplate': 'Model=%{y}
Model size (B params)=%{x}',
+ 'legendgroup': 'LLaMA-2 Chat (7B)',
'marker': {'color': '#B6E880', 'pattern': {'shape': ''}},
- 'name': 'chat-bison-32k',
- 'offsetgroup': 'chat-bison-32k',
- 'orientation': 'v',
+ 'name': 'LLaMA-2 Chat (7B)',
+ 'offsetgroup': 'LLaMA-2 Chat (7B)',
+ 'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['chat-bison-32k'], dtype=object),
+ 'x': array([7.]),
'xaxis': 'x',
- 'y': array([4.12817777]),
+ 'y': array(['LLaMA-2 Chat (7B)'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'chat-bison-32k (PaLM 2 32K)',
+ 'hovertemplate': 'Model=%{y}
Model size (B params)=%{x}',
+ 'legendgroup': 'Mistral (7B) Instruct v0.2 (Together AI)',
'marker': {'color': '#FF97FF', 'pattern': {'shape': ''}},
- 'name': 'chat-bison-32k (PaLM 2 32K)',
- 'offsetgroup': 'chat-bison-32k (PaLM 2 32K)',
- 'orientation': 'v',
+ 'name': 'Mistral (7B) Instruct v0.2 (Together AI)',
+ 'offsetgroup': 'Mistral (7B) Instruct v0.2 (Together AI)',
+ 'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['chat-bison-32k (PaLM 2 32K)'], dtype=object),
+ 'x': array([7.]),
'xaxis': 'x',
- 'y': array([6.73886213]),
+ 'y': array(['Mistral (7B) Instruct v0.2 (Together AI)'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'gemini-pro',
+ 'hovertemplate': 'Model=%{y}
Model size (B params)=%{x}',
+ 'legendgroup': 'Qwen 1.5 Chat (7B)',
'marker': {'color': '#FECB52', 'pattern': {'shape': ''}},
- 'name': 'gemini-pro',
- 'offsetgroup': 'gemini-pro',
- 'orientation': 'v',
+ 'name': 'Qwen 1.5 Chat (7B)',
+ 'offsetgroup': 'Qwen 1.5 Chat (7B)',
+ 'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['gemini-pro'], dtype=object),
+ 'x': array([7.]),
'xaxis': 'x',
- 'y': array([3.03678889]),
+ 'y': array(['Qwen 1.5 Chat (7B)'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'gpt-3.5-turbo',
+ 'hovertemplate': 'Model=%{y}
Model size (B params)=%{x}',
+ 'legendgroup': 'RedPajama-INCITE Chat (7B)',
'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
- 'name': 'gpt-3.5-turbo',
- 'offsetgroup': 'gpt-3.5-turbo',
- 'orientation': 'v',
+ 'name': 'RedPajama-INCITE Chat (7B)',
+ 'offsetgroup': 'RedPajama-INCITE Chat (7B)',
+ 'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['gpt-3.5-turbo'], dtype=object),
+ 'x': array([7.]),
'xaxis': 'x',
- 'y': array([4.29307391]),
+ 'y': array(['RedPajama-INCITE Chat (7B)'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'gpt-4',
+ 'hovertemplate': 'Model=%{y}
Model size (B params)=%{x}',
+ 'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
- 'name': 'gpt-4',
- 'offsetgroup': 'gpt-4',
- 'orientation': 'v',
- 'showlegend': True,
- 'textposition': 'auto',
- 'type': 'bar',
- 'x': array(['gpt-4'], dtype=object),
- 'xaxis': 'x',
- 'y': array([14.32373439]),
- 'yaxis': 'y'},
- {'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'gpt-4-turbo',
- 'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
- 'name': 'gpt-4-turbo',
- 'offsetgroup': 'gpt-4-turbo',
- 'orientation': 'v',
- 'showlegend': True,
- 'textposition': 'auto',
- 'type': 'bar',
- 'x': array(['gpt-4-turbo'], dtype=object),
- 'xaxis': 'x',
- 'y': array([15.5354884]),
- 'yaxis': 'y'},
- {'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'llama-2-70b-chat',
- 'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
- 'name': 'llama-2-70b-chat',
- 'offsetgroup': 'llama-2-70b-chat',
- 'orientation': 'v',
- 'showlegend': True,
- 'textposition': 'auto',
- 'type': 'bar',
- 'x': array(['llama-2-70b-chat'], dtype=object),
- 'xaxis': 'x',
- 'y': array([3.6742315]),
- 'yaxis': 'y'},
- {'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'zephyr-7b-beta',
- 'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
- 'name': 'zephyr-7b-beta',
- 'offsetgroup': 'zephyr-7b-beta',
- 'orientation': 'v',
+ 'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+ 'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+ 'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['zephyr-7b-beta'], dtype=object),
+ 'x': array([1.1]),
'xaxis': 'x',
- 'y': array([3.6370095]),
+ 'y': array(['TinyLlama/TinyLlama-1.1B-Chat-v1.0'], dtype=object),
'yaxis': 'y'}],
'layout': {'barmode': 'relative',
'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
'template': '...',
- 'title': {'text': 'Model endpoints response times'},
- 'xaxis': {'anchor': 'y',
- 'categoryarray': [01-ai Yi Chat (34B), Chronos Hermes
- (13B), Falcon Instruct (7B), LLaMA-2
+ 'title': {'text': 'Model sizes in billions of parameters'},
+ 'xaxis': {'anchor': 'y', 'domain': [0.0, 1.0], 'title': {'text': 'Model size (B params)'}},
+ 'yaxis': {'anchor': 'x',
+ 'categoryarray': [TinyLlama/TinyLlama-1.1B-Chat-v1.0,
+ RedPajama-INCITE Chat (7B), Qwen 1.5
Chat (7B), Mistral (7B) Instruct v0.2
- (Together AI), Mistral-7B-Instruct-v0.2,
+ (Together AI), LLaMA-2 Chat (7B), Vicuna
+ v1.5 (7B), Mistral-7B-Instruct-v0.2,
+ zephyr-7b-beta, Upstage SOLAR Instruct
+ v1 (11B), WizardLM v1.2 (13B),
Mixtral-8x7B-Instruct-v0.1,
- OpenHermes-2.5-Mistral (7B), Qwen 1.5
- Chat (7B), RedPajama-INCITE Chat (7B),
- Snorkel Mistral PairRM DPO (7B),
- TinyLlama/TinyLlama-1.1B-Chat-v1.0,
- Upstage SOLAR Instruct v1 (11B), Vicuna
- v1.5 (7B), WizardLM v1.2 (13B), chat-
- bison, chat-bison (PaLM 2), chat-
- bison-32k, chat-bison-32k (PaLM 2 32K),
- gemini-pro, gpt-3.5-turbo, gpt-4,
- gpt-4-turbo, llama-2-70b-chat,
- zephyr-7b-beta],
+ llama-2-70b-chat],
'categoryorder': 'array',
'domain': [0.0, 1.0],
- 'title': {'text': 'Model'}},
- 'yaxis': {'anchor': 'x', 'domain': [0.0, 1.0], 'title': {'text': 'Execution time (s)'}}}
-})",Model endpoints response times,"This plot shows averaged response time for one query for each model.
-
-We used `time.time()` result saved to variable before making the call to API and compared it to `time.time()` result after receiving the results.
-
-It is also important to note that we used litellm python library for all of the models which adds some overhead compared to pure curl calls.","{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""01-ai Yi Chat (34B)"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""01-ai Yi Chat (34B)"",""offsetgroup"":""01-ai Yi Chat (34B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""01-ai Yi Chat (34B)""],""xaxis"":""x"",""y"":[null],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Chronos Hermes (13B)"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""Chronos Hermes (13B)"",""offsetgroup"":""Chronos Hermes (13B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Chronos Hermes (13B)""],""xaxis"":""x"",""y"":[4.168874144554138],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Falcon Instruct (7B)"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""Falcon Instruct (7B)"",""offsetgroup"":""Falcon Instruct (7B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Falcon Instruct (7B)""],""xaxis"":""x"",""y"":[1.0781378746032715],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""LLaMA-2 Chat (7B)"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""LLaMA-2 Chat (7B)"",""offsetgroup"":""LLaMA-2 Chat (7B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""LLaMA-2 Chat (7B)""],""xaxis"":""x"",""y"":[1.686687707901001],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral (7B) Instruct v0.2 (Together AI)"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""Mistral (7B) Instruct v0.2 (Together AI)"",""offsetgroup"":""Mistral (7B) Instruct v0.2 (Together AI)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mistral (7B) Instruct v0.2 (Together AI)""],""xaxis"":""x"",""y"":[1.7709604501724243],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mistral-7B-Instruct-v0.2""],""xaxis"":""x"",""y"":[3.271026806036631],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mixtral-8x7B-Instruct-v0.1""],""xaxis"":""x"",""y"":[4.835006467343727],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""OpenHermes-2.5-Mistral (7B)"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""OpenHermes-2.5-Mistral (7B)"",""offsetgroup"":""OpenHermes-2.5-Mistral (7B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""OpenHermes-2.5-Mistral (7B)""],""xaxis"":""x"",""y"":[3.330963373184204],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Qwen 1.5 Chat (7B)"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""Qwen 1.5 Chat (7B)"",""offsetgroup"":""Qwen 1.5 Chat (7B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Qwen 1.5 Chat (7B)""],""xaxis"":""x"",""y"":[null],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""RedPajama-INCITE Chat (7B)"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""RedPajama-INCITE Chat (7B)"",""offsetgroup"":""RedPajama-INCITE Chat (7B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""RedPajama-INCITE Chat (7B)""],""xaxis"":""x"",""y"":[1.3475391864776611],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Snorkel Mistral PairRM DPO (7B)"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""Snorkel Mistral PairRM DPO (7B)"",""offsetgroup"":""Snorkel Mistral PairRM DPO (7B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Snorkel Mistral PairRM DPO (7B)""],""xaxis"":""x"",""y"":[4.399647831916809],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""xaxis"":""x"",""y"":[1.7927383464436197],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Upstage SOLAR Instruct v1 (11B)"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""Upstage SOLAR Instruct v1 (11B)"",""offsetgroup"":""Upstage SOLAR Instruct v1 (11B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Upstage SOLAR Instruct v1 (11B)""],""xaxis"":""x"",""y"":[null],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Vicuna v1.5 (7B)"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""Vicuna v1.5 (7B)"",""offsetgroup"":""Vicuna v1.5 (7B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Vicuna v1.5 (7B)""],""xaxis"":""x"",""y"":[null],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""WizardLM v1.2 (13B)"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""WizardLM v1.2 (13B)"",""offsetgroup"":""WizardLM v1.2 (13B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""WizardLM v1.2 (13B)""],""xaxis"":""x"",""y"":[3.350038528442383],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""chat-bison"",""offsetgroup"":""chat-bison"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison""],""xaxis"":""x"",""y"":[3.890243631601334],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison (PaLM 2)"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""chat-bison (PaLM 2)"",""offsetgroup"":""chat-bison (PaLM 2)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison (PaLM 2)""],""xaxis"":""x"",""y"":[2.668303858522159],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k"",""offsetgroup"":""chat-bison-32k"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k""],""xaxis"":""x"",""y"":[4.128177767992019],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k (PaLM 2 32K)"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k (PaLM 2 32K)"",""offsetgroup"":""chat-bison-32k (PaLM 2 32K)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k (PaLM 2 32K)""],""xaxis"":""x"",""y"":[6.738862129823485],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gemini-pro"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""gemini-pro"",""offsetgroup"":""gemini-pro"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gemini-pro""],""xaxis"":""x"",""y"":[3.036788886637026],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-3.5-turbo""],""xaxis"":""x"",""y"":[4.293073908173212],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4""],""xaxis"":""x"",""y"":[14.323734385105828],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4-turbo""],""xaxis"":""x"",""y"":[15.535488395149827],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""llama-2-70b-chat""],""xaxis"":""x"",""y"":[3.6742314998615178],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""zephyr-7b-beta""],""xaxis"":""x"",""y"":[3.6370095004383316],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model""},""categoryorder"":""array"",""categoryarray"":[""01-ai Yi Chat (34B)"",""Chronos Hermes (13B)"",""Falcon Instruct (7B)"",""LLaMA-2 Chat (7B)"",""Mistral (7B) Instruct v0.2 (Together AI)"",""Mistral-7B-Instruct-v0.2"",""Mixtral-8x7B-Instruct-v0.1"",""OpenHermes-2.5-Mistral (7B)"",""Qwen 1.5 Chat (7B)"",""RedPajama-INCITE Chat (7B)"",""Snorkel Mistral PairRM DPO (7B)"",""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""Upstage SOLAR Instruct v1 (11B)"",""Vicuna v1.5 (7B)"",""WizardLM v1.2 (13B)"",""chat-bison"",""chat-bison (PaLM 2)"",""chat-bison-32k"",""chat-bison-32k (PaLM 2 32K)"",""gemini-pro"",""gpt-3.5-turbo"",""gpt-4"",""gpt-4-turbo"",""llama-2-70b-chat"",""zephyr-7b-beta""]},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Execution time (s)""}},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Model endpoints response times""},""barmode"":""relative""}}","{""x"": ""model"", ""y"": ""execution_time"", ""color"": ""model"", ""title"": ""Model endpoints response times"", ""labels"": {""model"": ""Model"", ""execution_time"": ""Execution time (s)""}}",",model,execution_time
-0,01-ai Yi Chat (34B),
-1,Chronos Hermes (13B),4.168874144554138
-2,Falcon Instruct (7B),1.0781378746032715
-3,LLaMA-2 Chat (7B),1.686687707901001
-4,Mistral (7B) Instruct v0.2 (Together AI),1.7709604501724243
-5,Mistral-7B-Instruct-v0.2,3.271026806036631
-6,Mixtral-8x7B-Instruct-v0.1,4.835006467343727
-7,OpenHermes-2.5-Mistral (7B),3.330963373184204
-8,Qwen 1.5 Chat (7B),
-9,RedPajama-INCITE Chat (7B),1.3475391864776611
-10,Snorkel Mistral PairRM DPO (7B),4.399647831916809
-11,TinyLlama/TinyLlama-1.1B-Chat-v1.0,1.7927383464436197
-12,Upstage SOLAR Instruct v1 (11B),
-13,Vicuna v1.5 (7B),
-14,WizardLM v1.2 (13B),3.350038528442383
-15,chat-bison,3.890243631601334
-16,chat-bison (PaLM 2),2.668303858522159
-17,chat-bison-32k,4.128177767992019
-18,chat-bison-32k (PaLM 2 32K),6.738862129823485
-19,gemini-pro,3.036788886637026
-20,gpt-3.5-turbo,4.293073908173212
-21,gpt-4,14.323734385105828
-22,gpt-4-turbo,15.535488395149827
-23,llama-2-70b-chat,3.6742314998615178
-24,zephyr-7b-beta,3.6370095004383316
+ 'title': {'text': 'Model'}}}
+})",Model sizes in billions of parameters,,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[70.0],""xaxis"":""x"",""y"":[""llama-2-70b-chat""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[56.0],""xaxis"":""x"",""y"":[""Mixtral-8x7B-Instruct-v0.1""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""WizardLM v1.2 (13B)"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""WizardLM v1.2 (13B)"",""offsetgroup"":""WizardLM v1.2 (13B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[13.0],""xaxis"":""x"",""y"":[""WizardLM v1.2 (13B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Upstage SOLAR Instruct v1 (11B)"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""Upstage SOLAR Instruct v1 (11B)"",""offsetgroup"":""Upstage SOLAR Instruct v1 (11B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[11.0],""xaxis"":""x"",""y"":[""Upstage SOLAR Instruct v1 (11B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[7.0],""xaxis"":""x"",""y"":[""zephyr-7b-beta""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[7.0],""xaxis"":""x"",""y"":[""Mistral-7B-Instruct-v0.2""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Vicuna v1.5 (7B)"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""Vicuna v1.5 (7B)"",""offsetgroup"":""Vicuna v1.5 (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[7.0],""xaxis"":""x"",""y"":[""Vicuna v1.5 (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""LLaMA-2 Chat (7B)"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""LLaMA-2 Chat (7B)"",""offsetgroup"":""LLaMA-2 Chat (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[7.0],""xaxis"":""x"",""y"":[""LLaMA-2 Chat (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral (7B) Instruct v0.2 (Together AI)"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""Mistral (7B) Instruct v0.2 (Together AI)"",""offsetgroup"":""Mistral (7B) Instruct v0.2 (Together AI)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[7.0],""xaxis"":""x"",""y"":[""Mistral (7B) Instruct v0.2 (Together AI)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Qwen 1.5 Chat (7B)"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""Qwen 1.5 Chat (7B)"",""offsetgroup"":""Qwen 1.5 Chat (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[7.0],""xaxis"":""x"",""y"":[""Qwen 1.5 Chat (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""RedPajama-INCITE Chat (7B)"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""RedPajama-INCITE Chat (7B)"",""offsetgroup"":""RedPajama-INCITE Chat (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[7.0],""xaxis"":""x"",""y"":[""RedPajama-INCITE Chat (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[1.1],""xaxis"":""x"",""y"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model size (B params)""}},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Model""},""categoryorder"":""array"",""categoryarray"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""RedPajama-INCITE Chat (7B)"",""Qwen 1.5 Chat (7B)"",""Mistral (7B) Instruct v0.2 (Together AI)"",""LLaMA-2 Chat (7B)"",""Vicuna v1.5 (7B)"",""Mistral-7B-Instruct-v0.2"",""zephyr-7b-beta"",""Upstage SOLAR Instruct v1 (11B)"",""WizardLM v1.2 (13B)"",""Mixtral-8x7B-Instruct-v0.1"",""llama-2-70b-chat""]},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Model sizes in billions of parameters""},""barmode"":""relative""}}","{""x"": ""model_sizes"", ""y"": ""model"", ""color"": ""model"", ""orientation"": ""h"", ""title"": ""Model sizes in billions of parameters"", ""labels"": {""model"": ""Model"", ""model_sizes"": ""Model size (B params)""}}",",model_sizes,model
+0,70.0,llama-2-70b-chat
+1,56.0,Mixtral-8x7B-Instruct-v0.1
+10,13.0,WizardLM v1.2 (13B)
+11,11.0,Upstage SOLAR Instruct v1 (11B)
+2,7.0,zephyr-7b-beta
+3,7.0,Mistral-7B-Instruct-v0.2
+5,7.0,Vicuna v1.5 (7B)
+6,7.0,LLaMA-2 Chat (7B)
+7,7.0,Mistral (7B) Instruct v0.2 (Together AI)
+8,7.0,Qwen 1.5 Chat (7B)
+9,7.0,RedPajama-INCITE Chat (7B)
+4,1.1,TinyLlama/TinyLlama-1.1B-Chat-v1.0
"
-./html/plots/execution_costs.html,"Figure({
+model_input_sizes,./html/plots/model_input_sizes.html,"Figure({
'data': [{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{y}
Execution cost ($)=%{x}',
- 'legendgroup': 'gpt-3.5-turbo',
+ 'hovertemplate': 'Model=%{y}
Model input size (tokens)=%{x}',
+ 'legendgroup': 'gpt-4-turbo',
'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
- 'name': 'gpt-3.5-turbo',
- 'offsetgroup': 'gpt-3.5-turbo',
+ 'name': 'gpt-4-turbo',
+ 'offsetgroup': 'gpt-4-turbo',
'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array([0.00105]),
+ 'x': array([128000]),
'xaxis': 'x',
- 'y': array(['gpt-3.5-turbo'], dtype=object),
+ 'y': array(['gpt-4-turbo'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{y}
Execution cost ($)=%{x}',
- 'legendgroup': 'gpt-4-turbo',
+ 'hovertemplate': 'Model=%{y}
Model input size (tokens)=%{x}',
+ 'legendgroup': 'Mistral (7B) Instruct v0.2 (Together AI)',
'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
- 'name': 'gpt-4-turbo',
- 'offsetgroup': 'gpt-4-turbo',
+ 'name': 'Mistral (7B) Instruct v0.2 (Together AI)',
+ 'offsetgroup': 'Mistral (7B) Instruct v0.2 (Together AI)',
'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array([0.0125]),
+ 'x': array([32768]),
'xaxis': 'x',
- 'y': array(['gpt-4-turbo'], dtype=object),
+ 'y': array(['Mistral (7B) Instruct v0.2 (Together AI)'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{y}
Execution cost ($)=%{x}',
- 'legendgroup': 'gpt-4',
+ 'hovertemplate': 'Model=%{y}
Model input size (tokens)=%{x}',
+ 'legendgroup': 'Snorkel Mistral PairRM DPO (7B)',
'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
- 'name': 'gpt-4',
- 'offsetgroup': 'gpt-4',
+ 'name': 'Snorkel Mistral PairRM DPO (7B)',
+ 'offsetgroup': 'Snorkel Mistral PairRM DPO (7B)',
'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array([0.0315]),
+ 'x': array([32768]),
'xaxis': 'x',
- 'y': array(['gpt-4'], dtype=object),
+ 'y': array(['Snorkel Mistral PairRM DPO (7B)'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{y}
Execution cost ($)=%{x}',
- 'legendgroup': 'llama-2-70b-chat',
+ 'hovertemplate': 'Model=%{y}
Model input size (tokens)=%{x}',
+ 'legendgroup': 'Qwen 1.5 Chat (7B)',
'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
- 'name': 'llama-2-70b-chat',
- 'offsetgroup': 'llama-2-70b-chat',
+ 'name': 'Qwen 1.5 Chat (7B)',
+ 'offsetgroup': 'Qwen 1.5 Chat (7B)',
'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array([0.002529]),
+ 'x': array([32768]),
'xaxis': 'x',
- 'y': array(['llama-2-70b-chat'], dtype=object),
+ 'y': array(['Qwen 1.5 Chat (7B)'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{y}
Execution cost ($)=%{x}',
- 'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
+ 'hovertemplate': 'Model=%{y}
Model input size (tokens)=%{x}',
+ 'legendgroup': 'gpt-4',
'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
- 'name': 'Mixtral-8x7B-Instruct-v0.1',
- 'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
+ 'name': 'gpt-4',
+ 'offsetgroup': 'gpt-4',
'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array([0.000855]),
+ 'x': array([32000]),
'xaxis': 'x',
- 'y': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
+ 'y': array(['gpt-4'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{y}
Execution cost ($)=%{x}',
- 'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+ 'hovertemplate': 'Model=%{y}
Model input size (tokens)=%{x}',
+ 'legendgroup': 'chat-bison (PaLM 2)',
'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
- 'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
- 'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+ 'name': 'chat-bison (PaLM 2)',
+ 'offsetgroup': 'chat-bison (PaLM 2)',
'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array([0.00031001]),
+ 'x': array([8196]),
'xaxis': 'x',
- 'y': array(['TinyLlama/TinyLlama-1.1B-Chat-v1.0'], dtype=object),
+ 'y': array(['chat-bison (PaLM 2)'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{y}
Execution cost ($)=%{x}',
- 'legendgroup': 'gemini-pro',
+ 'hovertemplate': 'Model=%{y}
Model input size (tokens)=%{x}',
+ 'legendgroup': 'OpenHermes-2.5-Mistral (7B)',
'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
- 'name': 'gemini-pro',
- 'offsetgroup': 'gemini-pro',
+ 'name': 'OpenHermes-2.5-Mistral (7B)',
+ 'offsetgroup': 'OpenHermes-2.5-Mistral (7B)',
'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array([0.0002825]),
+ 'x': array([8192]),
'xaxis': 'x',
- 'y': array(['gemini-pro'], dtype=object),
+ 'y': array(['OpenHermes-2.5-Mistral (7B)'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{y}
Execution cost ($)=%{x}',
- 'legendgroup': 'chat-bison (PaLM 2)',
+ 'hovertemplate': 'Model=%{y}
Model input size (tokens)=%{x}',
+ 'legendgroup': 'gpt-3.5-turbo',
'marker': {'color': '#B6E880', 'pattern': {'shape': ''}},
- 'name': 'chat-bison (PaLM 2)',
- 'offsetgroup': 'chat-bison (PaLM 2)',
+ 'name': 'gpt-3.5-turbo',
+ 'offsetgroup': 'gpt-3.5-turbo',
'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array([0.0002825]),
+ 'x': array([4096]),
'xaxis': 'x',
- 'y': array(['chat-bison (PaLM 2)'], dtype=object),
+ 'y': array(['gpt-3.5-turbo'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{y}
Execution cost ($)=%{x}',
- 'legendgroup': 'chat-bison-32k (PaLM 2 32K)',
+ 'hovertemplate': 'Model=%{y}
Model input size (tokens)=%{x}',
+ 'legendgroup': 'WizardLM v1.2 (13B)',
'marker': {'color': '#FF97FF', 'pattern': {'shape': ''}},
- 'name': 'chat-bison-32k (PaLM 2 32K)',
- 'offsetgroup': 'chat-bison-32k (PaLM 2 32K)',
+ 'name': 'WizardLM v1.2 (13B)',
+ 'offsetgroup': 'WizardLM v1.2 (13B)',
'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array([0.0002825]),
+ 'x': array([4096]),
'xaxis': 'x',
- 'y': array(['chat-bison-32k (PaLM 2 32K)'], dtype=object),
- 'yaxis': 'y'}],
- 'layout': {'barmode': 'relative',
- 'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
- 'template': '...',
- 'title': {'text': 'Costs of execution of 20 test queries per model'},
- 'xaxis': {'anchor': 'y', 'domain': [0.0, 1.0], 'title': {'text': 'Execution cost ($)'}},
- 'yaxis': {'anchor': 'x',
- 'categoryarray': [chat-bison-32k (PaLM 2 32K), chat-bison
- (PaLM 2), gemini-pro,
- TinyLlama/TinyLlama-1.1B-Chat-v1.0,
- Mixtral-8x7B-Instruct-v0.1,
- llama-2-70b-chat, gpt-4, gpt-4-turbo,
- gpt-3.5-turbo],
- 'categoryorder': 'array',
- 'domain': [0.0, 1.0],
- 'title': {'text': 'Model'}}}
-})",Costs of execution of 20 test queries per model,"Costs of experiments carried out to create this table, 20 test queries per model.
-
-Note that pause and resume time cost was not included in the cost calculation for the models billed hourly.
-
-Also although Gemini-Pro model is free for under 60 queries per minute we calculated the full cost for generating the tokens.","{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.00105],""xaxis"":""x"",""y"":[""gpt-3.5-turbo""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.0125],""xaxis"":""x"",""y"":[""gpt-4-turbo""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.0315],""xaxis"":""x"",""y"":[""gpt-4""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.002529],""xaxis"":""x"",""y"":[""llama-2-70b-chat""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.000855],""xaxis"":""x"",""y"":[""Mixtral-8x7B-Instruct-v0.1""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.0003100131352742513],""xaxis"":""x"",""y"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gemini-pro"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""gemini-pro"",""offsetgroup"":""gemini-pro"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.0002825],""xaxis"":""x"",""y"":[""gemini-pro""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison (PaLM 2)"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""chat-bison (PaLM 2)"",""offsetgroup"":""chat-bison (PaLM 2)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.0002825],""xaxis"":""x"",""y"":[""chat-bison (PaLM 2)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eExecution cost ($)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k (PaLM 2 32K)"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k (PaLM 2 32K)"",""offsetgroup"":""chat-bison-32k (PaLM 2 32K)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[0.0002825],""xaxis"":""x"",""y"":[""chat-bison-32k (PaLM 2 32K)""],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Execution cost ($)""}},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Model""},""categoryorder"":""array"",""categoryarray"":[""chat-bison-32k (PaLM 2 32K)"",""chat-bison (PaLM 2)"",""gemini-pro"",""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""Mixtral-8x7B-Instruct-v0.1"",""llama-2-70b-chat"",""gpt-4"",""gpt-4-turbo"",""gpt-3.5-turbo""]},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Costs of execution of 20 test queries per model""},""barmode"":""relative""}}","{""y"": ""model"", ""x"": ""model_query_costs"", ""color"": ""model"", ""orientation"": ""h"", ""title"": ""Costs of execution of 20 test queries per model"", ""labels"": {""model"": ""Model"", ""model_query_costs"": ""Execution cost ($)""}}",",model_query_costs,model
-0,0.00105,gpt-3.5-turbo
-1,0.0125,gpt-4-turbo
-2,0.0315,gpt-4
-3,0.002529,llama-2-70b-chat
-4,0.000855,Mixtral-8x7B-Instruct-v0.1
-7,0.0003100131352742513,TinyLlama/TinyLlama-1.1B-Chat-v1.0
-8,0.0002825,gemini-pro
-9,0.0002825,chat-bison (PaLM 2)
-10,0.0002825,chat-bison-32k (PaLM 2 32K)
-"
-./html/plots/model_sizes.html,"Figure({
- 'data': [{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{y}
Model size (B params)=%{x}',
- 'legendgroup': 'llama-2-70b-chat',
+ 'y': array(['WizardLM v1.2 (13B)'], dtype=object),
+ 'yaxis': 'y'},
+ {'alignmentgroup': 'True',
+ 'hovertemplate': 'Model=%{y}
Model input size (tokens)=%{x}',
+ 'legendgroup': 'LLaMA-2 Chat (7B)',
+ 'marker': {'color': '#FECB52', 'pattern': {'shape': ''}},
+ 'name': 'LLaMA-2 Chat (7B)',
+ 'offsetgroup': 'LLaMA-2 Chat (7B)',
+ 'orientation': 'h',
+ 'showlegend': True,
+ 'textposition': 'auto',
+ 'type': 'bar',
+ 'x': array([4096]),
+ 'xaxis': 'x',
+ 'y': array(['LLaMA-2 Chat (7B)'], dtype=object),
+ 'yaxis': 'y'},
+ {'alignmentgroup': 'True',
+ 'hovertemplate': 'Model=%{y}
Model input size (tokens)=%{x}',
+ 'legendgroup': 'Vicuna v1.5 (7B)',
'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
- 'name': 'llama-2-70b-chat',
- 'offsetgroup': 'llama-2-70b-chat',
+ 'name': 'Vicuna v1.5 (7B)',
+ 'offsetgroup': 'Vicuna v1.5 (7B)',
'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array([70.]),
+ 'x': array([4096]),
'xaxis': 'x',
- 'y': array(['llama-2-70b-chat'], dtype=object),
+ 'y': array(['Vicuna v1.5 (7B)'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{y}
Model size (B params)=%{x}',
- 'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
+ 'hovertemplate': 'Model=%{y}
Model input size (tokens)=%{x}',
+ 'legendgroup': '01-ai Yi Chat (34B)',
'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
- 'name': 'Mixtral-8x7B-Instruct-v0.1',
- 'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
+ 'name': '01-ai Yi Chat (34B)',
+ 'offsetgroup': '01-ai Yi Chat (34B)',
'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array([56.]),
+ 'x': array([4096]),
'xaxis': 'x',
- 'y': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
+ 'y': array(['01-ai Yi Chat (34B)'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{y}
Model size (B params)=%{x}',
- 'legendgroup': 'zephyr-7b-beta',
+ 'hovertemplate': 'Model=%{y}
Model input size (tokens)=%{x}',
+ 'legendgroup': 'Upstage SOLAR Instruct v1 (11B)',
'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
- 'name': 'zephyr-7b-beta',
- 'offsetgroup': 'zephyr-7b-beta',
+ 'name': 'Upstage SOLAR Instruct v1 (11B)',
+ 'offsetgroup': 'Upstage SOLAR Instruct v1 (11B)',
'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array([7.]),
+ 'x': array([4096]),
'xaxis': 'x',
- 'y': array(['zephyr-7b-beta'], dtype=object),
+ 'y': array(['Upstage SOLAR Instruct v1 (11B)'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{y}
Model size (B params)=%{x}',
- 'legendgroup': 'Mistral-7B-Instruct-v0.2',
+ 'hovertemplate': 'Model=%{y}
Model input size (tokens)=%{x}',
+ 'legendgroup': 'Chronos Hermes (13B)',
'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
- 'name': 'Mistral-7B-Instruct-v0.2',
- 'offsetgroup': 'Mistral-7B-Instruct-v0.2',
+ 'name': 'Chronos Hermes (13B)',
+ 'offsetgroup': 'Chronos Hermes (13B)',
'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array([7.]),
+ 'x': array([2048]),
'xaxis': 'x',
- 'y': array(['Mistral-7B-Instruct-v0.2'], dtype=object),
+ 'y': array(['Chronos Hermes (13B)'], dtype=object),
'yaxis': 'y'},
{'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{y}
Model size (B params)=%{x}',
- 'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+ 'hovertemplate': 'Model=%{y}
Model input size (tokens)=%{x}',
+ 'legendgroup': 'Falcon Instruct (7B)',
'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
- 'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
- 'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+ 'name': 'Falcon Instruct (7B)',
+ 'offsetgroup': 'Falcon Instruct (7B)',
'orientation': 'h',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array([1.1]),
+ 'x': array([2048]),
'xaxis': 'x',
- 'y': array(['TinyLlama/TinyLlama-1.1B-Chat-v1.0'], dtype=object),
+ 'y': array(['Falcon Instruct (7B)'], dtype=object),
+ 'yaxis': 'y'},
+ {'alignmentgroup': 'True',
+ 'hovertemplate': 'Model=%{y}
Model input size (tokens)=%{x}',
+ 'legendgroup': 'RedPajama-INCITE Chat (7B)',
+ 'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
+ 'name': 'RedPajama-INCITE Chat (7B)',
+ 'offsetgroup': 'RedPajama-INCITE Chat (7B)',
+ 'orientation': 'h',
+ 'showlegend': True,
+ 'textposition': 'auto',
+ 'type': 'bar',
+ 'x': array([2048]),
+ 'xaxis': 'x',
+ 'y': array(['RedPajama-INCITE Chat (7B)'], dtype=object),
+ 'yaxis': 'y'},
+ {'alignmentgroup': 'True',
+ 'hovertemplate': 'Model=%{y}
Model input size (tokens)=%{x}',
+ 'legendgroup': 'llama-2-70b-chat',
+ 'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
+ 'name': 'llama-2-70b-chat',
+ 'offsetgroup': 'llama-2-70b-chat',
+ 'orientation': 'h',
+ 'showlegend': True,
+ 'textposition': 'auto',
+ 'type': 'bar',
+ 'x': array([2048]),
+ 'xaxis': 'x',
+ 'y': array(['llama-2-70b-chat'], dtype=object),
'yaxis': 'y'}],
'layout': {'barmode': 'relative',
'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
'template': '...',
- 'title': {'text': 'Model sizes in billions of parameters'},
- 'xaxis': {'anchor': 'y', 'domain': [0.0, 1.0], 'title': {'text': 'Model size (B params)'}},
+ 'title': {'text': 'Model input sizes in tokens'},
+ 'xaxis': {'anchor': 'y', 'domain': [0.0, 1.0], 'title': {'text': 'Model input size (tokens)'}},
'yaxis': {'anchor': 'x',
- 'categoryarray': [TinyLlama/TinyLlama-1.1B-Chat-v1.0,
- Mistral-7B-Instruct-v0.2,
- zephyr-7b-beta,
- Mixtral-8x7B-Instruct-v0.1,
- llama-2-70b-chat],
+ 'categoryarray': [llama-2-70b-chat, RedPajama-INCITE Chat
+ (7B), Falcon Instruct (7B), Chronos
+ Hermes (13B), Upstage SOLAR Instruct v1
+ (11B), 01-ai Yi Chat (34B), Vicuna v1.5
+ (7B), LLaMA-2 Chat (7B), WizardLM v1.2
+ (13B), gpt-3.5-turbo,
+ OpenHermes-2.5-Mistral (7B), chat-bison
+ (PaLM 2), gpt-4, Qwen 1.5 Chat (7B),
+ Snorkel Mistral PairRM DPO (7B), Mistral
+ (7B) Instruct v0.2 (Together AI),
+ gpt-4-turbo],
'categoryorder': 'array',
'domain': [0.0, 1.0],
'title': {'text': 'Model'}}}
-})",Model sizes in billions of parameters,"Model sizes based on release blogs and documentation of their respective developers.
-
-Note that models, which sizes were not disclosed publicly, were omitted from this comparison.","{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[70.0],""xaxis"":""x"",""y"":[""llama-2-70b-chat""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[56.0],""xaxis"":""x"",""y"":[""Mixtral-8x7B-Instruct-v0.1""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[7.0],""xaxis"":""x"",""y"":[""zephyr-7b-beta""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[7.0],""xaxis"":""x"",""y"":[""Mistral-7B-Instruct-v0.2""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel size (B params)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[1.1],""xaxis"":""x"",""y"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model size (B params)""}},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Model""},""categoryorder"":""array"",""categoryarray"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""Mistral-7B-Instruct-v0.2"",""zephyr-7b-beta"",""Mixtral-8x7B-Instruct-v0.1"",""llama-2-70b-chat""]},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Model sizes in billions of parameters""},""barmode"":""relative""}}","{""x"": ""model_sizes"", ""y"": ""model"", ""color"": ""model"", ""orientation"": ""h"", ""title"": ""Model sizes in billions of parameters"", ""labels"": {""model"": ""Model"", ""model_sizes"": ""Model size (B params)""}}",",model_sizes,model
-0,70.0,llama-2-70b-chat
-1,56.0,Mixtral-8x7B-Instruct-v0.1
-2,7.0,zephyr-7b-beta
-3,7.0,Mistral-7B-Instruct-v0.2
-4,1.1,TinyLlama/TinyLlama-1.1B-Chat-v1.0
+})",Model input sizes in tokens,,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[128000],""xaxis"":""x"",""y"":[""gpt-4-turbo""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral (7B) Instruct v0.2 (Together AI)"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""Mistral (7B) Instruct v0.2 (Together AI)"",""offsetgroup"":""Mistral (7B) Instruct v0.2 (Together AI)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[32768],""xaxis"":""x"",""y"":[""Mistral (7B) Instruct v0.2 (Together AI)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Snorkel Mistral PairRM DPO (7B)"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""Snorkel Mistral PairRM DPO (7B)"",""offsetgroup"":""Snorkel Mistral PairRM DPO (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[32768],""xaxis"":""x"",""y"":[""Snorkel Mistral PairRM DPO (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Qwen 1.5 Chat (7B)"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""Qwen 1.5 Chat (7B)"",""offsetgroup"":""Qwen 1.5 Chat (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[32768],""xaxis"":""x"",""y"":[""Qwen 1.5 Chat (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[32000],""xaxis"":""x"",""y"":[""gpt-4""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison (PaLM 2)"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""chat-bison (PaLM 2)"",""offsetgroup"":""chat-bison (PaLM 2)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[8196],""xaxis"":""x"",""y"":[""chat-bison (PaLM 2)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""OpenHermes-2.5-Mistral (7B)"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""OpenHermes-2.5-Mistral (7B)"",""offsetgroup"":""OpenHermes-2.5-Mistral (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[8192],""xaxis"":""x"",""y"":[""OpenHermes-2.5-Mistral (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[4096],""xaxis"":""x"",""y"":[""gpt-3.5-turbo""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""WizardLM v1.2 (13B)"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""WizardLM v1.2 (13B)"",""offsetgroup"":""WizardLM v1.2 (13B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[4096],""xaxis"":""x"",""y"":[""WizardLM v1.2 (13B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""LLaMA-2 Chat (7B)"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""LLaMA-2 Chat (7B)"",""offsetgroup"":""LLaMA-2 Chat (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[4096],""xaxis"":""x"",""y"":[""LLaMA-2 Chat (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Vicuna v1.5 (7B)"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""Vicuna v1.5 (7B)"",""offsetgroup"":""Vicuna v1.5 (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[4096],""xaxis"":""x"",""y"":[""Vicuna v1.5 (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""01-ai Yi Chat (34B)"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""01-ai Yi Chat (34B)"",""offsetgroup"":""01-ai Yi Chat (34B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[4096],""xaxis"":""x"",""y"":[""01-ai Yi Chat (34B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Upstage SOLAR Instruct v1 (11B)"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""Upstage SOLAR Instruct v1 (11B)"",""offsetgroup"":""Upstage SOLAR Instruct v1 (11B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[4096],""xaxis"":""x"",""y"":[""Upstage SOLAR Instruct v1 (11B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Chronos Hermes (13B)"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""Chronos Hermes (13B)"",""offsetgroup"":""Chronos Hermes (13B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[2048],""xaxis"":""x"",""y"":[""Chronos Hermes (13B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Falcon Instruct (7B)"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""Falcon Instruct (7B)"",""offsetgroup"":""Falcon Instruct (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[2048],""xaxis"":""x"",""y"":[""Falcon Instruct (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""RedPajama-INCITE Chat (7B)"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""RedPajama-INCITE Chat (7B)"",""offsetgroup"":""RedPajama-INCITE Chat (7B)"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[2048],""xaxis"":""x"",""y"":[""RedPajama-INCITE Chat (7B)""],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{y}\u003cbr\u003eModel input size (tokens)=%{x}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""h"",""showlegend"":true,""textposition"":""auto"",""x"":[2048],""xaxis"":""x"",""y"":[""llama-2-70b-chat""],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model input size (tokens)""}},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Model""},""categoryorder"":""array"",""categoryarray"":[""llama-2-70b-chat"",""RedPajama-INCITE Chat (7B)"",""Falcon Instruct (7B)"",""Chronos Hermes (13B)"",""Upstage SOLAR Instruct v1 (11B)"",""01-ai Yi Chat (34B)"",""Vicuna v1.5 (7B)"",""LLaMA-2 Chat (7B)"",""WizardLM v1.2 (13B)"",""gpt-3.5-turbo"",""OpenHermes-2.5-Mistral (7B)"",""chat-bison (PaLM 2)"",""gpt-4"",""Qwen 1.5 Chat (7B)"",""Snorkel Mistral PairRM DPO (7B)"",""Mistral (7B) Instruct v0.2 (Together AI)"",""gpt-4-turbo""]},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Model input sizes in tokens""},""barmode"":""relative""}}","{""x"": ""model_input_sizes"", ""y"": ""model"", ""color"": ""model"", ""orientation"": ""h"", ""title"": ""Model input sizes in tokens"", ""labels"": {""model"": ""Model"", ""model_input_sizes"": ""Model input size (tokens)""}}",",model_input_sizes,model
+1,128000,gpt-4-turbo
+9,32768,Mistral (7B) Instruct v0.2 (Together AI)
+11,32768,Snorkel Mistral PairRM DPO (7B)
+10,32768,Qwen 1.5 Chat (7B)
+2,32000,gpt-4
+4,8196,chat-bison (PaLM 2)
+12,8192,OpenHermes-2.5-Mistral (7B)
+0,4096,gpt-3.5-turbo
+15,4096,WizardLM v1.2 (13B)
+8,4096,LLaMA-2 Chat (7B)
+7,4096,Vicuna v1.5 (7B)
+5,4096,01-ai Yi Chat (34B)
+16,4096,Upstage SOLAR Instruct v1 (11B)
+6,2048,Chronos Hermes (13B)
+13,2048,Falcon Instruct (7B)
+14,2048,RedPajama-INCITE Chat (7B)
+3,2048,llama-2-70b-chat
"