diff --git "a/data/output_plots.csv" "b/data/output_plots.csv"
--- "a/data/output_plots.csv"
+++ "b/data/output_plots.csv"
@@ -2,120 +2,134 @@ plot_object,header,plot_json,df,arguments,description
"Figure({
'data': [{'alignmentgroup': 'True',
'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'Mistral-7B-Instruct-v0.2',
+ 'legendgroup': 'Snorkel Mistral PairRM DPO (7B)',
'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
- 'name': 'Mistral-7B-Instruct-v0.2',
- 'offsetgroup': 'Mistral-7B-Instruct-v0.2',
+ 'name': 'Snorkel Mistral PairRM DPO (7B)',
+ 'offsetgroup': 'Snorkel Mistral PairRM DPO (7B)',
'orientation': 'v',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['Mistral-7B-Instruct-v0.2'], dtype=object),
+ 'x': array(['Snorkel Mistral PairRM DPO (7B)'], dtype=object),
'xaxis': 'x',
- 'y': array([3.1836577]),
+ 'y': array([25.61907138]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
+ 'legendgroup': 'gpt-4',
'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
- 'name': 'Mixtral-8x7B-Instruct-v0.1',
- 'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
+ 'name': 'gpt-4',
+ 'offsetgroup': 'gpt-4',
'orientation': 'v',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
+ 'x': array(['gpt-4'], dtype=object),
'xaxis': 'x',
- 'y': array([6.88927143]),
+ 'y': array([17.68367981]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+ 'legendgroup': 'gpt-4-turbo',
'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
- 'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
- 'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+ 'name': 'gpt-4-turbo',
+ 'offsetgroup': 'gpt-4-turbo',
'orientation': 'v',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['TinyLlama/TinyLlama-1.1B-Chat-v1.0'], dtype=object),
+ 'x': array(['gpt-4-turbo'], dtype=object),
'xaxis': 'x',
- 'y': array([1.59549656]),
+ 'y': array([15.85205264]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'chat-bison',
+ 'legendgroup': '01-ai Yi Chat (34B)',
'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
- 'name': 'chat-bison',
- 'offsetgroup': 'chat-bison',
+ 'name': '01-ai Yi Chat (34B)',
+ 'offsetgroup': '01-ai Yi Chat (34B)',
'orientation': 'v',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['chat-bison'], dtype=object),
+ 'x': array(['01-ai Yi Chat (34B)'], dtype=object),
'xaxis': 'x',
- 'y': array([3.89024363]),
+ 'y': array([10.45530375]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'chat-bison (PaLM 2)',
+ 'legendgroup': 'Chronos Hermes (13B)',
'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
- 'name': 'chat-bison (PaLM 2)',
- 'offsetgroup': 'chat-bison (PaLM 2)',
+ 'name': 'Chronos Hermes (13B)',
+ 'offsetgroup': 'Chronos Hermes (13B)',
'orientation': 'v',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['chat-bison (PaLM 2)'], dtype=object),
+ 'x': array(['Chronos Hermes (13B)'], dtype=object),
'xaxis': 'x',
- 'y': array([3.30885241]),
+ 'y': array([8.26602804]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'chat-bison-32k',
+ 'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
- 'name': 'chat-bison-32k',
- 'offsetgroup': 'chat-bison-32k',
+ 'name': 'Mixtral-8x7B-Instruct-v0.1',
+ 'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
'orientation': 'v',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['chat-bison-32k'], dtype=object),
+ 'x': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
'xaxis': 'x',
- 'y': array([4.12817777]),
+ 'y': array([6.55972525]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'chat-bison-32k (PaLM 2 32K)',
+ 'legendgroup': 'llama-2-70b-chat',
'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
- 'name': 'chat-bison-32k (PaLM 2 32K)',
- 'offsetgroup': 'chat-bison-32k (PaLM 2 32K)',
+ 'name': 'llama-2-70b-chat',
+ 'offsetgroup': 'llama-2-70b-chat',
'orientation': 'v',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['chat-bison-32k (PaLM 2 32K)'], dtype=object),
+ 'x': array(['llama-2-70b-chat'], dtype=object),
'xaxis': 'x',
- 'y': array([5.46443628]),
+ 'y': array([6.06536523]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'gemini-pro',
+ 'legendgroup': 'WizardLM v1.2 (13B)',
'marker': {'color': '#B6E880', 'pattern': {'shape': ''}},
- 'name': 'gemini-pro',
- 'offsetgroup': 'gemini-pro',
+ 'name': 'WizardLM v1.2 (13B)',
+ 'offsetgroup': 'WizardLM v1.2 (13B)',
'orientation': 'v',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['gemini-pro'], dtype=object),
+ 'x': array(['WizardLM v1.2 (13B)'], dtype=object),
'xaxis': 'x',
- 'y': array([4.11187035]),
+ 'y': array([5.47183404]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'gpt-3.5-turbo',
+ 'legendgroup': 'chat-bison-32k (PaLM 2 32K)',
'marker': {'color': '#FF97FF', 'pattern': {'shape': ''}},
+ 'name': 'chat-bison-32k (PaLM 2 32K)',
+ 'offsetgroup': 'chat-bison-32k (PaLM 2 32K)',
+ 'orientation': 'v',
+ 'showlegend': True,
+ 'textposition': 'auto',
+ 'type': 'bar',
+ 'x': array(['chat-bison-32k (PaLM 2 32K)'], dtype=object),
+ 'xaxis': 'x',
+ 'y': array([5.46443628]),
+ 'yaxis': 'y'},
+ {'alignmentgroup': 'True',
+ 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
+ 'legendgroup': 'gpt-3.5-turbo',
+ 'marker': {'color': '#FECB52', 'pattern': {'shape': ''}},
'name': 'gpt-3.5-turbo',
'offsetgroup': 'gpt-3.5-turbo',
'orientation': 'v',
@@ -128,50 +142,50 @@ plot_object,header,plot_json,df,arguments,description
'yaxis': 'y'},
{'alignmentgroup': 'True',
'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'gpt-4',
- 'marker': {'color': '#FECB52', 'pattern': {'shape': ''}},
- 'name': 'gpt-4',
- 'offsetgroup': 'gpt-4',
+ 'legendgroup': 'gemini-pro',
+ 'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
+ 'name': 'gemini-pro',
+ 'offsetgroup': 'gemini-pro',
'orientation': 'v',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['gpt-4'], dtype=object),
+ 'x': array(['gemini-pro'], dtype=object),
'xaxis': 'x',
- 'y': array([17.68367981]),
+ 'y': array([4.11187035]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'gpt-4-turbo',
- 'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
- 'name': 'gpt-4-turbo',
- 'offsetgroup': 'gpt-4-turbo',
+ 'legendgroup': 'Mistral (7B) Instruct v0.2 (Together AI)',
+ 'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
+ 'name': 'Mistral (7B) Instruct v0.2 (Together AI)',
+ 'offsetgroup': 'Mistral (7B) Instruct v0.2 (Together AI)',
'orientation': 'v',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['gpt-4-turbo'], dtype=object),
+ 'x': array(['Mistral (7B) Instruct v0.2 (Together AI)'], dtype=object),
'xaxis': 'x',
- 'y': array([15.85205264]),
+ 'y': array([4.00486071]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
- 'legendgroup': 'llama-2-70b-chat',
- 'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
- 'name': 'llama-2-70b-chat',
- 'offsetgroup': 'llama-2-70b-chat',
+ 'legendgroup': 'Vicuna v1.5 (7B)',
+ 'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
+ 'name': 'Vicuna v1.5 (7B)',
+ 'offsetgroup': 'Vicuna v1.5 (7B)',
'orientation': 'v',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['llama-2-70b-chat'], dtype=object),
+ 'x': array(['Vicuna v1.5 (7B)'], dtype=object),
'xaxis': 'x',
- 'y': array([6.06536523]),
+ 'y': array([3.85941075]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
'legendgroup': 'zephyr-7b-beta',
- 'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
+ 'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
'name': 'zephyr-7b-beta',
'offsetgroup': 'zephyr-7b-beta',
'orientation': 'v',
@@ -180,39 +194,167 @@ plot_object,header,plot_json,df,arguments,description
'type': 'bar',
'x': array(['zephyr-7b-beta'], dtype=object),
'xaxis': 'x',
- 'y': array([3.38989453]),
+ 'y': array([3.57674572]),
+ 'yaxis': 'y'},
+ {'alignmentgroup': 'True',
+ 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
+ 'legendgroup': 'LLaMA-2 Chat (7B)',
+ 'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
+ 'name': 'LLaMA-2 Chat (7B)',
+ 'offsetgroup': 'LLaMA-2 Chat (7B)',
+ 'orientation': 'v',
+ 'showlegend': True,
+ 'textposition': 'auto',
+ 'type': 'bar',
+ 'x': array(['LLaMA-2 Chat (7B)'], dtype=object),
+ 'xaxis': 'x',
+ 'y': array([3.44595034]),
+ 'yaxis': 'y'},
+ {'alignmentgroup': 'True',
+ 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
+ 'legendgroup': 'chat-bison (PaLM 2)',
+ 'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
+ 'name': 'chat-bison (PaLM 2)',
+ 'offsetgroup': 'chat-bison (PaLM 2)',
+ 'orientation': 'v',
+ 'showlegend': True,
+ 'textposition': 'auto',
+ 'type': 'bar',
+ 'x': array(['chat-bison (PaLM 2)'], dtype=object),
+ 'xaxis': 'x',
+ 'y': array([3.30885241]),
+ 'yaxis': 'y'},
+ {'alignmentgroup': 'True',
+ 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
+ 'legendgroup': 'Mistral-7B-Instruct-v0.2',
+ 'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
+ 'name': 'Mistral-7B-Instruct-v0.2',
+ 'offsetgroup': 'Mistral-7B-Instruct-v0.2',
+ 'orientation': 'v',
+ 'showlegend': True,
+ 'textposition': 'auto',
+ 'type': 'bar',
+ 'x': array(['Mistral-7B-Instruct-v0.2'], dtype=object),
+ 'xaxis': 'x',
+ 'y': array([3.1836577]),
+ 'yaxis': 'y'},
+ {'alignmentgroup': 'True',
+ 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
+ 'legendgroup': 'OpenHermes-2.5-Mistral (7B)',
+ 'marker': {'color': '#B6E880', 'pattern': {'shape': ''}},
+ 'name': 'OpenHermes-2.5-Mistral (7B)',
+ 'offsetgroup': 'OpenHermes-2.5-Mistral (7B)',
+ 'orientation': 'v',
+ 'showlegend': True,
+ 'textposition': 'auto',
+ 'type': 'bar',
+ 'x': array(['OpenHermes-2.5-Mistral (7B)'], dtype=object),
+ 'xaxis': 'x',
+ 'y': array([2.83201321]),
+ 'yaxis': 'y'},
+ {'alignmentgroup': 'True',
+ 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
+ 'legendgroup': 'Qwen 1.5 Chat (7B)',
+ 'marker': {'color': '#FF97FF', 'pattern': {'shape': ''}},
+ 'name': 'Qwen 1.5 Chat (7B)',
+ 'offsetgroup': 'Qwen 1.5 Chat (7B)',
+ 'orientation': 'v',
+ 'showlegend': True,
+ 'textposition': 'auto',
+ 'type': 'bar',
+ 'x': array(['Qwen 1.5 Chat (7B)'], dtype=object),
+ 'xaxis': 'x',
+ 'y': array([2.22289769]),
+ 'yaxis': 'y'},
+ {'alignmentgroup': 'True',
+ 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
+ 'legendgroup': 'RedPajama-INCITE Chat (7B)',
+ 'marker': {'color': '#FECB52', 'pattern': {'shape': ''}},
+ 'name': 'RedPajama-INCITE Chat (7B)',
+ 'offsetgroup': 'RedPajama-INCITE Chat (7B)',
+ 'orientation': 'v',
+ 'showlegend': True,
+ 'textposition': 'auto',
+ 'type': 'bar',
+ 'x': array(['RedPajama-INCITE Chat (7B)'], dtype=object),
+ 'xaxis': 'x',
+ 'y': array([1.93642128]),
+ 'yaxis': 'y'},
+ {'alignmentgroup': 'True',
+ 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
+ 'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+ 'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
+ 'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+ 'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+ 'orientation': 'v',
+ 'showlegend': True,
+ 'textposition': 'auto',
+ 'type': 'bar',
+ 'x': array(['TinyLlama/TinyLlama-1.1B-Chat-v1.0'], dtype=object),
+ 'xaxis': 'x',
+ 'y': array([1.59549656]),
+ 'yaxis': 'y'},
+ {'alignmentgroup': 'True',
+ 'hovertemplate': 'Model=%{x}
Execution time (s)=%{y}',
+ 'legendgroup': 'Falcon Instruct (7B)',
+ 'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
+ 'name': 'Falcon Instruct (7B)',
+ 'offsetgroup': 'Falcon Instruct (7B)',
+ 'orientation': 'v',
+ 'showlegend': True,
+ 'textposition': 'auto',
+ 'type': 'bar',
+ 'x': array(['Falcon Instruct (7B)'], dtype=object),
+ 'xaxis': 'x',
+ 'y': array([1.35819112]),
'yaxis': 'y'}],
'layout': {'barmode': 'relative',
'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
'template': '...',
'title': {'text': 'Plot of execution time given the same input for summarization task'},
'xaxis': {'anchor': 'y',
- 'categoryarray': [Mistral-7B-Instruct-v0.2,
+ 'categoryarray': [Snorkel Mistral PairRM DPO (7B), gpt-4,
+ gpt-4-turbo, 01-ai Yi Chat (34B),
+ Chronos Hermes (13B),
Mixtral-8x7B-Instruct-v0.1,
+ llama-2-70b-chat, WizardLM v1.2 (13B),
+ chat-bison-32k (PaLM 2 32K),
+ gpt-3.5-turbo, gemini-pro, Mistral (7B)
+ Instruct v0.2 (Together AI), Vicuna v1.5
+ (7B), zephyr-7b-beta, LLaMA-2 Chat (7B),
+ chat-bison (PaLM 2),
+ Mistral-7B-Instruct-v0.2,
+ OpenHermes-2.5-Mistral (7B), Qwen 1.5
+ Chat (7B), RedPajama-INCITE Chat (7B),
TinyLlama/TinyLlama-1.1B-Chat-v1.0,
- chat-bison, chat-bison (PaLM 2), chat-
- bison-32k, chat-bison-32k (PaLM 2 32K),
- gemini-pro, gpt-3.5-turbo, gpt-4,
- gpt-4-turbo, llama-2-70b-chat,
- zephyr-7b-beta],
+ Falcon Instruct (7B)],
'categoryorder': 'array',
'domain': [0.0, 1.0],
'title': {'text': 'Model'}},
'yaxis': {'anchor': 'x', 'domain': [0.0, 1.0], 'title': {'text': 'Execution time (s)'}}}
-})",Plot of execution time given the same input for summarization task,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mistral-7B-Instruct-v0.2""],""xaxis"":""x"",""y"":[3.183657696204526],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mixtral-8x7B-Instruct-v0.1""],""xaxis"":""x"",""y"":[6.889271434409023],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""xaxis"":""x"",""y"":[1.5954965559732053],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""chat-bison"",""offsetgroup"":""chat-bison"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison""],""xaxis"":""x"",""y"":[3.890243631601334],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison (PaLM 2)"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""chat-bison (PaLM 2)"",""offsetgroup"":""chat-bison (PaLM 2)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison (PaLM 2)""],""xaxis"":""x"",""y"":[3.3088524077087644],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k"",""offsetgroup"":""chat-bison-32k"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k""],""xaxis"":""x"",""y"":[4.128177767992019],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k (PaLM 2 32K)"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k (PaLM 2 32K)"",""offsetgroup"":""chat-bison-32k (PaLM 2 32K)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k (PaLM 2 32K)""],""xaxis"":""x"",""y"":[5.464436282749687],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gemini-pro"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""gemini-pro"",""offsetgroup"":""gemini-pro"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gemini-pro""],""xaxis"":""x"",""y"":[4.111870346541831],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-3.5-turbo""],""xaxis"":""x"",""y"":[4.599214611078473],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4""],""xaxis"":""x"",""y"":[17.683679809433446],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4-turbo""],""xaxis"":""x"",""y"":[15.852052640759334],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""llama-2-70b-chat""],""xaxis"":""x"",""y"":[6.065365233652719],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""zephyr-7b-beta""],""xaxis"":""x"",""y"":[3.389894526934474],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model""},""categoryorder"":""array"",""categoryarray"":[""Mistral-7B-Instruct-v0.2"",""Mixtral-8x7B-Instruct-v0.1"",""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""chat-bison"",""chat-bison (PaLM 2)"",""chat-bison-32k"",""chat-bison-32k (PaLM 2 32K)"",""gemini-pro"",""gpt-3.5-turbo"",""gpt-4"",""gpt-4-turbo"",""llama-2-70b-chat"",""zephyr-7b-beta""]},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Execution time (s)""}},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Plot of execution time given the same input for summarization task""},""barmode"":""relative""}}",",model,execution_time
-0,Mistral-7B-Instruct-v0.2,3.183657696204526
-1,Mixtral-8x7B-Instruct-v0.1,6.889271434409023
-2,TinyLlama/TinyLlama-1.1B-Chat-v1.0,1.5954965559732053
-3,chat-bison,3.890243631601334
-4,chat-bison (PaLM 2),3.3088524077087644
-5,chat-bison-32k,4.128177767992019
-6,chat-bison-32k (PaLM 2 32K),5.464436282749687
-7,gemini-pro,4.111870346541831
-8,gpt-3.5-turbo,4.599214611078473
-9,gpt-4,17.683679809433446
-10,gpt-4-turbo,15.852052640759334
-11,llama-2-70b-chat,6.065365233652719
-12,zephyr-7b-beta,3.389894526934474
+})",Plot of execution time given the same input for summarization task,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Snorkel Mistral PairRM DPO (7B)"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""Snorkel Mistral PairRM DPO (7B)"",""offsetgroup"":""Snorkel Mistral PairRM DPO (7B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Snorkel Mistral PairRM DPO (7B)""],""xaxis"":""x"",""y"":[25.61907137927748],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4""],""xaxis"":""x"",""y"":[17.683679809433446],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4-turbo""],""xaxis"":""x"",""y"":[15.852052640759334],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""01-ai Yi Chat (34B)"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""01-ai Yi Chat (34B)"",""offsetgroup"":""01-ai Yi Chat (34B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""01-ai Yi Chat (34B)""],""xaxis"":""x"",""y"":[10.45530375040444],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Chronos Hermes (13B)"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""Chronos Hermes (13B)"",""offsetgroup"":""Chronos Hermes (13B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Chronos Hermes (13B)""],""xaxis"":""x"",""y"":[8.266028038975072],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mixtral-8x7B-Instruct-v0.1""],""xaxis"":""x"",""y"":[6.559725254549839],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""llama-2-70b-chat""],""xaxis"":""x"",""y"":[6.065365233652719],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""WizardLM v1.2 (13B)"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""WizardLM v1.2 (13B)"",""offsetgroup"":""WizardLM v1.2 (13B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""WizardLM v1.2 (13B)""],""xaxis"":""x"",""y"":[5.471834043704625],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k (PaLM 2 32K)"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k (PaLM 2 32K)"",""offsetgroup"":""chat-bison-32k (PaLM 2 32K)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k (PaLM 2 32K)""],""xaxis"":""x"",""y"":[5.464436282749687],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-3.5-turbo""],""xaxis"":""x"",""y"":[4.599214611078473],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gemini-pro"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""gemini-pro"",""offsetgroup"":""gemini-pro"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gemini-pro""],""xaxis"":""x"",""y"":[4.111870346541831],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral (7B) Instruct v0.2 (Together AI)"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""Mistral (7B) Instruct v0.2 (Together AI)"",""offsetgroup"":""Mistral (7B) Instruct v0.2 (Together AI)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mistral (7B) Instruct v0.2 (Together AI)""],""xaxis"":""x"",""y"":[4.004860708039667],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Vicuna v1.5 (7B)"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""Vicuna v1.5 (7B)"",""offsetgroup"":""Vicuna v1.5 (7B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Vicuna v1.5 (7B)""],""xaxis"":""x"",""y"":[3.8594107521168954],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""zephyr-7b-beta""],""xaxis"":""x"",""y"":[3.5767457232334547],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""LLaMA-2 Chat (7B)"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""LLaMA-2 Chat (7B)"",""offsetgroup"":""LLaMA-2 Chat (7B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""LLaMA-2 Chat (7B)""],""xaxis"":""x"",""y"":[3.44595034401415],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison (PaLM 2)"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""chat-bison (PaLM 2)"",""offsetgroup"":""chat-bison (PaLM 2)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison (PaLM 2)""],""xaxis"":""x"",""y"":[3.3088524077087644],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mistral-7B-Instruct-v0.2""],""xaxis"":""x"",""y"":[3.183657696204526],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""OpenHermes-2.5-Mistral (7B)"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""OpenHermes-2.5-Mistral (7B)"",""offsetgroup"":""OpenHermes-2.5-Mistral (7B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""OpenHermes-2.5-Mistral (7B)""],""xaxis"":""x"",""y"":[2.8320132098211745],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Qwen 1.5 Chat (7B)"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""Qwen 1.5 Chat (7B)"",""offsetgroup"":""Qwen 1.5 Chat (7B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Qwen 1.5 Chat (7B)""],""xaxis"":""x"",""y"":[2.222897692003517],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""RedPajama-INCITE Chat (7B)"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""RedPajama-INCITE Chat (7B)"",""offsetgroup"":""RedPajama-INCITE Chat (7B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""RedPajama-INCITE Chat (7B)""],""xaxis"":""x"",""y"":[1.9364212772293492],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""xaxis"":""x"",""y"":[1.5954965559732053],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eExecution time (s)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Falcon Instruct (7B)"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""Falcon Instruct (7B)"",""offsetgroup"":""Falcon Instruct (7B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Falcon Instruct (7B)""],""xaxis"":""x"",""y"":[1.3581911159155498],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model""},""categoryorder"":""array"",""categoryarray"":[""Snorkel Mistral PairRM DPO (7B)"",""gpt-4"",""gpt-4-turbo"",""01-ai Yi Chat (34B)"",""Chronos Hermes (13B)"",""Mixtral-8x7B-Instruct-v0.1"",""llama-2-70b-chat"",""WizardLM v1.2 (13B)"",""chat-bison-32k (PaLM 2 32K)"",""gpt-3.5-turbo"",""gemini-pro"",""Mistral (7B) Instruct v0.2 (Together AI)"",""Vicuna v1.5 (7B)"",""zephyr-7b-beta"",""LLaMA-2 Chat (7B)"",""chat-bison (PaLM 2)"",""Mistral-7B-Instruct-v0.2"",""OpenHermes-2.5-Mistral (7B)"",""Qwen 1.5 Chat (7B)"",""RedPajama-INCITE Chat (7B)"",""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""Falcon Instruct (7B)""]},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Execution time (s)""}},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Plot of execution time given the same input for summarization task""},""barmode"":""relative""}}",",model,execution_time
+10,Snorkel Mistral PairRM DPO (7B),25.61907137927748
+18,gpt-4,17.683679809433446
+19,gpt-4-turbo,15.852052640759334
+0,01-ai Yi Chat (34B),10.45530375040444
+1,Chronos Hermes (13B),8.266028038975072
+6,Mixtral-8x7B-Instruct-v0.1,6.559725254549839
+20,llama-2-70b-chat,6.065365233652719
+13,WizardLM v1.2 (13B),5.471834043704625
+15,chat-bison-32k (PaLM 2 32K),5.464436282749687
+17,gpt-3.5-turbo,4.599214611078473
+16,gemini-pro,4.111870346541831
+4,Mistral (7B) Instruct v0.2 (Together AI),4.004860708039667
+12,Vicuna v1.5 (7B),3.8594107521168954
+21,zephyr-7b-beta,3.5767457232334547
+3,LLaMA-2 Chat (7B),3.44595034401415
+14,chat-bison (PaLM 2),3.3088524077087644
+5,Mistral-7B-Instruct-v0.2,3.183657696204526
+7,OpenHermes-2.5-Mistral (7B),2.8320132098211745
+8,Qwen 1.5 Chat (7B),2.222897692003517
+9,RedPajama-INCITE Chat (7B),1.9364212772293492
+11,TinyLlama/TinyLlama-1.1B-Chat-v1.0,1.5954965559732053
+2,Falcon Instruct (7B),1.3581911159155498
","{""x"": ""model"", ""y"": ""execution_time"", ""color"": ""model"", ""labels"": {""model"": ""Model"", ""execution_time"": ""Execution time (s)""}, ""title"": ""Plot of execution time given the same input for summarization task""}","This plot shows averaged response time for one query for each model.
We used `time.time()` result saved to variable before making the call to API and compared it to `time.time()` result after receiving the results.
@@ -221,106 +363,120 @@ It is also important to note that we used litellm python library for all of the
"Figure({
'data': [{'alignmentgroup': 'True',
'hovertemplate': 'Model=%{x}
Output words count=%{y}',
- 'legendgroup': 'Mistral-7B-Instruct-v0.2',
+ 'legendgroup': 'Snorkel Mistral PairRM DPO (7B)',
'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
- 'name': 'Mistral-7B-Instruct-v0.2',
- 'offsetgroup': 'Mistral-7B-Instruct-v0.2',
+ 'name': 'Snorkel Mistral PairRM DPO (7B)',
+ 'offsetgroup': 'Snorkel Mistral PairRM DPO (7B)',
'orientation': 'v',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['Mistral-7B-Instruct-v0.2'], dtype=object),
+ 'x': array(['Snorkel Mistral PairRM DPO (7B)'], dtype=object),
'xaxis': 'x',
- 'y': array([29.32127976]),
+ 'y': array([235.58421355]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
'hovertemplate': 'Model=%{x}
Output words count=%{y}',
- 'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
+ 'legendgroup': 'llama-2-70b-chat',
'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
- 'name': 'Mixtral-8x7B-Instruct-v0.1',
- 'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
+ 'name': 'llama-2-70b-chat',
+ 'offsetgroup': 'llama-2-70b-chat',
'orientation': 'v',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
+ 'x': array(['llama-2-70b-chat'], dtype=object),
'xaxis': 'x',
- 'y': array([54.6203696]),
+ 'y': array([95.35066814]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
'hovertemplate': 'Model=%{x}
Output words count=%{y}',
- 'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+ 'legendgroup': 'Chronos Hermes (13B)',
'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
- 'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
- 'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+ 'name': 'Chronos Hermes (13B)',
+ 'offsetgroup': 'Chronos Hermes (13B)',
'orientation': 'v',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['TinyLlama/TinyLlama-1.1B-Chat-v1.0'], dtype=object),
+ 'x': array(['Chronos Hermes (13B)'], dtype=object),
'xaxis': 'x',
- 'y': array([32.55470546]),
+ 'y': array([72.67356172]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
'hovertemplate': 'Model=%{x}
Output words count=%{y}',
- 'legendgroup': 'chat-bison',
+ 'legendgroup': 'Mistral (7B) Instruct v0.2 (Together AI)',
'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
- 'name': 'chat-bison',
- 'offsetgroup': 'chat-bison',
+ 'name': 'Mistral (7B) Instruct v0.2 (Together AI)',
+ 'offsetgroup': 'Mistral (7B) Instruct v0.2 (Together AI)',
'orientation': 'v',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['chat-bison'], dtype=object),
+ 'x': array(['Mistral (7B) Instruct v0.2 (Together AI)'], dtype=object),
'xaxis': 'x',
- 'y': array([49.]),
+ 'y': array([68.3833169]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
'hovertemplate': 'Model=%{x}
Output words count=%{y}',
- 'legendgroup': 'chat-bison (PaLM 2)',
+ 'legendgroup': 'LLaMA-2 Chat (7B)',
'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
- 'name': 'chat-bison (PaLM 2)',
- 'offsetgroup': 'chat-bison (PaLM 2)',
+ 'name': 'LLaMA-2 Chat (7B)',
+ 'offsetgroup': 'LLaMA-2 Chat (7B)',
'orientation': 'v',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['chat-bison (PaLM 2)'], dtype=object),
+ 'x': array(['LLaMA-2 Chat (7B)'], dtype=object),
'xaxis': 'x',
- 'y': array([40.9371875]),
+ 'y': array([62.62490104]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
'hovertemplate': 'Model=%{x}
Output words count=%{y}',
- 'legendgroup': 'chat-bison-32k',
+ 'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
- 'name': 'chat-bison-32k',
- 'offsetgroup': 'chat-bison-32k',
+ 'name': 'Mixtral-8x7B-Instruct-v0.1',
+ 'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
'orientation': 'v',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['chat-bison-32k'], dtype=object),
+ 'x': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
'xaxis': 'x',
- 'y': array([48.025]),
+ 'y': array([54.16817689]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
'hovertemplate': 'Model=%{x}
Output words count=%{y}',
- 'legendgroup': 'chat-bison-32k (PaLM 2 32K)',
+ 'legendgroup': '01-ai Yi Chat (34B)',
'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
- 'name': 'chat-bison-32k (PaLM 2 32K)',
- 'offsetgroup': 'chat-bison-32k (PaLM 2 32K)',
+ 'name': '01-ai Yi Chat (34B)',
+ 'offsetgroup': '01-ai Yi Chat (34B)',
'orientation': 'v',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['chat-bison-32k (PaLM 2 32K)'], dtype=object),
+ 'x': array(['01-ai Yi Chat (34B)'], dtype=object),
'xaxis': 'x',
- 'y': array([38.72477679]),
+ 'y': array([44.55932188]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
'hovertemplate': 'Model=%{x}
Output words count=%{y}',
- 'legendgroup': 'gemini-pro',
+ 'legendgroup': 'chat-bison (PaLM 2)',
'marker': {'color': '#B6E880', 'pattern': {'shape': ''}},
+ 'name': 'chat-bison (PaLM 2)',
+ 'offsetgroup': 'chat-bison (PaLM 2)',
+ 'orientation': 'v',
+ 'showlegend': True,
+ 'textposition': 'auto',
+ 'type': 'bar',
+ 'x': array(['chat-bison (PaLM 2)'], dtype=object),
+ 'xaxis': 'x',
+ 'y': array([40.9371875]),
+ 'yaxis': 'y'},
+ {'alignmentgroup': 'True',
+ 'hovertemplate': 'Model=%{x}
Output words count=%{y}',
+ 'legendgroup': 'gemini-pro',
+ 'marker': {'color': '#FF97FF', 'pattern': {'shape': ''}},
'name': 'gemini-pro',
'offsetgroup': 'gemini-pro',
'orientation': 'v',
@@ -334,7 +490,7 @@ It is also important to note that we used litellm python library for all of the
{'alignmentgroup': 'True',
'hovertemplate': 'Model=%{x}
Output words count=%{y}',
'legendgroup': 'gpt-3.5-turbo',
- 'marker': {'color': '#FF97FF', 'pattern': {'shape': ''}},
+ 'marker': {'color': '#FECB52', 'pattern': {'shape': ''}},
'name': 'gpt-3.5-turbo',
'offsetgroup': 'gpt-3.5-turbo',
'orientation': 'v',
@@ -347,22 +503,22 @@ It is also important to note that we used litellm python library for all of the
'yaxis': 'y'},
{'alignmentgroup': 'True',
'hovertemplate': 'Model=%{x}
Output words count=%{y}',
- 'legendgroup': 'gpt-4',
- 'marker': {'color': '#FECB52', 'pattern': {'shape': ''}},
- 'name': 'gpt-4',
- 'offsetgroup': 'gpt-4',
+ 'legendgroup': 'chat-bison-32k (PaLM 2 32K)',
+ 'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
+ 'name': 'chat-bison-32k (PaLM 2 32K)',
+ 'offsetgroup': 'chat-bison-32k (PaLM 2 32K)',
'orientation': 'v',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['gpt-4'], dtype=object),
+ 'x': array(['chat-bison-32k (PaLM 2 32K)'], dtype=object),
'xaxis': 'x',
- 'y': array([32.90737496]),
+ 'y': array([38.72477679]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
'hovertemplate': 'Model=%{x}
Output words count=%{y}',
'legendgroup': 'gpt-4-turbo',
- 'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
+ 'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
'name': 'gpt-4-turbo',
'offsetgroup': 'gpt-4-turbo',
'orientation': 'v',
@@ -375,181 +531,295 @@ It is also important to note that we used litellm python library for all of the
'yaxis': 'y'},
{'alignmentgroup': 'True',
'hovertemplate': 'Model=%{x}
Output words count=%{y}',
- 'legendgroup': 'llama-2-70b-chat',
- 'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
- 'name': 'llama-2-70b-chat',
- 'offsetgroup': 'llama-2-70b-chat',
+ 'legendgroup': 'Vicuna v1.5 (7B)',
+ 'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
+ 'name': 'Vicuna v1.5 (7B)',
+ 'offsetgroup': 'Vicuna v1.5 (7B)',
'orientation': 'v',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['llama-2-70b-chat'], dtype=object),
+ 'x': array(['Vicuna v1.5 (7B)'], dtype=object),
'xaxis': 'x',
- 'y': array([95.35066814]),
+ 'y': array([37.18623469]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
'hovertemplate': 'Model=%{x}
Output words count=%{y}',
- 'legendgroup': 'zephyr-7b-beta',
- 'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
- 'name': 'zephyr-7b-beta',
- 'offsetgroup': 'zephyr-7b-beta',
+ 'legendgroup': 'OpenHermes-2.5-Mistral (7B)',
+ 'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
+ 'name': 'OpenHermes-2.5-Mistral (7B)',
+ 'offsetgroup': 'OpenHermes-2.5-Mistral (7B)',
+ 'orientation': 'v',
+ 'showlegend': True,
+ 'textposition': 'auto',
+ 'type': 'bar',
+ 'x': array(['OpenHermes-2.5-Mistral (7B)'], dtype=object),
+ 'xaxis': 'x',
+ 'y': array([32.90882368]),
+ 'yaxis': 'y'},
+ {'alignmentgroup': 'True',
+ 'hovertemplate': 'Model=%{x}
Output words count=%{y}',
+ 'legendgroup': 'gpt-4',
+ 'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
+ 'name': 'gpt-4',
+ 'offsetgroup': 'gpt-4',
+ 'orientation': 'v',
+ 'showlegend': True,
+ 'textposition': 'auto',
+ 'type': 'bar',
+ 'x': array(['gpt-4'], dtype=object),
+ 'xaxis': 'x',
+ 'y': array([32.90737496]),
+ 'yaxis': 'y'},
+ {'alignmentgroup': 'True',
+ 'hovertemplate': 'Model=%{x}
Output words count=%{y}',
+ 'legendgroup': 'Qwen 1.5 Chat (7B)',
+ 'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
+ 'name': 'Qwen 1.5 Chat (7B)',
+ 'offsetgroup': 'Qwen 1.5 Chat (7B)',
+ 'orientation': 'v',
+ 'showlegend': True,
+ 'textposition': 'auto',
+ 'type': 'bar',
+ 'x': array(['Qwen 1.5 Chat (7B)'], dtype=object),
+ 'xaxis': 'x',
+ 'y': array([32.61154087]),
+ 'yaxis': 'y'},
+ {'alignmentgroup': 'True',
+ 'hovertemplate': 'Model=%{x}
Output words count=%{y}',
+ 'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+ 'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
+ 'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+ 'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+ 'orientation': 'v',
+ 'showlegend': True,
+ 'textposition': 'auto',
+ 'type': 'bar',
+ 'x': array(['TinyLlama/TinyLlama-1.1B-Chat-v1.0'], dtype=object),
+ 'xaxis': 'x',
+ 'y': array([32.55470546]),
+ 'yaxis': 'y'},
+ {'alignmentgroup': 'True',
+ 'hovertemplate': 'Model=%{x}
Output words count=%{y}',
+ 'legendgroup': 'zephyr-7b-beta',
+ 'marker': {'color': '#B6E880', 'pattern': {'shape': ''}},
+ 'name': 'zephyr-7b-beta',
+ 'offsetgroup': 'zephyr-7b-beta',
'orientation': 'v',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
'x': array(['zephyr-7b-beta'], dtype=object),
'xaxis': 'x',
- 'y': array([31.15925708]),
+ 'y': array([32.25400169]),
+ 'yaxis': 'y'},
+ {'alignmentgroup': 'True',
+ 'hovertemplate': 'Model=%{x}
Output words count=%{y}',
+ 'legendgroup': 'WizardLM v1.2 (13B)',
+ 'marker': {'color': '#FF97FF', 'pattern': {'shape': ''}},
+ 'name': 'WizardLM v1.2 (13B)',
+ 'offsetgroup': 'WizardLM v1.2 (13B)',
+ 'orientation': 'v',
+ 'showlegend': True,
+ 'textposition': 'auto',
+ 'type': 'bar',
+ 'x': array(['WizardLM v1.2 (13B)'], dtype=object),
+ 'xaxis': 'x',
+ 'y': array([30.37805458]),
+ 'yaxis': 'y'},
+ {'alignmentgroup': 'True',
+ 'hovertemplate': 'Model=%{x}
Output words count=%{y}',
+ 'legendgroup': 'Mistral-7B-Instruct-v0.2',
+ 'marker': {'color': '#FECB52', 'pattern': {'shape': ''}},
+ 'name': 'Mistral-7B-Instruct-v0.2',
+ 'offsetgroup': 'Mistral-7B-Instruct-v0.2',
+ 'orientation': 'v',
+ 'showlegend': True,
+ 'textposition': 'auto',
+ 'type': 'bar',
+ 'x': array(['Mistral-7B-Instruct-v0.2'], dtype=object),
+ 'xaxis': 'x',
+ 'y': array([29.32127976]),
+ 'yaxis': 'y'},
+ {'alignmentgroup': 'True',
+ 'hovertemplate': 'Model=%{x}
Output words count=%{y}',
+ 'legendgroup': 'Falcon Instruct (7B)',
+ 'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
+ 'name': 'Falcon Instruct (7B)',
+ 'offsetgroup': 'Falcon Instruct (7B)',
+ 'orientation': 'v',
+ 'showlegend': True,
+ 'textposition': 'auto',
+ 'type': 'bar',
+ 'x': array(['Falcon Instruct (7B)'], dtype=object),
+ 'xaxis': 'x',
+ 'y': array([10.03811449]),
+ 'yaxis': 'y'},
+ {'alignmentgroup': 'True',
+ 'hovertemplate': 'Model=%{x}
Output words count=%{y}',
+ 'legendgroup': 'RedPajama-INCITE Chat (7B)',
+ 'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
+ 'name': 'RedPajama-INCITE Chat (7B)',
+ 'offsetgroup': 'RedPajama-INCITE Chat (7B)',
+ 'orientation': 'v',
+ 'showlegend': True,
+ 'textposition': 'auto',
+ 'type': 'bar',
+ 'x': array(['RedPajama-INCITE Chat (7B)'], dtype=object),
+ 'xaxis': 'x',
+ 'y': array([5.25619503]),
'yaxis': 'y'}],
'layout': {'barmode': 'relative',
'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
'template': '...',
'title': {'text': 'Plot of output words count given the same input for summarization task'},
'xaxis': {'anchor': 'y',
- 'categoryarray': [Mistral-7B-Instruct-v0.2,
- Mixtral-8x7B-Instruct-v0.1,
+ 'categoryarray': [Snorkel Mistral PairRM DPO (7B),
+ llama-2-70b-chat, Chronos Hermes (13B),
+ Mistral (7B) Instruct v0.2 (Together
+ AI), LLaMA-2 Chat (7B),
+ Mixtral-8x7B-Instruct-v0.1, 01-ai Yi
+ Chat (34B), chat-bison (PaLM 2), gemini-
+ pro, gpt-3.5-turbo, chat-bison-32k (PaLM
+ 2 32K), gpt-4-turbo, Vicuna v1.5 (7B),
+ OpenHermes-2.5-Mistral (7B), gpt-4, Qwen
+ 1.5 Chat (7B),
TinyLlama/TinyLlama-1.1B-Chat-v1.0,
- chat-bison, chat-bison (PaLM 2), chat-
- bison-32k, chat-bison-32k (PaLM 2 32K),
- gemini-pro, gpt-3.5-turbo, gpt-4,
- gpt-4-turbo, llama-2-70b-chat,
- zephyr-7b-beta],
+ zephyr-7b-beta, WizardLM v1.2 (13B),
+ Mistral-7B-Instruct-v0.2, Falcon
+ Instruct (7B), RedPajama-INCITE Chat
+ (7B)],
'categoryorder': 'array',
'domain': [0.0, 1.0],
'title': {'text': 'Model'}},
'yaxis': {'anchor': 'x', 'domain': [0.0, 1.0], 'title': {'text': 'Output words count'}}}
-})",Plot of output words count given the same input for summarization task,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mistral-7B-Instruct-v0.2""],""xaxis"":""x"",""y"":[29.321279761904762],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mixtral-8x7B-Instruct-v0.1""],""xaxis"":""x"",""y"":[54.620369597277694],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""xaxis"":""x"",""y"":[32.55470545977011],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""chat-bison"",""offsetgroup"":""chat-bison"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison""],""xaxis"":""x"",""y"":[49.0],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison (PaLM 2)"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""chat-bison (PaLM 2)"",""offsetgroup"":""chat-bison (PaLM 2)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison (PaLM 2)""],""xaxis"":""x"",""y"":[40.9371875],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k"",""offsetgroup"":""chat-bison-32k"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k""],""xaxis"":""x"",""y"":[48.025],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k (PaLM 2 32K)"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k (PaLM 2 32K)"",""offsetgroup"":""chat-bison-32k (PaLM 2 32K)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k (PaLM 2 32K)""],""xaxis"":""x"",""y"":[38.72477678571428],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gemini-pro"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""gemini-pro"",""offsetgroup"":""gemini-pro"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gemini-pro""],""xaxis"":""x"",""y"":[39.69346182236394],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-3.5-turbo""],""xaxis"":""x"",""y"":[39.25851283971182],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4""],""xaxis"":""x"",""y"":[32.907374959094994],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4-turbo""],""xaxis"":""x"",""y"":[38.36012473477448],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""llama-2-70b-chat""],""xaxis"":""x"",""y"":[95.35066814260509],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""zephyr-7b-beta""],""xaxis"":""x"",""y"":[31.159257075471697],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model""},""categoryorder"":""array"",""categoryarray"":[""Mistral-7B-Instruct-v0.2"",""Mixtral-8x7B-Instruct-v0.1"",""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""chat-bison"",""chat-bison (PaLM 2)"",""chat-bison-32k"",""chat-bison-32k (PaLM 2 32K)"",""gemini-pro"",""gpt-3.5-turbo"",""gpt-4"",""gpt-4-turbo"",""llama-2-70b-chat"",""zephyr-7b-beta""]},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Output words count""}},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Plot of output words count given the same input for summarization task""},""barmode"":""relative""}}",",model,words_count
-0,Mistral-7B-Instruct-v0.2,29.321279761904762
-1,Mixtral-8x7B-Instruct-v0.1,54.620369597277694
-2,TinyLlama/TinyLlama-1.1B-Chat-v1.0,32.55470545977011
-3,chat-bison,49.0
-4,chat-bison (PaLM 2),40.9371875
-5,chat-bison-32k,48.025
-6,chat-bison-32k (PaLM 2 32K),38.72477678571428
-7,gemini-pro,39.69346182236394
-8,gpt-3.5-turbo,39.25851283971182
-9,gpt-4,32.907374959094994
-10,gpt-4-turbo,38.36012473477448
-11,llama-2-70b-chat,95.35066814260509
-12,zephyr-7b-beta,31.159257075471697
+})",Plot of output words count given the same input for summarization task,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Snorkel Mistral PairRM DPO (7B)"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""Snorkel Mistral PairRM DPO (7B)"",""offsetgroup"":""Snorkel Mistral PairRM DPO (7B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Snorkel Mistral PairRM DPO (7B)""],""xaxis"":""x"",""y"":[235.584213548992],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""llama-2-70b-chat""],""xaxis"":""x"",""y"":[95.35066814260509],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Chronos Hermes (13B)"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""Chronos Hermes (13B)"",""offsetgroup"":""Chronos Hermes (13B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Chronos Hermes (13B)""],""xaxis"":""x"",""y"":[72.6735617165285],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral (7B) Instruct v0.2 (Together AI)"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""Mistral (7B) Instruct v0.2 (Together AI)"",""offsetgroup"":""Mistral (7B) Instruct v0.2 (Together AI)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mistral (7B) Instruct v0.2 (Together AI)""],""xaxis"":""x"",""y"":[68.38331690448811],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""LLaMA-2 Chat (7B)"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""LLaMA-2 Chat (7B)"",""offsetgroup"":""LLaMA-2 Chat (7B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""LLaMA-2 Chat (7B)""],""xaxis"":""x"",""y"":[62.624901040740085],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mixtral-8x7B-Instruct-v0.1""],""xaxis"":""x"",""y"":[54.16817688738913],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""01-ai Yi Chat (34B)"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""01-ai Yi Chat (34B)"",""offsetgroup"":""01-ai Yi Chat (34B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""01-ai Yi Chat (34B)""],""xaxis"":""x"",""y"":[44.559321875420764],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison (PaLM 2)"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""chat-bison (PaLM 2)"",""offsetgroup"":""chat-bison (PaLM 2)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison (PaLM 2)""],""xaxis"":""x"",""y"":[40.9371875],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gemini-pro"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""gemini-pro"",""offsetgroup"":""gemini-pro"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gemini-pro""],""xaxis"":""x"",""y"":[39.69346182236394],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-3.5-turbo""],""xaxis"":""x"",""y"":[39.25851283971182],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k (PaLM 2 32K)"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k (PaLM 2 32K)"",""offsetgroup"":""chat-bison-32k (PaLM 2 32K)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k (PaLM 2 32K)""],""xaxis"":""x"",""y"":[38.72477678571428],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4-turbo""],""xaxis"":""x"",""y"":[38.36012473477448],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Vicuna v1.5 (7B)"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""Vicuna v1.5 (7B)"",""offsetgroup"":""Vicuna v1.5 (7B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Vicuna v1.5 (7B)""],""xaxis"":""x"",""y"":[37.186234691924994],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""OpenHermes-2.5-Mistral (7B)"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""OpenHermes-2.5-Mistral (7B)"",""offsetgroup"":""OpenHermes-2.5-Mistral (7B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""OpenHermes-2.5-Mistral (7B)""],""xaxis"":""x"",""y"":[32.90882367725072],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4""],""xaxis"":""x"",""y"":[32.907374959094994],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Qwen 1.5 Chat (7B)"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""Qwen 1.5 Chat (7B)"",""offsetgroup"":""Qwen 1.5 Chat (7B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Qwen 1.5 Chat (7B)""],""xaxis"":""x"",""y"":[32.611540865760276],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""xaxis"":""x"",""y"":[32.55470545977011],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""zephyr-7b-beta""],""xaxis"":""x"",""y"":[32.254001687967204],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""WizardLM v1.2 (13B)"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""WizardLM v1.2 (13B)"",""offsetgroup"":""WizardLM v1.2 (13B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""WizardLM v1.2 (13B)""],""xaxis"":""x"",""y"":[30.37805458105951],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mistral-7B-Instruct-v0.2""],""xaxis"":""x"",""y"":[29.321279761904762],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Falcon Instruct (7B)"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""Falcon Instruct (7B)"",""offsetgroup"":""Falcon Instruct (7B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Falcon Instruct (7B)""],""xaxis"":""x"",""y"":[10.038114488057422],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eOutput words count=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""RedPajama-INCITE Chat (7B)"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""RedPajama-INCITE Chat (7B)"",""offsetgroup"":""RedPajama-INCITE Chat (7B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""RedPajama-INCITE Chat (7B)""],""xaxis"":""x"",""y"":[5.256195028382509],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model""},""categoryorder"":""array"",""categoryarray"":[""Snorkel Mistral PairRM DPO (7B)"",""llama-2-70b-chat"",""Chronos Hermes (13B)"",""Mistral (7B) Instruct v0.2 (Together AI)"",""LLaMA-2 Chat (7B)"",""Mixtral-8x7B-Instruct-v0.1"",""01-ai Yi Chat (34B)"",""chat-bison (PaLM 2)"",""gemini-pro"",""gpt-3.5-turbo"",""chat-bison-32k (PaLM 2 32K)"",""gpt-4-turbo"",""Vicuna v1.5 (7B)"",""OpenHermes-2.5-Mistral (7B)"",""gpt-4"",""Qwen 1.5 Chat (7B)"",""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""zephyr-7b-beta"",""WizardLM v1.2 (13B)"",""Mistral-7B-Instruct-v0.2"",""Falcon Instruct (7B)"",""RedPajama-INCITE Chat (7B)""]},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Output words count""}},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Plot of output words count given the same input for summarization task""},""barmode"":""relative""}}",",model,words_count
+10,Snorkel Mistral PairRM DPO (7B),235.584213548992
+20,llama-2-70b-chat,95.35066814260509
+1,Chronos Hermes (13B),72.6735617165285
+4,Mistral (7B) Instruct v0.2 (Together AI),68.38331690448811
+3,LLaMA-2 Chat (7B),62.624901040740085
+6,Mixtral-8x7B-Instruct-v0.1,54.16817688738913
+0,01-ai Yi Chat (34B),44.559321875420764
+14,chat-bison (PaLM 2),40.9371875
+16,gemini-pro,39.69346182236394
+17,gpt-3.5-turbo,39.25851283971182
+15,chat-bison-32k (PaLM 2 32K),38.72477678571428
+19,gpt-4-turbo,38.36012473477448
+12,Vicuna v1.5 (7B),37.186234691924994
+7,OpenHermes-2.5-Mistral (7B),32.90882367725072
+18,gpt-4,32.907374959094994
+8,Qwen 1.5 Chat (7B),32.611540865760276
+11,TinyLlama/TinyLlama-1.1B-Chat-v1.0,32.55470545977011
+21,zephyr-7b-beta,32.254001687967204
+13,WizardLM v1.2 (13B),30.37805458105951
+5,Mistral-7B-Instruct-v0.2,29.321279761904762
+2,Falcon Instruct (7B),10.038114488057422
+9,RedPajama-INCITE Chat (7B),5.256195028382509
","{""x"": ""model"", ""y"": ""words_count"", ""color"": ""model"", ""labels"": {""model"": ""Model"", ""words_count"": ""Output words count""}, ""title"": ""Plot of output words count given the same input for summarization task""}",
"Figure({
'data': [{'alignmentgroup': 'True',
'hovertemplate': 'Model=%{x}
Chunks mean size (characters)=%{y}',
- 'legendgroup': 'Mistral-7B-Instruct-v0.2',
+ 'legendgroup': 'gemini-pro',
'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
- 'name': 'Mistral-7B-Instruct-v0.2',
- 'offsetgroup': 'Mistral-7B-Instruct-v0.2',
+ 'name': 'gemini-pro',
+ 'offsetgroup': 'gemini-pro',
'orientation': 'v',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['Mistral-7B-Instruct-v0.2'], dtype=object),
+ 'x': array(['gemini-pro'], dtype=object),
'xaxis': 'x',
- 'y': array([2.69413923]),
+ 'y': array([142.50301526]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
'hovertemplate': 'Model=%{x}
Chunks mean size (characters)=%{y}',
- 'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
+ 'legendgroup': 'chat-bison (PaLM 2)',
'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
- 'name': 'Mixtral-8x7B-Instruct-v0.1',
- 'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
+ 'name': 'chat-bison (PaLM 2)',
+ 'offsetgroup': 'chat-bison (PaLM 2)',
'orientation': 'v',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
+ 'x': array(['chat-bison (PaLM 2)'], dtype=object),
'xaxis': 'x',
- 'y': array([2.95690172]),
+ 'y': array([57.12081731]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
'hovertemplate': 'Model=%{x}
Chunks mean size (characters)=%{y}',
- 'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+ 'legendgroup': 'chat-bison-32k (PaLM 2 32K)',
'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
- 'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
- 'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+ 'name': 'chat-bison-32k (PaLM 2 32K)',
+ 'offsetgroup': 'chat-bison-32k (PaLM 2 32K)',
'orientation': 'v',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['TinyLlama/TinyLlama-1.1B-Chat-v1.0'], dtype=object),
+ 'x': array(['chat-bison-32k (PaLM 2 32K)'], dtype=object),
'xaxis': 'x',
- 'y': array([3.14663951]),
+ 'y': array([56.57639874]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
'hovertemplate': 'Model=%{x}
Chunks mean size (characters)=%{y}',
- 'legendgroup': 'chat-bison',
+ 'legendgroup': 'Falcon Instruct (7B)',
'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
- 'name': 'chat-bison',
- 'offsetgroup': 'chat-bison',
+ 'name': 'Falcon Instruct (7B)',
+ 'offsetgroup': 'Falcon Instruct (7B)',
'orientation': 'v',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['chat-bison'], dtype=object),
+ 'x': array(['Falcon Instruct (7B)'], dtype=object),
'xaxis': 'x',
- 'y': array([61.46100879]),
+ 'y': array([3.98255342]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
'hovertemplate': 'Model=%{x}
Chunks mean size (characters)=%{y}',
- 'legendgroup': 'chat-bison (PaLM 2)',
+ 'legendgroup': 'llama-2-70b-chat',
'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
- 'name': 'chat-bison (PaLM 2)',
- 'offsetgroup': 'chat-bison (PaLM 2)',
+ 'name': 'llama-2-70b-chat',
+ 'offsetgroup': 'llama-2-70b-chat',
'orientation': 'v',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['chat-bison (PaLM 2)'], dtype=object),
+ 'x': array(['llama-2-70b-chat'], dtype=object),
'xaxis': 'x',
- 'y': array([57.12081731]),
+ 'y': array([3.20032418]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
'hovertemplate': 'Model=%{x}
Chunks mean size (characters)=%{y}',
- 'legendgroup': 'chat-bison-32k',
+ 'legendgroup': 'RedPajama-INCITE Chat (7B)',
'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
- 'name': 'chat-bison-32k',
- 'offsetgroup': 'chat-bison-32k',
+ 'name': 'RedPajama-INCITE Chat (7B)',
+ 'offsetgroup': 'RedPajama-INCITE Chat (7B)',
'orientation': 'v',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['chat-bison-32k'], dtype=object),
+ 'x': array(['RedPajama-INCITE Chat (7B)'], dtype=object),
'xaxis': 'x',
- 'y': array([62.01633273]),
+ 'y': array([3.17457251]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
'hovertemplate': 'Model=%{x}
Chunks mean size (characters)=%{y}',
- 'legendgroup': 'chat-bison-32k (PaLM 2 32K)',
+ 'legendgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
- 'name': 'chat-bison-32k (PaLM 2 32K)',
- 'offsetgroup': 'chat-bison-32k (PaLM 2 32K)',
- 'orientation': 'v',
- 'showlegend': True,
- 'textposition': 'auto',
- 'type': 'bar',
- 'x': array(['chat-bison-32k (PaLM 2 32K)'], dtype=object),
- 'xaxis': 'x',
- 'y': array([56.57639874]),
- 'yaxis': 'y'},
- {'alignmentgroup': 'True',
- 'hovertemplate': 'Model=%{x}
Chunks mean size (characters)=%{y}',
- 'legendgroup': 'gemini-pro',
- 'marker': {'color': '#B6E880', 'pattern': {'shape': ''}},
- 'name': 'gemini-pro',
- 'offsetgroup': 'gemini-pro',
+ 'name': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
+ 'offsetgroup': 'TinyLlama/TinyLlama-1.1B-Chat-v1.0',
'orientation': 'v',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['gemini-pro'], dtype=object),
+ 'x': array(['TinyLlama/TinyLlama-1.1B-Chat-v1.0'], dtype=object),
'xaxis': 'x',
- 'y': array([142.50301526]),
+ 'y': array([3.14663951]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
'hovertemplate': 'Model=%{x}
Chunks mean size (characters)=%{y}',
'legendgroup': 'gpt-3.5-turbo',
- 'marker': {'color': '#FF97FF', 'pattern': {'shape': ''}},
+ 'marker': {'color': '#B6E880', 'pattern': {'shape': ''}},
'name': 'gpt-3.5-turbo',
'offsetgroup': 'gpt-3.5-turbo',
'orientation': 'v',
@@ -563,7 +833,7 @@ It is also important to note that we used litellm python library for all of the
{'alignmentgroup': 'True',
'hovertemplate': 'Model=%{x}
Chunks mean size (characters)=%{y}',
'legendgroup': 'gpt-4',
- 'marker': {'color': '#FECB52', 'pattern': {'shape': ''}},
+ 'marker': {'color': '#FF97FF', 'pattern': {'shape': ''}},
'name': 'gpt-4',
'offsetgroup': 'gpt-4',
'orientation': 'v',
@@ -577,7 +847,7 @@ It is also important to note that we used litellm python library for all of the
{'alignmentgroup': 'True',
'hovertemplate': 'Model=%{x}
Chunks mean size (characters)=%{y}',
'legendgroup': 'gpt-4-turbo',
- 'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
+ 'marker': {'color': '#FECB52', 'pattern': {'shape': ''}},
'name': 'gpt-4-turbo',
'offsetgroup': 'gpt-4-turbo',
'orientation': 'v',
@@ -590,22 +860,78 @@ It is also important to note that we used litellm python library for all of the
'yaxis': 'y'},
{'alignmentgroup': 'True',
'hovertemplate': 'Model=%{x}
Chunks mean size (characters)=%{y}',
- 'legendgroup': 'llama-2-70b-chat',
+ 'legendgroup': 'Mixtral-8x7B-Instruct-v0.1',
+ 'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
+ 'name': 'Mixtral-8x7B-Instruct-v0.1',
+ 'offsetgroup': 'Mixtral-8x7B-Instruct-v0.1',
+ 'orientation': 'v',
+ 'showlegend': True,
+ 'textposition': 'auto',
+ 'type': 'bar',
+ 'x': array(['Mixtral-8x7B-Instruct-v0.1'], dtype=object),
+ 'xaxis': 'x',
+ 'y': array([2.95880337]),
+ 'yaxis': 'y'},
+ {'alignmentgroup': 'True',
+ 'hovertemplate': 'Model=%{x}
Chunks mean size (characters)=%{y}',
+ 'legendgroup': 'LLaMA-2 Chat (7B)',
'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
- 'name': 'llama-2-70b-chat',
- 'offsetgroup': 'llama-2-70b-chat',
+ 'name': 'LLaMA-2 Chat (7B)',
+ 'offsetgroup': 'LLaMA-2 Chat (7B)',
'orientation': 'v',
'showlegend': True,
'textposition': 'auto',
'type': 'bar',
- 'x': array(['llama-2-70b-chat'], dtype=object),
+ 'x': array(['LLaMA-2 Chat (7B)'], dtype=object),
'xaxis': 'x',
- 'y': array([3.20032418]),
+ 'y': array([2.7744863]),
'yaxis': 'y'},
{'alignmentgroup': 'True',
'hovertemplate': 'Model=%{x}
Chunks mean size (characters)=%{y}',
- 'legendgroup': 'zephyr-7b-beta',
+ 'legendgroup': 'Qwen 1.5 Chat (7B)',
'marker': {'color': '#00cc96', 'pattern': {'shape': ''}},
+ 'name': 'Qwen 1.5 Chat (7B)',
+ 'offsetgroup': 'Qwen 1.5 Chat (7B)',
+ 'orientation': 'v',
+ 'showlegend': True,
+ 'textposition': 'auto',
+ 'type': 'bar',
+ 'x': array(['Qwen 1.5 Chat (7B)'], dtype=object),
+ 'xaxis': 'x',
+ 'y': array([2.76407305]),
+ 'yaxis': 'y'},
+ {'alignmentgroup': 'True',
+ 'hovertemplate': 'Model=%{x}
Chunks mean size (characters)=%{y}',
+ 'legendgroup': 'Snorkel Mistral PairRM DPO (7B)',
+ 'marker': {'color': '#ab63fa', 'pattern': {'shape': ''}},
+ 'name': 'Snorkel Mistral PairRM DPO (7B)',
+ 'offsetgroup': 'Snorkel Mistral PairRM DPO (7B)',
+ 'orientation': 'v',
+ 'showlegend': True,
+ 'textposition': 'auto',
+ 'type': 'bar',
+ 'x': array(['Snorkel Mistral PairRM DPO (7B)'], dtype=object),
+ 'xaxis': 'x',
+ 'y': array([2.71983728]),
+ 'yaxis': 'y'},
+ {'alignmentgroup': 'True',
+ 'hovertemplate': 'Model=%{x}
Chunks mean size (characters)=%{y}',
+ 'legendgroup': 'Chronos Hermes (13B)',
+ 'marker': {'color': '#FFA15A', 'pattern': {'shape': ''}},
+ 'name': 'Chronos Hermes (13B)',
+ 'offsetgroup': 'Chronos Hermes (13B)',
+ 'orientation': 'v',
+ 'showlegend': True,
+ 'textposition': 'auto',
+ 'type': 'bar',
+ 'x': array(['Chronos Hermes (13B)'], dtype=object),
+ 'xaxis': 'x',
+ 'y': array([2.71485241]),
+ 'yaxis': 'y'},
+ {'alignmentgroup': 'True',
+ 'hovertemplate': 'Model=%{x}
Chunks mean size (characters)=%{y}',
+ 'legendgroup': 'zephyr-7b-beta',
+ 'marker': {'color': '#19d3f3', 'pattern': {'shape': ''}},
'name': 'zephyr-7b-beta',
'offsetgroup': 'zephyr-7b-beta',
'orientation': 'v',
@@ -614,37 +940,136 @@ It is also important to note that we used litellm python library for all of the
'type': 'bar',
'x': array(['zephyr-7b-beta'], dtype=object),
'xaxis': 'x',
- 'y': array([2.71100327]),
+ 'y': array([2.70854999]),
+ 'yaxis': 'y'},
+ {'alignmentgroup': 'True',
+ 'hovertemplate': 'Model=%{x}
Chunks mean size (characters)=%{y}',
+ 'legendgroup': 'Mistral-7B-Instruct-v0.2',
+ 'marker': {'color': '#FF6692', 'pattern': {'shape': ''}},
+ 'name': 'Mistral-7B-Instruct-v0.2',
+ 'offsetgroup': 'Mistral-7B-Instruct-v0.2',
+ 'orientation': 'v',
+ 'showlegend': True,
+ 'textposition': 'auto',
+ 'type': 'bar',
+ 'x': array(['Mistral-7B-Instruct-v0.2'], dtype=object),
+ 'xaxis': 'x',
+ 'y': array([2.69413923]),
+ 'yaxis': 'y'},
+ {'alignmentgroup': 'True',
+ 'hovertemplate': 'Model=%{x}
Chunks mean size (characters)=%{y}',
+ 'legendgroup': 'Mistral (7B) Instruct v0.2 (Together AI)',
+ 'marker': {'color': '#B6E880', 'pattern': {'shape': ''}},
+ 'name': 'Mistral (7B) Instruct v0.2 (Together AI)',
+ 'offsetgroup': 'Mistral (7B) Instruct v0.2 (Together AI)',
+ 'orientation': 'v',
+ 'showlegend': True,
+ 'textposition': 'auto',
+ 'type': 'bar',
+ 'x': array(['Mistral (7B) Instruct v0.2 (Together AI)'], dtype=object),
+ 'xaxis': 'x',
+ 'y': array([2.63865224]),
+ 'yaxis': 'y'},
+ {'alignmentgroup': 'True',
+ 'hovertemplate': 'Model=%{x}
Chunks mean size (characters)=%{y}',
+ 'legendgroup': '01-ai Yi Chat (34B)',
+ 'marker': {'color': '#FF97FF', 'pattern': {'shape': ''}},
+ 'name': '01-ai Yi Chat (34B)',
+ 'offsetgroup': '01-ai Yi Chat (34B)',
+ 'orientation': 'v',
+ 'showlegend': True,
+ 'textposition': 'auto',
+ 'type': 'bar',
+ 'x': array(['01-ai Yi Chat (34B)'], dtype=object),
+ 'xaxis': 'x',
+ 'y': array([2.60737454]),
+ 'yaxis': 'y'},
+ {'alignmentgroup': 'True',
+ 'hovertemplate': 'Model=%{x}
Chunks mean size (characters)=%{y}',
+ 'legendgroup': 'OpenHermes-2.5-Mistral (7B)',
+ 'marker': {'color': '#FECB52', 'pattern': {'shape': ''}},
+ 'name': 'OpenHermes-2.5-Mistral (7B)',
+ 'offsetgroup': 'OpenHermes-2.5-Mistral (7B)',
+ 'orientation': 'v',
+ 'showlegend': True,
+ 'textposition': 'auto',
+ 'type': 'bar',
+ 'x': array(['OpenHermes-2.5-Mistral (7B)'], dtype=object),
+ 'xaxis': 'x',
+ 'y': array([2.41139151]),
+ 'yaxis': 'y'},
+ {'alignmentgroup': 'True',
+ 'hovertemplate': 'Model=%{x}
Chunks mean size (characters)=%{y}',
+ 'legendgroup': 'Vicuna v1.5 (7B)',
+ 'marker': {'color': '#636efa', 'pattern': {'shape': ''}},
+ 'name': 'Vicuna v1.5 (7B)',
+ 'offsetgroup': 'Vicuna v1.5 (7B)',
+ 'orientation': 'v',
+ 'showlegend': True,
+ 'textposition': 'auto',
+ 'type': 'bar',
+ 'x': array(['Vicuna v1.5 (7B)'], dtype=object),
+ 'xaxis': 'x',
+ 'y': array([2.38783449]),
+ 'yaxis': 'y'},
+ {'alignmentgroup': 'True',
+ 'hovertemplate': 'Model=%{x}
Chunks mean size (characters)=%{y}',
+ 'legendgroup': 'WizardLM v1.2 (13B)',
+ 'marker': {'color': '#EF553B', 'pattern': {'shape': ''}},
+ 'name': 'WizardLM v1.2 (13B)',
+ 'offsetgroup': 'WizardLM v1.2 (13B)',
+ 'orientation': 'v',
+ 'showlegend': True,
+ 'textposition': 'auto',
+ 'type': 'bar',
+ 'x': array(['WizardLM v1.2 (13B)'], dtype=object),
+ 'xaxis': 'x',
+ 'y': array([2.29383365]),
'yaxis': 'y'}],
'layout': {'barmode': 'relative',
'legend': {'title': {'text': 'Model'}, 'tracegroupgap': 0},
'template': '...',
'title': {'text': 'Plot of chunks mean size given the same input for summarization task'},
'xaxis': {'anchor': 'y',
- 'categoryarray': [Mistral-7B-Instruct-v0.2,
- Mixtral-8x7B-Instruct-v0.1,
+ 'categoryarray': [gemini-pro, chat-bison (PaLM 2), chat-
+ bison-32k (PaLM 2 32K), Falcon Instruct
+ (7B), llama-2-70b-chat, RedPajama-INCITE
+ Chat (7B),
TinyLlama/TinyLlama-1.1B-Chat-v1.0,
- chat-bison, chat-bison (PaLM 2), chat-
- bison-32k, chat-bison-32k (PaLM 2 32K),
- gemini-pro, gpt-3.5-turbo, gpt-4,
- gpt-4-turbo, llama-2-70b-chat,
- zephyr-7b-beta],
+ gpt-3.5-turbo, gpt-4, gpt-4-turbo,
+ Mixtral-8x7B-Instruct-v0.1, LLaMA-2 Chat
+ (7B), Qwen 1.5 Chat (7B), Snorkel
+ Mistral PairRM DPO (7B), Chronos Hermes
+ (13B), zephyr-7b-beta,
+ Mistral-7B-Instruct-v0.2, Mistral (7B)
+ Instruct v0.2 (Together AI), 01-ai Yi
+ Chat (34B), OpenHermes-2.5-Mistral (7B),
+ Vicuna v1.5 (7B), WizardLM v1.2 (13B)],
'categoryorder': 'array',
'domain': [0.0, 1.0],
'title': {'text': 'Model'}},
'yaxis': {'anchor': 'x', 'domain': [0.0, 1.0], 'title': {'text': 'Chunks mean size (characters)'}}}
-})",Plot of chunks mean size given the same input for summarization task,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mistral-7B-Instruct-v0.2""],""xaxis"":""x"",""y"":[2.6941392346739055],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mixtral-8x7B-Instruct-v0.1""],""xaxis"":""x"",""y"":[2.9569017240855695],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""xaxis"":""x"",""y"":[3.146639511119026],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""chat-bison"",""offsetgroup"":""chat-bison"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison""],""xaxis"":""x"",""y"":[61.46100879376141],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison (PaLM 2)"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""chat-bison (PaLM 2)"",""offsetgroup"":""chat-bison (PaLM 2)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison (PaLM 2)""],""xaxis"":""x"",""y"":[57.12081730998353],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k"",""offsetgroup"":""chat-bison-32k"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k""],""xaxis"":""x"",""y"":[62.01633272616879],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k (PaLM 2 32K)"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k (PaLM 2 32K)"",""offsetgroup"":""chat-bison-32k (PaLM 2 32K)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k (PaLM 2 32K)""],""xaxis"":""x"",""y"":[56.57639873825073],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gemini-pro"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""gemini-pro"",""offsetgroup"":""gemini-pro"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gemini-pro""],""xaxis"":""x"",""y"":[142.5030152620613],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-3.5-turbo""],""xaxis"":""x"",""y"":[3.0439941593593898],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4""],""xaxis"":""x"",""y"":[3.018755174356351],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4-turbo""],""xaxis"":""x"",""y"":[3.0126559039321035],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""llama-2-70b-chat""],""xaxis"":""x"",""y"":[3.2003241836714382],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""zephyr-7b-beta""],""xaxis"":""x"",""y"":[2.711003271075019],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model""},""categoryorder"":""array"",""categoryarray"":[""Mistral-7B-Instruct-v0.2"",""Mixtral-8x7B-Instruct-v0.1"",""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""chat-bison"",""chat-bison (PaLM 2)"",""chat-bison-32k"",""chat-bison-32k (PaLM 2 32K)"",""gemini-pro"",""gpt-3.5-turbo"",""gpt-4"",""gpt-4-turbo"",""llama-2-70b-chat"",""zephyr-7b-beta""]},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Chunks mean size (characters)""}},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Plot of chunks mean size given the same input for summarization task""},""barmode"":""relative""}}",",model,chunk_sizes_mean
-0,Mistral-7B-Instruct-v0.2,2.6941392346739055
-1,Mixtral-8x7B-Instruct-v0.1,2.9569017240855695
-2,TinyLlama/TinyLlama-1.1B-Chat-v1.0,3.146639511119026
-3,chat-bison,61.46100879376141
-4,chat-bison (PaLM 2),57.12081730998353
-5,chat-bison-32k,62.01633272616879
-6,chat-bison-32k (PaLM 2 32K),56.57639873825073
-7,gemini-pro,142.5030152620613
-8,gpt-3.5-turbo,3.0439941593593898
-9,gpt-4,3.018755174356351
-10,gpt-4-turbo,3.0126559039321035
-11,llama-2-70b-chat,3.2003241836714382
-12,zephyr-7b-beta,2.711003271075019
+})",Plot of chunks mean size given the same input for summarization task,"{""data"":[{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gemini-pro"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""gemini-pro"",""offsetgroup"":""gemini-pro"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gemini-pro""],""xaxis"":""x"",""y"":[142.5030152620613],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison (PaLM 2)"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""chat-bison (PaLM 2)"",""offsetgroup"":""chat-bison (PaLM 2)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison (PaLM 2)""],""xaxis"":""x"",""y"":[57.12081730998353],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""chat-bison-32k (PaLM 2 32K)"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""chat-bison-32k (PaLM 2 32K)"",""offsetgroup"":""chat-bison-32k (PaLM 2 32K)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""chat-bison-32k (PaLM 2 32K)""],""xaxis"":""x"",""y"":[56.57639873825073],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Falcon Instruct (7B)"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""Falcon Instruct (7B)"",""offsetgroup"":""Falcon Instruct (7B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Falcon Instruct (7B)""],""xaxis"":""x"",""y"":[3.9825534184447133],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""llama-2-70b-chat"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""llama-2-70b-chat"",""offsetgroup"":""llama-2-70b-chat"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""llama-2-70b-chat""],""xaxis"":""x"",""y"":[3.2003241836714382],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""RedPajama-INCITE Chat (7B)"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""RedPajama-INCITE Chat (7B)"",""offsetgroup"":""RedPajama-INCITE Chat (7B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""RedPajama-INCITE Chat (7B)""],""xaxis"":""x"",""y"":[3.1745725066288473],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""offsetgroup"":""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0""],""xaxis"":""x"",""y"":[3.146639511119026],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-3.5-turbo"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""gpt-3.5-turbo"",""offsetgroup"":""gpt-3.5-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-3.5-turbo""],""xaxis"":""x"",""y"":[3.0439941593593898],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""gpt-4"",""offsetgroup"":""gpt-4"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4""],""xaxis"":""x"",""y"":[3.018755174356351],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""gpt-4-turbo"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""gpt-4-turbo"",""offsetgroup"":""gpt-4-turbo"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""gpt-4-turbo""],""xaxis"":""x"",""y"":[3.0126559039321035],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mixtral-8x7B-Instruct-v0.1"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""Mixtral-8x7B-Instruct-v0.1"",""offsetgroup"":""Mixtral-8x7B-Instruct-v0.1"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mixtral-8x7B-Instruct-v0.1""],""xaxis"":""x"",""y"":[2.9588033651203722],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""LLaMA-2 Chat (7B)"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""LLaMA-2 Chat (7B)"",""offsetgroup"":""LLaMA-2 Chat (7B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""LLaMA-2 Chat (7B)""],""xaxis"":""x"",""y"":[2.7744863004419584],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Qwen 1.5 Chat (7B)"",""marker"":{""color"":""#00cc96"",""pattern"":{""shape"":""""}},""name"":""Qwen 1.5 Chat (7B)"",""offsetgroup"":""Qwen 1.5 Chat (7B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Qwen 1.5 Chat (7B)""],""xaxis"":""x"",""y"":[2.7640730521621997],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Snorkel Mistral PairRM DPO (7B)"",""marker"":{""color"":""#ab63fa"",""pattern"":{""shape"":""""}},""name"":""Snorkel Mistral PairRM DPO (7B)"",""offsetgroup"":""Snorkel Mistral PairRM DPO (7B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Snorkel Mistral PairRM DPO (7B)""],""xaxis"":""x"",""y"":[2.7198372838312563],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Chronos Hermes (13B)"",""marker"":{""color"":""#FFA15A"",""pattern"":{""shape"":""""}},""name"":""Chronos Hermes (13B)"",""offsetgroup"":""Chronos Hermes (13B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Chronos Hermes (13B)""],""xaxis"":""x"",""y"":[2.7148524063962256],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""zephyr-7b-beta"",""marker"":{""color"":""#19d3f3"",""pattern"":{""shape"":""""}},""name"":""zephyr-7b-beta"",""offsetgroup"":""zephyr-7b-beta"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""zephyr-7b-beta""],""xaxis"":""x"",""y"":[2.7085499882907946],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral-7B-Instruct-v0.2"",""marker"":{""color"":""#FF6692"",""pattern"":{""shape"":""""}},""name"":""Mistral-7B-Instruct-v0.2"",""offsetgroup"":""Mistral-7B-Instruct-v0.2"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mistral-7B-Instruct-v0.2""],""xaxis"":""x"",""y"":[2.6941392346739055],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Mistral (7B) Instruct v0.2 (Together AI)"",""marker"":{""color"":""#B6E880"",""pattern"":{""shape"":""""}},""name"":""Mistral (7B) Instruct v0.2 (Together AI)"",""offsetgroup"":""Mistral (7B) Instruct v0.2 (Together AI)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Mistral (7B) Instruct v0.2 (Together AI)""],""xaxis"":""x"",""y"":[2.638652243574353],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""01-ai Yi Chat (34B)"",""marker"":{""color"":""#FF97FF"",""pattern"":{""shape"":""""}},""name"":""01-ai Yi Chat (34B)"",""offsetgroup"":""01-ai Yi Chat (34B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""01-ai Yi Chat (34B)""],""xaxis"":""x"",""y"":[2.6073745366880123],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""OpenHermes-2.5-Mistral (7B)"",""marker"":{""color"":""#FECB52"",""pattern"":{""shape"":""""}},""name"":""OpenHermes-2.5-Mistral (7B)"",""offsetgroup"":""OpenHermes-2.5-Mistral (7B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""OpenHermes-2.5-Mistral (7B)""],""xaxis"":""x"",""y"":[2.4113915137304467],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""Vicuna v1.5 (7B)"",""marker"":{""color"":""#636efa"",""pattern"":{""shape"":""""}},""name"":""Vicuna v1.5 (7B)"",""offsetgroup"":""Vicuna v1.5 (7B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""Vicuna v1.5 (7B)""],""xaxis"":""x"",""y"":[2.387834485473196],""yaxis"":""y"",""type"":""bar""},{""alignmentgroup"":""True"",""hovertemplate"":""Model=%{x}\u003cbr\u003eChunks mean size (characters)=%{y}\u003cextra\u003e\u003c\u002fextra\u003e"",""legendgroup"":""WizardLM v1.2 (13B)"",""marker"":{""color"":""#EF553B"",""pattern"":{""shape"":""""}},""name"":""WizardLM v1.2 (13B)"",""offsetgroup"":""WizardLM v1.2 (13B)"",""orientation"":""v"",""showlegend"":true,""textposition"":""auto"",""x"":[""WizardLM v1.2 (13B)""],""xaxis"":""x"",""y"":[2.2938336528446786],""yaxis"":""y"",""type"":""bar""}],""layout"":{""template"":{""data"":{""histogram2dcontour"":[{""type"":""histogram2dcontour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""choropleth"":[{""type"":""choropleth"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""histogram2d"":[{""type"":""histogram2d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmap"":[{""type"":""heatmap"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""heatmapgl"":[{""type"":""heatmapgl"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""contourcarpet"":[{""type"":""contourcarpet"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""contour"":[{""type"":""contour"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""surface"":[{""type"":""surface"",""colorbar"":{""outlinewidth"":0,""ticks"":""""},""colorscale"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]]}],""mesh3d"":[{""type"":""mesh3d"",""colorbar"":{""outlinewidth"":0,""ticks"":""""}}],""scatter"":[{""fillpattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2},""type"":""scatter""}],""parcoords"":[{""type"":""parcoords"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolargl"":[{""type"":""scatterpolargl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""bar"":[{""error_x"":{""color"":""#2a3f5f""},""error_y"":{""color"":""#2a3f5f""},""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""bar""}],""scattergeo"":[{""type"":""scattergeo"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterpolar"":[{""type"":""scatterpolar"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""histogram"":[{""marker"":{""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""histogram""}],""scattergl"":[{""type"":""scattergl"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatter3d"":[{""type"":""scatter3d"",""line"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattermapbox"":[{""type"":""scattermapbox"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scatterternary"":[{""type"":""scatterternary"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""scattercarpet"":[{""type"":""scattercarpet"",""marker"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}}}],""carpet"":[{""aaxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""baxis"":{""endlinecolor"":""#2a3f5f"",""gridcolor"":""white"",""linecolor"":""white"",""minorgridcolor"":""white"",""startlinecolor"":""#2a3f5f""},""type"":""carpet""}],""table"":[{""cells"":{""fill"":{""color"":""#EBF0F8""},""line"":{""color"":""white""}},""header"":{""fill"":{""color"":""#C8D4E3""},""line"":{""color"":""white""}},""type"":""table""}],""barpolar"":[{""marker"":{""line"":{""color"":""#E5ECF6"",""width"":0.5},""pattern"":{""fillmode"":""overlay"",""size"":10,""solidity"":0.2}},""type"":""barpolar""}],""pie"":[{""automargin"":true,""type"":""pie""}]},""layout"":{""autotypenumbers"":""strict"",""colorway"":[""#636efa"",""#EF553B"",""#00cc96"",""#ab63fa"",""#FFA15A"",""#19d3f3"",""#FF6692"",""#B6E880"",""#FF97FF"",""#FECB52""],""font"":{""color"":""#2a3f5f""},""hovermode"":""closest"",""hoverlabel"":{""align"":""left""},""paper_bgcolor"":""white"",""plot_bgcolor"":""#E5ECF6"",""polar"":{""bgcolor"":""#E5ECF6"",""angularaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""radialaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""ternary"":{""bgcolor"":""#E5ECF6"",""aaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""baxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""},""caxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":""""}},""coloraxis"":{""colorbar"":{""outlinewidth"":0,""ticks"":""""}},""colorscale"":{""sequential"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""sequentialminus"":[[0.0,""#0d0887""],[0.1111111111111111,""#46039f""],[0.2222222222222222,""#7201a8""],[0.3333333333333333,""#9c179e""],[0.4444444444444444,""#bd3786""],[0.5555555555555556,""#d8576b""],[0.6666666666666666,""#ed7953""],[0.7777777777777778,""#fb9f3a""],[0.8888888888888888,""#fdca26""],[1.0,""#f0f921""]],""diverging"":[[0,""#8e0152""],[0.1,""#c51b7d""],[0.2,""#de77ae""],[0.3,""#f1b6da""],[0.4,""#fde0ef""],[0.5,""#f7f7f7""],[0.6,""#e6f5d0""],[0.7,""#b8e186""],[0.8,""#7fbc41""],[0.9,""#4d9221""],[1,""#276419""]]},""xaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""yaxis"":{""gridcolor"":""white"",""linecolor"":""white"",""ticks"":"""",""title"":{""standoff"":15},""zerolinecolor"":""white"",""automargin"":true,""zerolinewidth"":2},""scene"":{""xaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""yaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2},""zaxis"":{""backgroundcolor"":""#E5ECF6"",""gridcolor"":""white"",""linecolor"":""white"",""showbackground"":true,""ticks"":"""",""zerolinecolor"":""white"",""gridwidth"":2}},""shapedefaults"":{""line"":{""color"":""#2a3f5f""}},""annotationdefaults"":{""arrowcolor"":""#2a3f5f"",""arrowhead"":0,""arrowwidth"":1},""geo"":{""bgcolor"":""white"",""landcolor"":""#E5ECF6"",""subunitcolor"":""white"",""showland"":true,""showlakes"":true,""lakecolor"":""white""},""title"":{""x"":0.05},""mapbox"":{""style"":""light""}}},""xaxis"":{""anchor"":""y"",""domain"":[0.0,1.0],""title"":{""text"":""Model""},""categoryorder"":""array"",""categoryarray"":[""gemini-pro"",""chat-bison (PaLM 2)"",""chat-bison-32k (PaLM 2 32K)"",""Falcon Instruct (7B)"",""llama-2-70b-chat"",""RedPajama-INCITE Chat (7B)"",""TinyLlama\u002fTinyLlama-1.1B-Chat-v1.0"",""gpt-3.5-turbo"",""gpt-4"",""gpt-4-turbo"",""Mixtral-8x7B-Instruct-v0.1"",""LLaMA-2 Chat (7B)"",""Qwen 1.5 Chat (7B)"",""Snorkel Mistral PairRM DPO (7B)"",""Chronos Hermes (13B)"",""zephyr-7b-beta"",""Mistral-7B-Instruct-v0.2"",""Mistral (7B) Instruct v0.2 (Together AI)"",""01-ai Yi Chat (34B)"",""OpenHermes-2.5-Mistral (7B)"",""Vicuna v1.5 (7B)"",""WizardLM v1.2 (13B)""]},""yaxis"":{""anchor"":""x"",""domain"":[0.0,1.0],""title"":{""text"":""Chunks mean size (characters)""}},""legend"":{""title"":{""text"":""Model""},""tracegroupgap"":0},""title"":{""text"":""Plot of chunks mean size given the same input for summarization task""},""barmode"":""relative""}}",",model,chunk_sizes_mean
+16,gemini-pro,142.5030152620613
+14,chat-bison (PaLM 2),57.12081730998353
+15,chat-bison-32k (PaLM 2 32K),56.57639873825073
+2,Falcon Instruct (7B),3.9825534184447133
+20,llama-2-70b-chat,3.2003241836714382
+9,RedPajama-INCITE Chat (7B),3.1745725066288473
+11,TinyLlama/TinyLlama-1.1B-Chat-v1.0,3.146639511119026
+17,gpt-3.5-turbo,3.0439941593593898
+18,gpt-4,3.018755174356351
+19,gpt-4-turbo,3.0126559039321035
+6,Mixtral-8x7B-Instruct-v0.1,2.9588033651203722
+3,LLaMA-2 Chat (7B),2.7744863004419584
+8,Qwen 1.5 Chat (7B),2.7640730521621997
+10,Snorkel Mistral PairRM DPO (7B),2.7198372838312563
+1,Chronos Hermes (13B),2.7148524063962256
+21,zephyr-7b-beta,2.7085499882907946
+5,Mistral-7B-Instruct-v0.2,2.6941392346739055
+4,Mistral (7B) Instruct v0.2 (Together AI),2.638652243574353
+0,01-ai Yi Chat (34B),2.6073745366880123
+7,OpenHermes-2.5-Mistral (7B),2.4113915137304467
+12,Vicuna v1.5 (7B),2.387834485473196
+13,WizardLM v1.2 (13B),2.2938336528446786
","{""x"": ""model"", ""y"": ""chunk_sizes_mean"", ""color"": ""model"", ""labels"": {""model"": ""Model"", ""chunk_sizes_mean"": ""Chunks mean size (characters)""}, ""title"": ""Plot of chunks mean size given the same input for summarization task""}",