updated

.gitignore

@@ -0,0 +1,8 @@
+__pycache__/
+*.py[cod]
+*$py.class
+.env
+.venv
+env/
+venv/
+.streamlit/

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "window.title": "${dirty}${activeEditorShort}${separator}${rootName}${separator}${profileName}${separator}${appName}${separator}[Branch: main]"
+}

README.md

@@ -1,13 +1,42 @@
 ---
-title: Max Output Tokens Analysis 0225
-emoji: 🏢
-colorFrom: yellow
-colorTo: yellow
+title: Max Output Tokens Analysis
+emoji: 📊
+colorFrom: blue
+colorTo: green
 sdk: streamlit
-sdk_version: 1.42.0
+sdk_version: 1.28.0
 app_file: app.py
 pinned: false
-short_description: Max output tokens by model over time
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# Max Output Tokens Analysis
+
+This Streamlit app visualizes the evolution of maximum output tokens across different AI companies and their language models. The visualization includes:
+
+1. An interactive line chart showing how max output tokens have evolved over time for different companies
+2. A detailed data table showing the maximum output tokens for each model
+
+## Data
+
+The data includes information about models from:
+- OpenAI
+- Anthropic
+- Google
+- Cohere
+- Deep Seek
+
+Each model entry contains:
+- Model name
+- Maximum output tokens
+- Company
+- Launch date
+
+## Usage
+
+The chart is interactive - you can:
+- Hover over lines to see detailed information
+- Click and drag to zoom
+- Double click to reset the view
+- Use the legend to toggle different companies
+
+The data table below the chart can be sorted by clicking on column headers.

app.py

@@ -0,0 +1,104 @@
+import streamlit as st
+import pandas as pd
+import plotly.express as px
+
+# Page config
+st.set_page_config(
+    page_title="Max Output Tokens Analysis",
+    layout="wide"
+)
+
+# Load data
+@st.cache_data
+def load_data():
+    df = pd.read_csv('data/max-tokens-by-model.csv')
+    df['launch_date'] = pd.to_datetime(df['launch_date'])
+    return df
+
+df = load_data()
+
+# Title
+st.title("LLM Max Output Tokens Analysis")
+
+# Company selection
+companies = sorted(df['company'].unique())
+selected_companies = st.multiselect(
+    "Select companies to display:",
+    options=companies,
+    default=companies,
+    key='company_filter'
+)
+
+# Filter data based on selection
+filtered_df = df[df['company'].isin(selected_companies)]
+
+# Create the evolution chart
+fig = px.line(filtered_df, 
+              x='launch_date', 
+              y='max_output_tokens',
+              color='company',
+              hover_data=['model_name', 'max_output_tokens'],
+              title='Evolution of Max Output Tokens by Company',
+              labels={
+                  'launch_date': 'Launch Date',
+                  'max_output_tokens': 'Max Output Tokens',
+                  'company': 'Company'
+              },
+              markers=True)  # Add markers to make trends clearer
+
+fig.update_layout(
+    hovermode='x unified',
+    xaxis_title="Launch Date",
+    yaxis_title="Max Output Tokens",
+    yaxis_type="log",  # Using log scale for better visualization
+    height=600,  # Make chart taller
+    showlegend=True,
+    legend=dict(
+        yanchor="top",
+        y=0.99,
+        xanchor="left",
+        x=0.01
+    ),
+    margin=dict(l=20, r=20, t=40, b=20)
+)
+
+fig.update_traces(
+    line=dict(width=2),  # Make lines thicker
+    marker=dict(size=8)  # Make markers more visible
+)
+
+# Display the chart
+st.plotly_chart(fig, use_container_width=True)
+
+# Display the data table
+st.subheader("Max Output Tokens by Model")
+
+# Prepare the data with better formatting
+display_df = (
+    filtered_df[['model_name', 'company', 'max_output_tokens', 'launch_date']]
+    .sort_values('max_output_tokens', ascending=False)
+    .assign(
+        launch_date=lambda x: x['launch_date'].dt.strftime('%Y-%m-%d'),
+        max_output_tokens=lambda x: x['max_output_tokens'].apply(lambda v: f"{v:,}")
+    )
+    .rename(columns={
+        'model_name': 'Model Name',
+        'company': 'Company',
+        'max_output_tokens': 'Max Output Tokens',
+        'launch_date': 'Launch Date'
+    })
+)
+
+# Display the styled table
+st.dataframe(
+    display_df,
+    use_container_width=True,
+    hide_index=True
+)
+
+# Attribution
+st.markdown("---")
+st.markdown(
+    "By: [Daniel Rosehill](https://danielrosehill.com) | "
+    "Data sourced from public sources on February 8, 2025"
+)

data/data-sources/cohere/cohere-models.csv

@@ -0,0 +1,3 @@
+model-name,max-output-tokens
+command-r7b-12-2024,4000
+command-r-plus-08-2024,4000

data/data-sources/data-sources.md

@@ -0,0 +1,16 @@
+# Data Sources
+
+Feb 08
+
+## Google / Gemini
+
+Gemini API Docs, 08-Feb-2025
+
+https://ai.google.dev/gemini-api/docs/models/gemini
+
+## OpenAI
+
+OpenAI Docs, 08-Feb
+
+https://platform.openai.com/docs/models
+

data/data-sources/google/google-models.csv

@@ -0,0 +1,6 @@
+model-name,max-output-tokens
+Gemini 2.0 Flash,"8,192"
+Gemini 2.0 Flash-Lite Preview,"8,192"
+Gemini 1.5 Flash,"8,192"
+Gemini 1.5 Flash 8B,"8,192"
+Gemini 1.5 Pro,"8,192"

data/data-sources/openai/max-tokens-and-prices.csv

@@ -0,0 +1,12 @@
+model name,max output tokens
+gpt-4o,16384
+chatgpt-4o-latest,16384
+gpt-4o-mini,16384
+o1,100000
+o1-mini,65536
+o1-preview,32768
+ gpt-4-turbo,4096
+gpt-4-turbo-preview,4096
+gpt-4,8192
+gpt-4-0613,8192
+gpt-4-0314,8192

data/data-sources/openai/openai-models-by-max-tokens-no-snapshots.csv

@@ -0,0 +1,12 @@
+model name,max output tokens
+gpt-4o,16384
+chatgpt-4o-latest,16384
+gpt-4o-mini,16384
+o1,100000
+o1-mini,65536
+o1-preview,32768
+ gpt-4-turbo,4096
+gpt-4-turbo-preview,4096
+gpt-4,8192
+gpt-4-0613,8192
+gpt-4-0314,8192

data/data-sources/openai/openai-models-by-max-tokens.csv

@@ -0,0 +1,41 @@
+model name,max output tokens
+gpt-4o,16384
+gpt-4o-2024-08-06,16384
+gpt-4o-2024-11-20,16384
+gpt-4o-2024-08-06,16384
+gpt-4o-2024-05-13,4096
+chatgpt-4o-latest,16384
+gpt-4o-mini,16384
+gpt-4o-mini-2024-07-18,16384
+gpt-4o-mini-2024-07-18,16384
+o1,100000
+o1-2024-12-17,100000
+o1-mini,65536
+o1-mini-2024-09-12,65536
+o1-mini-2024-09-12,65536
+o1-preview,32768
+o1-preview-2024-09-12,32768
+o1-preview-2024-09-12,32768
+gpt-4o-realtime-preview,4096
+gpt-4o-realtime-preview-2024-12-17,4096
+gpt-4o-realtime-preview-2024-10-01,4096
+gpt-4o-mini-realtime-preview,4096
+gpt-4o-mini-realtime-preview-2024-12-17,4096
+gpt-4o-mini-realtime-preview-2024-12-17,4096
+gpt-4o-audio-preview,16384
+gpt-4o-audio-preview-2024-12-17,16384
+gpt-4o-audio-preview-2024-10-01,16384
+gpt-4o-mini-audio-preview,16384
+gpt-4o-mini-audio-preview-2024-12-17,16384
+gpt-4o-mini-audio-preview-2024-12-17,16384
+gpt-4-turbo,4096
+gpt-4-turbo-2024-04-09,4096
+gpt-4-turbo-2024-04-09,4096
+gpt-4-turbo-preview,4096
+gpt-4-0125-preview,4096
+gpt-4-0125-preview,4096
+gpt-4-1106-preview,4096
+gpt-4,8192
+gpt-4-0613,8192
+gpt-4-0613,8192
+gpt-4-0314,8192

data/max-tokens-by-model.csv

@@ -0,0 +1,26 @@
+model_name,max_output_tokens,company,launch_date,max_input_tokens,max_output_as_percentage_of_max_input,release-type
+Claude 3.5 Sonnet,8192,Anthropic,2024-06-20,200000,4.10,Fixed
+Claude 3.5 Haiku,8192,Anthropic,2024-10-22,200000,4.10,Fixed
+Claude 3 Opus,4096,Anthropic,2024-03-14,200000,2.05,Fixed
+Claude 3 Sonnet,4096,Anthropic,2024-03-14,200000,2.05,Fixed
+Claude 3 Haiku,4096,Anthropic,2024-03-07,200000,2.05,Fixed
+command-r7b-12-2024,4000,Cohere,2024-12-01,128000,3.13,Fixed
+command-r-plus-08-2024,4000,Cohere,2024-08-01,128000,3.13,Fixed
+Gemini 2.0 Flash,8192,Google,2025-02-05,1000000,0.82,Fixed
+Gemini 2.0 Flash-Lite Preview,8192,Google,2025-02-05,1000000,0.82,Fixed
+Gemini 1.5 Flash,8192,Google,2024-05-14,1000000,0.82,Fixed
+Gemini 1.5 Flash 8B,8192,Google,2024-10-08,1000000,0.82,Fixed
+Gemini 1.5 Pro,8192,Google,2024-05-23,1000000,0.82,Fixed
+gpt-4o,16384,OpenAI,2024-11-06,128000,12.80,Fixed
+chatgpt-4o-latest,16384,OpenAI,2024-11-06,128000,12.80,Fixed
+gpt-4o-mini,16384,OpenAI,2024-11-06,128000,12.80,Fixed
+o1,100000,OpenAI,2024-11-06,1000000,10.00,Fixed
+o1-mini,65536,OpenAI,2024-11-06,1000000,6.55,Fixed
+o1-preview,32768,OpenAI,2024-11-06,1000000,3.28,Fixed
+gpt-4-turbo,4096,OpenAI,2023-11-06,128000,3.20,Fixed
+gpt-4-turbo-preview,4096,OpenAI,2023-11-06,128000,3.20,Fixed
+gpt-4,8192,OpenAI,2023-03-14,8192,100.00,Fixed
+gpt-4-0613,8192,OpenAI,2023-06-13,8192,100.00,Fixed
+gpt-4-0314,8192,OpenAI,2023-03-14,8192,100.00,Fixed
+Deep Seek V3,8000,Deep Seek,2025-01-20,32768,24.41,Rolling
+Deep Seek Reasoner,32768,Deep Seek,2025-01-20,32768,100,Rolling

data/table.txt

@@ -0,0 +1,27 @@
+| model_name                    |   max_output_tokens | company   | launch_date         |   max_input_tokens |   max_output_as_percentage_of_max_input | release-type   |
+|:------------------------------|--------------------:|:----------|:--------------------|-------------------:|----------------------------------------:|:---------------|
+| o1                            |              100000 | OpenAI    | 2024-11-06 00:00:00 |            1000000 |                                   10    | Fixed          |
+| o1-mini                       |               65536 | OpenAI    | 2024-11-06 00:00:00 |            1000000 |                                    6.55 | Fixed          |
+| Deep Seek Reasoner            |               32768 | Deep Seek | 2025-01-20 00:00:00 |              32768 |                                  100    | Rolling        |
+| o1-preview                    |               32768 | OpenAI    | 2024-11-06 00:00:00 |            1000000 |                                    3.28 | Fixed          |
+| chatgpt-4o-latest             |               16384 | OpenAI    | 2024-11-06 00:00:00 |             128000 |                                   12.8  | Fixed          |
+| gpt-4o-mini                   |               16384 | OpenAI    | 2024-11-06 00:00:00 |             128000 |                                   12.8  | Fixed          |
+| gpt-4o                        |               16384 | OpenAI    | 2024-11-06 00:00:00 |             128000 |                                   12.8  | Fixed          |
+| Gemini 2.0 Flash              |                8192 | Google    | 2025-02-08 00:00:00 |            1000000 |                                    0.82 | Fixed          |
+| Gemini 2.0 Flash-Lite Preview |                8192 | Google    | 2025-02-08 00:00:00 |            1000000 |                                    0.82 | Fixed          |
+| gpt-4-0314                    |                8192 | OpenAI    | 2023-03-14 00:00:00 |               8192 |                                  100    | Fixed          |
+| gpt-4-0613                    |                8192 | OpenAI    | 2023-06-13 00:00:00 |               8192 |                                  100    | Fixed          |
+| gpt-4                         |                8192 | OpenAI    | 2023-03-14 00:00:00 |               8192 |                                  100    | Fixed          |
+| Claude 3.5 Sonnet             |                8192 | Anthropic | 2024-06-20 00:00:00 |             200000 |                                    4.1  | Fixed          |
+| Claude 3.5 Haiku              |                8192 | Anthropic | 2024-10-22 00:00:00 |             200000 |                                    4.1  | Fixed          |
+| Gemini 1.5 Pro                |                8192 | Google    | 2025-02-08 00:00:00 |            1000000 |                                    0.82 | Fixed          |
+| Gemini 1.5 Flash 8B           |                8192 | Google    | 2025-02-08 00:00:00 |            1000000 |                                    0.82 | Fixed          |
+| Gemini 1.5 Flash              |                8192 | Google    | 2025-02-08 00:00:00 |            1000000 |                                    0.82 | Fixed          |
+| Deep Seek V3                  |                8000 | Deep Seek | 2025-01-20 00:00:00 |              32768 |                                   24.41 | Rolling        |
+| Claude 3 Haiku                |                4096 | Anthropic | 2024-03-07 00:00:00 |             200000 |                                    2.05 | Fixed          |
+| Claude 3 Opus                 |                4096 | Anthropic | 2024-03-14 00:00:00 |             200000 |                                    2.05 | Fixed          |
+| gpt-4-turbo                   |                4096 | OpenAI    | 2023-11-06 00:00:00 |             128000 |                                    3.2  | Fixed          |
+| gpt-4-turbo-preview           |                4096 | OpenAI    | 2023-11-06 00:00:00 |             128000 |                                    3.2  | Fixed          |
+| Claude 3 Sonnet               |                4096 | Anthropic | 2024-03-14 00:00:00 |             200000 |                                    2.05 | Fixed          |
+| command-r-plus-08-2024        |                4000 | Cohere    | 2024-08-01 00:00:00 |             128000 |                                    3.13 | Fixed          |
+| command-r7b-12-2024           |                4000 | Cohere    | 2024-12-01 00:00:00 |             128000 |                                    3.13 | Fixed          |

requirements.txt

@@ -0,0 +1,3 @@
+streamlit
+pandas
+plotly