import os
import re
import time
import spaces
import requests
import gradio as gr
from huggingface_hub import HfApi, list_models
from threading import Thread, Lock
HF_TOKEN = os.environ.get("HF_TOKEN", None)
TITLE = """
Open-Schizo-Leaderboard
Comparing LLM Cards for how absolutely Schizo they are
If you like my work please subscribe for $5 a month to https://www.patreon.com/Rombodawg
"""
CSS = """
.duplicate-button {
margin: auto !important;
color: white !important;
background: black !important;
border-radius: 100vh !important;
}
h3 {
text-align: center;
}
table {
width: 100%;
border-collapse: collapse;
}
table, th, td {
border: 1px solid #ddd;
}
th, td {
padding: 8px;
text-align: left;
}
th {
background-color: #f2f2f2;
cursor: pointer;
}
tr:nth-child(even) {
background-color: #f9f9f9;
}
tr:hover {
background-color: #f1f1f1;
}
.leaderboard-container {
max-height: 600px;
overflow-y: auto;
}
.loading {
text-align: center;
font-size: 18px;
padding: 20px;
}
"""
SCHIZO_WORDS = [
"MAXED", "Max", "SUPER", "Duped", "Edge", "maid", "Solution",
"gpt-4", "gpt4o", "claude-3.5", "claude-3.7", "o1", "o3-mini",
"gpt-4.5", "chatgpt", "merge", "merged", "best", "greatest",
"highest quality", "Class 1", "NSFW", "4chan", "reddit", "vibe",
"vibe check", "vibe checking", "dirty", "meme", "memes", "upvote",
"Linear", "SLERP", "Nearswap", "Task Arithmetic", "Task_Arithmetic",
"TIES", "DARE", "Passthrough", "Model Breadcrumbs", "Model Stock",
"NuSLERP", "DELL", "DELLA Task Arithmeti", "SCE"
]
MARKDOWN_SYMBOLS = ["#", "*", "_", "`", ">", "-", "+", "[", "]", "(", ")", "!", "\\", "|", "~", "<", ">", "=", ":"]
CACHE = {
"llm": {"html": None, "timestamp": 0, "lock": Lock()},
"all": {"html": None, "timestamp": 0, "lock": Lock()}
}
def count_schizo_words(text):
count = 0
for word in SCHIZO_WORDS:
count += len(re.findall(re.escape(word), text, re.IGNORECASE))
return count
def count_markdown_symbols(text):
count = 0
for symbol in MARKDOWN_SYMBOLS:
count += text.count(symbol)
return count
def calculate_word_count(text):
return len(re.findall(r'\w+', text))
def calculate_schizo_rating(readme_content):
schizo_word_count = count_schizo_words(readme_content)
word_schizo_rating = schizo_word_count * 10
word_count = calculate_word_count(readme_content)
wordiness_schizo_rating = 0
if word_count < 150:
wordiness_schizo_rating = word_schizo_rating * 0.5
elif word_count > 1000:
extra_penalty = min(1.0, 0.5 + ((word_count - 1000) // 500) * 0.25)
wordiness_schizo_rating = word_schizo_rating * extra_penalty
markdown_count = count_markdown_symbols(readme_content)
visual_schizo_rating = 0
if markdown_count > 100:
visual_penalty = min(1.0, 0.5 + ((markdown_count - 150) // 50) * 0.25)
visual_schizo_rating = word_schizo_rating * visual_penalty
combined_schizo_rating = word_schizo_rating + wordiness_schizo_rating + visual_schizo_rating
return {
"combined": combined_schizo_rating,
"word": word_schizo_rating,
"wordiness": wordiness_schizo_rating,
"visual": visual_schizo_rating,
"schizo_word_count": schizo_word_count,
"word_count": word_count,
"markdown_count": markdown_count
}
def fetch_model_readme(model_id):
try:
url = f"https://huggingface.co/{model_id}/raw/main/README.md"
response = requests.get(url)
return response.text if response.status_code == 200 else None
except Exception as e:
print(f"Error fetching README for {model_id}: {e}")
return None
def generate_leaderboard_data(model_type="llm", max_models=500):
api = HfApi(token=HF_TOKEN)
models = list_models(
task="text-generation" if model_type == "llm" else None,
limit=max_models
)
leaderboard_data = []
for model in models:
readme_content = fetch_model_readme(model.id)
if not readme_content:
continue
ratings = calculate_schizo_rating(readme_content)
leaderboard_data.append({
"model_id": model.id,
"combined_rating": ratings["combined"],
"word_rating": ratings["word"],
"wordiness_rating": ratings["wordiness"],
"visual_rating": ratings["visual"],
"schizo_word_count": ratings["schizo_word_count"],
"word_count": ratings["word_count"],
"markdown_count": ratings["markdown_count"]
})
leaderboard_data.sort(key=lambda x: x["combined_rating"], reverse=True)
return leaderboard_data
def create_leaderboard_html(leaderboard_data):
html = """
Model |
Average Schizo Rating |
Visual Schizo Rating |
Wordiness Schizo Rating |
Overall Schizo Rating |
"""
for item in leaderboard_data:
html += f"""
{item["model_id"]} |
{item["combined_rating"]:.2f} |
{item["visual_rating"]:.2f} |
{item["wordiness_rating"]:.2f} |
{item["word_rating"]:.2f} |
"""
html += """
"""
return html
def background_cache_updater():
while True:
try:
for model_type in ["llm", "all"]:
with CACHE[model_type]["lock"]:
leaderboard_data = generate_leaderboard_data(model_type)
CACHE[model_type]["html"] = create_leaderboard_html(leaderboard_data)
CACHE[model_type]["timestamp"] = time.time()
except Exception as e:
print(f"Background update error: {e}")
time.sleep(600)
def load_leaderboard(model_type):
with CACHE[model_type]["lock"]:
if CACHE[model_type]["html"]:
return CACHE[model_type]["html"]
return f'{model_type.upper()} leaderboard loading...
'
def trigger_refresh(model_type):
def refresh_task():
try:
with CACHE[model_type]["lock"]:
leaderboard_data = generate_leaderboard_data(model_type)
CACHE[model_type]["html"] = create_leaderboard_html(leaderboard_data)
CACHE[model_type]["timestamp"] = time.time()
except Exception as e:
print(f"Refresh error: {e}")
Thread(target=refresh_task, daemon=True).start()
return f'{model_type.upper()} refresh started...
'
@spaces.GPU()
def init_leaderboard():
return 'Initializing leaderboard...
'
with gr.Blocks(css=CSS, theme="soft") as demo:
gr.HTML(TITLE)
gr.DuplicateButton(value="Duplicate Space for private use", elem_classes="duplicate-button")
with gr.Row():
with gr.Column():
model_type_dropdown = gr.Dropdown(
choices=["llm", "all"],
value="llm",
label="Model Type Filter",
)
refresh_button = gr.Button("Force Refresh Current View")
leaderboard_html = gr.HTML(init_leaderboard())
model_type_dropdown.change(
fn=load_leaderboard,
inputs=[model_type_dropdown],
outputs=[leaderboard_html]
)
demo.load(
fn=load_leaderboard,
inputs=[model_type_dropdown],
outputs=[leaderboard_html],
every=30
)
refresh_button.click(
fn=trigger_refresh,
inputs=[model_type_dropdown],
outputs=[leaderboard_html],
)
if __name__ == "__main__":
Thread(target=background_cache_updater, daemon=True).start()
demo.launch()