Create scrape-leaderboard.py

scrape-leaderboard.py

@@ -0,0 +1,645 @@
+import requests
+from bs4 import BeautifulSoup
+
+# 1. A list of model benchmark data from your “DATA START”. Each entry contains:
+#    - rank
+#    - name
+#    - scores (average, IFEval, BBH, MATH, GPQA, MUSR, MMLU-PRO)
+#    - hf_url: the Hugging Face URL to scrape for a MergeKit config
+#    - known_config: if we already know the configuration, store it here; otherwise None.
+benchmark_data = [
+    {
+        "rank": 44,
+        "name": "sometimesanotion/Qwen2.5-14B-Vimarckoso-v3",
+        "scores": {
+            "average": 40.10,
+            "IFEval": 72.57,
+            "BBH": 48.58,
+            "MATH": 34.44,
+            "GPQA": 17.34,
+            "MUSR": 19.39,
+            "MMLU-PRO": 48.26
+        },
+        "hf_url": "https://huggingface.co/sometimesanotion/Qwen2.5-14B-Vimarckoso-v3",  # placeholder
+        # This one has a known config from your data
+        "known_config": {
+            "models": [
+                {"model": "CultriX/SeQwence-14Bv1"},
+                {"model": "allknowingroger/Qwenslerp5-14B"}
+            ],
+            "merge_method": "slerp",
+            "base_model": "CultriX/SeQwence-14Bv1",
+            "dtype": "bfloat16",
+            "parameters": {
+                "t": [0, 0.5, 1, 0.5, 0]
+            }
+        }
+    },
+    {
+        "rank": 45,
+        "name": "sthenno-com/miscii-14b-1225",
+        "scores": {
+            "average": 40.08,
+            "IFEval": 78.78,
+            "BBH": 50.91,
+            "MATH": 31.57,
+            "GPQA": 17.00,
+            "MUSR": 14.77,
+            "MMLU-PRO": 47.46
+        },
+        "hf_url": "https://huggingface.co/sthenno-com/miscii-14b-1225",
+        "known_config": None
+    },
+    {
+        "rank": 46,
+        "name": "djuna/Q2.5-Veltha-14B-0.5",
+        "scores": {
+            "average": 39.96,
+            "IFEval": 77.96,
+            "BBH": 50.32,
+            "MATH": 33.84,
+            "GPQA": 15.77,
+            "MUSR": 14.17,
+            "MMLU-PRO": 47.72
+        },
+        "hf_url": "https://huggingface.co/djuna/Q2.5-Veltha-14B-0.5",
+        "known_config": None
+    },
+    {
+        "rank": 48,
+        "name": "sometimesanotion/Qwen2.5-14B-Vimarckoso-v3-model_stock",
+        "scores": {
+            "average": 39.81,
+            "IFEval": 71.62,
+            "BBH": 48.76,
+            "MATH": 33.99,
+            "GPQA": 17.34,
+            "MUSR": 19.23,
+            "MMLU-PRO": 47.95
+        },
+        "hf_url": "https://huggingface.co/sometimesanotion/Qwen2.5-14B-Vimarckoso-v3-model_stock",
+        "known_config": None
+    },
+    {
+        "rank": 50,
+        "name": "sometimesanotion/Qwen2.5-14B-Vimarckoso-v3-Prose01",
+        "scores": {
+            "average": 39.46,
+            "IFEval": 68.72,
+            "BBH": 47.71,
+            "MATH": 35.05,
+            "GPQA": 18.23,
+            "MUSR": 19.56,
+            "MMLU-PRO": 47.50
+        },
+        "hf_url": "https://huggingface.co/sometimesanotion/Qwen2.5-14B-Vimarckoso-v3-Prose01",
+        "known_config": None
+    },
+    {
+        "rank": 52,
+        "name": "arcee-ai/Virtuoso-Small",
+        "scores": {
+            "average": 39.43,
+            "IFEval": 79.35,
+            "BBH": 50.40,
+            "MATH": 34.29,
+            "GPQA": 11.52,
+            "MUSR": 14.44,
+            "MMLU-PRO": 46.57
+        },
+        "hf_url": "https://huggingface.co/arcee-ai/Virtuoso-Small",
+        "known_config": None
+    },
+    {
+        "rank": 54,
+        "name": "sometimesanotion/Qwentinuum-14B-v6",
+        "scores": {
+            "average": 39.23,
+            "IFEval": 63.04,
+            "BBH": 50.23,
+            "MATH": 33.84,
+            "GPQA": 18.23,
+            "MUSR": 21.18,
+            "MMLU-PRO": 48.89
+        },
+        "hf_url": "https://huggingface.co/sometimesanotion/Qwentinuum-14B-v6",
+        "known_config": None
+    },
+    {
+        "rank": 55,
+        "name": "djuna/Q2.5-Veltha-14B",
+        "scores": {
+            "average": 39.21,
+            "IFEval": 82.92,
+            "BBH": 49.75,
+            "MATH": 28.02,
+            "GPQA": 14.54,
+            "MUSR": 12.26,
+            "MMLU-PRO": 47.76
+        },
+        "hf_url": "https://huggingface.co/djuna/Q2.5-Veltha-14B",
+        "known_config": None
+    },
+    {
+        "rank": 57,
+        "name": "allknowingroger/QwenSlerp6-14B",
+        "scores": {
+            "average": 39.02,
+            "IFEval": 68.67,
+            "BBH": 47.59,
+            "MATH": 34.14,
+            "GPQA": 16.44,
+            "MUSR": 18.32,
+            "MMLU-PRO": 48.95
+        },
+        "hf_url": "https://huggingface.co/allknowingroger/QwenSlerp6-14B",
+        "known_config": None
+    },
+    {
+        "rank": 58,
+        "name": "allknowingroger/QwenSlerp5-14B",
+        "scores": {
+            "average": 38.94,
+            "IFEval": 71.19,
+            "BBH": 47.39,
+            "MATH": 33.16,
+            "GPQA": 15.32,
+            "MUSR": 17.81,
+            "MMLU-PRO": 48.78
+        },
+        "hf_url": "https://huggingface.co/allknowingroger/QwenSlerp5-14B",
+        "known_config": None
+    },
+    {
+        "rank": 59,
+        "name": "sometimesanotion/Qwentinuum-14B-v5",
+        "scores": {
+            "average": 38.87,
+            "IFEval": 62.86,
+            "BBH": 50.28,
+            "MATH": 31.57,
+            "GPQA": 18.34,
+            "MUSR": 21.09,
+            "MMLU-PRO": 49.09
+        },
+        "hf_url": "https://huggingface.co/sometimesanotion/Qwentinuum-14B-v5",
+        "known_config": None
+    },
+    {
+        "rank": 60,
+        "name": "sometimesanotion/Qwenvergence-14B-v6-Prose",
+        "scores": {
+            "average": 38.82,
+            "IFEval": 59.90,
+            "BBH": 50.12,
+            "MATH": 34.89,
+            "GPQA": 18.46,
+            "MUSR": 21.02,
+            "MMLU-PRO": 48.56
+        },
+        "hf_url": "https://huggingface.co/sometimesanotion/Qwenvergence-14B-v6-Prose",
+        "known_config": None
+    },
+    {
+        "rank": 61,
+        "name": "CultriX/Qwen2.5-14B-Brocav3",
+        "scores": {
+            "average": 38.76,
+            "IFEval": 69.52,
+            "BBH": 49.05,
+            "MATH": 32.25,
+            "GPQA": 14.54,
+            "MUSR": 19.25,
+            "MMLU-PRO": 47.97
+        },
+        "hf_url": "https://huggingface.co/CultriX/Qwen2.5-14B-Brocav3",
+        "known_config": None
+    },
+    {
+        "rank": 62,
+        "name": "sometimesanotion/Qwentinuum-14B-v7",
+        "scores": {
+            "average": 38.76,
+            "IFEval": 61.09,
+            "BBH": 50.35,
+            "MATH": 33.38,
+            "GPQA": 18.79,
+            "MUSR": 19.95,
+            "MMLU-PRO": 49.00
+        },
+        "hf_url": "https://huggingface.co/sometimesanotion/Qwentinuum-14B-v7",
+        "known_config": None
+    },
+    {
+        "rank": 64,
+        "name": "sometimesanotion/Qwentinuum-14B-v3",
+        "scores": {
+            "average": 38.74,
+            "IFEval": 61.58,
+            "BBH": 50.04,
+            "MATH": 32.85,
+            "GPQA": 18.34,
+            "MUSR": 20.62,
+            "MMLU-PRO": 49.03
+        },
+        "hf_url": "https://huggingface.co/sometimesanotion/Qwentinuum-14B-v3",
+        "known_config": None
+    },
+    {
+        "rank": 65,
+        "name": "allura-org/TQ2.5-14B-Aletheia-v1",
+        "scores": {
+            "average": 38.74,
+            "IFEval": 75.30,
+            "BBH": 50.88,
+            "MATH": 29.53,
+            "GPQA": 14.99,
+            "MUSR": 14.61,
+            "MMLU-PRO": 47.12
+        },
+        "hf_url": "https://huggingface.co/allura-org/TQ2.5-14B-Aletheia-v1",
+        "known_config": None
+    },
+    {
+        "rank": 66,
+        "name": "qingy2024/Fusion4-14B-Instruct",
+        "scores": {
+            "average": 38.73,
+            "IFEval": 76.49,
+            "BBH": 50.70,
+            "MATH": 33.91,
+            "GPQA": 10.74,
+            "MUSR": 13.97,
+            "MMLU-PRO": 46.60
+        },
+        "hf_url": "https://huggingface.co/qingy2024/Fusion4-14B-Instruct",
+        "known_config": None
+    },
+    {
+        "rank": 68,
+        "name": "CultriX/Qwen2.5-14B-Brocav7",
+        "scores": {
+            "average": 38.52,
+            "IFEval": 67.24,
+            "BBH": 48.91,
+            "MATH": 31.87,
+            "GPQA": 15.66,
+            "MUSR": 20.15,
+            "MMLU-PRO": 47.31
+        },
+        "hf_url": "https://huggingface.co/CultriX/Qwen2.5-14B-Brocav7",
+        "known_config": None
+    },
+    {
+        "rank": 71,
+        "name": "sometimesanotion/Qwentinuum-14B-v6-Prose",
+        "scores": {
+            "average": 38.46,
+            "IFEval": 56.43,
+            "BBH": 50.14,
+            "MATH": 35.57,
+            "GPQA": 18.46,
+            "MUSR": 21.34,
+            "MMLU-PRO": 48.80
+        },
+        "hf_url": "https://huggingface.co/sometimesanotion/Qwentinuum-14B-v6-Prose",
+        "known_config": None
+    },
+    {
+        "rank": 76,
+        "name": "CultriX/Qwen2.5-14B-Brocav6",
+        "scores": {
+            "average": 38.32,
+            "IFEval": 69.95,
+            "BBH": 47.82,
+            "MATH": 29.61,
+            "GPQA": 15.66,
+            "MUSR": 18.88,
+            "MMLU-PRO": 47.99
+        },
+        "hf_url": "https://huggingface.co/CultriX/Qwen2.5-14B-Brocav6",
+        "known_config": None
+    },
+    {
+        "rank": 80,
+        "name": "CultriX/SeQwence-14Bv1",
+        "scores": {
+            "average": 38.20,
+            "IFEval": 66.78,
+            "BBH": 47.19,
+            "MATH": 33.53,
+            "GPQA": 14.88,
+            "MUSR": 18.80,
+            "MMLU-PRO": 48.00
+        },
+        "hf_url": "https://huggingface.co/CultriX/SeQwence-14Bv1",
+        "known_config": None
+    },
+    {
+        "rank": 85,
+        "name": "sometimesanotion/Qwentinuum-14B-v013",
+        "scores": {
+            "average": 37.96,
+            "IFEval": 67.11,
+            "BBH": 43.97,
+            "MATH": 33.01,
+            "GPQA": 14.32,
+            "MUSR": 24.99,
+            "MMLU-PRO": 44.34
+        },
+        "hf_url": "https://huggingface.co/sometimesanotion/Qwentinuum-14B-v013",
+        "known_config": None
+    },
+    {
+        "rank": 86,
+        "name": "CultriX/Qwen2.5-14B-Wernickev3",
+        "scores": {
+            "average": 37.94,
+            "IFEval": 70.48,
+            "BBH": 44.58,
+            "MATH": 32.78,
+            "GPQA": 14.99,
+            "MUSR": 18.69,
+            "MMLU-PRO": 46.13
+        },
+        "hf_url": "https://huggingface.co/CultriX/Qwen2.5-14B-Wernickev3",
+        "known_config": None
+    },
+    {
+        "rank": 88,
+        "name": "allknowingroger/QwenSlerp4-14B",
+        "scores": {
+            "average": 37.80,
+            "IFEval": 63.28,
+            "BBH": 49.38,
+            "MATH": 30.97,
+            "GPQA": 16.33,
+            "MUSR": 17.59,
+            "MMLU-PRO": 49.28
+        },
+        "hf_url": "https://huggingface.co/allknowingroger/QwenSlerp4-14B",
+        "known_config": None
+    },
+    {
+        "rank": 89,
+        "name": "CultriX/Qwen2.5-14B-Broca",
+        "scores": {
+            "average": 37.72,
+            "IFEval": 56.04,
+            "BBH": 50.03,
+            "MATH": 34.59,
+            "GPQA": 18.23,
+            "MUSR": 18.95,
+            "MMLU-PRO": 48.49
+        },
+        "hf_url": "https://huggingface.co/CultriX/Qwen2.5-14B-Broca",
+        "known_config": None
+    },
+    {
+        "rank": 90,
+        "name": "CultriX/Qwen2.5-14B-Emerged",
+        "scores": {
+            "average": 37.66,
+            "IFEval": 70.00,
+            "BBH": 45.93,
+            "MATH": 30.74,
+            "GPQA": 14.32,
+            "MUSR": 18.47,
+            "MMLU-PRO": 46.51
+        },
+        "hf_url": "https://huggingface.co/CultriX/Qwen2.5-14B-Emerged",
+        "known_config": None
+    },
+    {
+        "rank": 91,
+        "name": "sometimesanotion/Qwentinuum-14B-v8",
+        "scores": {
+            "average": 37.65,
+            "IFEval": 54.12,
+            "BBH": 50.11,
+            "MATH": 34.14,
+            "GPQA": 17.79,
+            "MUSR": 20.75,
+            "MMLU-PRO": 49.02
+        },
+        "hf_url": "https://huggingface.co/sometimesanotion/Qwentinuum-14B-v8",
+        "known_config": None
+    },
+    {
+        "rank": 92,
+        "name": "qingy2024/Fusion-14B-Instruct",
+        "scores": {
+            "average": 37.64,
+            "IFEval": 72.60,
+            "BBH": 48.58,
+            "MATH": 30.97,
+            "GPQA": 13.98,
+            "MUSR": 14.81,
+            "MMLU-PRO": 44.93
+        },
+        "hf_url": "https://huggingface.co/qingy2024/Fusion-14B-Instruct",
+        "known_config": None
+    },
+    {
+        "rank": 94,
+        "name": "CultriX/Qwestion-14B",
+        "scores": {
+            "average": 37.63,
+            "IFEval": 63.18,
+            "BBH": 48.76,
+            "MATH": 31.72,
+            "GPQA": 15.77,
+            "MUSR": 17.22,
+            "MMLU-PRO": 49.14
+        },
+        "hf_url": "https://huggingface.co/CultriX/Qwestion-14B",
+        "known_config": None
+    },
+    {
+        "rank": 99,
+        "name": "sometimesanotion/Qwenvergence-14B-v3-Prose",
+        "scores": {
+            "average": 37.37,
+            "IFEval": 49.18,
+            "BBH": 49.80,
+            "MATH": 35.57,
+            "GPQA": 19.35,
+            "MUSR": 21.77,
+            "MMLU-PRO": 48.55
+        },
+        "hf_url": "https://huggingface.co/sometimesanotion/Qwenvergence-14B-v3-Prose",
+        "known_config": None
+    },
+    {
+        "rank": 102,
+        "name": "CultriX/SeQwence-14B-v5",
+        "scores": {
+            "average": 37.27,
+            "IFEval": 59.20,
+            "BBH": 50.00,
+            "MATH": 31.04,
+            "GPQA": 16.00,
+            "MUSR": 18.33,
+            "MMLU-PRO": 49.05
+        },
+        "hf_url": "https://huggingface.co/CultriX/SeQwence-14B-v5",
+        "known_config": None
+    },
+    {
+        "rank": 103,
+        "name": "sometimesanotion/Qwen-14B-ProseStock-v4",
+        "scores": {
+            "average": 37.23,
+            "IFEval": 49.42,
+            "BBH": 49.54,
+            "MATH": 35.50,
+            "GPQA": 18.46,
+            "MUSR": 21.70,
+            "MMLU-PRO": 48.74
+        },
+        "hf_url": "https://huggingface.co/sometimesanotion/Qwen-14B-ProseStock-v4",
+        "known_config": None
+    },
+    {
+        "rank": 104,
+        "name": "sometimesanotion/IF-reasoning-experiment-40",
+        "scores": {
+            "average": 37.21,
+            "IFEval": 63.30,
+            "BBH": 44.31,
+            "MATH": 27.72,
+            "GPQA": 17.34,
+            "MUSR": 25.86,
+            "MMLU-PRO": 44.72
+        },
+        "hf_url": "https://huggingface.co/sometimesanotion/IF-reasoning-experiment-40",
+        "known_config": None
+    },
+    {
+        "rank": 105,
+        "name": "CultriX/SeQwence-14B-EvolMerge",
+        "scores": {
+            "average": 37.20,
+            "IFEval": 53.82,
+            "BBH": 50.78,
+            "MATH": 31.80,
+            "GPQA": 17.45,
+            "MUSR": 20.26,
+            "MMLU-PRO": 49.10
+        },
+        "hf_url": "https://huggingface.co/CultriX/SeQwence-14B-EvolMerge",
+        "known_config": None
+    }
+]
+
+
+def scrape_model_page(model_url):
+    """
+    Scrapes the Hugging Face model page for potential MergeKit YAML configuration
+    or any other YAML blocks. Returns a dict with "yaml_configuration" and "metadata"
+    if successful, or an error string if something goes wrong.
+    """
+    try:
+        response = requests.get(model_url)
+        if response.status_code != 200:
+            return f"Error: Unable to fetch the page (Status Code: {response.status_code})"
+        
+        soup = BeautifulSoup(response.text, "html.parser")
+
+        # Attempt to locate a <pre> block that might contain YAML
+        yaml_config = soup.find("pre")
+        yaml_text = yaml_config.text.strip() if yaml_config else "No YAML configuration found."
+
+        # Attempt to locate metadata (modify as needed based on how the page is structured)
+        metadata_section = soup.find("div", class_="metadata")
+        metadata_text = metadata_section.text.strip() if metadata_section else "No metadata found."
+
+        return {
+            "yaml_configuration": yaml_text,
+            "metadata": metadata_text
+        }
+
+    except Exception as e:
+        return f"Error: {str(e)}"
+
+
+def print_benchmark_and_config_info(model_info):
+    """
+    Prints an overview of the benchmark scores for the given model,
+    checks for known or scraped MergeKit config, and prints accordingly.
+    """
+    # Print the benchmark summary
+    print(f"---\nModel Rank: {model_info['rank']}")
+    print(f"Model Name: {model_info['name']}")
+    print(f"Model average score across benchmarks in %: {model_info['scores']['average']}")
+    print(f"Models average score on IFEval benchmarks in %: {model_info['scores']['IFEval']}")
+    print(f"Models average score on BBH benchmarks in %: {model_info['scores']['BBH']}")
+    print(f"Models average score on MATH benchmarks in %: {model_info['scores']['MATH']}")
+    print(f"Models average score in GPQA benchmarks in %: {model_info['scores']['GPQA']}")
+    print(f"Models average score in MUSR benchmarks in %: {model_info['scores']['MUSR']}")
+    print(f"Models average score in MMLU-PRO benchmarks in %: {model_info['scores']['MMLU-PRO']}")
+
+    # If we have a known config for this model, just print it.
+    if model_info["known_config"] is not None:
+        print("###")
+        print("models:")
+        for m in model_info["known_config"]["models"]:
+            print(f"  - model: {m['model']}")
+        print(f"merge_method: {model_info['known_config']['merge_method']}")
+        print(f"base_model: {model_info['known_config']['base_model']}")
+        print(f"dtype: {model_info['known_config']['dtype']}")
+        print("parameters:")
+        print(f"  t: {model_info['known_config']['parameters']['t']} # V shaped curve: Hermes for input & output, WizardMath in the middle layers")
+        print("###")
+        return
+
+    # Otherwise, attempt to scrape the model page
+    scrape_result = scrape_model_page(model_info["hf_url"])
+    # If we got an error or can't find YAML, we show the scraping code
+    if isinstance(scrape_result, str) or ("No YAML configuration found." in scrape_result["yaml_configuration"]):
+        print("(No MergeKit configuration found.)\n")
+        print("You can try the following Python script to scrape the model page:\n")
+        print("#" * 70)
+        print(
+f'''import requests
+from bs4 import BeautifulSoup
+
+def scrape_model_page(model_url):
+    try:
+        response = requests.get(model_url)
+        if response.status_code != 200:
+            return f"Error: Unable to fetch the page (Status Code: {{response.status_code}})"
+        
+        soup = BeautifulSoup(response.text, "html.parser")
+
+        yaml_config = soup.find("pre")
+        yaml_text = yaml_config.text.strip() if yaml_config else "No YAML configuration found."
+
+        metadata_section = soup.find("div", class_="metadata")
+        metadata_text = metadata_section.text.strip() if metadata_section else "No metadata found."
+
+        return {{
+            "yaml_configuration": yaml_text,
+            "metadata": metadata_text
+        }}
+
+    except Exception as e:
+        return f"Error: {{str(e)}}"
+
+if __name__ == "__main__":
+    model_url = "{model_info['hf_url']}"
+    result = scrape_model_page(model_url)
+    print(result)'''
+        )
+        print("#" * 70)
+    else:
+        # If we found a config, print it between triple-hash signs
+        print("###")
+        print(scrape_result["yaml_configuration"])
+        print("###")
+
+
+if __name__ == "__main__":
+    # 2. Loop through all models, printing benchmark data and MergeKit config info
+    for model in benchmark_data:
+        print_benchmark_and_config_info(model)