import requests from bs4 import BeautifulSoup #!/usr/bin/env python3 def main(): print("""### INSTRUCTION ### Read the instructions below: 1. You will be presented with benchmark scores by various LLM's. 2. The layout of the data presented to you is as follows: >>> START LAYOUT EXAMPLE <<< --- (start of a new model marker) Model ranking Model name Model average score across benchmarks in % Models average score on IFEval benchmarks in % Models average score on BBH benchmarks in % Models average score on MATH benchmarks in % Models average score in GPQA benchmarks in % Models average score in MUSR benchmarks in % Models average score in MMLU-PRO benchmarks in % ### (start of YAML-configuration marker) The YAML-configuration file that was used to create the model in mergekit. Note that this part is only available for certain models and not for all models on the list! The configuration starts and ends with '###' ### (end of YAML-configuration marker) >>> END LAYOUT EXAMPLE <<< For example, the following input could be possible: >>> START INPUT EXAMPLE <<< --- 44 sometimesanotion/Qwen2.5-14B-Vimarckoso-v3 40.10 % 72.57 % 48.58 % 34.44 % 17.34 % 19.39 % 48.26 % ### models: - model: CultriX/SeQwence-14Bv1 - model: allknowingroger/Qwenslerp5-14B merge_method: slerp base_model: CultriX/SeQwence-14Bv1 dtype: bfloat16 parameters: t: [0, 0.5, 1, 0.5, 0] # V shaped curve: Hermes for input & output, WizardMath in the middle layers ### --- 45 sthenno-com/miscii-14b-1225 40.08 % 78.78 % 50.91 % 31.57 % 17.00 % 14.77 % 47.46 % --- >>> END INPUT EXAMPLE <<< >>> START INTERPRETATION OF INPUT EXAMPLE <<< --- Model Rank: 44 Model Name: sometimesanotion/Qwen2.5-14B-Vimarckoso-v3 Model average score across benchmarks in %: 40.10 Models average score on IFEval benchmarks in %: 72.57 Models average score on BBH benchmarks in %: 48.58 Models average score on MATH benchmarks in % 34.44 Models average score in GPQA benchmarks in % 17.34 Models average score in MUSR benchmarks in % 19.39 Models average score in MMLU-PRO benchmarks in % 48.26 ### (THE CONFIGURATION FOR MERGING THE MODEL: sometimesanotion/Qwen2.5-14B-Vimarckoso-v3 was found) models: - model: CultriX/SeQwence-14Bv1 - model: allknowingroger/Qwenslerp5-14B merge_method: slerp base_model: CultriX/SeQwence-14Bv1 dtype: bfloat16 parameters: t: [0, 0.5, 1, 0.5, 0] # V shaped curve: Hermes for input & output, WizardMath in the middle layers ### --- Model Rank: 45 Model Name: sthenno-com/miscii-14b-1225 Model average score across benchmarks in %: 40.08 Models average score on IFEval benchmarks in %: 78.78 Models average score on BBH benchmarks in %: 50.91 Models average score on MATH benchmarks in % 31.57 Models average score in GPQA benchmarks in % 17.00 Models average score in MUSR benchmarks in % 14.77 Models average score in MMLU-PRO benchmarks in % 47.46 ### THE MERGEKIT CONFIGURATION FOR MODEL sthenno-com/miscii-14b-1225 WAS NOT FOUND SO IT IS SKIPPED ### --- (next model etc...) >>> END INTERPRETATION OF INPUT EXAMPLE <<< 4. >>> INSTRUCTIONS <<< Below follows the scraped data from the leaderboard. >>> DATA START <<< """ benchmark_data = [ { "rank": 44, "name": "sometimesanotion/Qwen2.5-14B-Vimarckoso-v3", "scores": { "average": 40.10, "IFEval": 72.57, "BBH": 48.58, "MATH": 34.44, "GPQA": 17.34, "MUSR": 19.39, "MMLU-PRO": 48.26 }, "hf_url": "https://huggingface.co/sometimesanotion/Qwen2.5-14B-Vimarckoso-v3", # placeholder # This one has a known config from your data "known_config": { "models": [ {"model": "CultriX/SeQwence-14Bv1"}, {"model": "allknowingroger/Qwenslerp5-14B"} ], "merge_method": "slerp", "base_model": "CultriX/SeQwence-14Bv1", "dtype": "bfloat16", "parameters": { "t": [0, 0.5, 1, 0.5, 0] } } }, { "rank": 45, "name": "sthenno-com/miscii-14b-1225", "scores": { "average": 40.08, "IFEval": 78.78, "BBH": 50.91, "MATH": 31.57, "GPQA": 17.00, "MUSR": 14.77, "MMLU-PRO": 47.46 }, "hf_url": "https://huggingface.co/sthenno-com/miscii-14b-1225", "known_config": None }, { "rank": 46, "name": "djuna/Q2.5-Veltha-14B-0.5", "scores": { "average": 39.96, "IFEval": 77.96, "BBH": 50.32, "MATH": 33.84, "GPQA": 15.77, "MUSR": 14.17, "MMLU-PRO": 47.72 }, "hf_url": "https://huggingface.co/djuna/Q2.5-Veltha-14B-0.5", "known_config": None }, { "rank": 48, "name": "sometimesanotion/Qwen2.5-14B-Vimarckoso-v3-model_stock", "scores": { "average": 39.81, "IFEval": 71.62, "BBH": 48.76, "MATH": 33.99, "GPQA": 17.34, "MUSR": 19.23, "MMLU-PRO": 47.95 }, "hf_url": "https://huggingface.co/sometimesanotion/Qwen2.5-14B-Vimarckoso-v3-model_stock", "known_config": None }, { "rank": 50, "name": "sometimesanotion/Qwen2.5-14B-Vimarckoso-v3-Prose01", "scores": { "average": 39.46, "IFEval": 68.72, "BBH": 47.71, "MATH": 35.05, "GPQA": 18.23, "MUSR": 19.56, "MMLU-PRO": 47.50 }, "hf_url": "https://huggingface.co/sometimesanotion/Qwen2.5-14B-Vimarckoso-v3-Prose01", "known_config": None }, { "rank": 52, "name": "arcee-ai/Virtuoso-Small", "scores": { "average": 39.43, "IFEval": 79.35, "BBH": 50.40, "MATH": 34.29, "GPQA": 11.52, "MUSR": 14.44, "MMLU-PRO": 46.57 }, "hf_url": "https://huggingface.co/arcee-ai/Virtuoso-Small", "known_config": None }, { "rank": 54, "name": "sometimesanotion/Qwentinuum-14B-v6", "scores": { "average": 39.23, "IFEval": 63.04, "BBH": 50.23, "MATH": 33.84, "GPQA": 18.23, "MUSR": 21.18, "MMLU-PRO": 48.89 }, "hf_url": "https://huggingface.co/sometimesanotion/Qwentinuum-14B-v6", "known_config": None }, { "rank": 55, "name": "djuna/Q2.5-Veltha-14B", "scores": { "average": 39.21, "IFEval": 82.92, "BBH": 49.75, "MATH": 28.02, "GPQA": 14.54, "MUSR": 12.26, "MMLU-PRO": 47.76 }, "hf_url": "https://huggingface.co/djuna/Q2.5-Veltha-14B", "known_config": None }, { "rank": 57, "name": "allknowingroger/QwenSlerp6-14B", "scores": { "average": 39.02, "IFEval": 68.67, "BBH": 47.59, "MATH": 34.14, "GPQA": 16.44, "MUSR": 18.32, "MMLU-PRO": 48.95 }, "hf_url": "https://huggingface.co/allknowingroger/QwenSlerp6-14B", "known_config": None }, { "rank": 58, "name": "allknowingroger/QwenSlerp5-14B", "scores": { "average": 38.94, "IFEval": 71.19, "BBH": 47.39, "MATH": 33.16, "GPQA": 15.32, "MUSR": 17.81, "MMLU-PRO": 48.78 }, "hf_url": "https://huggingface.co/allknowingroger/QwenSlerp5-14B", "known_config": None }, { "rank": 59, "name": "sometimesanotion/Qwentinuum-14B-v5", "scores": { "average": 38.87, "IFEval": 62.86, "BBH": 50.28, "MATH": 31.57, "GPQA": 18.34, "MUSR": 21.09, "MMLU-PRO": 49.09 }, "hf_url": "https://huggingface.co/sometimesanotion/Qwentinuum-14B-v5", "known_config": None }, { "rank": 60, "name": "sometimesanotion/Qwenvergence-14B-v6-Prose", "scores": { "average": 38.82, "IFEval": 59.90, "BBH": 50.12, "MATH": 34.89, "GPQA": 18.46, "MUSR": 21.02, "MMLU-PRO": 48.56 }, "hf_url": "https://huggingface.co/sometimesanotion/Qwenvergence-14B-v6-Prose", "known_config": None }, { "rank": 61, "name": "CultriX/Qwen2.5-14B-Brocav3", "scores": { "average": 38.76, "IFEval": 69.52, "BBH": 49.05, "MATH": 32.25, "GPQA": 14.54, "MUSR": 19.25, "MMLU-PRO": 47.97 }, "hf_url": "https://huggingface.co/CultriX/Qwen2.5-14B-Brocav3", "known_config": None }, { "rank": 62, "name": "sometimesanotion/Qwentinuum-14B-v7", "scores": { "average": 38.76, "IFEval": 61.09, "BBH": 50.35, "MATH": 33.38, "GPQA": 18.79, "MUSR": 19.95, "MMLU-PRO": 49.00 }, "hf_url": "https://huggingface.co/sometimesanotion/Qwentinuum-14B-v7", "known_config": None }, { "rank": 64, "name": "sometimesanotion/Qwentinuum-14B-v3", "scores": { "average": 38.74, "IFEval": 61.58, "BBH": 50.04, "MATH": 32.85, "GPQA": 18.34, "MUSR": 20.62, "MMLU-PRO": 49.03 }, "hf_url": "https://huggingface.co/sometimesanotion/Qwentinuum-14B-v3", "known_config": None }, { "rank": 65, "name": "allura-org/TQ2.5-14B-Aletheia-v1", "scores": { "average": 38.74, "IFEval": 75.30, "BBH": 50.88, "MATH": 29.53, "GPQA": 14.99, "MUSR": 14.61, "MMLU-PRO": 47.12 }, "hf_url": "https://huggingface.co/allura-org/TQ2.5-14B-Aletheia-v1", "known_config": None }, { "rank": 66, "name": "qingy2024/Fusion4-14B-Instruct", "scores": { "average": 38.73, "IFEval": 76.49, "BBH": 50.70, "MATH": 33.91, "GPQA": 10.74, "MUSR": 13.97, "MMLU-PRO": 46.60 }, "hf_url": "https://huggingface.co/qingy2024/Fusion4-14B-Instruct", "known_config": None }, { "rank": 68, "name": "CultriX/Qwen2.5-14B-Brocav7", "scores": { "average": 38.52, "IFEval": 67.24, "BBH": 48.91, "MATH": 31.87, "GPQA": 15.66, "MUSR": 20.15, "MMLU-PRO": 47.31 }, "hf_url": "https://huggingface.co/CultriX/Qwen2.5-14B-Brocav7", "known_config": None }, { "rank": 71, "name": "sometimesanotion/Qwentinuum-14B-v6-Prose", "scores": { "average": 38.46, "IFEval": 56.43, "BBH": 50.14, "MATH": 35.57, "GPQA": 18.46, "MUSR": 21.34, "MMLU-PRO": 48.80 }, "hf_url": "https://huggingface.co/sometimesanotion/Qwentinuum-14B-v6-Prose", "known_config": None }, { "rank": 76, "name": "CultriX/Qwen2.5-14B-Brocav6", "scores": { "average": 38.32, "IFEval": 69.95, "BBH": 47.82, "MATH": 29.61, "GPQA": 15.66, "MUSR": 18.88, "MMLU-PRO": 47.99 }, "hf_url": "https://huggingface.co/CultriX/Qwen2.5-14B-Brocav6", "known_config": None }, { "rank": 80, "name": "CultriX/SeQwence-14Bv1", "scores": { "average": 38.20, "IFEval": 66.78, "BBH": 47.19, "MATH": 33.53, "GPQA": 14.88, "MUSR": 18.80, "MMLU-PRO": 48.00 }, "hf_url": "https://huggingface.co/CultriX/SeQwence-14Bv1", "known_config": None }, { "rank": 85, "name": "sometimesanotion/Qwentinuum-14B-v013", "scores": { "average": 37.96, "IFEval": 67.11, "BBH": 43.97, "MATH": 33.01, "GPQA": 14.32, "MUSR": 24.99, "MMLU-PRO": 44.34 }, "hf_url": "https://huggingface.co/sometimesanotion/Qwentinuum-14B-v013", "known_config": None }, { "rank": 86, "name": "CultriX/Qwen2.5-14B-Wernickev3", "scores": { "average": 37.94, "IFEval": 70.48, "BBH": 44.58, "MATH": 32.78, "GPQA": 14.99, "MUSR": 18.69, "MMLU-PRO": 46.13 }, "hf_url": "https://huggingface.co/CultriX/Qwen2.5-14B-Wernickev3", "known_config": None }, { "rank": 88, "name": "allknowingroger/QwenSlerp4-14B", "scores": { "average": 37.80, "IFEval": 63.28, "BBH": 49.38, "MATH": 30.97, "GPQA": 16.33, "MUSR": 17.59, "MMLU-PRO": 49.28 }, "hf_url": "https://huggingface.co/allknowingroger/QwenSlerp4-14B", "known_config": None }, { "rank": 89, "name": "CultriX/Qwen2.5-14B-Broca", "scores": { "average": 37.72, "IFEval": 56.04, "BBH": 50.03, "MATH": 34.59, "GPQA": 18.23, "MUSR": 18.95, "MMLU-PRO": 48.49 }, "hf_url": "https://huggingface.co/CultriX/Qwen2.5-14B-Broca", "known_config": None }, { "rank": 90, "name": "CultriX/Qwen2.5-14B-Emerged", "scores": { "average": 37.66, "IFEval": 70.00, "BBH": 45.93, "MATH": 30.74, "GPQA": 14.32, "MUSR": 18.47, "MMLU-PRO": 46.51 }, "hf_url": "https://huggingface.co/CultriX/Qwen2.5-14B-Emerged", "known_config": None }, { "rank": 91, "name": "sometimesanotion/Qwentinuum-14B-v8", "scores": { "average": 37.65, "IFEval": 54.12, "BBH": 50.11, "MATH": 34.14, "GPQA": 17.79, "MUSR": 20.75, "MMLU-PRO": 49.02 }, "hf_url": "https://huggingface.co/sometimesanotion/Qwentinuum-14B-v8", "known_config": None }, { "rank": 92, "name": "qingy2024/Fusion-14B-Instruct", "scores": { "average": 37.64, "IFEval": 72.60, "BBH": 48.58, "MATH": 30.97, "GPQA": 13.98, "MUSR": 14.81, "MMLU-PRO": 44.93 }, "hf_url": "https://huggingface.co/qingy2024/Fusion-14B-Instruct", "known_config": None }, { "rank": 94, "name": "CultriX/Qwestion-14B", "scores": { "average": 37.63, "IFEval": 63.18, "BBH": 48.76, "MATH": 31.72, "GPQA": 15.77, "MUSR": 17.22, "MMLU-PRO": 49.14 }, "hf_url": "https://huggingface.co/CultriX/Qwestion-14B", "known_config": None }, { "rank": 99, "name": "sometimesanotion/Qwenvergence-14B-v3-Prose", "scores": { "average": 37.37, "IFEval": 49.18, "BBH": 49.80, "MATH": 35.57, "GPQA": 19.35, "MUSR": 21.77, "MMLU-PRO": 48.55 }, "hf_url": "https://huggingface.co/sometimesanotion/Qwenvergence-14B-v3-Prose", "known_config": None }, { "rank": 102, "name": "CultriX/SeQwence-14B-v5", "scores": { "average": 37.27, "IFEval": 59.20, "BBH": 50.00, "MATH": 31.04, "GPQA": 16.00, "MUSR": 18.33, "MMLU-PRO": 49.05 }, "hf_url": "https://huggingface.co/CultriX/SeQwence-14B-v5", "known_config": None }, { "rank": 103, "name": "sometimesanotion/Qwen-14B-ProseStock-v4", "scores": { "average": 37.23, "IFEval": 49.42, "BBH": 49.54, "MATH": 35.50, "GPQA": 18.46, "MUSR": 21.70, "MMLU-PRO": 48.74 }, "hf_url": "https://huggingface.co/sometimesanotion/Qwen-14B-ProseStock-v4", "known_config": None }, { "rank": 104, "name": "sometimesanotion/IF-reasoning-experiment-40", "scores": { "average": 37.21, "IFEval": 63.30, "BBH": 44.31, "MATH": 27.72, "GPQA": 17.34, "MUSR": 25.86, "MMLU-PRO": 44.72 }, "hf_url": "https://huggingface.co/sometimesanotion/IF-reasoning-experiment-40", "known_config": None }, { "rank": 105, "name": "CultriX/SeQwence-14B-EvolMerge", "scores": { "average": 37.20, "IFEval": 53.82, "BBH": 50.78, "MATH": 31.80, "GPQA": 17.45, "MUSR": 20.26, "MMLU-PRO": 49.10 }, "hf_url": "https://huggingface.co/CultriX/SeQwence-14B-EvolMerge", "known_config": None } ] def scrape_model_page(model_url): """ Scrapes the Hugging Face model page for potential MergeKit YAML configuration or any other YAML blocks. Returns a dict with "yaml_configuration" and "metadata" if successful, or an error string if something goes wrong. """ try: response = requests.get(model_url) if response.status_code != 200: return f"Error: Unable to fetch the page (Status Code: {response.status_code})" soup = BeautifulSoup(response.text, "html.parser") # Attempt to locate a 
 block that might contain YAML
        yaml_config = soup.find("pre")
        yaml_text = yaml_config.text.strip() if yaml_config else "No YAML configuration found."

        # Attempt to locate metadata (modify as needed based on how the page is structured)
        metadata_section = soup.find("div", class_="metadata")
        metadata_text = metadata_section.text.strip() if metadata_section else "No metadata found."

        return {
            "yaml_configuration": yaml_text,
            "metadata": metadata_text
        }

    except Exception as e:
        return f"Error: {str(e)}"


def print_benchmark_and_config_info(model_info):
    """
    Prints an overview of the benchmark scores for the given model,
    checks for known or scraped MergeKit config, and prints accordingly.
    """
    # Print the benchmark summary
    print(f"---\nModel Rank: {model_info['rank']}")
    print(f"Model Name: {model_info['name']}")
    print(f"Model average score across benchmarks in %: {model_info['scores']['average']}")
    print(f"Models average score on IFEval benchmarks in %: {model_info['scores']['IFEval']}")
    print(f"Models average score on BBH benchmarks in %: {model_info['scores']['BBH']}")
    print(f"Models average score on MATH benchmarks in %: {model_info['scores']['MATH']}")
    print(f"Models average score in GPQA benchmarks in %: {model_info['scores']['GPQA']}")
    print(f"Models average score in MUSR benchmarks in %: {model_info['scores']['MUSR']}")
    print(f"Models average score in MMLU-PRO benchmarks in %: {model_info['scores']['MMLU-PRO']}")

    # If we have a known config for this model, just print it.
    if model_info["known_config"] is not None:
        print("###")
        print("models:")
        for m in model_info["known_config"]["models"]:
            print(f"  - model: {m['model']}")
        print(f"merge_method: {model_info['known_config']['merge_method']}")
        print(f"base_model: {model_info['known_config']['base_model']}")
        print(f"dtype: {model_info['known_config']['dtype']}")
        print("parameters:")
        print(f"  t: {model_info['known_config']['parameters']['t']} # V shaped curve: Hermes for input & output, WizardMath in the middle layers")
        print("###")
        return

    # Otherwise, attempt to scrape the model page
    scrape_result = scrape_model_page(model_info["hf_url"])
    # If we got an error or can't find YAML, we show the scraping code
    if isinstance(scrape_result, str) or ("No YAML configuration found." in scrape_result["yaml_configuration"]):
        print("(No MergeKit configuration found.)\n")
        print("You can try the following Python script to scrape the model page:\n")
        print("#" * 70)
        print(
f'''import requests
from bs4 import BeautifulSoup

def scrape_model_page(model_url):
    try:
        response = requests.get(model_url)
        if response.status_code != 200:
            return f"Error: Unable to fetch the page (Status Code: {{response.status_code}})"
        
        soup = BeautifulSoup(response.text, "html.parser")

        yaml_config = soup.find("pre")
        yaml_text = yaml_config.text.strip() if yaml_config else "No YAML configuration found."

        metadata_section = soup.find("div", class_="metadata")
        metadata_text = metadata_section.text.strip() if metadata_section else "No metadata found."

        return {{
            "yaml_configuration": yaml_text,
            "metadata": metadata_text
        }}

    except Exception as e:
        return f"Error: {{str(e)}}"

if __name__ == "__main__":
    model_url = "{model_info['hf_url']}"
    result = scrape_model_page(model_url)
    print(result)'''
        )
        print("#" * 70)
    else:
        # If we found a config, print it between triple-hash signs
        print("###")
        print(scrape_result["yaml_configuration"])
        print("###")


if __name__ == "__main__":
    # 2. Loop through all models, printing benchmark data and MergeKit config info
    for model in benchmark_data:
        print_benchmark_and_config_info(model)