src/evaluation/dynamic_eval.py

import json
import os
from datetime import datetime
from src.evaluation.perplexity_eval import evaluate_perplexity, create_perplexity_result
from src.envs import EVAL_RESULTS_PATH, API, RESULTS_REPO

def run_dynamic_perplexity_eval(model_name, revision="main", precision="float16"):
    """
    Run perplexity evaluation and save results.
    """
    try:
        # Run evaluation
        perplexity_score = evaluate_perplexity(model_name, revision)
        
        # Create result structure
        result = create_perplexity_result(model_name, revision, precision, perplexity_score)
        
        # Save result file
        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
        result_filename = f"results_{model_name.replace('/', '_')}_{timestamp}.json"
        
        # Create directory structure
        org, model = model_name.split("/") if "/" in model_name else ("", model_name)
        result_dir = os.path.join(EVAL_RESULTS_PATH, org) if org else EVAL_RESULTS_PATH
        os.makedirs(result_dir, exist_ok=True)
        
        result_path = os.path.join(result_dir, result_filename)
        
        with open(result_path, "w") as f:
            json.dump(result, f, indent=2)
        
        # Upload to Hugging Face dataset
        API.upload_file(
            path_or_fileobj=result_path,
            path_in_repo=result_path.split("eval-results/")[1],
            repo_id=RESULTS_REPO,
            repo_type="dataset",
            commit_message=f"Add perplexity results for {model_name}",
        )
        
        return True, perplexity_score
        
    except Exception as e:
        return False, str(e)