src/utils.py

import yaml
from transformers import pipeline


def load_pipelines_from_config(config_path: str):
    with open(config_path, "r") as f:
        model_config = yaml.safe_load(f.read())

    models = {}
    for model, config in model_config.items():
        models[model] = pipeline(
            task=config["task"],
            model=config["model_name"],
            revision=config["hf_commit_hash"],
        )

    return models