| { | |
| "batch_size": 128, | |
| "grad_accum_steps": 1, | |
| "epochs": 10, | |
| "lr": 0.0003, | |
| "betas": [ | |
| 0.9, | |
| 0.95 | |
| ], | |
| "weight_decay": 0.1, | |
| "warmup_steps": 500, | |
| "max_steps": null, | |
| "clip_grad": 1.0, | |
| "min_lr": 1e-06, | |
| "label_smoothing": 0.0, | |
| "mixed_precision": "bf16", | |
| "log_dir": "./runs/crystal_beeper", | |
| "log_interval": 50, | |
| "ckpt_dir": "./checkpoints_crystal", | |
| "export_dir": "./export_crystal", | |
| "resume": true, | |
| "resume_strict": false, | |
| "resume_tag": "best_model.safetensors", | |
| "hf_repo": "AbstractPhil/beeper-ascii-v1", | |
| "upload_to_hub": true, | |
| "add_bos_eos": true, | |
| "span_corrupt_frac": 0.0, | |
| "val_ratio": 0.01, | |
| "test_ratio": 0.01, | |
| "max_rows_per_dataset": null, | |
| "dataset_cache_verbose": true, | |
| "lambda_route": 0.2, | |
| "route_topk": 32, | |
| "lambda_geom": 0.3, | |
| "lambda_geom_angle": 0.8, | |
| "lambda_geom_var": 0.3, | |
| "lambda_geom_edge": 0.3, | |
| "lambda_geom_vol": 0.6, | |
| "lambda_geom_minrel": 1.0, | |
| "geom_min_edge_rel": 0.6, | |
| "geom_vol_lower_frac": 0.85, | |
| "geom_sample_classes": 64, | |
| "lambda_rose": 0.1, | |
| "rose_scale": 1.8, | |
| "contrast_warmup": 800, | |
| "pent_temp": 0.1, | |
| "lambda_contrast": 0.25, | |
| "punctuation": { | |
| "enable": true, | |
| "chars": [ | |
| ".", | |
| ",", | |
| ";", | |
| ":", | |
| "!", | |
| "?", | |
| "'", | |
| "\"", | |
| "(", | |
| ")", | |
| "[", | |
| "]", | |
| "{", | |
| "}", | |
| "-", | |
| "\u2014", | |
| "\u2026" | |
| ], | |
| "alpha_soft": 0.6, | |
| "hard_mask_gate": false, | |
| "apply_to_coarse_ids": "ALL" | |
| }, | |
| "harmony": { | |
| "apply": true, | |
| "system": "You are Crystal-Beeper, a helpful, honest, precise assistant.", | |
| "style": "concise" | |
| }, | |
| "stages": [ | |
| { | |
| "name": "bootstrap", | |
| "epochs": 1, | |
| "lambda_route": 0.05, | |
| "lambda_geom": 0.2, | |
| "gate_tau": 0.1, | |
| "punct_alpha": 0.6, | |
| "hard_mask_gate": false, | |
| "mix_sdpa": 1.0 | |
| }, | |
| { | |
| "name": "crystal_warmup", | |
| "epochs": 2, | |
| "lambda_route": 0.2, | |
| "lambda_geom": 0.3, | |
| "gate_tau": 0.08, | |
| "punct_alpha": 0.7, | |
| "hard_mask_gate": false, | |
| "mix_sdpa": 1.0 | |
| }, | |
| { | |
| "name": "dictionary_crystals", | |
| "epochs": 2, | |
| "lambda_route": 0.25, | |
| "lambda_geom": 0.35, | |
| "gate_tau": 0.06, | |
| "punct_alpha": 0.85, | |
| "hard_mask_gate": false, | |
| "mix_sdpa": 0.9 | |
| }, | |
| { | |
| "name": "stability_tuning", | |
| "epochs": 3, | |
| "lambda_route": 0.3, | |
| "lambda_geom": 0.4, | |
| "gate_tau": 0.05, | |
| "punct_alpha": 1.0, | |
| "hard_mask_gate": false, | |
| "mix_sdpa": 0.8 | |
| } | |
| ], | |
| "corpus": [ | |
| { | |
| "name": "TinyStories", | |
| "path": "roneneldan/TinyStories", | |
| "split": "train[30%:50%]", | |
| "weight": 0.1, | |
| "dialect": [ | |
| 0.6, | |
| 0.1, | |
| 0.05, | |
| 0.05, | |
| 0.2 | |
| ] | |
| }, | |
| { | |
| "name": "WikipediaEN", | |
| "path": "wikimedia/wikipedia", | |
| "config": "20231101.en", | |
| "split": "train[5%:15%]", | |
| "weight": 0.5, | |
| "dialect": [ | |
| 0.12, | |
| 0.58, | |
| 0.1, | |
| 0.1, | |
| 0.1 | |
| ] | |
| }, | |
| { | |
| "name": "AGNews", | |
| "path": "ag_news", | |
| "split": "train[:]", | |
| "weight": 0.1, | |
| "dialect": [ | |
| 0.2, | |
| 0.5, | |
| 0.1, | |
| 0.1, | |
| 0.1 | |
| ] | |
| }, | |
| { | |
| "name": "GSM8K", | |
| "path": "openai/gsm8k", | |
| "config": "main", | |
| "split": "train[40%:60%]", | |
| "weight": 0.6, | |
| "dialect": [ | |
| 0.1, | |
| 0.15, | |
| 0.5, | |
| 0.15, | |
| 0.1 | |
| ] | |
| }, | |
| { | |
| "name": "AI2-ARC-Easy", | |
| "path": "allenai/ai2_arc", | |
| "config": "ARC-Easy", | |
| "split": "train[30%:60%]", | |
| "weight": 0.6, | |
| "dialect": [ | |
| 0.05, | |
| 0.15, | |
| 0.4, | |
| 0.25, | |
| 0.15 | |
| ] | |
| }, | |
| { | |
| "name": "HH-RLHF", | |
| "path": "Anthropic/hh-rlhf", | |
| "split": "train[5%:10%]", | |
| "weight": 0.5, | |
| "dialect": [ | |
| 0.1, | |
| 0.25, | |
| 0.2, | |
| 0.25, | |
| 0.2 | |
| ] | |
| }, | |
| { | |
| "name": "SVAMP", | |
| "path": "ChilleD/SVAMP", | |
| "split": "train", | |
| "weight": 0.25, | |
| "dialect": [ | |
| 0.1, | |
| 0.15, | |
| 0.55, | |
| 0.15, | |
| 0.05 | |
| ] | |
| }, | |
| { | |
| "name": "MATH-500", | |
| "path": "HuggingFaceH4/MATH-500", | |
| "split": "test", | |
| "weight": 0.25, | |
| "dialect": [ | |
| 0.05, | |
| 0.15, | |
| 0.6, | |
| 0.15, | |
| 0.05 | |
| ] | |
| }, | |
| { | |
| "name": "SEP", | |
| "path": "AiresPucrs/stanford-encyclopedia-philosophy", | |
| "split": "train", | |
| "weight": 0.3, | |
| "dialect": [ | |
| 0.05, | |
| 0.45, | |
| 0.18, | |
| 0.22, | |
| 0.1 | |
| ] | |
| } | |
| ], | |
| "_alive_entries": [ | |
| { | |
| "name": "TinyStories", | |
| "path": "roneneldan/TinyStories", | |
| "split": "train[30%:50%]", | |
| "weight": 0.1, | |
| "dialect": [ | |
| 0.6000000238418579, | |
| 0.10000000149011612, | |
| 0.05000000074505806, | |
| 0.05000000074505806, | |
| 0.20000000298023224 | |
| ], | |
| "class_id": 0, | |
| "p": 0.03125000000000001 | |
| }, | |
| { | |
| "name": "WikipediaEN", | |
| "path": "wikimedia/wikipedia", | |
| "config": "20231101.en", | |
| "split": "train[5%:15%]", | |
| "weight": 0.5, | |
| "dialect": [ | |
| 0.11999999731779099, | |
| 0.5799999833106995, | |
| 0.10000000149011612, | |
| 0.10000000149011612, | |
| 0.10000000149011612 | |
| ], | |
| "class_id": 1, | |
| "p": 0.15625 | |
| }, | |
| { | |
| "name": "AGNews", | |
| "path": "ag_news", | |
| "split": "train[:]", | |
| "weight": 0.1, | |
| "dialect": [ | |
| 0.20000000298023224, | |
| 0.5, | |
| 0.10000000149011612, | |
| 0.10000000149011612, | |
| 0.10000000149011612 | |
| ], | |
| "class_id": 2, | |
| "p": 0.03125000000000001 | |
| }, | |
| { | |
| "name": "GSM8K", | |
| "path": "openai/gsm8k", | |
| "config": "main", | |
| "split": "train[40%:60%]", | |
| "weight": 0.6, | |
| "dialect": [ | |
| 0.10000000149011612, | |
| 0.15000000596046448, | |
| 0.5, | |
| 0.15000000596046448, | |
| 0.10000000149011612 | |
| ], | |
| "class_id": 3, | |
| "p": 0.1875 | |
| }, | |
| { | |
| "name": "AI2-ARC-Easy", | |
| "path": "allenai/ai2_arc", | |
| "config": "ARC-Easy", | |
| "split": "train[30%:60%]", | |
| "weight": 0.6, | |
| "dialect": [ | |
| 0.05000000074505806, | |
| 0.15000000596046448, | |
| 0.4000000059604645, | |
| 0.25, | |
| 0.15000000596046448 | |
| ], | |
| "class_id": 4, | |
| "p": 0.1875 | |
| }, | |
| { | |
| "name": "HH-RLHF", | |
| "path": "Anthropic/hh-rlhf", | |
| "split": "train[5%:10%]", | |
| "weight": 0.5, | |
| "dialect": [ | |
| 0.10000000149011612, | |
| 0.25, | |
| 0.20000000298023224, | |
| 0.25, | |
| 0.20000000298023224 | |
| ], | |
| "class_id": 5, | |
| "p": 0.15625 | |
| }, | |
| { | |
| "name": "SVAMP", | |
| "path": "ChilleD/SVAMP", | |
| "split": "train", | |
| "weight": 0.25, | |
| "dialect": [ | |
| 0.10000000149011612, | |
| 0.15000000596046448, | |
| 0.550000011920929, | |
| 0.15000000596046448, | |
| 0.05000000074505806 | |
| ], | |
| "class_id": 6, | |
| "p": 0.078125 | |
| }, | |
| { | |
| "name": "MATH-500", | |
| "path": "HuggingFaceH4/MATH-500", | |
| "split": "test", | |
| "weight": 0.25, | |
| "dialect": [ | |
| 0.05000000074505806, | |
| 0.15000000596046448, | |
| 0.6000000238418579, | |
| 0.15000000596046448, | |
| 0.05000000074505806 | |
| ], | |
| "class_id": 7, | |
| "p": 0.078125 | |
| }, | |
| { | |
| "name": "SEP", | |
| "path": "AiresPucrs/stanford-encyclopedia-philosophy", | |
| "split": "train", | |
| "weight": 0.3, | |
| "dialect": [ | |
| 0.05000000074505806, | |
| 0.44999998807907104, | |
| 0.18000000715255737, | |
| 0.2199999988079071, | |
| 0.10000000149011612 | |
| ], | |
| "class_id": 8, | |
| "p": 0.09375 | |
| } | |
| ] | |
| } |