Delete files models/OLMo-1B/wandb/ with huggingface_hub

models/OLMo-1B/wandb/wandb/debug-internal.log

@@ -1,13 +0,0 @@
-{"time":"2025-12-19T08:35:08.828028876+01:00","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
-{"time":"2025-12-19T08:35:09.083424379+01:00","level":"INFO","msg":"stream: created new stream","id":"zvk9qxz6"}
-{"time":"2025-12-19T08:35:09.084141654+01:00","level":"INFO","msg":"handler: started","stream_id":"zvk9qxz6"}
-{"time":"2025-12-19T08:35:09.086154012+01:00","level":"INFO","msg":"stream: started","id":"zvk9qxz6"}
-{"time":"2025-12-19T08:35:09.086168743+01:00","level":"INFO","msg":"writer: started","stream_id":"zvk9qxz6"}
-{"time":"2025-12-19T08:35:09.086173112+01:00","level":"INFO","msg":"sender: started","stream_id":"zvk9qxz6"}
-{"time":"2025-12-20T01:19:49.769458459+01:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/marksmans/olmo-debug/zvk9qxz6/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:51827->127.0.0.53:53: i/o timeout"}
-{"time":"2025-12-20T06:02:10.195866407+01:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
-{"time":"2025-12-20T06:02:10.390657483+01:00","level":"INFO","msg":"handler: operation stats","stats":{}}
-{"time":"2025-12-20T06:02:10.434932054+01:00","level":"INFO","msg":"stream: closing","id":"zvk9qxz6"}
-{"time":"2025-12-20T06:02:10.445057599+01:00","level":"INFO","msg":"handler: closed","stream_id":"zvk9qxz6"}
-{"time":"2025-12-20T06:02:10.457659306+01:00","level":"INFO","msg":"sender: closed","stream_id":"zvk9qxz6"}
-{"time":"2025-12-20T06:02:10.464136678+01:00","level":"INFO","msg":"stream: closed","id":"zvk9qxz6"}

models/OLMo-1B/wandb/wandb/debug.log

@@ -1,26 +0,0 @@
-2025-12-19 08:35:08,585 INFO    MainThread:264281 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
-2025-12-19 08:35:08,587 INFO    MainThread:264281 [wandb_setup.py:_flush():80] Configure stats pid to 264281
-2025-12-19 08:35:08,587 INFO    MainThread:264281 [wandb_setup.py:_flush():80] Loading settings from /root/.config/wandb/settings
-2025-12-19 08:35:08,588 INFO    MainThread:264281 [wandb_setup.py:_flush():80] Loading settings from /netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/wandb/settings
-2025-12-19 08:35:08,589 INFO    MainThread:264281 [wandb_setup.py:_flush():80] Loading settings from environment variables
-2025-12-19 08:35:08,589 INFO    MainThread:264281 [wandb_init.py:setup_run_log_directory():714] Logging user logs to checkpoints/OLMo-1B-as_fm3_omi2/wandb/wandb/run-20251219_083508-zvk9qxz6/logs/debug.log
-2025-12-19 08:35:08,590 INFO    MainThread:264281 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to checkpoints/OLMo-1B-as_fm3_omi2/wandb/wandb/run-20251219_083508-zvk9qxz6/logs/debug-internal.log
-2025-12-19 08:35:08,591 INFO    MainThread:264281 [wandb_init.py:init():841] calling init triggers
-2025-12-19 08:35:08,591 INFO    MainThread:264281 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
-config: {'run_name': 'OLMo-1B-as_fm3_omi2', 'seed': 6198, 'epoch': None, 'dry_run': False, 'model': {'d_model': 2048, 'n_heads': 16, 'n_kv_heads': None, 'clip_qkv': None, 'n_layers': 16, 'mlp_ratio': 8, 'mlp_hidden_size': None, 'activation_type': 'swiglu', 'block_type': 'sequential', 'block_group_size': 1, 'alibi': False, 'alibi_bias_max': 8.0, 'rope': True, 'rope_full_precision': True, 'rope_theta': 10000, 'flash_attention': False, 'attention_dropout': 0.0, 'multi_query_attention': False, 'attention_layer_norm': False, 'residual_dropout': 0.0, 'embedding_dropout': 0.0, 'embedding_layer_norm': False, 'layer_norm_type': 'default', 'layer_norm_with_affine': False, 'layer_norm_eps': 1e-05, 'attention_layer_norm_with_affine': False, 'max_sequence_length': 2048, 'include_bias': False, 'bias_for_layer_norm': False, 'scale_logits': False, 'vocab_size': 32000, 'embedding_size': 32000, 'weight_tying': True, 'eos_token_id': 0, 'pad_token_id': 1, 'init_device': 'cuda', 'init_fn': 'normal', 'init_std': 0.02, 'init_cutoff_factor': 3.0, 'precision': 'amp_bf16', 'scale_emb_init': False, 'emb_init_std': None, 'norm_after': False}, 'optimizer': {'name': 'adamw', 'learning_rate': 0.0005, 'weight_decay': 0.1, 'betas': (0.9, 0.95), 'eps': 1e-08, 'no_decay_norm_and_bias': None, 'selective_updates': False, 'decay_norm_and_bias': True, 'decay_embeddings': True, 'metrics_log_interval': 10, 'record_update_metrics': False}, 'scheduler': {'name': 'cosine_with_warmup', 'units': 'steps', 't_warmup': 2000, 't_max': None, 'alpha_f': 0.1, 'grad_clip_warmup_steps': None, 'grad_clip_warmup_factor': None, 'warmup_min_lr': 0.0}, 'data': {'paths': ['data_token/as_fm3_omi2/algebraic-stack-tokenized/00000_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00001_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00002_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00003_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00004_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00005_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00006_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00007_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00008_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00009_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00010_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00011_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00012_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00016_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00019_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00020_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00021_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00022_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00023_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00024_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00025_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00026_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00027_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00028_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00029_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00030_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00031_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00032_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00033_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00034_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00035_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00036_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00037_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00038_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00039_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00040_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00041_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00042_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00043_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00044_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00045_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00046_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00047_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00048_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00049_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00050_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00051_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00052_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00053_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00054_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00055_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00056_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00057_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00058_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00059_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00060_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00061_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00062_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00063_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00064_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00065_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00066_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00067_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00068_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00069_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00070_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00071_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00072_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00073_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00074_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00075_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00076_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00077_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00078_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00000_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00001_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00002_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00003_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00004_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00005_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00006_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00007_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00008_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00009_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00010_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00011_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00012_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00013_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00014_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00015_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00016_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00017_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00018_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00019_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00020_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00021_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00022_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00023_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00024_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00025_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00026_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00027_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00028_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00029_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00030_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00031_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00032_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00033_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00034_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00035_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00036_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00037_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00038_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00039_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00040_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00041_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00042_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00043_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00044_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00045_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00046_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00047_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00048_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00049_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00050_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00051_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00052_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00053_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00054_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00055_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00056_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00057_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00058_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00059_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00060_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00061_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00062_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00063_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00064_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00065_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00066_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00067_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00068_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00069_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00070_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00071_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00072_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00073_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00074_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00075_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00076_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00077_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00078_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00079_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00080_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00081_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00082_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00083_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00084_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00085_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00086_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00087_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00088_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00089_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00090_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00091_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00092_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00093_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00094_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00095_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00096_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00097_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00098_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00099_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct1-tokenized/00000_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00000_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00001_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00002_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00003_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00004_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00005_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00006_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00007_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00008_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00009_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00010_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00011_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00012_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00013_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00014_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00015_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00016_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00017_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00018_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00019_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00020_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00021_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00022_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00023_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00024_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00025_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00026_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00027_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00028_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00029_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00030_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00031_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00032_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00033_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00034_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00035_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00036_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00037_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00038_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00039_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00040_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00041_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00042_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00043_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00044_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00045_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00046_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00047_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00048_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00049_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00050_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00051_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00052_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00053_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00054_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/tinygsm-tokenized/00000_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/tinygsm-tokenized/00001_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/tinygsm-tokenized/00002_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/tinygsm-tokenized/00003_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/tinygsm-tokenized/00004_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/tinygsm-tokenized/00005_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/tinygsm-tokenized/00006_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/tinygsm-tokenized/00007_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/tinygsm-tokenized/00008_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/tinygsm-tokenized/00009_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/tinygsm-tokenized/00010_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/tinygsm-tokenized/00011_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/tinygsm-tokenized/00012_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/tinygsm-tokenized/00013_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/tinygsm-tokenized/00014_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/tinygsm-tokenized/00015_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/tinygsm-tokenized/00016_00000_doc_shuffled.ds'], 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 32, 'drop_last': True, 'pin_memory': True, 'prefetch_factor': 8, 'persistent_workers': True, 'timeout': 0, 'seed': None, 'instance_filter': None, 'custom_dataset': None}, 'restore_dataloader': True, 'fast_forward_batches': None, 'evaluators': [], 'eval_interval': 5000, 'tokenizer': {'identifier': 'meta-llama/Llama-2-7b-hf', 'truncate_direction': 'right'}, 'save_folder': 'checkpoints/OLMo-1B-as_fm3_omi2', 'remote_save_folder': None, 'canceled_check_interval': 6000, 'save_interval': 3000, 'save_interval_unsharded': 3000, 'save_interval_ephemeral': None, 'save_num_checkpoints_to_keep': -1, 'save_num_unsharded_checkpoints_to_keep': -1, 'save_overwrite': True, 'force_save_unsharded': False, 'no_pre_train_checkpoint': False, 'load_path': 'checkpoints/OLMo-1B-as_fm3_omi2/step51000-unsharded', 'load_path_sharded_checkpointer': None, 'try_load_latest_save': False, 'reset_optimizer_state': False, 'reset_trainer_state': False, 'sharded_checkpointer': 'torch_legacy', 'new_style_checkpoints': None, 'max_duration': '1ep', 'global_train_batch_size': 512, 'device_train_batch_size': 128, 'device_train_microbatch_size': 16, 'device_eval_batch_size': 16, 'eval_subset_num_batches': -1, 'eval_on_load': False, 'device_train_grad_accum': 8, 'max_grad_norm': 1.0, 'max_grad_norm_ratio': None, 'precision': 'amp_bf16', 'speed_monitor': {'window_size': 20, 'gpu_flops_available': None}, 'console_log_interval': 1, 'gen1_gc_interval': 1, 'compile': None, 'distributed_strategy': 'fsdp', 'fsdp': {'use_orig_params': True, 'sharding_strategy': <ShardingStrategy.FULL_SHARD: 1>, 'wrapping_strategy': None, 'precision': 'mixed', 'hybrid_sharding_num_model_replicas': None}, 'ddp': {'grad_sync_mode': 'batch', 'find_unused_params': False}, 'single': {'device': 'auto'}, 'softmax_auxiliary_loss': False, 'auxiliary_loss_multiplier': 0.0001, 'time_limit': None, 'extra_steps_after_cancel': 10, 'early_stopping_factor': None, 'save_data_indices': True, 'python_profiling': False, 'torch_profiling': False, 'stop_at': None, 'stop_after': None, 'activation_checkpointing': None, 'fused_loss': None, 'hf_datasets_cache_dir': None, 'module_outputs_save_steps': None, '_wandb': {}}
-2025-12-19 08:35:08,592 INFO    MainThread:264281 [wandb_init.py:init():889] starting backend
-2025-12-19 08:35:08,822 INFO    MainThread:264281 [wandb_init.py:init():892] sending inform_init request
-2025-12-19 08:35:08,826 INFO    MainThread:264281 [wandb_init.py:init():900] backend started and connected
-2025-12-19 08:35:08,829 INFO    MainThread:264281 [wandb_init.py:init():970] updated telemetry
-2025-12-19 08:35:08,830 INFO    MainThread:264281 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
-2025-12-19 08:35:09,331 INFO    MainThread:264281 [wandb_init.py:init():1041] starting run threads in backend
-2025-12-19 08:35:09,390 INFO    MainThread:264281 [wandb_run.py:_console_start():2521] atexit reg
-2025-12-19 08:35:09,391 INFO    MainThread:264281 [wandb_run.py:_redirect():2369] redirect: wrap_raw
-2025-12-19 08:35:09,391 INFO    MainThread:264281 [wandb_run.py:_redirect():2438] Wrapping output streams.
-2025-12-19 08:35:09,392 INFO    MainThread:264281 [wandb_run.py:_redirect():2461] Redirects installed.
-2025-12-19 08:35:09,394 INFO    MainThread:264281 [wandb_init.py:init():1081] run started, returning control to user process
-2025-12-20 06:02:08,253 INFO    MainThread:264281 [wandb_run.py:_finish():2287] finishing run marksmans/olmo-debug/zvk9qxz6
-2025-12-20 06:02:08,293 INFO    MainThread:264281 [wandb_run.py:_atexit_cleanup():2486] got exitcode: 0
-2025-12-20 06:02:08,312 INFO    MainThread:264281 [wandb_run.py:_restore():2468] restore
-2025-12-20 06:02:08,315 INFO    MainThread:264281 [wandb_run.py:_restore():2474] restore done
-2025-12-20 06:02:10,416 INFO    MainThread:264281 [wandb_run.py:_footer_sync_info():3862] logging synced files

models/OLMo-1B/wandb/wandb/run-20251214_153041-8qjie2ty/files/config.yaml

@@ -1,575 +0,0 @@
-_wandb:
-    value:
-        cli_version: 0.23.1
-        e:
-            gat9u7yjnpjadjdd0jcwog0os0b9blu3:
-                args:
-                    - pretraining/configs/RL-1B.yaml
-                codePath: OLMo/scripts/train.py
-                codePathLocal: OLMo/scripts/train.py
-                cpu_count: 112
-                cpu_count_logical: 224
-                cudaVersion: "13.0"
-                disk:
-                    /:
-                        total: "2055141851136"
-                        used: "49254445056"
-                email: [email protected]
-                executable: /opt/conda/bin/python
-                gpu: NVIDIA H100 80GB HBM3
-                gpu_count: 1
-                gpu_nvidia:
-                    - architecture: Hopper
-                      cudaCores: 16896
-                      memoryTotal: "85520809984"
-                      name: NVIDIA H100 80GB HBM3
-                      uuid: GPU-4c999b2a-2578-9e62-0539-4b826d85fda8
-                host: serv-3342
-                memory:
-                    total: "2164176814080"
-                os: Linux-6.8.0-1043-nvidia-x86_64-with-glibc2.35
-                program: /netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/scripts/train.py
-                python: CPython 3.11.11
-                root: checkpoints/OLMo-1B-as_fm3_omi2/wandb
-                slurm:
-                    cluster_name: pegasus
-                    conf: /etc/slurm/slurm.conf
-                    cpu_bind: quiet,mask_cpu:0x000000000001FE00000000000000000000000001FE00000000000000
-                    cpu_bind_list: 0x000000000001FE00000000000000000000000001FE00000000000000
-                    cpu_bind_type: 'mask_cpu:'
-                    cpu_bind_verbose: quiet
-                    cpus_on_node: "16"
-                    cpus_per_task: "16"
-                    distribution: cyclic
-                    gpus: "1"
-                    gpus_on_node: "1"
-                    gtids: "0"
-                    job_cpus_per_node: "16"
-                    job_end_time: "1765734524"
-                    job_gid: "8000"
-                    job_group: iml
-                    job_id: "2383756"
-                    job_name: bash
-                    job_nodelist: serv-3342
-                    job_num_nodes: "1"
-                    job_partition: H100
-                    job_qos: normal
-                    job_start_time: "1765720124"
-                    job_uid: "13262"
-                    job_user: nguyen
-                    jobid: "2383756"
-                    launch_node_ipaddr: 192.168.33.114
-                    localid: "0"
-                    mem_per_cpu: "16384"
-                    mpi_type: pmix
-                    nnodes: "1"
-                    nodeid: "0"
-                    nodelist: serv-3342
-                    nprocs: "1"
-                    ntasks: "1"
-                    oom_kill_step: "0"
-                    pmix_mapping_serv: (vector,(0,1,1))
-                    pmixp_abort_agent_port: "33735"
-                    prio_process: "1"
-                    procid: "0"
-                    pty_port: "45219"
-                    pty_win_col: "156"
-                    pty_win_row: "41"
-                    srun_comm_host: 192.168.33.114
-                    srun_comm_port: "35153"
-                    step_gpus: "5"
-                    step_id: "0"
-                    step_launcher_port: "35153"
-                    step_nodelist: serv-3342
-                    step_num_nodes: "1"
-                    step_num_tasks: "1"
-                    step_tasks_per_node: "1"
-                    stepid: "0"
-                    submit_dir: /netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain
-                    submit_host: login1
-                    task_pid: "3684902"
-                    tasks_per_node: "1"
-                    topology_addr: serv-3342
-                    topology_addr_pattern: node
-                    tres_bind: gres/gpu:per_task:1
-                    tres_per_task: cpu=16,gres/gpu=1
-                    umask: "0022"
-                startedAt: "2025-12-14T14:30:41.667717Z"
-                writerId: gat9u7yjnpjadjdd0jcwog0os0b9blu3
-        m: []
-        python_version: 3.11.11
-        t:
-            "1":
-                - 1
-                - 5
-                - 11
-                - 41
-                - 49
-                - 51
-                - 53
-            "2":
-                - 1
-                - 5
-                - 11
-                - 41
-                - 49
-                - 51
-                - 53
-            "3":
-                - 13
-                - 15
-                - 16
-            "4": 3.11.11
-            "5": 0.23.1
-            "6": 4.57.3
-            "12": 0.23.1
-            "13": linux-x86_64
-activation_checkpointing:
-    value: null
-auxiliary_loss_multiplier:
-    value: 0.0001
-canceled_check_interval:
-    value: 6000
-compile:
-    value: null
-console_log_interval:
-    value: 1
-data:
-    value:
-        custom_dataset: null
-        datasets: null
-        drop_last: true
-        generate_attention_mask: false
-        generate_doc_lengths: false
-        instance_filter: null
-        label_mask_paths: null
-        memmap_dtype: uint16
-        num_workers: 32
-        pad_direction: right
-        paths:
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00000_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00001_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00002_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00003_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00004_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00005_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00006_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00007_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00008_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00009_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00010_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00011_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00012_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00016_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00019_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00020_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00021_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00022_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00023_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00024_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00025_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00026_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00027_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00028_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00029_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00030_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00031_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00032_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00033_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00034_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00035_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00036_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00037_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00038_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00039_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00040_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00041_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00042_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00043_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00044_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00045_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00046_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00047_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00048_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00049_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00050_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00051_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00052_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00053_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00054_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00055_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00056_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00057_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00058_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00059_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00060_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00061_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00062_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00063_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00064_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00065_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00066_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00067_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00068_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00069_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00070_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00071_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00072_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00073_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00074_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00075_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00076_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00077_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00078_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00000_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00001_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00002_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00003_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00004_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00005_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00006_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00007_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00008_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00009_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00010_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00011_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00012_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00013_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00014_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00015_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00016_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00017_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00018_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00019_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00020_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00021_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00022_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00023_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00024_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00025_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00026_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00027_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00028_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00029_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00030_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00031_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00032_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00033_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00034_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00035_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00036_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00037_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00038_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00039_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00040_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00041_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00042_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00043_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00044_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00045_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00046_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00047_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00048_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00049_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00050_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00051_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00052_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00053_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00054_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00055_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00056_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00057_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00058_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00059_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00060_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00061_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00062_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00063_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00064_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00065_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00066_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00067_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00068_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00069_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00070_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00071_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00072_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00073_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00074_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00075_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00076_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00077_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00078_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00079_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00080_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00081_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00082_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00083_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00084_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00085_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00086_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00087_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00088_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00089_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00090_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00091_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00092_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00093_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00094_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00095_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00096_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00097_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00098_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00099_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00000_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00001_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00002_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00003_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00004_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00005_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00006_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00007_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00008_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00009_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00010_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00011_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00012_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00013_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00014_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00015_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00016_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00017_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00018_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00019_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00020_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00021_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00022_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00023_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00024_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00025_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00026_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00027_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00028_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00029_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00030_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00031_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00032_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00033_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00034_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00035_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00036_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00037_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00038_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00039_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00040_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00041_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00042_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00043_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00044_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00045_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00046_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00047_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00048_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00049_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00050_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00051_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00052_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00053_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00054_00000_doc_shuffled.ds
-        persistent_workers: true
-        pin_memory: true
-        prefetch_factor: 8
-        seed: null
-        timeout: 0
-ddp:
-    value:
-        find_unused_params: false
-        grad_sync_mode: batch
-device_eval_batch_size:
-    value: 16
-device_train_batch_size:
-    value: 512
-device_train_grad_accum:
-    value: 32
-device_train_microbatch_size:
-    value: 16
-distributed_strategy:
-    value: fsdp
-dry_run:
-    value: false
-early_stopping_factor:
-    value: null
-epoch:
-    value: null
-eval_interval:
-    value: 5000
-eval_on_load:
-    value: false
-eval_subset_num_batches:
-    value: -1
-evaluators:
-    value: []
-extra_steps_after_cancel:
-    value: 10
-fast_forward_batches:
-    value: null
-force_save_unsharded:
-    value: false
-fsdp:
-    value:
-        hybrid_sharding_num_model_replicas: null
-        precision: mixed
-        sharding_strategy: FULL_SHARD
-        use_orig_params: true
-        wrapping_strategy: null
-fused_loss:
-    value: null
-gen1_gc_interval:
-    value: 1
-global_train_batch_size:
-    value: 512
-hf_datasets_cache_dir:
-    value: null
-load_path:
-    value: null
-load_path_sharded_checkpointer:
-    value: null
-max_duration:
-    value: 1ep
-max_grad_norm:
-    value: 1
-max_grad_norm_ratio:
-    value: null
-model:
-    value:
-        activation_type: swiglu
-        alibi: false
-        alibi_bias_max: 8
-        attention_dropout: 0
-        attention_layer_norm: false
-        attention_layer_norm_with_affine: false
-        bias_for_layer_norm: false
-        block_group_size: 1
-        block_type: sequential
-        clip_qkv: null
-        d_model: 2048
-        emb_init_std: null
-        embedding_dropout: 0
-        embedding_layer_norm: false
-        embedding_size: 32000
-        eos_token_id: 0
-        flash_attention: false
-        include_bias: false
-        init_cutoff_factor: 3
-        init_device: cuda
-        init_fn: normal
-        init_std: 0.02
-        layer_norm_eps: 1e-05
-        layer_norm_type: default
-        layer_norm_with_affine: false
-        max_sequence_length: 2048
-        mlp_hidden_size: null
-        mlp_ratio: 8
-        multi_query_attention: false
-        n_heads: 16
-        n_kv_heads: null
-        n_layers: 16
-        norm_after: false
-        pad_token_id: 1
-        precision: amp_bf16
-        residual_dropout: 0
-        rope: true
-        rope_full_precision: true
-        rope_theta: 10000
-        scale_emb_init: false
-        scale_logits: false
-        vocab_size: 32000
-        weight_tying: true
-module_outputs_save_steps:
-    value: null
-new_style_checkpoints:
-    value: null
-no_pre_train_checkpoint:
-    value: false
-optimizer:
-    value:
-        betas:
-            - 0.9
-            - 0.95
-        decay_embeddings: true
-        decay_norm_and_bias: true
-        eps: 1e-08
-        learning_rate: 0.0005
-        metrics_log_interval: 10
-        name: adamw
-        no_decay_norm_and_bias: null
-        record_update_metrics: false
-        selective_updates: false
-        weight_decay: 0.1
-precision:
-    value: amp_bf16
-python_profiling:
-    value: false
-remote_save_folder:
-    value: null
-reset_optimizer_state:
-    value: false
-reset_trainer_state:
-    value: false
-restore_dataloader:
-    value: true
-run_name:
-    value: OLMo-1B-as_fm3_omi2
-save_data_indices:
-    value: true
-save_folder:
-    value: checkpoints/OLMo-1B-as_fm3_omi2
-save_interval:
-    value: 3000
-save_interval_ephemeral:
-    value: null
-save_interval_unsharded:
-    value: 3000
-save_num_checkpoints_to_keep:
-    value: -1
-save_num_unsharded_checkpoints_to_keep:
-    value: -1
-save_overwrite:
-    value: true
-scheduler:
-    value:
-        alpha_f: 0.1
-        grad_clip_warmup_factor: null
-        grad_clip_warmup_steps: null
-        name: cosine_with_warmup
-        t_max: null
-        t_warmup: 2000
-        units: steps
-        warmup_min_lr: 0
-seed:
-    value: 6198
-sharded_checkpointer:
-    value: torch_legacy
-single:
-    value:
-        device: auto
-softmax_auxiliary_loss:
-    value: false
-speed_monitor:
-    value:
-        gpu_flops_available: null
-        window_size: 20
-stop_after:
-    value: null
-stop_at:
-    value: null
-time_limit:
-    value: null
-tokenizer:
-    value:
-        identifier: meta-llama/Llama-2-7b-hf
-        truncate_direction: right
-torch_profiling:
-    value: false
-try_load_latest_save:
-    value: false

models/OLMo-1B/wandb/wandb/run-20251214_153041-8qjie2ty/files/output.log

@@ -1,144 +0,0 @@
-Traceback (most recent call last):
-  File "/netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/scripts/train.py", line 436, in <module>
-    main(cfg)
-  File "/netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/scripts/train.py", line 132, in main
-    train_loader = build_train_dataloader(cfg)
-                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/olmo/data/__init__.py", line 156, in build_train_dataloader
-    dataset = IterableDataset(
-              ^^^^^^^^^^^^^^^^
-  File "/netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/olmo/data/iterable_dataset.py", line 57, in __init__
-    if self.drop_last and len(self.dataset) % self.world_size != 0:  # type: ignore[arg-type]
-                          ^^^^^^^^^^^^^^^^^
-  File "/netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/olmo/data/memmap_dataset.py", line 176, in __len__
-    self._num_instances = self.offsets[-1][1]
-                          ^^^^^^^^^^^^
-  File "/netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/olmo/data/memmap_dataset.py", line 138, in offsets
-    path, length = future.result()
-                   ^^^^^^^^^^^^^^^
-  File "/opt/conda/lib/python3.11/concurrent/futures/_base.py", line 449, in result
-    return self.__get_result()
-           ^^^^^^^^^^^^^^^^^^^
-  File "/opt/conda/lib/python3.11/concurrent/futures/_base.py", line 401, in __get_result
-    raise self._exception
-  File "/opt/conda/lib/python3.11/concurrent/futures/thread.py", line 58, in run
-    result = self.fn(*self.args, **self.kwargs)
-             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/olmo/data/memmap_dataset.py", line 172, in _get_file_length
-    return path, file_size(path) // (item_size * self._chunk_size)
-                 ^^^^^^^^^^^^^^^
-  File "/netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/olmo/util.py", line 350, in file_size
-    return os.stat(path).st_size
-           ^^^^^^^^^^^^^
-FileNotFoundError: [Errno 2] No such file or directory: 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00006_00000_doc_shuffled.ds'
-[2025-12-14 15:30:44] CRITICAL [olmo.util:168, rank=0] Uncaught FileNotFoundError: [Errno 2] No such file or directory: 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00006_00000_doc_shuffled.ds'
-╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮
-│ /netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/scripts/train.py:436 │
-│                                                                                                  │
-│   433 │   │   log.info("Device is CPU. Updating config...")                                      │
-│   434 │   │   cfg.model.init_device = "cpu"                                                      │
-│   435 │   │   cfg.distributed_strategy = "single"  # type: ignore                                │
-│ ❱ 436 │   main(cfg)                                                                              │
-│   437                                                                                            │
-│                                                                                                  │
-│ /netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/scripts/train.py:132 │
-│                                                                                                  │
-│   129 │   seed_all(cfg.seed)                                                                     │
-│   130 │                                                                                          │
-│   131 │   # Construct data loader.                                                               │
-│ ❱ 132 │   train_loader = build_train_dataloader(cfg)                                             │
-│   133 │                                                                                          │
-│   134 │   # Construct evaluators.                                                                │
-│   135 │   evaluators = build_evaluators(cfg, device)                                             │
-│                                                                                                  │
-│ /netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/olmo/data/__init__.p │
-│                                                                                                  │
-│   153 │   │   │   )                                                                              │
-│   154 │   │   else:                                                                              │
-│   155 │   │   │   work_dir.mkdir(exist_ok=True, parents=True)                                    │
-│ ❱ 156 │   dataset = IterableDataset(                                                             │
-│   157 │   │   dataset,  # type: ignore                                                           │
-│   158 │   │   train_config.global_train_batch_size,                                              │
-│   159 │   │   seed=seed,                                                                         │
-│                                                                                                  │
-│ /netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/olmo/data/iterable_d │
-│                                                                                                  │
-│    54 │   │   self.world_size = world_size if world_size is not None else get_world_size()       │
-│    55 │   │   # If the dataset length is evenly divisible by # of replicas, then there           │
-│    56 │   │   # is no need to drop any data, since the dataset will be split equally.            │
-│ ❱  57 │   │   if self.drop_last and len(self.dataset) % self.world_size != 0:  # type: ignore[ar │
-│    58 │   │   │   # Split to nearest available length that is evenly divisible by world size.    │
-│    59 │   │   │   # This is to ensure each rank receives the same amount of data.                │
-│    60 │   │   │   num_samples = math.ceil(                                                       │
-│                                                                                                  │
-│ /netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/olmo/data/memmap_dat │
-│                                                                                                  │
-│   173 │                                                                                          │
-│   174 │   def __len__(self) -> int:                                                              │
-│   175 │   │   if self._num_instances is None:                                                    │
-│ ❱ 176 │   │   │   self._num_instances = self.offsets[-1][1]                                      │
-│   177 │   │   return self._num_instances                                                         │
-│   178 │                                                                                          │
-│   179 │   def __getitem__(self, index: int) -> Dict[str, Any]:                                   │
-│                                                                                                  │
-│ /netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/olmo/data/memmap_dat │
-│                                                                                                  │
-│   135 │   │   │   │   │   │   mask_path_futures.append(executor.submit(self._get_file_length, ma │
-│   136 │   │   │   │                                                                              │
-│   137 │   │   │   │   for future in concurrent.futures.as_completed(path_futures):               │
-│ ❱ 138 │   │   │   │   │   path, length = future.result()                                         │
-│   139 │   │   │   │   │   path_to_length[path] = length                                          │
-│   140 │   │   │   │                                                                              │
-│   141 │   │   │   │   for future in concurrent.futures.as_completed(mask_path_futures):          │
-│                                                                                                  │
-│ /opt/conda/lib/python3.11/concurrent/futures/_base.py:449 in result                              │
-│                                                                                                  │
-│   446 │   │   │   │   if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:                     │
-│   447 │   │   │   │   │   raise CancelledError()                                                 │
-│   448 │   │   │   │   elif self._state == FINISHED:                                              │
-│ ❱ 449 │   │   │   │   │   return self.__get_result()                                             │
-│   450 │   │   │   │                                                                              │
-│   451 │   │   │   │   self._condition.wait(timeout)                                              │
-│   452                                                                                            │
-│                                                                                                  │
-│ /opt/conda/lib/python3.11/concurrent/futures/_base.py:401 in __get_result                        │
-│                                                                                                  │
-│   398 │   def __get_result(self):                                                                │
-│   399 │   │   if self._exception:                                                                │
-│   400 │   │   │   try:                                                                           │
-│ ❱ 401 │   │   │   │   raise self._exception                                                      │
-│   402 │   │   │   finally:                                                                       │
-│   403 │   │   │   │   # Break a reference cycle with the exception in self._exception            │
-│   404 │   │   │   │   self = None                                                                │
-│                                                                                                  │
-│ /opt/conda/lib/python3.11/concurrent/futures/thread.py:58 in run                                 │
-│                                                                                                  │
-│    55 │   │   │   return                                                                         │
-│    56 │   │                                                                                      │
-│    57 │   │   try:                                                                               │
-│ ❱  58 │   │   │   result = self.fn(*self.args, **self.kwargs)                                    │
-│    59 │   │   except BaseException as exc:                                                       │
-│    60 │   │   │   self.future.set_exception(exc)                                                 │
-│    61 │   │   │   # Break a reference cycle with the exception 'exc'                             │
-│                                                                                                  │
-│ /netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/olmo/data/memmap_dat │
-│                                                                                                  │
-│   169 │   def _get_file_length(self, path, dtype=None) -> Tuple[PathOrStr, int]:                 │
-│   170 │   │   dtype = dtype or self.dtype                                                        │
-│   171 │   │   item_size = dtype(0).itemsize                                                      │
-│ ❱ 172 │   │   return path, file_size(path) // (item_size * self._chunk_size)                     │
-│   173 │                                                                                          │
-│   174 │   def __len__(self) -> int:                                                              │
-│   175 │   │   if self._num_instances is None:                                                    │
-│                                                                                                  │
-│ /netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/olmo/util.py:350 in  │
-│                                                                                                  │
-│   347 │   │   else:                                                                              │
-│   348 │   │   │   raise NotImplementedError(f"file size not implemented for '{parsed.scheme}' fi │
-│   349 │   else:                                                                                  │
-│ ❱ 350 │   │   return os.stat(path).st_size                                                       │
-│   351                                                                                            │
-│   352                                                                                            │
-│   353 def upload(source: PathOrStr, target: str, save_overwrite: bool = False):                  │
-╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
-FileNotFoundError: [Errno 2] No such file or directory: 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00006_00000_doc_shuffled.ds'

models/OLMo-1B/wandb/wandb/run-20251214_153041-8qjie2ty/files/requirements.txt

@@ -1,271 +0,0 @@
-scikit-learn==1.8.0
-joblib==1.5.2
-threadpoolctl==3.6.0
-torchmetrics==1.8.2
-lightning-utilities==0.15.2
-wandb==0.23.1
-GitPython==3.1.45
-gitdb==4.0.12
-smmap==5.0.2
-datasets==4.4.1
-pandas==2.3.3
-multiprocess==0.70.18
-pyarrow==22.0.0
-tzdata==2025.3
-xxhash==3.6.0
-ai2-olmo==0.6.0
-ai2-olmo-core==2.4.0
-cached_path==1.8.0
-google-cloud-storage==2.19.0
-google-cloud-core==2.5.0
-boto3==1.42.9
-google-api-core==2.28.1
-google-auth==2.43.0
-s3transfer==0.16.0
-botocore==1.42.9
-google-resumable-media==2.8.0
-pyasn1_modules==0.4.2
-rich==13.9.4
-rsa==4.9.1
-bettermap==1.3.1
-google-crc32c==1.7.1
-jmespath==1.0.1
-numpy==1.26.4
-omegaconf==2.3.0
-proto-plus==1.26.1
-pyasn1==0.6.1
-python-dateutil==2.9.0.post0
-antlr4-python3-runtime==4.9.3
-zstandard==0.23.0
-zipp==3.21.0
-yarl==1.22.0
-xgrammar==0.1.18
-xformers==0.0.29.post2
-wrapt==2.0.1
-wheel==0.45.1
-websockets==15.0.1
-wcwidth==0.2.13
-watchfiles==1.1.1
-vllm==0.8.5.post1
-uvloop==0.22.1
-uvicorn==0.38.0
-urllib3==2.3.0
-typing-inspection==0.4.2
-typing_extensions==4.15.0
-types-dataclasses==0.6.6
-typer==0.20.0
-truststore==0.10.0
-triton==3.2.0
-transformers==4.57.3
-traitlets==5.14.3
-tqdm==4.67.1
-torchvision==0.21.0+cu124
-torchelastic==0.2.2
-torchaudio==2.6.0+cu124
-torch==2.6.0+cu124
-tokenizers==0.22.1
-tiktoken==0.12.0
-sympy==1.13.1
-starlette==0.50.0
-stack_data==0.6.3
-soupsieve==2.5
-sortedcontainers==2.4.0
-sniffio==1.3.1
-six==1.17.0
-shellingham==1.5.4
-setuptools==75.8.0
-sentry-sdk==2.47.0
-sentencepiece==0.2.1
-scipy==1.16.3
-safetensors==0.7.0
-ruamel.yaml.clib==0.2.8
-ruamel.yaml==0.18.10
-rpds-py==0.22.3
-rignore==0.7.6
-rich-toolkit==0.17.0
-requests==2.32.3
-regex==2025.11.3
-referencing==0.36.2
-ray==2.52.1
-pyzmq==27.1.0
-pytz==2024.2
-python-multipart==0.0.20
-python-json-logger==4.0.0
-python-etcd==0.4.5
-python-dotenv==1.2.1
-Pygments==2.19.1
-pydantic-extra-types==2.10.6
-pydantic_core==2.41.5
-pydantic==2.12.5
-pycparser==2.22
-pycountry==24.6.1
-pycosat==0.6.6
-py-cpuinfo==9.0.0
-pure_eval==0.2.3
-ptyprocess==0.7.0
-psutil==6.1.1
-protobuf==4.25.8
-propcache==0.4.1
-prompt_toolkit==3.0.50
-prometheus-fastapi-instrumentator==7.1.0
-prometheus_client==0.23.1
-pluggy==1.5.0
-platformdirs==4.3.6
-pkgutil_resolve_name==1.3.10
-pkginfo==1.12.0
-pip==24.3.1
-pillow==11.0.0
-pickleshare==0.7.5
-pexpect==4.9.0
-partial-json-parser==0.2.1.1.post7
-parso==0.8.4
-packaging==24.2
-outlines_core==0.1.26
-outlines==0.1.11
-optree==0.14.0
-opentelemetry-semantic-conventions-ai==0.4.13
-opentelemetry-semantic-conventions==0.47b0
-opentelemetry-sdk==1.26.0
-opentelemetry-proto==1.26.0
-opentelemetry-exporter-otlp-proto-http==1.26.0
-opentelemetry-exporter-otlp-proto-grpc==1.26.0
-opentelemetry-exporter-otlp-proto-common==1.26.0
-opentelemetry-exporter-otlp==1.26.0
-opentelemetry-api==1.26.0
-opencv-python-headless==4.12.0.88
-openai==2.11.0
-nvidia-nvtx-cu12==12.4.127
-nvidia-nvjitlink-cu12==12.4.127
-nvidia-nccl-cu12==2.21.5
-nvidia-cusparselt-cu12==0.6.2
-nvidia-cusparse-cu12==12.3.1.170
-nvidia-cusolver-cu12==11.6.1.9
-nvidia-curand-cu12==10.3.5.147
-nvidia-cufft-cu12==11.2.1.3
-nvidia-cudnn-cu12==9.1.0.70
-nvidia-cuda-runtime-cu12==12.4.127
-nvidia-cuda-nvrtc-cu12==12.4.127
-nvidia-cuda-cupti-cu12==12.4.127
-nvidia-cublas-cu12==12.4.5.8
-numba==0.61.2
-ninja==1.11.1.3
-networkx==3.4.2
-nest-asyncio==1.6.0
-multidict==6.7.0
-msgspec==0.20.0
-msgpack==1.1.2
-mpmath==1.3.0
-more-itertools==10.6.0
-mistral_common==1.8.6
-menuinst==2.2.0
-mdurl==0.1.2
-matplotlib-inline==0.1.7
-math-verify==0.8.0
-markdown-it-py==4.0.0
-lm-format-enforcer==0.10.12
-llvmlite==0.44.0
-llguidance==0.7.30
-lintrunner==0.12.7
-lief==0.14.1
-libmambapy==2.0.5
-libarchive-c==5.1
-latex2sympy2_extended==1.10.2
-lark==1.2.2
-jsonschema-specifications==2024.10.1
-jsonschema==4.23.0
-jsonpointer==3.0.0
-jsonpatch==1.33
-jiter==0.12.0
-Jinja2==3.1.5
-jedi==0.19.2
-ipython==8.31.0
-interegular==0.3.3
-importlib_resources==6.5.2
-importlib_metadata==8.0.0
-idna==3.10
-hypothesis==6.124.7
-hyperframe==6.0.1
-huggingface-hub==0.36.0
-httpx==0.28.1
-httptools==0.7.1
-httpcore==1.0.9
-hpack==4.0.0
-hf-xet==1.2.0
-h2==4.1.0
-h11==0.16.0
-grpcio==1.76.0
-googleapis-common-protos==1.72.0
-gguf==0.17.1
-fsspec==2024.12.0
-frozenlist==1.8.0
-frozendict==2.4.6
-filelock==3.17.0
-fastrlock==0.8.3
-fastar==0.8.0
-fastapi-cloud-cli==0.6.0
-fastapi-cli==0.0.16
-fastapi==0.124.4
-expecttest==0.3.0
-executing==2.1.0
-exceptiongroup==1.2.2
-email-validator==2.3.0
-einops==0.8.1
-dnspython==2.7.0
-distro==1.9.0
-diskcache==5.6.3
-dill==0.4.0
-depyf==0.18.0
-Deprecated==1.3.1
-decorator==5.1.1
-cupy-cuda12x==13.6.0
-conda_package_streaming==0.11.0
-conda-package-handling==2.4.0
-conda-libmamba-solver==25.1.1
-conda_index==0.5.0
-conda-build==25.1.1
-conda==25.1.0
-compressed-tensors==0.9.3
-colorama==0.4.6
-cmake==3.31.4
-cloudpickle==3.1.2
-click==8.1.8
-charset-normalizer==3.4.1
-chardet==5.2.0
-cffi==1.17.1
-certifi==2024.12.14
-cachetools==6.2.3
-boltons==24.0.0
-blake3==1.0.8
-beautifulsoup4==4.12.3
-attrs==25.1.0
-astunparse==1.6.3
-asttokens==3.0.0
-astor==0.8.1
-archspec==0.2.5
-anyio==4.12.0
-annotated-types==0.7.0
-annotated-doc==0.0.4
-airportsdata==20250909
-aiosignal==1.4.0
-aiohttp==3.13.2
-aiohappyeyeballs==2.6.1
-PyYAML==6.0.2
-PySocks==1.7.1
-MarkupSafe==3.0.2
-Brotli==1.1.0
-zipp==3.19.2
-wheel==0.43.0
-typing_extensions==4.12.2
-typeguard==4.3.0
-tomli==2.0.1
-platformdirs==4.2.2
-packaging==24.2
-more-itertools==10.3.0
-jaraco.text==3.12.1
-jaraco.functools==4.0.1
-jaraco.context==5.3.0
-jaraco.collections==5.1.0
-inflect==7.3.1
-importlib_metadata==8.0.0
-backports.tarfile==1.2.0
-autocommand==2.2.2

models/OLMo-1B/wandb/wandb/run-20251214_153041-8qjie2ty/files/wandb-metadata.json

@@ -1,103 +0,0 @@
-{
-  "os":  "Linux-6.8.0-1043-nvidia-x86_64-with-glibc2.35",
-  "python":  "CPython 3.11.11",
-  "startedAt":  "2025-12-14T14:30:41.667717Z",
-  "args":  [
-    "pretraining/configs/RL-1B.yaml"
-  ],
-  "program":  "/netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/scripts/train.py",
-  "codePath":  "OLMo/scripts/train.py",
-  "codePathLocal":  "OLMo/scripts/train.py",
-  "email":  "[email protected]",
-  "root":  "checkpoints/OLMo-1B-as_fm3_omi2/wandb",
-  "host":  "serv-3342",
-  "executable":  "/opt/conda/bin/python",
-  "cpu_count":  112,
-  "cpu_count_logical":  224,
-  "gpu":  "NVIDIA H100 80GB HBM3",
-  "gpu_count":  1,
-  "disk":  {
-    "/":  {
-      "total":  "2055141851136",
-      "used":  "49254445056"
-    }
-  },
-  "memory":  {
-    "total":  "2164176814080"
-  },
-  "gpu_nvidia":  [
-    {
-      "name":  "NVIDIA H100 80GB HBM3",
-      "memoryTotal":  "85520809984",
-      "cudaCores":  16896,
-      "architecture":  "Hopper",
-      "uuid":  "GPU-4c999b2a-2578-9e62-0539-4b826d85fda8"
-    }
-  ],
-  "cudaVersion":  "13.0",
-  "slurm":  {
-    "cluster_name":  "pegasus",
-    "conf":  "/etc/slurm/slurm.conf",
-    "cpu_bind":  "quiet,mask_cpu:0x000000000001FE00000000000000000000000001FE00000000000000",
-    "cpu_bind_list":  "0x000000000001FE00000000000000000000000001FE00000000000000",
-    "cpu_bind_type":  "mask_cpu:",
-    "cpu_bind_verbose":  "quiet",
-    "cpus_on_node":  "16",
-    "cpus_per_task":  "16",
-    "distribution":  "cyclic",
-    "gpus":  "1",
-    "gpus_on_node":  "1",
-    "gtids":  "0",
-    "job_cpus_per_node":  "16",
-    "job_end_time":  "1765734524",
-    "job_gid":  "8000",
-    "job_group":  "iml",
-    "job_id":  "2383756",
-    "job_name":  "bash",
-    "job_nodelist":  "serv-3342",
-    "job_num_nodes":  "1",
-    "job_partition":  "H100",
-    "job_qos":  "normal",
-    "job_start_time":  "1765720124",
-    "job_uid":  "13262",
-    "job_user":  "nguyen",
-    "jobid":  "2383756",
-    "launch_node_ipaddr":  "192.168.33.114",
-    "localid":  "0",
-    "mem_per_cpu":  "16384",
-    "mpi_type":  "pmix",
-    "nnodes":  "1",
-    "nodeid":  "0",
-    "nodelist":  "serv-3342",
-    "nprocs":  "1",
-    "ntasks":  "1",
-    "oom_kill_step":  "0",
-    "pmix_mapping_serv":  "(vector,(0,1,1))",
-    "pmixp_abort_agent_port":  "33735",
-    "prio_process":  "1",
-    "procid":  "0",
-    "pty_port":  "45219",
-    "pty_win_col":  "156",
-    "pty_win_row":  "41",
-    "srun_comm_host":  "192.168.33.114",
-    "srun_comm_port":  "35153",
-    "step_gpus":  "5",
-    "step_id":  "0",
-    "step_launcher_port":  "35153",
-    "step_nodelist":  "serv-3342",
-    "step_num_nodes":  "1",
-    "step_num_tasks":  "1",
-    "step_tasks_per_node":  "1",
-    "stepid":  "0",
-    "submit_dir":  "/netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain",
-    "submit_host":  "login1",
-    "task_pid":  "3684902",
-    "tasks_per_node":  "1",
-    "topology_addr":  "serv-3342",
-    "topology_addr_pattern":  "node",
-    "tres_bind":  "gres/gpu:per_task:1",
-    "tres_per_task":  "cpu=16,gres/gpu=1",
-    "umask":  "0022"
-  },
-  "writerId":  "gat9u7yjnpjadjdd0jcwog0os0b9blu3"
-}

models/OLMo-1B/wandb/wandb/run-20251214_153041-8qjie2ty/files/wandb-summary.json

@@ -1 +0,0 @@
-{"_wandb":{"runtime":2},"_runtime":2}

models/OLMo-1B/wandb/wandb/run-20251214_153041-8qjie2ty/logs/debug-internal.log

@@ -1,11 +0,0 @@
-{"time":"2025-12-14T15:30:42.008201805+01:00","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
-{"time":"2025-12-14T15:30:42.27964618+01:00","level":"INFO","msg":"stream: created new stream","id":"8qjie2ty"}
-{"time":"2025-12-14T15:30:42.280781062+01:00","level":"INFO","msg":"handler: started","stream_id":"8qjie2ty"}
-{"time":"2025-12-14T15:30:42.299825499+01:00","level":"INFO","msg":"stream: started","id":"8qjie2ty"}
-{"time":"2025-12-14T15:30:42.300011321+01:00","level":"INFO","msg":"writer: started","stream_id":"8qjie2ty"}
-{"time":"2025-12-14T15:30:42.30003616+01:00","level":"INFO","msg":"sender: started","stream_id":"8qjie2ty"}
-{"time":"2025-12-14T15:30:45.045287757+01:00","level":"INFO","msg":"stream: closing","id":"8qjie2ty"}
-{"time":"2025-12-14T15:30:45.781741134+01:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
-{"time":"2025-12-14T15:30:45.993844409+01:00","level":"INFO","msg":"handler: closed","stream_id":"8qjie2ty"}
-{"time":"2025-12-14T15:30:46.002152318+01:00","level":"INFO","msg":"sender: closed","stream_id":"8qjie2ty"}
-{"time":"2025-12-14T15:30:46.010867789+01:00","level":"INFO","msg":"stream: closed","id":"8qjie2ty"}

models/OLMo-1B/wandb/wandb/run-20251214_153041-8qjie2ty/logs/debug.log

@@ -1,23 +0,0 @@
-2025-12-14 15:30:41,744 INFO    MainThread:3737058 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
-2025-12-14 15:30:41,746 INFO    MainThread:3737058 [wandb_setup.py:_flush():80] Configure stats pid to 3737058
-2025-12-14 15:30:41,748 INFO    MainThread:3737058 [wandb_setup.py:_flush():80] Loading settings from /root/.config/wandb/settings
-2025-12-14 15:30:41,749 INFO    MainThread:3737058 [wandb_setup.py:_flush():80] Loading settings from /netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/wandb/settings
-2025-12-14 15:30:41,750 INFO    MainThread:3737058 [wandb_setup.py:_flush():80] Loading settings from environment variables
-2025-12-14 15:30:41,751 INFO    MainThread:3737058 [wandb_init.py:setup_run_log_directory():714] Logging user logs to checkpoints/OLMo-1B-as_fm3_omi2/wandb/wandb/run-20251214_153041-8qjie2ty/logs/debug.log
-2025-12-14 15:30:41,752 INFO    MainThread:3737058 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to checkpoints/OLMo-1B-as_fm3_omi2/wandb/wandb/run-20251214_153041-8qjie2ty/logs/debug-internal.log
-2025-12-14 15:30:41,753 INFO    MainThread:3737058 [wandb_init.py:init():841] calling init triggers
-2025-12-14 15:30:41,754 INFO    MainThread:3737058 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
-config: {'run_name': 'OLMo-1B-as_fm3_omi2', 'seed': 6198, 'epoch': None, 'dry_run': False, 'model': {'d_model': 2048, 'n_heads': 16, 'n_kv_heads': None, 'clip_qkv': None, 'n_layers': 16, 'mlp_ratio': 8, 'mlp_hidden_size': None, 'activation_type': 'swiglu', 'block_type': 'sequential', 'block_group_size': 1, 'alibi': False, 'alibi_bias_max': 8.0, 'rope': True, 'rope_full_precision': True, 'rope_theta': 10000, 'flash_attention': False, 'attention_dropout': 0.0, 'multi_query_attention': False, 'attention_layer_norm': False, 'residual_dropout': 0.0, 'embedding_dropout': 0.0, 'embedding_layer_norm': False, 'layer_norm_type': 'default', 'layer_norm_with_affine': False, 'layer_norm_eps': 1e-05, 'attention_layer_norm_with_affine': False, 'max_sequence_length': 2048, 'include_bias': False, 'bias_for_layer_norm': False, 'scale_logits': False, 'vocab_size': 32000, 'embedding_size': 32000, 'weight_tying': True, 'eos_token_id': 0, 'pad_token_id': 1, 'init_device': 'cuda', 'init_fn': 'normal', 'init_std': 0.02, 'init_cutoff_factor': 3.0, 'precision': 'amp_bf16', 'scale_emb_init': False, 'emb_init_std': None, 'norm_after': False}, 'optimizer': {'name': 'adamw', 'learning_rate': 0.0005, 'weight_decay': 0.1, 'betas': (0.9, 0.95), 'eps': 1e-08, 'no_decay_norm_and_bias': None, 'selective_updates': False, 'decay_norm_and_bias': True, 'decay_embeddings': True, 'metrics_log_interval': 10, 'record_update_metrics': False}, 'scheduler': {'name': 'cosine_with_warmup', 'units': 'steps', 't_warmup': 2000, 't_max': None, 'alpha_f': 0.1, 'grad_clip_warmup_steps': None, 'grad_clip_warmup_factor': None, 'warmup_min_lr': 0.0}, 'data': {'paths': ['data_token/as_fm3_omi2/algebraic-stack-tokenized/00000_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00001_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00002_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00003_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00004_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00005_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00006_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00007_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00008_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00009_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00010_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00011_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00012_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00016_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00019_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00020_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00021_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00022_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00023_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00024_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00025_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00026_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00027_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00028_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00029_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00030_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00031_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00032_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00033_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00034_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00035_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00036_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00037_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00038_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00039_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00040_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00041_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00042_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00043_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00044_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00045_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00046_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00047_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00048_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00049_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00050_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00051_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00052_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00053_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00054_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00055_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00056_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00057_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00058_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00059_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00060_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00061_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00062_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00063_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00064_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00065_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00066_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00067_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00068_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00069_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00070_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00071_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00072_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00073_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00074_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00075_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00076_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00077_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00078_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00000_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00001_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00002_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00003_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00004_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00005_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00006_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00007_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00008_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00009_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00010_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00011_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00012_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00013_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00014_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00015_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00016_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00017_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00018_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00019_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00020_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00021_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00022_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00023_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00024_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00025_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00026_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00027_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00028_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00029_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00030_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00031_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00032_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00033_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00034_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00035_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00036_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00037_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00038_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00039_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00040_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00041_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00042_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00043_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00044_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00045_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00046_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00047_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00048_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00049_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00050_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00051_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00052_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00053_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00054_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00055_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00056_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00057_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00058_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00059_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00060_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00061_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00062_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00063_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00064_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00065_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00066_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00067_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00068_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00069_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00070_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00071_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00072_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00073_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00074_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00075_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00076_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00077_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00078_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00079_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00080_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00081_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00082_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00083_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00084_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00085_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00086_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00087_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00088_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00089_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00090_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00091_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00092_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00093_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00094_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00095_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00096_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00097_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00098_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00099_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00000_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00001_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00002_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00003_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00004_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00005_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00006_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00007_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00008_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00009_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00010_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00011_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00012_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00013_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00014_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00015_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00016_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00017_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00018_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00019_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00020_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00021_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00022_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00023_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00024_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00025_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00026_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00027_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00028_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00029_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00030_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00031_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00032_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00033_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00034_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00035_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00036_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00037_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00038_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00039_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00040_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00041_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00042_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00043_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00044_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00045_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00046_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00047_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00048_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00049_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00050_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00051_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00052_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00053_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00054_00000_doc_shuffled.ds'], 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 32, 'drop_last': True, 'pin_memory': True, 'prefetch_factor': 8, 'persistent_workers': True, 'timeout': 0, 'seed': None, 'instance_filter': None, 'custom_dataset': None}, 'restore_dataloader': True, 'fast_forward_batches': None, 'evaluators': [], 'eval_interval': 5000, 'tokenizer': {'identifier': 'meta-llama/Llama-2-7b-hf', 'truncate_direction': 'right'}, 'save_folder': 'checkpoints/OLMo-1B-as_fm3_omi2', 'remote_save_folder': None, 'canceled_check_interval': 6000, 'save_interval': 3000, 'save_interval_unsharded': 3000, 'save_interval_ephemeral': None, 'save_num_checkpoints_to_keep': -1, 'save_num_unsharded_checkpoints_to_keep': -1, 'save_overwrite': True, 'force_save_unsharded': False, 'no_pre_train_checkpoint': False, 'load_path': None, 'load_path_sharded_checkpointer': None, 'try_load_latest_save': False, 'reset_optimizer_state': False, 'reset_trainer_state': False, 'sharded_checkpointer': 'torch_legacy', 'new_style_checkpoints': None, 'max_duration': '1ep', 'global_train_batch_size': 512, 'device_train_batch_size': 512, 'device_train_microbatch_size': 16, 'device_eval_batch_size': 16, 'eval_subset_num_batches': -1, 'eval_on_load': False, 'device_train_grad_accum': 32, 'max_grad_norm': 1.0, 'max_grad_norm_ratio': None, 'precision': 'amp_bf16', 'speed_monitor': {'window_size': 20, 'gpu_flops_available': None}, 'console_log_interval': 1, 'gen1_gc_interval': 1, 'compile': None, 'distributed_strategy': 'fsdp', 'fsdp': {'use_orig_params': True, 'sharding_strategy': <ShardingStrategy.FULL_SHARD: 1>, 'wrapping_strategy': None, 'precision': 'mixed', 'hybrid_sharding_num_model_replicas': None}, 'ddp': {'grad_sync_mode': 'batch', 'find_unused_params': False}, 'single': {'device': 'auto'}, 'softmax_auxiliary_loss': False, 'auxiliary_loss_multiplier': 0.0001, 'time_limit': None, 'extra_steps_after_cancel': 10, 'early_stopping_factor': None, 'save_data_indices': True, 'python_profiling': False, 'torch_profiling': False, 'stop_at': None, 'stop_after': None, 'activation_checkpointing': None, 'fused_loss': None, 'hf_datasets_cache_dir': None, 'module_outputs_save_steps': None, '_wandb': {}}
-2025-12-14 15:30:41,755 INFO    MainThread:3737058 [wandb_init.py:init():889] starting backend
-2025-12-14 15:30:42,000 INFO    MainThread:3737058 [wandb_init.py:init():892] sending inform_init request
-2025-12-14 15:30:42,006 INFO    MainThread:3737058 [wandb_init.py:init():900] backend started and connected
-2025-12-14 15:30:42,009 INFO    MainThread:3737058 [wandb_init.py:init():970] updated telemetry
-2025-12-14 15:30:42,010 INFO    MainThread:3737058 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
-2025-12-14 15:30:42,684 INFO    MainThread:3737058 [wandb_init.py:init():1041] starting run threads in backend
-2025-12-14 15:30:42,776 INFO    MainThread:3737058 [wandb_run.py:_console_start():2521] atexit reg
-2025-12-14 15:30:42,778 INFO    MainThread:3737058 [wandb_run.py:_redirect():2369] redirect: wrap_raw
-2025-12-14 15:30:42,780 INFO    MainThread:3737058 [wandb_run.py:_redirect():2438] Wrapping output streams.
-2025-12-14 15:30:42,781 INFO    MainThread:3737058 [wandb_run.py:_redirect():2461] Redirects installed.
-2025-12-14 15:30:42,783 INFO    MainThread:3737058 [wandb_init.py:init():1081] run started, returning control to user process
-2025-12-14 15:30:45,045 INFO    wandb-AsyncioManager-main:3737058 [service_client.py:_forward_responses():80] Reached EOF.
-2025-12-14 15:30:45,049 INFO    wandb-AsyncioManager-main:3737058 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.

models/OLMo-1B/wandb/wandb/run-20251214_153618-kgru6t23/files/config.yaml

@@ -1,575 +0,0 @@
-_wandb:
-    value:
-        cli_version: 0.23.1
-        e:
-            yq3dhquxezgxgixc3uqa3megunlrc2ta:
-                args:
-                    - pretraining/configs/RL-1B.yaml
-                codePath: OLMo/scripts/train.py
-                codePathLocal: OLMo/scripts/train.py
-                cpu_count: 112
-                cpu_count_logical: 224
-                cudaVersion: "13.0"
-                disk:
-                    /:
-                        total: "2055141851136"
-                        used: "49254449152"
-                email: [email protected]
-                executable: /opt/conda/bin/python
-                gpu: NVIDIA H100 80GB HBM3
-                gpu_count: 1
-                gpu_nvidia:
-                    - architecture: Hopper
-                      cudaCores: 16896
-                      memoryTotal: "85520809984"
-                      name: NVIDIA H100 80GB HBM3
-                      uuid: GPU-4c999b2a-2578-9e62-0539-4b826d85fda8
-                host: serv-3342
-                memory:
-                    total: "2164176814080"
-                os: Linux-6.8.0-1043-nvidia-x86_64-with-glibc2.35
-                program: /netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/scripts/train.py
-                python: CPython 3.11.11
-                root: checkpoints/OLMo-1B-as_fm3_omi2/wandb
-                slurm:
-                    cluster_name: pegasus
-                    conf: /etc/slurm/slurm.conf
-                    cpu_bind: quiet,mask_cpu:0x000000000001FE00000000000000000000000001FE00000000000000
-                    cpu_bind_list: 0x000000000001FE00000000000000000000000001FE00000000000000
-                    cpu_bind_type: 'mask_cpu:'
-                    cpu_bind_verbose: quiet
-                    cpus_on_node: "16"
-                    cpus_per_task: "16"
-                    distribution: cyclic
-                    gpus: "1"
-                    gpus_on_node: "1"
-                    gtids: "0"
-                    job_cpus_per_node: "16"
-                    job_end_time: "1765734524"
-                    job_gid: "8000"
-                    job_group: iml
-                    job_id: "2383756"
-                    job_name: bash
-                    job_nodelist: serv-3342
-                    job_num_nodes: "1"
-                    job_partition: H100
-                    job_qos: normal
-                    job_start_time: "1765720124"
-                    job_uid: "13262"
-                    job_user: nguyen
-                    jobid: "2383756"
-                    launch_node_ipaddr: 192.168.33.114
-                    localid: "0"
-                    mem_per_cpu: "16384"
-                    mpi_type: pmix
-                    nnodes: "1"
-                    nodeid: "0"
-                    nodelist: serv-3342
-                    nprocs: "1"
-                    ntasks: "1"
-                    oom_kill_step: "0"
-                    pmix_mapping_serv: (vector,(0,1,1))
-                    pmixp_abort_agent_port: "33735"
-                    prio_process: "1"
-                    procid: "0"
-                    pty_port: "45219"
-                    pty_win_col: "156"
-                    pty_win_row: "41"
-                    srun_comm_host: 192.168.33.114
-                    srun_comm_port: "35153"
-                    step_gpus: "5"
-                    step_id: "0"
-                    step_launcher_port: "35153"
-                    step_nodelist: serv-3342
-                    step_num_nodes: "1"
-                    step_num_tasks: "1"
-                    step_tasks_per_node: "1"
-                    stepid: "0"
-                    submit_dir: /netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain
-                    submit_host: login1
-                    task_pid: "3684902"
-                    tasks_per_node: "1"
-                    topology_addr: serv-3342
-                    topology_addr_pattern: node
-                    tres_bind: gres/gpu:per_task:1
-                    tres_per_task: cpu=16,gres/gpu=1
-                    umask: "0022"
-                startedAt: "2025-12-14T14:36:18.700317Z"
-                writerId: yq3dhquxezgxgixc3uqa3megunlrc2ta
-        m: []
-        python_version: 3.11.11
-        t:
-            "1":
-                - 1
-                - 5
-                - 11
-                - 41
-                - 49
-                - 51
-                - 53
-            "2":
-                - 1
-                - 5
-                - 11
-                - 41
-                - 49
-                - 51
-                - 53
-            "3":
-                - 13
-                - 15
-                - 16
-            "4": 3.11.11
-            "5": 0.23.1
-            "6": 4.57.3
-            "12": 0.23.1
-            "13": linux-x86_64
-activation_checkpointing:
-    value: null
-auxiliary_loss_multiplier:
-    value: 0.0001
-canceled_check_interval:
-    value: 6000
-compile:
-    value: null
-console_log_interval:
-    value: 1
-data:
-    value:
-        custom_dataset: null
-        datasets: null
-        drop_last: true
-        generate_attention_mask: false
-        generate_doc_lengths: false
-        instance_filter: null
-        label_mask_paths: null
-        memmap_dtype: uint16
-        num_workers: 32
-        pad_direction: right
-        paths:
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00000_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00001_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00002_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00003_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00004_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00005_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00006_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00007_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00008_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00009_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00010_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00011_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00012_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00016_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00019_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00020_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00021_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00022_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00023_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00024_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00025_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00026_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00027_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00028_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00029_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00030_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00031_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00032_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00033_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00034_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00035_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00036_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00037_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00038_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00039_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00040_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00041_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00042_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00043_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00044_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00045_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00046_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00047_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00048_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00049_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00050_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00051_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00052_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00053_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00054_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00055_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00056_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00057_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00058_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00059_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00060_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00061_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00062_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00063_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00064_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00065_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00066_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00067_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00068_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00069_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00070_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00071_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00072_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00073_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00074_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00075_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00076_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00077_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00078_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00000_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00001_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00002_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00003_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00004_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00005_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00006_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00007_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00008_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00009_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00010_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00011_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00012_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00013_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00014_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00015_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00016_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00017_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00018_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00019_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00020_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00021_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00022_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00023_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00024_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00025_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00026_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00027_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00028_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00029_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00030_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00031_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00032_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00033_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00034_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00035_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00036_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00037_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00038_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00039_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00040_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00041_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00042_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00043_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00044_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00045_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00046_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00047_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00048_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00049_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00050_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00051_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00052_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00053_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00054_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00055_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00056_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00057_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00058_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00059_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00060_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00061_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00062_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00063_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00064_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00065_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00066_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00067_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00068_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00069_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00070_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00071_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00072_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00073_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00074_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00075_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00076_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00077_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00078_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00079_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00080_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00081_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00082_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00083_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00084_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00085_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00086_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00087_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00088_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00089_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00090_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00091_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00092_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00093_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00094_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00095_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00096_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00097_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00098_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00099_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00000_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00001_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00002_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00003_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00004_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00005_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00006_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00007_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00008_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00009_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00010_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00011_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00012_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00013_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00014_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00015_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00016_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00017_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00018_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00019_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00020_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00021_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00022_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00023_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00024_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00025_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00026_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00027_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00028_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00029_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00030_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00031_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00032_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00033_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00034_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00035_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00036_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00037_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00038_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00039_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00040_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00041_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00042_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00043_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00044_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00045_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00046_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00047_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00048_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00049_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00050_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00051_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00052_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00053_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00054_00000_doc_shuffled.ds
-        persistent_workers: true
-        pin_memory: true
-        prefetch_factor: 8
-        seed: null
-        timeout: 0
-ddp:
-    value:
-        find_unused_params: false
-        grad_sync_mode: batch
-device_eval_batch_size:
-    value: 16
-device_train_batch_size:
-    value: 512
-device_train_grad_accum:
-    value: 32
-device_train_microbatch_size:
-    value: 16
-distributed_strategy:
-    value: fsdp
-dry_run:
-    value: false
-early_stopping_factor:
-    value: null
-epoch:
-    value: null
-eval_interval:
-    value: 5000
-eval_on_load:
-    value: false
-eval_subset_num_batches:
-    value: -1
-evaluators:
-    value: []
-extra_steps_after_cancel:
-    value: 10
-fast_forward_batches:
-    value: null
-force_save_unsharded:
-    value: false
-fsdp:
-    value:
-        hybrid_sharding_num_model_replicas: null
-        precision: mixed
-        sharding_strategy: FULL_SHARD
-        use_orig_params: true
-        wrapping_strategy: null
-fused_loss:
-    value: null
-gen1_gc_interval:
-    value: 1
-global_train_batch_size:
-    value: 512
-hf_datasets_cache_dir:
-    value: null
-load_path:
-    value: null
-load_path_sharded_checkpointer:
-    value: null
-max_duration:
-    value: 1ep
-max_grad_norm:
-    value: 1
-max_grad_norm_ratio:
-    value: null
-model:
-    value:
-        activation_type: swiglu
-        alibi: false
-        alibi_bias_max: 8
-        attention_dropout: 0
-        attention_layer_norm: false
-        attention_layer_norm_with_affine: false
-        bias_for_layer_norm: false
-        block_group_size: 1
-        block_type: sequential
-        clip_qkv: null
-        d_model: 2048
-        emb_init_std: null
-        embedding_dropout: 0
-        embedding_layer_norm: false
-        embedding_size: 32000
-        eos_token_id: 0
-        flash_attention: false
-        include_bias: false
-        init_cutoff_factor: 3
-        init_device: cuda
-        init_fn: normal
-        init_std: 0.02
-        layer_norm_eps: 1e-05
-        layer_norm_type: default
-        layer_norm_with_affine: false
-        max_sequence_length: 2048
-        mlp_hidden_size: null
-        mlp_ratio: 8
-        multi_query_attention: false
-        n_heads: 16
-        n_kv_heads: null
-        n_layers: 16
-        norm_after: false
-        pad_token_id: 1
-        precision: amp_bf16
-        residual_dropout: 0
-        rope: true
-        rope_full_precision: true
-        rope_theta: 10000
-        scale_emb_init: false
-        scale_logits: false
-        vocab_size: 32000
-        weight_tying: true
-module_outputs_save_steps:
-    value: null
-new_style_checkpoints:
-    value: null
-no_pre_train_checkpoint:
-    value: false
-optimizer:
-    value:
-        betas:
-            - 0.9
-            - 0.95
-        decay_embeddings: true
-        decay_norm_and_bias: true
-        eps: 1e-08
-        learning_rate: 0.0005
-        metrics_log_interval: 10
-        name: adamw
-        no_decay_norm_and_bias: null
-        record_update_metrics: false
-        selective_updates: false
-        weight_decay: 0.1
-precision:
-    value: amp_bf16
-python_profiling:
-    value: false
-remote_save_folder:
-    value: null
-reset_optimizer_state:
-    value: false
-reset_trainer_state:
-    value: false
-restore_dataloader:
-    value: true
-run_name:
-    value: OLMo-1B-as_fm3_omi2
-save_data_indices:
-    value: true
-save_folder:
-    value: checkpoints/OLMo-1B-as_fm3_omi2
-save_interval:
-    value: 3000
-save_interval_ephemeral:
-    value: null
-save_interval_unsharded:
-    value: 3000
-save_num_checkpoints_to_keep:
-    value: -1
-save_num_unsharded_checkpoints_to_keep:
-    value: -1
-save_overwrite:
-    value: true
-scheduler:
-    value:
-        alpha_f: 0.1
-        grad_clip_warmup_factor: null
-        grad_clip_warmup_steps: null
-        name: cosine_with_warmup
-        t_max: null
-        t_warmup: 2000
-        units: steps
-        warmup_min_lr: 0
-seed:
-    value: 6198
-sharded_checkpointer:
-    value: torch_legacy
-single:
-    value:
-        device: auto
-softmax_auxiliary_loss:
-    value: false
-speed_monitor:
-    value:
-        gpu_flops_available: null
-        window_size: 20
-stop_after:
-    value: null
-stop_at:
-    value: null
-time_limit:
-    value: null
-tokenizer:
-    value:
-        identifier: meta-llama/Llama-2-7b-hf
-        truncate_direction: right
-torch_profiling:
-    value: false
-try_load_latest_save:
-    value: false

models/OLMo-1B/wandb/wandb/run-20251214_153618-kgru6t23/files/output.log

@@ -1,241 +0,0 @@
-[2025-12-14 15:36:21] INFO     [olmo.data.iterable_dataset:79, rank=0] Saving global data order indices...
-[2025-12-14 15:36:23] INFO     [olmo.data.iterable_dataset:88, rank=0] Global data order indices saved to 'checkpoints/OLMo-1B-as_fm3_omi2/train_data/global_indices.npy'
-[2025-12-14 15:36:23] WARNING  [py.warnings:110, rank=0] /opt/conda/lib/python3.11/site-packages/torch/utils/data/dataloader.py:624: UserWarning: This DataLoader will create 32 worker processes in total. Our suggested max number of worker in current system is 16, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.
-  warnings.warn(
-Traceback (most recent call last):
-  File "/opt/conda/lib/python3.11/site-packages/huggingface_hub/utils/_http.py", line 402, in hf_raise_for_status
-    response.raise_for_status()
-  File "/opt/conda/lib/python3.11/site-packages/requests/models.py", line 1024, in raise_for_status
-    raise HTTPError(http_error_msg, response=self)
-requests.exceptions.HTTPError: 401 Client Error: Unauthorized for url: https://huggingface.co/meta-llama/Llama-2-7b-hf/resolve/main/tokenizer.json
-
-The above exception was the direct cause of the following exception:
-
-Traceback (most recent call last):
-  File "/netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/scripts/train.py", line 436, in <module>
-    main(cfg)
-  File "/netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/scripts/train.py", line 135, in main
-    evaluators = build_evaluators(cfg, device)
-                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/olmo/eval/__init__.py", line 111, in build_evaluators
-    tokenizer = Tokenizer.from_train_config(cfg)
-                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/olmo/tokenizer.py", line 75, in from_train_config
-    tokenizer = cls.from_pretrained(
-                ^^^^^^^^^^^^^^^^^^^^
-  File "/netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/olmo/tokenizer.py", line 93, in from_pretrained
-    base_tokenizer = BaseTokenizer.from_pretrained(identifier)
-                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/opt/conda/lib/python3.11/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn
-    return fn(*args, **kwargs)
-           ^^^^^^^^^^^^^^^^^^^
-  File "/opt/conda/lib/python3.11/site-packages/huggingface_hub/file_download.py", line 1007, in hf_hub_download
-    return _hf_hub_download_to_cache_dir(
-           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/opt/conda/lib/python3.11/site-packages/huggingface_hub/file_download.py", line 1114, in _hf_hub_download_to_cache_dir
-    _raise_on_head_call_error(head_call_error, force_download, local_files_only)
-  File "/opt/conda/lib/python3.11/site-packages/huggingface_hub/file_download.py", line 1655, in _raise_on_head_call_error
-    raise head_call_error
-  File "/opt/conda/lib/python3.11/site-packages/huggingface_hub/file_download.py", line 1543, in _get_metadata_or_catch_error
-    metadata = get_hf_file_metadata(
-               ^^^^^^^^^^^^^^^^^^^^^
-  File "/opt/conda/lib/python3.11/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn
-    return fn(*args, **kwargs)
-           ^^^^^^^^^^^^^^^^^^^
-  File "/opt/conda/lib/python3.11/site-packages/huggingface_hub/file_download.py", line 1460, in get_hf_file_metadata
-    r = _request_wrapper(
-        ^^^^^^^^^^^^^^^^^
-  File "/opt/conda/lib/python3.11/site-packages/huggingface_hub/file_download.py", line 283, in _request_wrapper
-    response = _request_wrapper(
-               ^^^^^^^^^^^^^^^^^
-  File "/opt/conda/lib/python3.11/site-packages/huggingface_hub/file_download.py", line 307, in _request_wrapper
-    hf_raise_for_status(response)
-  File "/opt/conda/lib/python3.11/site-packages/huggingface_hub/utils/_http.py", line 419, in hf_raise_for_status
-    raise _format(GatedRepoError, message, response) from e
-huggingface_hub.errors.GatedRepoError: 401 Client Error. (Request ID: Root=1-693ecb67-4dcf6cbc46f1f55a2357bb92;7f173849-a847-492f-a0d0-f552bb475103)
-
-Cannot access gated repo for url https://huggingface.co/meta-llama/Llama-2-7b-hf/resolve/main/tokenizer.json.
-Access to model meta-llama/Llama-2-7b-hf is restricted. You must have access to it and be authenticated to access it. Please log in.
-
-[2025-12-14 15:36:23] CRITICAL [olmo.util:168, rank=0] Uncaught GatedRepoError: 401 Client Error. (Request ID: Root=1-693ecb67-4dcf6cbc46f1f55a2357bb92;7f173849-a847-492f-a0d0-f552bb475103)
-
-Cannot access gated repo for url https://huggingface.co/meta-llama/Llama-2-7b-hf/resolve/main/tokenizer.json.
-Access to model meta-llama/Llama-2-7b-hf is restricted. You must have access to it and be authenticated to access it. Please log in.
-╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮
-│ /opt/conda/lib/python3.11/site-packages/huggingface_hub/utils/_http.py:402 in hf_raise_for_statu │
-│                                                                                                  │
-│   399 │   >         If request failed for a reason not listed above.                             │
-│   400 │   """                                                                                    │
-│   401 │   try:                                                                                   │
-│ ❱ 402 │   │   response.raise_for_status()                                                        │
-│   403 │   except HTTPError as e:                                                                 │
-│   404 │   │   error_code = response.headers.get("X-Error-Code")                                  │
-│   405 │   │   error_message = response.headers.get("X-Error-Message")                            │
-│                                                                                                  │
-│ /opt/conda/lib/python3.11/site-packages/requests/models.py:1024 in raise_for_status              │
-│                                                                                                  │
-│   1021 │   │   │   )                                                                             │
-│   1022 │   │                                                                                     │
-│   1023 │   │   if http_error_msg:                                                                │
-│ ❱ 1024 │   │   │   raise HTTPError(http_error_msg, response=self)                                │
-│   1025 │                                                                                         │
-│   1026 │   def close(self):                                                                      │
-│   1027 │   │   """Releases the connection back to the pool. Once this method has been            │
-╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
-HTTPError: 401 Client Error: Unauthorized for url: https://huggingface.co/meta-llama/Llama-2-7b-hf/resolve/main/tokenizer.json
-
-The above exception was the direct cause of the following exception:
-
-╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮
-│ /netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/scripts/train.py:436 │
-│                                                                                                  │
-│   433 │   │   log.info("Device is CPU. Updating config...")                                      │
-│   434 │   │   cfg.model.init_device = "cpu"                                                      │
-│   435 │   │   cfg.distributed_strategy = "single"  # type: ignore                                │
-│ ❱ 436 │   main(cfg)                                                                              │
-│   437                                                                                            │
-│                                                                                                  │
-│ /netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/scripts/train.py:135 │
-│                                                                                                  │
-│   132 │   train_loader = build_train_dataloader(cfg)                                             │
-│   133 │                                                                                          │
-│   134 │   # Construct evaluators.                                                                │
-│ ❱ 135 │   evaluators = build_evaluators(cfg, device)                                             │
-│   136 │   barrier()                                                                              │
-│   137 │                                                                                          │
-│   138 │   # Initialize the model.                                                                │
-│                                                                                                  │
-│ /netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/olmo/eval/__init__.p │
-│                                                                                                  │
-│   108                                                                                            │
-│   109 def build_evaluators(cfg: TrainConfig, device: torch.device) -> List[Evaluator]:           │
-│   110 │   evaluators = []                                                                        │
-│ ❱ 111 │   tokenizer = Tokenizer.from_train_config(cfg)                                           │
-│   112 │   for eval_cfg in cfg.evaluators:                                                        │
-│   113 │   │   evaluators.append(build_evaluator(cfg, eval_cfg, tokenizer, device))               │
-│   114 │   return evaluators                                                                      │
-│                                                                                                  │
-│ /netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/olmo/tokenizer.py:75 │
-│                                                                                                  │
-│    72 │   │   │   │   │   pad_token_id=config.model.pad_token_id,                                │
-│    73 │   │   │   │   )                                                                          │
-│    74 │   │   else:                                                                              │
-│ ❱  75 │   │   │   tokenizer = cls.from_pretrained(                                               │
-│    76 │   │   │   │   tokenizer_identifier,                                                      │
-│    77 │   │   │   │   eos_token_id=config.model.eos_token_id,                                    │
-│    78 │   │   │   │   pad_token_id=config.model.pad_token_id,                                    │
-│                                                                                                  │
-│ /netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/olmo/tokenizer.py:93 │
-│                                                                                                  │
-│    90 │   │   │   ``tokenizer.json`` file.                                                       │
-│    91 │   │   :param kwargs: Other key word arguments passed to :class:`Tokenizer`.              │
-│    92 │   │   """                                                                                │
-│ ❱  93 │   │   base_tokenizer = BaseTokenizer.from_pretrained(identifier)                         │
-│    94 │   │   eos_token_id = kwargs.pop("eos_token_id", base_tokenizer.get_vocab_size() - 1)     │
-│    95 │   │   return cls(base_tokenizer, eos_token_id, **kwargs)                                 │
-│    96                                                                                            │
-│                                                                                                  │
-│ /opt/conda/lib/python3.11/site-packages/huggingface_hub/utils/_validators.py:114 in _inner_fn    │
-│                                                                                                  │
-│   111 │   │   if check_use_auth_token:                                                           │
-│   112 │   │   │   kwargs = smoothly_deprecate_use_auth_token(fn_name=fn.__name__, has_token=has_ │
-│   113 │   │                                                                                      │
-│ ❱ 114 │   │   return fn(*args, **kwargs)                                                         │
-│   115 │                                                                                          │
-│   116 │   return _inner_fn  # type: ignore                                                       │
-│   117                                                                                            │
-│                                                                                                  │
-│ /opt/conda/lib/python3.11/site-packages/huggingface_hub/file_download.py:1007 in hf_hub_download │
-│                                                                                                  │
-│   1004 │   │   │   local_files_only=local_files_only,                                            │
-│   1005 │   │   )                                                                                 │
-│   1006 │   else:                                                                                 │
-│ ❱ 1007 │   │   return _hf_hub_download_to_cache_dir(                                             │
-│   1008 │   │   │   # Destination                                                                 │
-│   1009 │   │   │   cache_dir=cache_dir,                                                          │
-│   1010 │   │   │   # File info                                                                   │
-│                                                                                                  │
-│ /opt/conda/lib/python3.11/site-packages/huggingface_hub/file_download.py:1114 in _hf_hub_downloa │
-│                                                                                                  │
-│   1111 │   │   │   │   │   return pointer_path                                                   │
-│   1112 │   │                                                                                     │
-│   1113 │   │   # Otherwise, raise appropriate error                                              │
-│ ❱ 1114 │   │   _raise_on_head_call_error(head_call_error, force_download, local_files_only)      │
-│   1115 │                                                                                         │
-│   1116 │   # From now on, etag, commit_hash, url and size are not None.                          │
-│   1117 │   assert etag is not None, "etag must have been retrieved from server"                  │
-│                                                                                                  │
-│ /opt/conda/lib/python3.11/site-packages/huggingface_hub/file_download.py:1655 in _raise_on_head_ │
-│                                                                                                  │
-│   1652 │   ):                                                                                    │
-│   1653 │   │   # Repo not found or gated => let's raise the actual error                         │
-│   1654 │   │   # Unauthorized => likely a token issue => let's raise the actual error            │
-│ ❱ 1655 │   │   raise head_call_error                                                             │
-│   1656 │   else:                                                                                 │
-│   1657 │   │   # Otherwise: most likely a connection issue or Hub downtime => let's warn the use │
-│   1658 │   │   raise LocalEntryNotFoundError(                                                    │
-│                                                                                                  │
-│ /opt/conda/lib/python3.11/site-packages/huggingface_hub/file_download.py:1543 in _get_metadata_o │
-│                                                                                                  │
-│   1540 │   if not local_files_only:                                                              │
-│   1541 │   │   try:                                                                              │
-│   1542 │   │   │   try:                                                                          │
-│ ❱ 1543 │   │   │   │   metadata = get_hf_file_metadata(                                          │
-│   1544 │   │   │   │   │   url=url, proxies=proxies, timeout=etag_timeout, headers=headers, toke │
-│   1545 │   │   │   │   )                                                                         │
-│   1546 │   │   │   except EntryNotFoundError as http_error:                                      │
-│                                                                                                  │
-│ /opt/conda/lib/python3.11/site-packages/huggingface_hub/utils/_validators.py:114 in _inner_fn    │
-│                                                                                                  │
-│   111 │   │   if check_use_auth_token:                                                           │
-│   112 │   │   │   kwargs = smoothly_deprecate_use_auth_token(fn_name=fn.__name__, has_token=has_ │
-│   113 │   │                                                                                      │
-│ ❱ 114 │   │   return fn(*args, **kwargs)                                                         │
-│   115 │                                                                                          │
-│   116 │   return _inner_fn  # type: ignore                                                       │
-│   117                                                                                            │
-│                                                                                                  │
-│ /opt/conda/lib/python3.11/site-packages/huggingface_hub/file_download.py:1460 in get_hf_file_met │
-│                                                                                                  │
-│   1457 │   hf_headers["Accept-Encoding"] = "identity"  # prevent any compression => we want to k │
-│   1458 │                                                                                         │
-│   1459 │   # Retrieve metadata                                                                   │
-│ ❱ 1460 │   r = _request_wrapper(                                                                 │
-│   1461 │   │   method="HEAD",                                                                    │
-│   1462 │   │   url=url,                                                                          │
-│   1463 │   │   headers=hf_headers,                                                               │
-│                                                                                                  │
-│ /opt/conda/lib/python3.11/site-packages/huggingface_hub/file_download.py:283 in _request_wrapper │
-│                                                                                                  │
-│    280 │   """                                                                                   │
-│    281 │   # Recursively follow relative redirects                                               │
-│    282 │   if follow_relative_redirects:                                                         │
-│ ❱  283 │   │   response = _request_wrapper(                                                      │
-│    284 │   │   │   method=method,                                                                │
-│    285 │   │   │   url=url,                                                                      │
-│    286 │   │   │   follow_relative_redirects=False,                                              │
-│                                                                                                  │
-│ /opt/conda/lib/python3.11/site-packages/huggingface_hub/file_download.py:307 in _request_wrapper │
-│                                                                                                  │
-│    304 │                                                                                         │
-│    305 │   # Perform request and return if status_code is not in the retry list.                 │
-│    306 │   response = http_backoff(method=method, url=url, **params)                             │
-│ ❱  307 │   hf_raise_for_status(response)                                                         │
-│    308 │   return response                                                                       │
-│    309                                                                                           │
-│    310                                                                                           │
-│                                                                                                  │
-│ /opt/conda/lib/python3.11/site-packages/huggingface_hub/utils/_http.py:419 in hf_raise_for_statu │
-│                                                                                                  │
-│   416 │   │   │   message = (                                                                    │
-│   417 │   │   │   │   f"{response.status_code} Client Error." + "\n\n" + f"Cannot access gated r │
-│   418 │   │   │   )                                                                              │
-│ ❱ 419 │   │   │   raise _format(GatedRepoError, message, response) from e                        │
-│   420 │   │                                                                                      │
-│   421 │   │   elif error_message == "Access to this resource is disabled.":                      │
-│   422 │   │   │   message = (                                                                    │
-╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
-GatedRepoError: 401 Client Error. (Request ID: Root=1-693ecb67-4dcf6cbc46f1f55a2357bb92;7f173849-a847-492f-a0d0-f552bb475103)
-
-Cannot access gated repo for url https://huggingface.co/meta-llama/Llama-2-7b-hf/resolve/main/tokenizer.json.
-Access to model meta-llama/Llama-2-7b-hf is restricted. You must have access to it and be authenticated to access it. Please log in.

models/OLMo-1B/wandb/wandb/run-20251214_153618-kgru6t23/files/requirements.txt

@@ -1,271 +0,0 @@
-scikit-learn==1.8.0
-joblib==1.5.2
-threadpoolctl==3.6.0
-torchmetrics==1.8.2
-lightning-utilities==0.15.2
-wandb==0.23.1
-GitPython==3.1.45
-gitdb==4.0.12
-smmap==5.0.2
-datasets==4.4.1
-pandas==2.3.3
-multiprocess==0.70.18
-pyarrow==22.0.0
-tzdata==2025.3
-xxhash==3.6.0
-ai2-olmo==0.6.0
-ai2-olmo-core==2.4.0
-cached_path==1.8.0
-google-cloud-storage==2.19.0
-google-cloud-core==2.5.0
-boto3==1.42.9
-google-api-core==2.28.1
-google-auth==2.43.0
-s3transfer==0.16.0
-botocore==1.42.9
-google-resumable-media==2.8.0
-pyasn1_modules==0.4.2
-rich==13.9.4
-rsa==4.9.1
-bettermap==1.3.1
-google-crc32c==1.7.1
-jmespath==1.0.1
-numpy==1.26.4
-omegaconf==2.3.0
-proto-plus==1.26.1
-pyasn1==0.6.1
-python-dateutil==2.9.0.post0
-antlr4-python3-runtime==4.9.3
-zstandard==0.23.0
-zipp==3.21.0
-yarl==1.22.0
-xgrammar==0.1.18
-xformers==0.0.29.post2
-wrapt==2.0.1
-wheel==0.45.1
-websockets==15.0.1
-wcwidth==0.2.13
-watchfiles==1.1.1
-vllm==0.8.5.post1
-uvloop==0.22.1
-uvicorn==0.38.0
-urllib3==2.3.0
-typing-inspection==0.4.2
-typing_extensions==4.15.0
-types-dataclasses==0.6.6
-typer==0.20.0
-truststore==0.10.0
-triton==3.2.0
-transformers==4.57.3
-traitlets==5.14.3
-tqdm==4.67.1
-torchvision==0.21.0+cu124
-torchelastic==0.2.2
-torchaudio==2.6.0+cu124
-torch==2.6.0+cu124
-tokenizers==0.22.1
-tiktoken==0.12.0
-sympy==1.13.1
-starlette==0.50.0
-stack_data==0.6.3
-soupsieve==2.5
-sortedcontainers==2.4.0
-sniffio==1.3.1
-six==1.17.0
-shellingham==1.5.4
-setuptools==75.8.0
-sentry-sdk==2.47.0
-sentencepiece==0.2.1
-scipy==1.16.3
-safetensors==0.7.0
-ruamel.yaml.clib==0.2.8
-ruamel.yaml==0.18.10
-rpds-py==0.22.3
-rignore==0.7.6
-rich-toolkit==0.17.0
-requests==2.32.3
-regex==2025.11.3
-referencing==0.36.2
-ray==2.52.1
-pyzmq==27.1.0
-pytz==2024.2
-python-multipart==0.0.20
-python-json-logger==4.0.0
-python-etcd==0.4.5
-python-dotenv==1.2.1
-Pygments==2.19.1
-pydantic-extra-types==2.10.6
-pydantic_core==2.41.5
-pydantic==2.12.5
-pycparser==2.22
-pycountry==24.6.1
-pycosat==0.6.6
-py-cpuinfo==9.0.0
-pure_eval==0.2.3
-ptyprocess==0.7.0
-psutil==6.1.1
-protobuf==4.25.8
-propcache==0.4.1
-prompt_toolkit==3.0.50
-prometheus-fastapi-instrumentator==7.1.0
-prometheus_client==0.23.1
-pluggy==1.5.0
-platformdirs==4.3.6
-pkgutil_resolve_name==1.3.10
-pkginfo==1.12.0
-pip==24.3.1
-pillow==11.0.0
-pickleshare==0.7.5
-pexpect==4.9.0
-partial-json-parser==0.2.1.1.post7
-parso==0.8.4
-packaging==24.2
-outlines_core==0.1.26
-outlines==0.1.11
-optree==0.14.0
-opentelemetry-semantic-conventions-ai==0.4.13
-opentelemetry-semantic-conventions==0.47b0
-opentelemetry-sdk==1.26.0
-opentelemetry-proto==1.26.0
-opentelemetry-exporter-otlp-proto-http==1.26.0
-opentelemetry-exporter-otlp-proto-grpc==1.26.0
-opentelemetry-exporter-otlp-proto-common==1.26.0
-opentelemetry-exporter-otlp==1.26.0
-opentelemetry-api==1.26.0
-opencv-python-headless==4.12.0.88
-openai==2.11.0
-nvidia-nvtx-cu12==12.4.127
-nvidia-nvjitlink-cu12==12.4.127
-nvidia-nccl-cu12==2.21.5
-nvidia-cusparselt-cu12==0.6.2
-nvidia-cusparse-cu12==12.3.1.170
-nvidia-cusolver-cu12==11.6.1.9
-nvidia-curand-cu12==10.3.5.147
-nvidia-cufft-cu12==11.2.1.3
-nvidia-cudnn-cu12==9.1.0.70
-nvidia-cuda-runtime-cu12==12.4.127
-nvidia-cuda-nvrtc-cu12==12.4.127
-nvidia-cuda-cupti-cu12==12.4.127
-nvidia-cublas-cu12==12.4.5.8
-numba==0.61.2
-ninja==1.11.1.3
-networkx==3.4.2
-nest-asyncio==1.6.0
-multidict==6.7.0
-msgspec==0.20.0
-msgpack==1.1.2
-mpmath==1.3.0
-more-itertools==10.6.0
-mistral_common==1.8.6
-menuinst==2.2.0
-mdurl==0.1.2
-matplotlib-inline==0.1.7
-math-verify==0.8.0
-markdown-it-py==4.0.0
-lm-format-enforcer==0.10.12
-llvmlite==0.44.0
-llguidance==0.7.30
-lintrunner==0.12.7
-lief==0.14.1
-libmambapy==2.0.5
-libarchive-c==5.1
-latex2sympy2_extended==1.10.2
-lark==1.2.2
-jsonschema-specifications==2024.10.1
-jsonschema==4.23.0
-jsonpointer==3.0.0
-jsonpatch==1.33
-jiter==0.12.0
-Jinja2==3.1.5
-jedi==0.19.2
-ipython==8.31.0
-interegular==0.3.3
-importlib_resources==6.5.2
-importlib_metadata==8.0.0
-idna==3.10
-hypothesis==6.124.7
-hyperframe==6.0.1
-huggingface-hub==0.36.0
-httpx==0.28.1
-httptools==0.7.1
-httpcore==1.0.9
-hpack==4.0.0
-hf-xet==1.2.0
-h2==4.1.0
-h11==0.16.0
-grpcio==1.76.0
-googleapis-common-protos==1.72.0
-gguf==0.17.1
-fsspec==2024.12.0
-frozenlist==1.8.0
-frozendict==2.4.6
-filelock==3.17.0
-fastrlock==0.8.3
-fastar==0.8.0
-fastapi-cloud-cli==0.6.0
-fastapi-cli==0.0.16
-fastapi==0.124.4
-expecttest==0.3.0
-executing==2.1.0
-exceptiongroup==1.2.2
-email-validator==2.3.0
-einops==0.8.1
-dnspython==2.7.0
-distro==1.9.0
-diskcache==5.6.3
-dill==0.4.0
-depyf==0.18.0
-Deprecated==1.3.1
-decorator==5.1.1
-cupy-cuda12x==13.6.0
-conda_package_streaming==0.11.0
-conda-package-handling==2.4.0
-conda-libmamba-solver==25.1.1
-conda_index==0.5.0
-conda-build==25.1.1
-conda==25.1.0
-compressed-tensors==0.9.3
-colorama==0.4.6
-cmake==3.31.4
-cloudpickle==3.1.2
-click==8.1.8
-charset-normalizer==3.4.1
-chardet==5.2.0
-cffi==1.17.1
-certifi==2024.12.14
-cachetools==6.2.3
-boltons==24.0.0
-blake3==1.0.8
-beautifulsoup4==4.12.3
-attrs==25.1.0
-astunparse==1.6.3
-asttokens==3.0.0
-astor==0.8.1
-archspec==0.2.5
-anyio==4.12.0
-annotated-types==0.7.0
-annotated-doc==0.0.4
-airportsdata==20250909
-aiosignal==1.4.0
-aiohttp==3.13.2
-aiohappyeyeballs==2.6.1
-PyYAML==6.0.2
-PySocks==1.7.1
-MarkupSafe==3.0.2
-Brotli==1.1.0
-zipp==3.19.2
-wheel==0.43.0
-typing_extensions==4.12.2
-typeguard==4.3.0
-tomli==2.0.1
-platformdirs==4.2.2
-packaging==24.2
-more-itertools==10.3.0
-jaraco.text==3.12.1
-jaraco.functools==4.0.1
-jaraco.context==5.3.0
-jaraco.collections==5.1.0
-inflect==7.3.1
-importlib_metadata==8.0.0
-backports.tarfile==1.2.0
-autocommand==2.2.2

models/OLMo-1B/wandb/wandb/run-20251214_153618-kgru6t23/files/wandb-metadata.json

@@ -1,103 +0,0 @@
-{
-  "os":  "Linux-6.8.0-1043-nvidia-x86_64-with-glibc2.35",
-  "python":  "CPython 3.11.11",
-  "startedAt":  "2025-12-14T14:36:18.700317Z",
-  "args":  [
-    "pretraining/configs/RL-1B.yaml"
-  ],
-  "program":  "/netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/scripts/train.py",
-  "codePath":  "OLMo/scripts/train.py",
-  "codePathLocal":  "OLMo/scripts/train.py",
-  "email":  "[email protected]",
-  "root":  "checkpoints/OLMo-1B-as_fm3_omi2/wandb",
-  "host":  "serv-3342",
-  "executable":  "/opt/conda/bin/python",
-  "cpu_count":  112,
-  "cpu_count_logical":  224,
-  "gpu":  "NVIDIA H100 80GB HBM3",
-  "gpu_count":  1,
-  "disk":  {
-    "/":  {
-      "total":  "2055141851136",
-      "used":  "49254449152"
-    }
-  },
-  "memory":  {
-    "total":  "2164176814080"
-  },
-  "gpu_nvidia":  [
-    {
-      "name":  "NVIDIA H100 80GB HBM3",
-      "memoryTotal":  "85520809984",
-      "cudaCores":  16896,
-      "architecture":  "Hopper",
-      "uuid":  "GPU-4c999b2a-2578-9e62-0539-4b826d85fda8"
-    }
-  ],
-  "cudaVersion":  "13.0",
-  "slurm":  {
-    "cluster_name":  "pegasus",
-    "conf":  "/etc/slurm/slurm.conf",
-    "cpu_bind":  "quiet,mask_cpu:0x000000000001FE00000000000000000000000001FE00000000000000",
-    "cpu_bind_list":  "0x000000000001FE00000000000000000000000001FE00000000000000",
-    "cpu_bind_type":  "mask_cpu:",
-    "cpu_bind_verbose":  "quiet",
-    "cpus_on_node":  "16",
-    "cpus_per_task":  "16",
-    "distribution":  "cyclic",
-    "gpus":  "1",
-    "gpus_on_node":  "1",
-    "gtids":  "0",
-    "job_cpus_per_node":  "16",
-    "job_end_time":  "1765734524",
-    "job_gid":  "8000",
-    "job_group":  "iml",
-    "job_id":  "2383756",
-    "job_name":  "bash",
-    "job_nodelist":  "serv-3342",
-    "job_num_nodes":  "1",
-    "job_partition":  "H100",
-    "job_qos":  "normal",
-    "job_start_time":  "1765720124",
-    "job_uid":  "13262",
-    "job_user":  "nguyen",
-    "jobid":  "2383756",
-    "launch_node_ipaddr":  "192.168.33.114",
-    "localid":  "0",
-    "mem_per_cpu":  "16384",
-    "mpi_type":  "pmix",
-    "nnodes":  "1",
-    "nodeid":  "0",
-    "nodelist":  "serv-3342",
-    "nprocs":  "1",
-    "ntasks":  "1",
-    "oom_kill_step":  "0",
-    "pmix_mapping_serv":  "(vector,(0,1,1))",
-    "pmixp_abort_agent_port":  "33735",
-    "prio_process":  "1",
-    "procid":  "0",
-    "pty_port":  "45219",
-    "pty_win_col":  "156",
-    "pty_win_row":  "41",
-    "srun_comm_host":  "192.168.33.114",
-    "srun_comm_port":  "35153",
-    "step_gpus":  "5",
-    "step_id":  "0",
-    "step_launcher_port":  "35153",
-    "step_nodelist":  "serv-3342",
-    "step_num_nodes":  "1",
-    "step_num_tasks":  "1",
-    "step_tasks_per_node":  "1",
-    "stepid":  "0",
-    "submit_dir":  "/netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain",
-    "submit_host":  "login1",
-    "task_pid":  "3684902",
-    "tasks_per_node":  "1",
-    "topology_addr":  "serv-3342",
-    "topology_addr_pattern":  "node",
-    "tres_bind":  "gres/gpu:per_task:1",
-    "tres_per_task":  "cpu=16,gres/gpu=1",
-    "umask":  "0022"
-  },
-  "writerId":  "yq3dhquxezgxgixc3uqa3megunlrc2ta"
-}

models/OLMo-1B/wandb/wandb/run-20251214_153618-kgru6t23/files/wandb-summary.json

@@ -1 +0,0 @@
-{"_wandb":{"runtime":4},"_runtime":4}

models/OLMo-1B/wandb/wandb/run-20251214_153618-kgru6t23/logs/debug-internal.log

@@ -1,11 +0,0 @@
-{"time":"2025-12-14T15:36:19.017560403+01:00","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
-{"time":"2025-12-14T15:36:19.283685097+01:00","level":"INFO","msg":"stream: created new stream","id":"kgru6t23"}
-{"time":"2025-12-14T15:36:19.28461896+01:00","level":"INFO","msg":"handler: started","stream_id":"kgru6t23"}
-{"time":"2025-12-14T15:36:19.286998194+01:00","level":"INFO","msg":"stream: started","id":"kgru6t23"}
-{"time":"2025-12-14T15:36:19.287018288+01:00","level":"INFO","msg":"writer: started","stream_id":"kgru6t23"}
-{"time":"2025-12-14T15:36:19.287033949+01:00","level":"INFO","msg":"sender: started","stream_id":"kgru6t23"}
-{"time":"2025-12-14T15:36:23.966894569+01:00","level":"INFO","msg":"stream: closing","id":"kgru6t23"}
-{"time":"2025-12-14T15:36:24.755431422+01:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
-{"time":"2025-12-14T15:36:24.947056958+01:00","level":"INFO","msg":"handler: closed","stream_id":"kgru6t23"}
-{"time":"2025-12-14T15:36:24.954143642+01:00","level":"INFO","msg":"sender: closed","stream_id":"kgru6t23"}
-{"time":"2025-12-14T15:36:24.955118036+01:00","level":"INFO","msg":"stream: closed","id":"kgru6t23"}

models/OLMo-1B/wandb/wandb/run-20251214_153618-kgru6t23/logs/debug.log

@@ -1,23 +0,0 @@
-2025-12-14 15:36:18,753 INFO    MainThread:3744013 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
-2025-12-14 15:36:18,754 INFO    MainThread:3744013 [wandb_setup.py:_flush():80] Configure stats pid to 3744013
-2025-12-14 15:36:18,755 INFO    MainThread:3744013 [wandb_setup.py:_flush():80] Loading settings from /root/.config/wandb/settings
-2025-12-14 15:36:18,756 INFO    MainThread:3744013 [wandb_setup.py:_flush():80] Loading settings from /netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/wandb/settings
-2025-12-14 15:36:18,757 INFO    MainThread:3744013 [wandb_setup.py:_flush():80] Loading settings from environment variables
-2025-12-14 15:36:18,758 INFO    MainThread:3744013 [wandb_init.py:setup_run_log_directory():714] Logging user logs to checkpoints/OLMo-1B-as_fm3_omi2/wandb/wandb/run-20251214_153618-kgru6t23/logs/debug.log
-2025-12-14 15:36:18,759 INFO    MainThread:3744013 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to checkpoints/OLMo-1B-as_fm3_omi2/wandb/wandb/run-20251214_153618-kgru6t23/logs/debug-internal.log
-2025-12-14 15:36:18,760 INFO    MainThread:3744013 [wandb_init.py:init():841] calling init triggers
-2025-12-14 15:36:18,761 INFO    MainThread:3744013 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
-config: {'run_name': 'OLMo-1B-as_fm3_omi2', 'seed': 6198, 'epoch': None, 'dry_run': False, 'model': {'d_model': 2048, 'n_heads': 16, 'n_kv_heads': None, 'clip_qkv': None, 'n_layers': 16, 'mlp_ratio': 8, 'mlp_hidden_size': None, 'activation_type': 'swiglu', 'block_type': 'sequential', 'block_group_size': 1, 'alibi': False, 'alibi_bias_max': 8.0, 'rope': True, 'rope_full_precision': True, 'rope_theta': 10000, 'flash_attention': False, 'attention_dropout': 0.0, 'multi_query_attention': False, 'attention_layer_norm': False, 'residual_dropout': 0.0, 'embedding_dropout': 0.0, 'embedding_layer_norm': False, 'layer_norm_type': 'default', 'layer_norm_with_affine': False, 'layer_norm_eps': 1e-05, 'attention_layer_norm_with_affine': False, 'max_sequence_length': 2048, 'include_bias': False, 'bias_for_layer_norm': False, 'scale_logits': False, 'vocab_size': 32000, 'embedding_size': 32000, 'weight_tying': True, 'eos_token_id': 0, 'pad_token_id': 1, 'init_device': 'cuda', 'init_fn': 'normal', 'init_std': 0.02, 'init_cutoff_factor': 3.0, 'precision': 'amp_bf16', 'scale_emb_init': False, 'emb_init_std': None, 'norm_after': False}, 'optimizer': {'name': 'adamw', 'learning_rate': 0.0005, 'weight_decay': 0.1, 'betas': (0.9, 0.95), 'eps': 1e-08, 'no_decay_norm_and_bias': None, 'selective_updates': False, 'decay_norm_and_bias': True, 'decay_embeddings': True, 'metrics_log_interval': 10, 'record_update_metrics': False}, 'scheduler': {'name': 'cosine_with_warmup', 'units': 'steps', 't_warmup': 2000, 't_max': None, 'alpha_f': 0.1, 'grad_clip_warmup_steps': None, 'grad_clip_warmup_factor': None, 'warmup_min_lr': 0.0}, 'data': {'paths': ['data_token/as_fm3_omi2/algebraic-stack-tokenized/00000_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00001_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00002_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00003_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00004_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00005_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00006_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00007_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00008_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00009_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00010_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00011_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00012_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00016_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00019_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00020_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00021_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00022_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00023_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00024_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00025_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00026_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00027_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00028_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00029_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00030_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00031_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00032_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00033_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00034_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00035_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00036_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00037_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00038_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00039_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00040_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00041_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00042_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00043_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00044_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00045_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00046_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00047_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00048_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00049_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00050_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00051_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00052_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00053_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00054_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00055_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00056_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00057_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00058_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00059_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00060_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00061_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00062_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00063_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00064_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00065_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00066_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00067_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00068_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00069_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00070_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00071_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00072_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00073_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00074_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00075_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00076_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00077_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00078_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00000_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00001_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00002_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00003_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00004_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00005_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00006_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00007_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00008_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00009_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00010_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00011_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00012_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00013_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00014_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00015_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00016_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00017_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00018_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00019_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00020_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00021_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00022_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00023_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00024_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00025_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00026_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00027_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00028_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00029_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00030_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00031_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00032_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00033_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00034_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00035_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00036_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00037_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00038_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00039_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00040_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00041_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00042_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00043_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00044_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00045_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00046_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00047_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00048_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00049_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00050_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00051_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00052_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00053_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00054_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00055_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00056_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00057_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00058_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00059_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00060_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00061_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00062_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00063_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00064_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00065_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00066_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00067_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00068_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00069_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00070_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00071_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00072_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00073_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00074_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00075_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00076_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00077_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00078_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00079_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00080_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00081_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00082_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00083_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00084_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00085_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00086_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00087_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00088_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00089_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00090_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00091_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00092_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00093_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00094_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00095_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00096_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00097_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00098_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00099_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00000_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00001_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00002_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00003_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00004_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00005_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00006_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00007_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00008_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00009_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00010_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00011_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00012_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00013_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00014_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00015_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00016_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00017_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00018_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00019_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00020_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00021_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00022_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00023_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00024_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00025_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00026_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00027_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00028_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00029_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00030_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00031_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00032_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00033_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00034_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00035_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00036_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00037_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00038_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00039_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00040_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00041_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00042_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00043_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00044_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00045_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00046_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00047_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00048_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00049_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00050_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00051_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00052_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00053_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00054_00000_doc_shuffled.ds'], 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 32, 'drop_last': True, 'pin_memory': True, 'prefetch_factor': 8, 'persistent_workers': True, 'timeout': 0, 'seed': None, 'instance_filter': None, 'custom_dataset': None}, 'restore_dataloader': True, 'fast_forward_batches': None, 'evaluators': [], 'eval_interval': 5000, 'tokenizer': {'identifier': 'meta-llama/Llama-2-7b-hf', 'truncate_direction': 'right'}, 'save_folder': 'checkpoints/OLMo-1B-as_fm3_omi2', 'remote_save_folder': None, 'canceled_check_interval': 6000, 'save_interval': 3000, 'save_interval_unsharded': 3000, 'save_interval_ephemeral': None, 'save_num_checkpoints_to_keep': -1, 'save_num_unsharded_checkpoints_to_keep': -1, 'save_overwrite': True, 'force_save_unsharded': False, 'no_pre_train_checkpoint': False, 'load_path': None, 'load_path_sharded_checkpointer': None, 'try_load_latest_save': False, 'reset_optimizer_state': False, 'reset_trainer_state': False, 'sharded_checkpointer': 'torch_legacy', 'new_style_checkpoints': None, 'max_duration': '1ep', 'global_train_batch_size': 512, 'device_train_batch_size': 512, 'device_train_microbatch_size': 16, 'device_eval_batch_size': 16, 'eval_subset_num_batches': -1, 'eval_on_load': False, 'device_train_grad_accum': 32, 'max_grad_norm': 1.0, 'max_grad_norm_ratio': None, 'precision': 'amp_bf16', 'speed_monitor': {'window_size': 20, 'gpu_flops_available': None}, 'console_log_interval': 1, 'gen1_gc_interval': 1, 'compile': None, 'distributed_strategy': 'fsdp', 'fsdp': {'use_orig_params': True, 'sharding_strategy': <ShardingStrategy.FULL_SHARD: 1>, 'wrapping_strategy': None, 'precision': 'mixed', 'hybrid_sharding_num_model_replicas': None}, 'ddp': {'grad_sync_mode': 'batch', 'find_unused_params': False}, 'single': {'device': 'auto'}, 'softmax_auxiliary_loss': False, 'auxiliary_loss_multiplier': 0.0001, 'time_limit': None, 'extra_steps_after_cancel': 10, 'early_stopping_factor': None, 'save_data_indices': True, 'python_profiling': False, 'torch_profiling': False, 'stop_at': None, 'stop_after': None, 'activation_checkpointing': None, 'fused_loss': None, 'hf_datasets_cache_dir': None, 'module_outputs_save_steps': None, '_wandb': {}}
-2025-12-14 15:36:18,762 INFO    MainThread:3744013 [wandb_init.py:init():889] starting backend
-2025-12-14 15:36:19,006 INFO    MainThread:3744013 [wandb_init.py:init():892] sending inform_init request
-2025-12-14 15:36:19,014 INFO    MainThread:3744013 [wandb_init.py:init():900] backend started and connected
-2025-12-14 15:36:19,018 INFO    MainThread:3744013 [wandb_init.py:init():970] updated telemetry
-2025-12-14 15:36:19,021 INFO    MainThread:3744013 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
-2025-12-14 15:36:19,655 INFO    MainThread:3744013 [wandb_init.py:init():1041] starting run threads in backend
-2025-12-14 15:36:19,746 INFO    MainThread:3744013 [wandb_run.py:_console_start():2521] atexit reg
-2025-12-14 15:36:19,747 INFO    MainThread:3744013 [wandb_run.py:_redirect():2369] redirect: wrap_raw
-2025-12-14 15:36:19,748 INFO    MainThread:3744013 [wandb_run.py:_redirect():2438] Wrapping output streams.
-2025-12-14 15:36:19,749 INFO    MainThread:3744013 [wandb_run.py:_redirect():2461] Redirects installed.
-2025-12-14 15:36:19,752 INFO    MainThread:3744013 [wandb_init.py:init():1081] run started, returning control to user process
-2025-12-14 15:36:23,966 INFO    wandb-AsyncioManager-main:3744013 [service_client.py:_forward_responses():80] Reached EOF.
-2025-12-14 15:36:23,967 INFO    wandb-AsyncioManager-main:3744013 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.

models/OLMo-1B/wandb/wandb/run-20251214_153700-c0ptjfpp/files/config.yaml

@@ -1,575 +0,0 @@
-_wandb:
-    value:
-        cli_version: 0.23.1
-        e:
-            sunp9aekjphrkaie8a5jhftzaeyt35vl:
-                args:
-                    - pretraining/configs/RL-1B.yaml
-                codePath: OLMo/scripts/train.py
-                codePathLocal: OLMo/scripts/train.py
-                cpu_count: 112
-                cpu_count_logical: 224
-                cudaVersion: "13.0"
-                disk:
-                    /:
-                        total: "2055141851136"
-                        used: "49254453248"
-                email: [email protected]
-                executable: /opt/conda/bin/python
-                gpu: NVIDIA H100 80GB HBM3
-                gpu_count: 1
-                gpu_nvidia:
-                    - architecture: Hopper
-                      cudaCores: 16896
-                      memoryTotal: "85520809984"
-                      name: NVIDIA H100 80GB HBM3
-                      uuid: GPU-4c999b2a-2578-9e62-0539-4b826d85fda8
-                host: serv-3342
-                memory:
-                    total: "2164176814080"
-                os: Linux-6.8.0-1043-nvidia-x86_64-with-glibc2.35
-                program: /netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/scripts/train.py
-                python: CPython 3.11.11
-                root: checkpoints/OLMo-1B-as_fm3_omi2/wandb
-                slurm:
-                    cluster_name: pegasus
-                    conf: /etc/slurm/slurm.conf
-                    cpu_bind: quiet,mask_cpu:0x000000000001FE00000000000000000000000001FE00000000000000
-                    cpu_bind_list: 0x000000000001FE00000000000000000000000001FE00000000000000
-                    cpu_bind_type: 'mask_cpu:'
-                    cpu_bind_verbose: quiet
-                    cpus_on_node: "16"
-                    cpus_per_task: "16"
-                    distribution: cyclic
-                    gpus: "1"
-                    gpus_on_node: "1"
-                    gtids: "0"
-                    job_cpus_per_node: "16"
-                    job_end_time: "1765734524"
-                    job_gid: "8000"
-                    job_group: iml
-                    job_id: "2383756"
-                    job_name: bash
-                    job_nodelist: serv-3342
-                    job_num_nodes: "1"
-                    job_partition: H100
-                    job_qos: normal
-                    job_start_time: "1765720124"
-                    job_uid: "13262"
-                    job_user: nguyen
-                    jobid: "2383756"
-                    launch_node_ipaddr: 192.168.33.114
-                    localid: "0"
-                    mem_per_cpu: "16384"
-                    mpi_type: pmix
-                    nnodes: "1"
-                    nodeid: "0"
-                    nodelist: serv-3342
-                    nprocs: "1"
-                    ntasks: "1"
-                    oom_kill_step: "0"
-                    pmix_mapping_serv: (vector,(0,1,1))
-                    pmixp_abort_agent_port: "33735"
-                    prio_process: "1"
-                    procid: "0"
-                    pty_port: "45219"
-                    pty_win_col: "156"
-                    pty_win_row: "41"
-                    srun_comm_host: 192.168.33.114
-                    srun_comm_port: "35153"
-                    step_gpus: "5"
-                    step_id: "0"
-                    step_launcher_port: "35153"
-                    step_nodelist: serv-3342
-                    step_num_nodes: "1"
-                    step_num_tasks: "1"
-                    step_tasks_per_node: "1"
-                    stepid: "0"
-                    submit_dir: /netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain
-                    submit_host: login1
-                    task_pid: "3684902"
-                    tasks_per_node: "1"
-                    topology_addr: serv-3342
-                    topology_addr_pattern: node
-                    tres_bind: gres/gpu:per_task:1
-                    tres_per_task: cpu=16,gres/gpu=1
-                    umask: "0022"
-                startedAt: "2025-12-14T14:37:00.974734Z"
-                writerId: sunp9aekjphrkaie8a5jhftzaeyt35vl
-        m: []
-        python_version: 3.11.11
-        t:
-            "1":
-                - 1
-                - 5
-                - 11
-                - 41
-                - 49
-                - 51
-                - 53
-            "2":
-                - 1
-                - 5
-                - 11
-                - 41
-                - 49
-                - 51
-                - 53
-            "3":
-                - 13
-                - 15
-                - 16
-            "4": 3.11.11
-            "5": 0.23.1
-            "6": 4.57.3
-            "12": 0.23.1
-            "13": linux-x86_64
-activation_checkpointing:
-    value: null
-auxiliary_loss_multiplier:
-    value: 0.0001
-canceled_check_interval:
-    value: 6000
-compile:
-    value: null
-console_log_interval:
-    value: 1
-data:
-    value:
-        custom_dataset: null
-        datasets: null
-        drop_last: true
-        generate_attention_mask: false
-        generate_doc_lengths: false
-        instance_filter: null
-        label_mask_paths: null
-        memmap_dtype: uint16
-        num_workers: 32
-        pad_direction: right
-        paths:
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00000_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00001_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00002_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00003_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00004_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00005_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00006_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00007_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00008_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00009_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00010_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00011_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00012_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00016_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00019_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00020_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00021_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00022_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00023_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00024_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00025_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00026_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00027_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00028_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00029_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00030_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00031_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00032_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00033_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00034_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00035_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00036_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00037_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00038_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00039_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00040_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00041_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00042_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00043_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00044_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00045_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00046_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00047_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00048_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00049_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00050_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00051_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00052_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00053_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00054_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00055_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00056_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00057_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00058_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00059_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00060_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00061_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00062_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00063_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00064_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00065_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00066_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00067_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00068_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00069_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00070_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00071_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00072_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00073_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00074_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00075_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00076_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00077_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00078_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00000_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00001_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00002_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00003_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00004_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00005_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00006_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00007_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00008_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00009_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00010_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00011_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00012_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00013_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00014_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00015_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00016_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00017_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00018_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00019_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00020_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00021_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00022_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00023_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00024_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00025_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00026_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00027_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00028_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00029_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00030_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00031_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00032_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00033_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00034_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00035_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00036_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00037_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00038_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00039_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00040_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00041_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00042_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00043_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00044_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00045_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00046_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00047_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00048_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00049_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00050_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00051_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00052_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00053_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00054_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00055_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00056_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00057_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00058_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00059_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00060_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00061_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00062_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00063_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00064_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00065_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00066_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00067_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00068_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00069_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00070_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00071_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00072_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00073_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00074_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00075_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00076_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00077_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00078_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00079_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00080_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00081_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00082_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00083_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00084_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00085_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00086_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00087_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00088_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00089_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00090_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00091_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00092_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00093_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00094_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00095_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00096_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00097_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00098_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00099_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00000_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00001_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00002_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00003_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00004_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00005_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00006_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00007_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00008_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00009_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00010_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00011_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00012_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00013_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00014_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00015_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00016_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00017_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00018_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00019_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00020_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00021_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00022_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00023_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00024_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00025_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00026_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00027_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00028_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00029_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00030_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00031_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00032_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00033_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00034_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00035_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00036_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00037_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00038_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00039_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00040_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00041_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00042_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00043_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00044_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00045_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00046_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00047_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00048_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00049_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00050_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00051_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00052_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00053_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00054_00000_doc_shuffled.ds
-        persistent_workers: true
-        pin_memory: true
-        prefetch_factor: 8
-        seed: null
-        timeout: 0
-ddp:
-    value:
-        find_unused_params: false
-        grad_sync_mode: batch
-device_eval_batch_size:
-    value: 16
-device_train_batch_size:
-    value: 512
-device_train_grad_accum:
-    value: 32
-device_train_microbatch_size:
-    value: 16
-distributed_strategy:
-    value: fsdp
-dry_run:
-    value: false
-early_stopping_factor:
-    value: null
-epoch:
-    value: null
-eval_interval:
-    value: 5000
-eval_on_load:
-    value: false
-eval_subset_num_batches:
-    value: -1
-evaluators:
-    value: []
-extra_steps_after_cancel:
-    value: 10
-fast_forward_batches:
-    value: null
-force_save_unsharded:
-    value: false
-fsdp:
-    value:
-        hybrid_sharding_num_model_replicas: null
-        precision: mixed
-        sharding_strategy: FULL_SHARD
-        use_orig_params: true
-        wrapping_strategy: null
-fused_loss:
-    value: null
-gen1_gc_interval:
-    value: 1
-global_train_batch_size:
-    value: 512
-hf_datasets_cache_dir:
-    value: null
-load_path:
-    value: null
-load_path_sharded_checkpointer:
-    value: null
-max_duration:
-    value: 1ep
-max_grad_norm:
-    value: 1
-max_grad_norm_ratio:
-    value: null
-model:
-    value:
-        activation_type: swiglu
-        alibi: false
-        alibi_bias_max: 8
-        attention_dropout: 0
-        attention_layer_norm: false
-        attention_layer_norm_with_affine: false
-        bias_for_layer_norm: false
-        block_group_size: 1
-        block_type: sequential
-        clip_qkv: null
-        d_model: 2048
-        emb_init_std: null
-        embedding_dropout: 0
-        embedding_layer_norm: false
-        embedding_size: 32000
-        eos_token_id: 0
-        flash_attention: false
-        include_bias: false
-        init_cutoff_factor: 3
-        init_device: cuda
-        init_fn: normal
-        init_std: 0.02
-        layer_norm_eps: 1e-05
-        layer_norm_type: default
-        layer_norm_with_affine: false
-        max_sequence_length: 2048
-        mlp_hidden_size: null
-        mlp_ratio: 8
-        multi_query_attention: false
-        n_heads: 16
-        n_kv_heads: null
-        n_layers: 16
-        norm_after: false
-        pad_token_id: 1
-        precision: amp_bf16
-        residual_dropout: 0
-        rope: true
-        rope_full_precision: true
-        rope_theta: 10000
-        scale_emb_init: false
-        scale_logits: false
-        vocab_size: 32000
-        weight_tying: true
-module_outputs_save_steps:
-    value: null
-new_style_checkpoints:
-    value: null
-no_pre_train_checkpoint:
-    value: false
-optimizer:
-    value:
-        betas:
-            - 0.9
-            - 0.95
-        decay_embeddings: true
-        decay_norm_and_bias: true
-        eps: 1e-08
-        learning_rate: 0.0005
-        metrics_log_interval: 10
-        name: adamw
-        no_decay_norm_and_bias: null
-        record_update_metrics: false
-        selective_updates: false
-        weight_decay: 0.1
-precision:
-    value: amp_bf16
-python_profiling:
-    value: false
-remote_save_folder:
-    value: null
-reset_optimizer_state:
-    value: false
-reset_trainer_state:
-    value: false
-restore_dataloader:
-    value: true
-run_name:
-    value: OLMo-1B-as_fm3_omi2
-save_data_indices:
-    value: true
-save_folder:
-    value: checkpoints/OLMo-1B-as_fm3_omi2
-save_interval:
-    value: 3000
-save_interval_ephemeral:
-    value: null
-save_interval_unsharded:
-    value: 3000
-save_num_checkpoints_to_keep:
-    value: -1
-save_num_unsharded_checkpoints_to_keep:
-    value: -1
-save_overwrite:
-    value: true
-scheduler:
-    value:
-        alpha_f: 0.1
-        grad_clip_warmup_factor: null
-        grad_clip_warmup_steps: null
-        name: cosine_with_warmup
-        t_max: null
-        t_warmup: 2000
-        units: steps
-        warmup_min_lr: 0
-seed:
-    value: 6198
-sharded_checkpointer:
-    value: torch_legacy
-single:
-    value:
-        device: auto
-softmax_auxiliary_loss:
-    value: false
-speed_monitor:
-    value:
-        gpu_flops_available: null
-        window_size: 20
-stop_after:
-    value: null
-stop_at:
-    value: null
-time_limit:
-    value: null
-tokenizer:
-    value:
-        identifier: meta-llama/Llama-2-7b-hf
-        truncate_direction: right
-torch_profiling:
-    value: false
-try_load_latest_save:
-    value: false

models/OLMo-1B/wandb/wandb/run-20251214_153700-c0ptjfpp/files/output.log

@@ -1,241 +0,0 @@
-[2025-12-14 15:37:04] INFO     [olmo.data.iterable_dataset:79, rank=0] Saving global data order indices...
-[2025-12-14 15:37:05] INFO     [olmo.data.iterable_dataset:88, rank=0] Global data order indices saved to 'checkpoints/OLMo-1B-as_fm3_omi2/train_data/global_indices.npy'
-[2025-12-14 15:37:05] WARNING  [py.warnings:110, rank=0] /opt/conda/lib/python3.11/site-packages/torch/utils/data/dataloader.py:624: UserWarning: This DataLoader will create 32 worker processes in total. Our suggested max number of worker in current system is 16, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.
-  warnings.warn(
-Traceback (most recent call last):
-  File "/opt/conda/lib/python3.11/site-packages/huggingface_hub/utils/_http.py", line 402, in hf_raise_for_status
-    response.raise_for_status()
-  File "/opt/conda/lib/python3.11/site-packages/requests/models.py", line 1024, in raise_for_status
-    raise HTTPError(http_error_msg, response=self)
-requests.exceptions.HTTPError: 401 Client Error: Unauthorized for url: https://huggingface.co/meta-llama/Llama-2-7b-hf/resolve/main/tokenizer.json
-
-The above exception was the direct cause of the following exception:
-
-Traceback (most recent call last):
-  File "/netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/scripts/train.py", line 436, in <module>
-    main(cfg)
-  File "/netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/scripts/train.py", line 135, in main
-    evaluators = build_evaluators(cfg, device)
-                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/olmo/eval/__init__.py", line 111, in build_evaluators
-    tokenizer = Tokenizer.from_train_config(cfg)
-                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/olmo/tokenizer.py", line 75, in from_train_config
-    tokenizer = cls.from_pretrained(
-                ^^^^^^^^^^^^^^^^^^^^
-  File "/netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/olmo/tokenizer.py", line 93, in from_pretrained
-    base_tokenizer = BaseTokenizer.from_pretrained(identifier)
-                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/opt/conda/lib/python3.11/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn
-    return fn(*args, **kwargs)
-           ^^^^^^^^^^^^^^^^^^^
-  File "/opt/conda/lib/python3.11/site-packages/huggingface_hub/file_download.py", line 1007, in hf_hub_download
-    return _hf_hub_download_to_cache_dir(
-           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/opt/conda/lib/python3.11/site-packages/huggingface_hub/file_download.py", line 1114, in _hf_hub_download_to_cache_dir
-    _raise_on_head_call_error(head_call_error, force_download, local_files_only)
-  File "/opt/conda/lib/python3.11/site-packages/huggingface_hub/file_download.py", line 1655, in _raise_on_head_call_error
-    raise head_call_error
-  File "/opt/conda/lib/python3.11/site-packages/huggingface_hub/file_download.py", line 1543, in _get_metadata_or_catch_error
-    metadata = get_hf_file_metadata(
-               ^^^^^^^^^^^^^^^^^^^^^
-  File "/opt/conda/lib/python3.11/site-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn
-    return fn(*args, **kwargs)
-           ^^^^^^^^^^^^^^^^^^^
-  File "/opt/conda/lib/python3.11/site-packages/huggingface_hub/file_download.py", line 1460, in get_hf_file_metadata
-    r = _request_wrapper(
-        ^^^^^^^^^^^^^^^^^
-  File "/opt/conda/lib/python3.11/site-packages/huggingface_hub/file_download.py", line 283, in _request_wrapper
-    response = _request_wrapper(
-               ^^^^^^^^^^^^^^^^^
-  File "/opt/conda/lib/python3.11/site-packages/huggingface_hub/file_download.py", line 307, in _request_wrapper
-    hf_raise_for_status(response)
-  File "/opt/conda/lib/python3.11/site-packages/huggingface_hub/utils/_http.py", line 419, in hf_raise_for_status
-    raise _format(GatedRepoError, message, response) from e
-huggingface_hub.errors.GatedRepoError: 401 Client Error. (Request ID: Root=1-693ecb91-6af595934595c5324944523c;f0cd17a6-52aa-4d39-8750-5d75e2cca80a)
-
-Cannot access gated repo for url https://huggingface.co/meta-llama/Llama-2-7b-hf/resolve/main/tokenizer.json.
-Access to model meta-llama/Llama-2-7b-hf is restricted. You must have access to it and be authenticated to access it. Please log in.
-
-[2025-12-14 15:37:05] CRITICAL [olmo.util:168, rank=0] Uncaught GatedRepoError: 401 Client Error. (Request ID: Root=1-693ecb91-6af595934595c5324944523c;f0cd17a6-52aa-4d39-8750-5d75e2cca80a)
-
-Cannot access gated repo for url https://huggingface.co/meta-llama/Llama-2-7b-hf/resolve/main/tokenizer.json.
-Access to model meta-llama/Llama-2-7b-hf is restricted. You must have access to it and be authenticated to access it. Please log in.
-╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮
-│ /opt/conda/lib/python3.11/site-packages/huggingface_hub/utils/_http.py:402 in hf_raise_for_statu │
-│                                                                                                  │
-│   399 │   >         If request failed for a reason not listed above.                             │
-│   400 │   """                                                                                    │
-│   401 │   try:                                                                                   │
-│ ❱ 402 │   │   response.raise_for_status()                                                        │
-│   403 │   except HTTPError as e:                                                                 │
-│   404 │   │   error_code = response.headers.get("X-Error-Code")                                  │
-│   405 │   │   error_message = response.headers.get("X-Error-Message")                            │
-│                                                                                                  │
-│ /opt/conda/lib/python3.11/site-packages/requests/models.py:1024 in raise_for_status              │
-│                                                                                                  │
-│   1021 │   │   │   )                                                                             │
-│   1022 │   │                                                                                     │
-│   1023 │   │   if http_error_msg:                                                                │
-│ ❱ 1024 │   │   │   raise HTTPError(http_error_msg, response=self)                                │
-│   1025 │                                                                                         │
-│   1026 │   def close(self):                                                                      │
-│   1027 │   │   """Releases the connection back to the pool. Once this method has been            │
-╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
-HTTPError: 401 Client Error: Unauthorized for url: https://huggingface.co/meta-llama/Llama-2-7b-hf/resolve/main/tokenizer.json
-
-The above exception was the direct cause of the following exception:
-
-╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮
-│ /netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/scripts/train.py:436 │
-│                                                                                                  │
-│   433 │   │   log.info("Device is CPU. Updating config...")                                      │
-│   434 │   │   cfg.model.init_device = "cpu"                                                      │
-│   435 │   │   cfg.distributed_strategy = "single"  # type: ignore                                │
-│ ❱ 436 │   main(cfg)                                                                              │
-│   437                                                                                            │
-│                                                                                                  │
-│ /netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/scripts/train.py:135 │
-│                                                                                                  │
-│   132 │   train_loader = build_train_dataloader(cfg)                                             │
-│   133 │                                                                                          │
-│   134 │   # Construct evaluators.                                                                │
-│ ❱ 135 │   evaluators = build_evaluators(cfg, device)                                             │
-│   136 │   barrier()                                                                              │
-│   137 │                                                                                          │
-│   138 │   # Initialize the model.                                                                │
-│                                                                                                  │
-│ /netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/olmo/eval/__init__.p │
-│                                                                                                  │
-│   108                                                                                            │
-│   109 def build_evaluators(cfg: TrainConfig, device: torch.device) -> List[Evaluator]:           │
-│   110 │   evaluators = []                                                                        │
-│ ❱ 111 │   tokenizer = Tokenizer.from_train_config(cfg)                                           │
-│   112 │   for eval_cfg in cfg.evaluators:                                                        │
-│   113 │   │   evaluators.append(build_evaluator(cfg, eval_cfg, tokenizer, device))               │
-│   114 │   return evaluators                                                                      │
-│                                                                                                  │
-│ /netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/olmo/tokenizer.py:75 │
-│                                                                                                  │
-│    72 │   │   │   │   │   pad_token_id=config.model.pad_token_id,                                │
-│    73 │   │   │   │   )                                                                          │
-│    74 │   │   else:                                                                              │
-│ ❱  75 │   │   │   tokenizer = cls.from_pretrained(                                               │
-│    76 │   │   │   │   tokenizer_identifier,                                                      │
-│    77 │   │   │   │   eos_token_id=config.model.eos_token_id,                                    │
-│    78 │   │   │   │   pad_token_id=config.model.pad_token_id,                                    │
-│                                                                                                  │
-│ /netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/olmo/tokenizer.py:93 │
-│                                                                                                  │
-│    90 │   │   │   ``tokenizer.json`` file.                                                       │
-│    91 │   │   :param kwargs: Other key word arguments passed to :class:`Tokenizer`.              │
-│    92 │   │   """                                                                                │
-│ ❱  93 │   │   base_tokenizer = BaseTokenizer.from_pretrained(identifier)                         │
-│    94 │   │   eos_token_id = kwargs.pop("eos_token_id", base_tokenizer.get_vocab_size() - 1)     │
-│    95 │   │   return cls(base_tokenizer, eos_token_id, **kwargs)                                 │
-│    96                                                                                            │
-│                                                                                                  │
-│ /opt/conda/lib/python3.11/site-packages/huggingface_hub/utils/_validators.py:114 in _inner_fn    │
-│                                                                                                  │
-│   111 │   │   if check_use_auth_token:                                                           │
-│   112 │   │   │   kwargs = smoothly_deprecate_use_auth_token(fn_name=fn.__name__, has_token=has_ │
-│   113 │   │                                                                                      │
-│ ❱ 114 │   │   return fn(*args, **kwargs)                                                         │
-│   115 │                                                                                          │
-│   116 │   return _inner_fn  # type: ignore                                                       │
-│   117                                                                                            │
-│                                                                                                  │
-│ /opt/conda/lib/python3.11/site-packages/huggingface_hub/file_download.py:1007 in hf_hub_download │
-│                                                                                                  │
-│   1004 │   │   │   local_files_only=local_files_only,                                            │
-│   1005 │   │   )                                                                                 │
-│   1006 │   else:                                                                                 │
-│ ❱ 1007 │   │   return _hf_hub_download_to_cache_dir(                                             │
-│   1008 │   │   │   # Destination                                                                 │
-│   1009 │   │   │   cache_dir=cache_dir,                                                          │
-│   1010 │   │   │   # File info                                                                   │
-│                                                                                                  │
-│ /opt/conda/lib/python3.11/site-packages/huggingface_hub/file_download.py:1114 in _hf_hub_downloa │
-│                                                                                                  │
-│   1111 │   │   │   │   │   return pointer_path                                                   │
-│   1112 │   │                                                                                     │
-│   1113 │   │   # Otherwise, raise appropriate error                                              │
-│ ❱ 1114 │   │   _raise_on_head_call_error(head_call_error, force_download, local_files_only)      │
-│   1115 │                                                                                         │
-│   1116 │   # From now on, etag, commit_hash, url and size are not None.                          │
-│   1117 │   assert etag is not None, "etag must have been retrieved from server"                  │
-│                                                                                                  │
-│ /opt/conda/lib/python3.11/site-packages/huggingface_hub/file_download.py:1655 in _raise_on_head_ │
-│                                                                                                  │
-│   1652 │   ):                                                                                    │
-│   1653 │   │   # Repo not found or gated => let's raise the actual error                         │
-│   1654 │   │   # Unauthorized => likely a token issue => let's raise the actual error            │
-│ ❱ 1655 │   │   raise head_call_error                                                             │
-│   1656 │   else:                                                                                 │
-│   1657 │   │   # Otherwise: most likely a connection issue or Hub downtime => let's warn the use │
-│   1658 │   │   raise LocalEntryNotFoundError(                                                    │
-│                                                                                                  │
-│ /opt/conda/lib/python3.11/site-packages/huggingface_hub/file_download.py:1543 in _get_metadata_o │
-│                                                                                                  │
-│   1540 │   if not local_files_only:                                                              │
-│   1541 │   │   try:                                                                              │
-│   1542 │   │   │   try:                                                                          │
-│ ❱ 1543 │   │   │   │   metadata = get_hf_file_metadata(                                          │
-│   1544 │   │   │   │   │   url=url, proxies=proxies, timeout=etag_timeout, headers=headers, toke │
-│   1545 │   │   │   │   )                                                                         │
-│   1546 │   │   │   except EntryNotFoundError as http_error:                                      │
-│                                                                                                  │
-│ /opt/conda/lib/python3.11/site-packages/huggingface_hub/utils/_validators.py:114 in _inner_fn    │
-│                                                                                                  │
-│   111 │   │   if check_use_auth_token:                                                           │
-│   112 │   │   │   kwargs = smoothly_deprecate_use_auth_token(fn_name=fn.__name__, has_token=has_ │
-│   113 │   │                                                                                      │
-│ ❱ 114 │   │   return fn(*args, **kwargs)                                                         │
-│   115 │                                                                                          │
-│   116 │   return _inner_fn  # type: ignore                                                       │
-│   117                                                                                            │
-│                                                                                                  │
-│ /opt/conda/lib/python3.11/site-packages/huggingface_hub/file_download.py:1460 in get_hf_file_met │
-│                                                                                                  │
-│   1457 │   hf_headers["Accept-Encoding"] = "identity"  # prevent any compression => we want to k │
-│   1458 │                                                                                         │
-│   1459 │   # Retrieve metadata                                                                   │
-│ ❱ 1460 │   r = _request_wrapper(                                                                 │
-│   1461 │   │   method="HEAD",                                                                    │
-│   1462 │   │   url=url,                                                                          │
-│   1463 │   │   headers=hf_headers,                                                               │
-│                                                                                                  │
-│ /opt/conda/lib/python3.11/site-packages/huggingface_hub/file_download.py:283 in _request_wrapper │
-│                                                                                                  │
-│    280 │   """                                                                                   │
-│    281 │   # Recursively follow relative redirects                                               │
-│    282 │   if follow_relative_redirects:                                                         │
-│ ❱  283 │   │   response = _request_wrapper(                                                      │
-│    284 │   │   │   method=method,                                                                │
-│    285 │   │   │   url=url,                                                                      │
-│    286 │   │   │   follow_relative_redirects=False,                                              │
-│                                                                                                  │
-│ /opt/conda/lib/python3.11/site-packages/huggingface_hub/file_download.py:307 in _request_wrapper │
-│                                                                                                  │
-│    304 │                                                                                         │
-│    305 │   # Perform request and return if status_code is not in the retry list.                 │
-│    306 │   response = http_backoff(method=method, url=url, **params)                             │
-│ ❱  307 │   hf_raise_for_status(response)                                                         │
-│    308 │   return response                                                                       │
-│    309                                                                                           │
-│    310                                                                                           │
-│                                                                                                  │
-│ /opt/conda/lib/python3.11/site-packages/huggingface_hub/utils/_http.py:419 in hf_raise_for_statu │
-│                                                                                                  │
-│   416 │   │   │   message = (                                                                    │
-│   417 │   │   │   │   f"{response.status_code} Client Error." + "\n\n" + f"Cannot access gated r │
-│   418 │   │   │   )                                                                              │
-│ ❱ 419 │   │   │   raise _format(GatedRepoError, message, response) from e                        │
-│   420 │   │                                                                                      │
-│   421 │   │   elif error_message == "Access to this resource is disabled.":                      │
-│   422 │   │   │   message = (                                                                    │
-╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
-GatedRepoError: 401 Client Error. (Request ID: Root=1-693ecb91-6af595934595c5324944523c;f0cd17a6-52aa-4d39-8750-5d75e2cca80a)
-
-Cannot access gated repo for url https://huggingface.co/meta-llama/Llama-2-7b-hf/resolve/main/tokenizer.json.
-Access to model meta-llama/Llama-2-7b-hf is restricted. You must have access to it and be authenticated to access it. Please log in.

models/OLMo-1B/wandb/wandb/run-20251214_153700-c0ptjfpp/files/requirements.txt

@@ -1,271 +0,0 @@
-scikit-learn==1.8.0
-joblib==1.5.2
-threadpoolctl==3.6.0
-torchmetrics==1.8.2
-lightning-utilities==0.15.2
-wandb==0.23.1
-GitPython==3.1.45
-gitdb==4.0.12
-smmap==5.0.2
-datasets==4.4.1
-pandas==2.3.3
-multiprocess==0.70.18
-pyarrow==22.0.0
-tzdata==2025.3
-xxhash==3.6.0
-ai2-olmo==0.6.0
-ai2-olmo-core==2.4.0
-cached_path==1.8.0
-google-cloud-storage==2.19.0
-google-cloud-core==2.5.0
-boto3==1.42.9
-google-api-core==2.28.1
-google-auth==2.43.0
-s3transfer==0.16.0
-botocore==1.42.9
-google-resumable-media==2.8.0
-pyasn1_modules==0.4.2
-rich==13.9.4
-rsa==4.9.1
-bettermap==1.3.1
-google-crc32c==1.7.1
-jmespath==1.0.1
-numpy==1.26.4
-omegaconf==2.3.0
-proto-plus==1.26.1
-pyasn1==0.6.1
-python-dateutil==2.9.0.post0
-antlr4-python3-runtime==4.9.3
-zstandard==0.23.0
-zipp==3.21.0
-yarl==1.22.0
-xgrammar==0.1.18
-xformers==0.0.29.post2
-wrapt==2.0.1
-wheel==0.45.1
-websockets==15.0.1
-wcwidth==0.2.13
-watchfiles==1.1.1
-vllm==0.8.5.post1
-uvloop==0.22.1
-uvicorn==0.38.0
-urllib3==2.3.0
-typing-inspection==0.4.2
-typing_extensions==4.15.0
-types-dataclasses==0.6.6
-typer==0.20.0
-truststore==0.10.0
-triton==3.2.0
-transformers==4.57.3
-traitlets==5.14.3
-tqdm==4.67.1
-torchvision==0.21.0+cu124
-torchelastic==0.2.2
-torchaudio==2.6.0+cu124
-torch==2.6.0+cu124
-tokenizers==0.22.1
-tiktoken==0.12.0
-sympy==1.13.1
-starlette==0.50.0
-stack_data==0.6.3
-soupsieve==2.5
-sortedcontainers==2.4.0
-sniffio==1.3.1
-six==1.17.0
-shellingham==1.5.4
-setuptools==75.8.0
-sentry-sdk==2.47.0
-sentencepiece==0.2.1
-scipy==1.16.3
-safetensors==0.7.0
-ruamel.yaml.clib==0.2.8
-ruamel.yaml==0.18.10
-rpds-py==0.22.3
-rignore==0.7.6
-rich-toolkit==0.17.0
-requests==2.32.3
-regex==2025.11.3
-referencing==0.36.2
-ray==2.52.1
-pyzmq==27.1.0
-pytz==2024.2
-python-multipart==0.0.20
-python-json-logger==4.0.0
-python-etcd==0.4.5
-python-dotenv==1.2.1
-Pygments==2.19.1
-pydantic-extra-types==2.10.6
-pydantic_core==2.41.5
-pydantic==2.12.5
-pycparser==2.22
-pycountry==24.6.1
-pycosat==0.6.6
-py-cpuinfo==9.0.0
-pure_eval==0.2.3
-ptyprocess==0.7.0
-psutil==6.1.1
-protobuf==4.25.8
-propcache==0.4.1
-prompt_toolkit==3.0.50
-prometheus-fastapi-instrumentator==7.1.0
-prometheus_client==0.23.1
-pluggy==1.5.0
-platformdirs==4.3.6
-pkgutil_resolve_name==1.3.10
-pkginfo==1.12.0
-pip==24.3.1
-pillow==11.0.0
-pickleshare==0.7.5
-pexpect==4.9.0
-partial-json-parser==0.2.1.1.post7
-parso==0.8.4
-packaging==24.2
-outlines_core==0.1.26
-outlines==0.1.11
-optree==0.14.0
-opentelemetry-semantic-conventions-ai==0.4.13
-opentelemetry-semantic-conventions==0.47b0
-opentelemetry-sdk==1.26.0
-opentelemetry-proto==1.26.0
-opentelemetry-exporter-otlp-proto-http==1.26.0
-opentelemetry-exporter-otlp-proto-grpc==1.26.0
-opentelemetry-exporter-otlp-proto-common==1.26.0
-opentelemetry-exporter-otlp==1.26.0
-opentelemetry-api==1.26.0
-opencv-python-headless==4.12.0.88
-openai==2.11.0
-nvidia-nvtx-cu12==12.4.127
-nvidia-nvjitlink-cu12==12.4.127
-nvidia-nccl-cu12==2.21.5
-nvidia-cusparselt-cu12==0.6.2
-nvidia-cusparse-cu12==12.3.1.170
-nvidia-cusolver-cu12==11.6.1.9
-nvidia-curand-cu12==10.3.5.147
-nvidia-cufft-cu12==11.2.1.3
-nvidia-cudnn-cu12==9.1.0.70
-nvidia-cuda-runtime-cu12==12.4.127
-nvidia-cuda-nvrtc-cu12==12.4.127
-nvidia-cuda-cupti-cu12==12.4.127
-nvidia-cublas-cu12==12.4.5.8
-numba==0.61.2
-ninja==1.11.1.3
-networkx==3.4.2
-nest-asyncio==1.6.0
-multidict==6.7.0
-msgspec==0.20.0
-msgpack==1.1.2
-mpmath==1.3.0
-more-itertools==10.6.0
-mistral_common==1.8.6
-menuinst==2.2.0
-mdurl==0.1.2
-matplotlib-inline==0.1.7
-math-verify==0.8.0
-markdown-it-py==4.0.0
-lm-format-enforcer==0.10.12
-llvmlite==0.44.0
-llguidance==0.7.30
-lintrunner==0.12.7
-lief==0.14.1
-libmambapy==2.0.5
-libarchive-c==5.1
-latex2sympy2_extended==1.10.2
-lark==1.2.2
-jsonschema-specifications==2024.10.1
-jsonschema==4.23.0
-jsonpointer==3.0.0
-jsonpatch==1.33
-jiter==0.12.0
-Jinja2==3.1.5
-jedi==0.19.2
-ipython==8.31.0
-interegular==0.3.3
-importlib_resources==6.5.2
-importlib_metadata==8.0.0
-idna==3.10
-hypothesis==6.124.7
-hyperframe==6.0.1
-huggingface-hub==0.36.0
-httpx==0.28.1
-httptools==0.7.1
-httpcore==1.0.9
-hpack==4.0.0
-hf-xet==1.2.0
-h2==4.1.0
-h11==0.16.0
-grpcio==1.76.0
-googleapis-common-protos==1.72.0
-gguf==0.17.1
-fsspec==2024.12.0
-frozenlist==1.8.0
-frozendict==2.4.6
-filelock==3.17.0
-fastrlock==0.8.3
-fastar==0.8.0
-fastapi-cloud-cli==0.6.0
-fastapi-cli==0.0.16
-fastapi==0.124.4
-expecttest==0.3.0
-executing==2.1.0
-exceptiongroup==1.2.2
-email-validator==2.3.0
-einops==0.8.1
-dnspython==2.7.0
-distro==1.9.0
-diskcache==5.6.3
-dill==0.4.0
-depyf==0.18.0
-Deprecated==1.3.1
-decorator==5.1.1
-cupy-cuda12x==13.6.0
-conda_package_streaming==0.11.0
-conda-package-handling==2.4.0
-conda-libmamba-solver==25.1.1
-conda_index==0.5.0
-conda-build==25.1.1
-conda==25.1.0
-compressed-tensors==0.9.3
-colorama==0.4.6
-cmake==3.31.4
-cloudpickle==3.1.2
-click==8.1.8
-charset-normalizer==3.4.1
-chardet==5.2.0
-cffi==1.17.1
-certifi==2024.12.14
-cachetools==6.2.3
-boltons==24.0.0
-blake3==1.0.8
-beautifulsoup4==4.12.3
-attrs==25.1.0
-astunparse==1.6.3
-asttokens==3.0.0
-astor==0.8.1
-archspec==0.2.5
-anyio==4.12.0
-annotated-types==0.7.0
-annotated-doc==0.0.4
-airportsdata==20250909
-aiosignal==1.4.0
-aiohttp==3.13.2
-aiohappyeyeballs==2.6.1
-PyYAML==6.0.2
-PySocks==1.7.1
-MarkupSafe==3.0.2
-Brotli==1.1.0
-zipp==3.19.2
-wheel==0.43.0
-typing_extensions==4.12.2
-typeguard==4.3.0
-tomli==2.0.1
-platformdirs==4.2.2
-packaging==24.2
-more-itertools==10.3.0
-jaraco.text==3.12.1
-jaraco.functools==4.0.1
-jaraco.context==5.3.0
-jaraco.collections==5.1.0
-inflect==7.3.1
-importlib_metadata==8.0.0
-backports.tarfile==1.2.0
-autocommand==2.2.2

models/OLMo-1B/wandb/wandb/run-20251214_153700-c0ptjfpp/files/wandb-metadata.json

@@ -1,103 +0,0 @@
-{
-  "os":  "Linux-6.8.0-1043-nvidia-x86_64-with-glibc2.35",
-  "python":  "CPython 3.11.11",
-  "startedAt":  "2025-12-14T14:37:00.974734Z",
-  "args":  [
-    "pretraining/configs/RL-1B.yaml"
-  ],
-  "program":  "/netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/scripts/train.py",
-  "codePath":  "OLMo/scripts/train.py",
-  "codePathLocal":  "OLMo/scripts/train.py",
-  "email":  "[email protected]",
-  "root":  "checkpoints/OLMo-1B-as_fm3_omi2/wandb",
-  "host":  "serv-3342",
-  "executable":  "/opt/conda/bin/python",
-  "cpu_count":  112,
-  "cpu_count_logical":  224,
-  "gpu":  "NVIDIA H100 80GB HBM3",
-  "gpu_count":  1,
-  "disk":  {
-    "/":  {
-      "total":  "2055141851136",
-      "used":  "49254453248"
-    }
-  },
-  "memory":  {
-    "total":  "2164176814080"
-  },
-  "gpu_nvidia":  [
-    {
-      "name":  "NVIDIA H100 80GB HBM3",
-      "memoryTotal":  "85520809984",
-      "cudaCores":  16896,
-      "architecture":  "Hopper",
-      "uuid":  "GPU-4c999b2a-2578-9e62-0539-4b826d85fda8"
-    }
-  ],
-  "cudaVersion":  "13.0",
-  "slurm":  {
-    "cluster_name":  "pegasus",
-    "conf":  "/etc/slurm/slurm.conf",
-    "cpu_bind":  "quiet,mask_cpu:0x000000000001FE00000000000000000000000001FE00000000000000",
-    "cpu_bind_list":  "0x000000000001FE00000000000000000000000001FE00000000000000",
-    "cpu_bind_type":  "mask_cpu:",
-    "cpu_bind_verbose":  "quiet",
-    "cpus_on_node":  "16",
-    "cpus_per_task":  "16",
-    "distribution":  "cyclic",
-    "gpus":  "1",
-    "gpus_on_node":  "1",
-    "gtids":  "0",
-    "job_cpus_per_node":  "16",
-    "job_end_time":  "1765734524",
-    "job_gid":  "8000",
-    "job_group":  "iml",
-    "job_id":  "2383756",
-    "job_name":  "bash",
-    "job_nodelist":  "serv-3342",
-    "job_num_nodes":  "1",
-    "job_partition":  "H100",
-    "job_qos":  "normal",
-    "job_start_time":  "1765720124",
-    "job_uid":  "13262",
-    "job_user":  "nguyen",
-    "jobid":  "2383756",
-    "launch_node_ipaddr":  "192.168.33.114",
-    "localid":  "0",
-    "mem_per_cpu":  "16384",
-    "mpi_type":  "pmix",
-    "nnodes":  "1",
-    "nodeid":  "0",
-    "nodelist":  "serv-3342",
-    "nprocs":  "1",
-    "ntasks":  "1",
-    "oom_kill_step":  "0",
-    "pmix_mapping_serv":  "(vector,(0,1,1))",
-    "pmixp_abort_agent_port":  "33735",
-    "prio_process":  "1",
-    "procid":  "0",
-    "pty_port":  "45219",
-    "pty_win_col":  "156",
-    "pty_win_row":  "41",
-    "srun_comm_host":  "192.168.33.114",
-    "srun_comm_port":  "35153",
-    "step_gpus":  "5",
-    "step_id":  "0",
-    "step_launcher_port":  "35153",
-    "step_nodelist":  "serv-3342",
-    "step_num_nodes":  "1",
-    "step_num_tasks":  "1",
-    "step_tasks_per_node":  "1",
-    "stepid":  "0",
-    "submit_dir":  "/netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain",
-    "submit_host":  "login1",
-    "task_pid":  "3684902",
-    "tasks_per_node":  "1",
-    "topology_addr":  "serv-3342",
-    "topology_addr_pattern":  "node",
-    "tres_bind":  "gres/gpu:per_task:1",
-    "tres_per_task":  "cpu=16,gres/gpu=1",
-    "umask":  "0022"
-  },
-  "writerId":  "sunp9aekjphrkaie8a5jhftzaeyt35vl"
-}

models/OLMo-1B/wandb/wandb/run-20251214_153700-c0ptjfpp/files/wandb-summary.json

@@ -1 +0,0 @@
-{"_runtime":4,"_wandb":{"runtime":4}}

models/OLMo-1B/wandb/wandb/run-20251214_153700-c0ptjfpp/logs/debug-internal.log

@@ -1,11 +0,0 @@
-{"time":"2025-12-14T15:37:01.275172336+01:00","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
-{"time":"2025-12-14T15:37:01.544361117+01:00","level":"INFO","msg":"stream: created new stream","id":"c0ptjfpp"}
-{"time":"2025-12-14T15:37:01.545813594+01:00","level":"INFO","msg":"handler: started","stream_id":"c0ptjfpp"}
-{"time":"2025-12-14T15:37:01.548802981+01:00","level":"INFO","msg":"stream: started","id":"c0ptjfpp"}
-{"time":"2025-12-14T15:37:01.548838014+01:00","level":"INFO","msg":"writer: started","stream_id":"c0ptjfpp"}
-{"time":"2025-12-14T15:37:01.548853437+01:00","level":"INFO","msg":"sender: started","stream_id":"c0ptjfpp"}
-{"time":"2025-12-14T15:37:06.118873288+01:00","level":"INFO","msg":"stream: closing","id":"c0ptjfpp"}
-{"time":"2025-12-14T15:37:06.807757435+01:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
-{"time":"2025-12-14T15:37:07.118752593+01:00","level":"INFO","msg":"handler: closed","stream_id":"c0ptjfpp"}
-{"time":"2025-12-14T15:37:07.122196378+01:00","level":"INFO","msg":"sender: closed","stream_id":"c0ptjfpp"}
-{"time":"2025-12-14T15:37:07.123213459+01:00","level":"INFO","msg":"stream: closed","id":"c0ptjfpp"}

models/OLMo-1B/wandb/wandb/run-20251214_153700-c0ptjfpp/logs/debug.log

@@ -1,23 +0,0 @@
-2025-12-14 15:37:01,006 INFO    MainThread:3745493 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
-2025-12-14 15:37:01,007 INFO    MainThread:3745493 [wandb_setup.py:_flush():80] Configure stats pid to 3745493
-2025-12-14 15:37:01,008 INFO    MainThread:3745493 [wandb_setup.py:_flush():80] Loading settings from /root/.config/wandb/settings
-2025-12-14 15:37:01,009 INFO    MainThread:3745493 [wandb_setup.py:_flush():80] Loading settings from /netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/wandb/settings
-2025-12-14 15:37:01,010 INFO    MainThread:3745493 [wandb_setup.py:_flush():80] Loading settings from environment variables
-2025-12-14 15:37:01,011 INFO    MainThread:3745493 [wandb_init.py:setup_run_log_directory():714] Logging user logs to checkpoints/OLMo-1B-as_fm3_omi2/wandb/wandb/run-20251214_153700-c0ptjfpp/logs/debug.log
-2025-12-14 15:37:01,013 INFO    MainThread:3745493 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to checkpoints/OLMo-1B-as_fm3_omi2/wandb/wandb/run-20251214_153700-c0ptjfpp/logs/debug-internal.log
-2025-12-14 15:37:01,014 INFO    MainThread:3745493 [wandb_init.py:init():841] calling init triggers
-2025-12-14 15:37:01,015 INFO    MainThread:3745493 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
-config: {'run_name': 'OLMo-1B-as_fm3_omi2', 'seed': 6198, 'epoch': None, 'dry_run': False, 'model': {'d_model': 2048, 'n_heads': 16, 'n_kv_heads': None, 'clip_qkv': None, 'n_layers': 16, 'mlp_ratio': 8, 'mlp_hidden_size': None, 'activation_type': 'swiglu', 'block_type': 'sequential', 'block_group_size': 1, 'alibi': False, 'alibi_bias_max': 8.0, 'rope': True, 'rope_full_precision': True, 'rope_theta': 10000, 'flash_attention': False, 'attention_dropout': 0.0, 'multi_query_attention': False, 'attention_layer_norm': False, 'residual_dropout': 0.0, 'embedding_dropout': 0.0, 'embedding_layer_norm': False, 'layer_norm_type': 'default', 'layer_norm_with_affine': False, 'layer_norm_eps': 1e-05, 'attention_layer_norm_with_affine': False, 'max_sequence_length': 2048, 'include_bias': False, 'bias_for_layer_norm': False, 'scale_logits': False, 'vocab_size': 32000, 'embedding_size': 32000, 'weight_tying': True, 'eos_token_id': 0, 'pad_token_id': 1, 'init_device': 'cuda', 'init_fn': 'normal', 'init_std': 0.02, 'init_cutoff_factor': 3.0, 'precision': 'amp_bf16', 'scale_emb_init': False, 'emb_init_std': None, 'norm_after': False}, 'optimizer': {'name': 'adamw', 'learning_rate': 0.0005, 'weight_decay': 0.1, 'betas': (0.9, 0.95), 'eps': 1e-08, 'no_decay_norm_and_bias': None, 'selective_updates': False, 'decay_norm_and_bias': True, 'decay_embeddings': True, 'metrics_log_interval': 10, 'record_update_metrics': False}, 'scheduler': {'name': 'cosine_with_warmup', 'units': 'steps', 't_warmup': 2000, 't_max': None, 'alpha_f': 0.1, 'grad_clip_warmup_steps': None, 'grad_clip_warmup_factor': None, 'warmup_min_lr': 0.0}, 'data': {'paths': ['data_token/as_fm3_omi2/algebraic-stack-tokenized/00000_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00001_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00002_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00003_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00004_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00005_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00006_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00007_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00008_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00009_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00010_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00011_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00012_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00016_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00019_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00020_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00021_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00022_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00023_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00024_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00025_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00026_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00027_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00028_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00029_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00030_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00031_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00032_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00033_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00034_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00035_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00036_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00037_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00038_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00039_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00040_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00041_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00042_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00043_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00044_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00045_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00046_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00047_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00048_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00049_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00050_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00051_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00052_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00053_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00054_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00055_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00056_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00057_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00058_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00059_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00060_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00061_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00062_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00063_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00064_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00065_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00066_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00067_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00068_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00069_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00070_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00071_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00072_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00073_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00074_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00075_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00076_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00077_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00078_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00000_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00001_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00002_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00003_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00004_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00005_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00006_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00007_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00008_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00009_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00010_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00011_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00012_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00013_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00014_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00015_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00016_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00017_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00018_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00019_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00020_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00021_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00022_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00023_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00024_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00025_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00026_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00027_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00028_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00029_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00030_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00031_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00032_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00033_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00034_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00035_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00036_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00037_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00038_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00039_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00040_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00041_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00042_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00043_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00044_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00045_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00046_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00047_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00048_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00049_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00050_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00051_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00052_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00053_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00054_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00055_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00056_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00057_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00058_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00059_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00060_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00061_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00062_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00063_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00064_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00065_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00066_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00067_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00068_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00069_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00070_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00071_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00072_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00073_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00074_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00075_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00076_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00077_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00078_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00079_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00080_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00081_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00082_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00083_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00084_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00085_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00086_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00087_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00088_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00089_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00090_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00091_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00092_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00093_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00094_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00095_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00096_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00097_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00098_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00099_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00000_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00001_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00002_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00003_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00004_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00005_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00006_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00007_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00008_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00009_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00010_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00011_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00012_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00013_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00014_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00015_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00016_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00017_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00018_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00019_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00020_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00021_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00022_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00023_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00024_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00025_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00026_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00027_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00028_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00029_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00030_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00031_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00032_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00033_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00034_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00035_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00036_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00037_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00038_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00039_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00040_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00041_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00042_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00043_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00044_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00045_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00046_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00047_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00048_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00049_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00050_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00051_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00052_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00053_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00054_00000_doc_shuffled.ds'], 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 32, 'drop_last': True, 'pin_memory': True, 'prefetch_factor': 8, 'persistent_workers': True, 'timeout': 0, 'seed': None, 'instance_filter': None, 'custom_dataset': None}, 'restore_dataloader': True, 'fast_forward_batches': None, 'evaluators': [], 'eval_interval': 5000, 'tokenizer': {'identifier': 'meta-llama/Llama-2-7b-hf', 'truncate_direction': 'right'}, 'save_folder': 'checkpoints/OLMo-1B-as_fm3_omi2', 'remote_save_folder': None, 'canceled_check_interval': 6000, 'save_interval': 3000, 'save_interval_unsharded': 3000, 'save_interval_ephemeral': None, 'save_num_checkpoints_to_keep': -1, 'save_num_unsharded_checkpoints_to_keep': -1, 'save_overwrite': True, 'force_save_unsharded': False, 'no_pre_train_checkpoint': False, 'load_path': None, 'load_path_sharded_checkpointer': None, 'try_load_latest_save': False, 'reset_optimizer_state': False, 'reset_trainer_state': False, 'sharded_checkpointer': 'torch_legacy', 'new_style_checkpoints': None, 'max_duration': '1ep', 'global_train_batch_size': 512, 'device_train_batch_size': 512, 'device_train_microbatch_size': 16, 'device_eval_batch_size': 16, 'eval_subset_num_batches': -1, 'eval_on_load': False, 'device_train_grad_accum': 32, 'max_grad_norm': 1.0, 'max_grad_norm_ratio': None, 'precision': 'amp_bf16', 'speed_monitor': {'window_size': 20, 'gpu_flops_available': None}, 'console_log_interval': 1, 'gen1_gc_interval': 1, 'compile': None, 'distributed_strategy': 'fsdp', 'fsdp': {'use_orig_params': True, 'sharding_strategy': <ShardingStrategy.FULL_SHARD: 1>, 'wrapping_strategy': None, 'precision': 'mixed', 'hybrid_sharding_num_model_replicas': None}, 'ddp': {'grad_sync_mode': 'batch', 'find_unused_params': False}, 'single': {'device': 'auto'}, 'softmax_auxiliary_loss': False, 'auxiliary_loss_multiplier': 0.0001, 'time_limit': None, 'extra_steps_after_cancel': 10, 'early_stopping_factor': None, 'save_data_indices': True, 'python_profiling': False, 'torch_profiling': False, 'stop_at': None, 'stop_after': None, 'activation_checkpointing': None, 'fused_loss': None, 'hf_datasets_cache_dir': None, 'module_outputs_save_steps': None, '_wandb': {}}
-2025-12-14 15:37:01,016 INFO    MainThread:3745493 [wandb_init.py:init():889] starting backend
-2025-12-14 15:37:01,265 INFO    MainThread:3745493 [wandb_init.py:init():892] sending inform_init request
-2025-12-14 15:37:01,273 INFO    MainThread:3745493 [wandb_init.py:init():900] backend started and connected
-2025-12-14 15:37:01,276 INFO    MainThread:3745493 [wandb_init.py:init():970] updated telemetry
-2025-12-14 15:37:01,277 INFO    MainThread:3745493 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
-2025-12-14 15:37:02,024 INFO    MainThread:3745493 [wandb_init.py:init():1041] starting run threads in backend
-2025-12-14 15:37:02,115 INFO    MainThread:3745493 [wandb_run.py:_console_start():2521] atexit reg
-2025-12-14 15:37:02,116 INFO    MainThread:3745493 [wandb_run.py:_redirect():2369] redirect: wrap_raw
-2025-12-14 15:37:02,117 INFO    MainThread:3745493 [wandb_run.py:_redirect():2438] Wrapping output streams.
-2025-12-14 15:37:02,118 INFO    MainThread:3745493 [wandb_run.py:_redirect():2461] Redirects installed.
-2025-12-14 15:37:02,120 INFO    MainThread:3745493 [wandb_init.py:init():1081] run started, returning control to user process
-2025-12-14 15:37:06,118 INFO    wandb-AsyncioManager-main:3745493 [service_client.py:_forward_responses():80] Reached EOF.
-2025-12-14 15:37:06,120 INFO    wandb-AsyncioManager-main:3745493 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.

models/OLMo-1B/wandb/wandb/run-20251214_153731-i0ugjt0v/files/config.yaml

@@ -1,579 +0,0 @@
-_wandb:
-    value:
-        cli_version: 0.23.1
-        e:
-            mots0eu19y85h2hcucy8g5oum3x38ji3:
-                args:
-                    - pretraining/configs/RL-1B.yaml
-                codePath: OLMo/scripts/train.py
-                codePathLocal: OLMo/scripts/train.py
-                cpu_count: 112
-                cpu_count_logical: 224
-                cudaVersion: "13.0"
-                disk:
-                    /:
-                        total: "2055141851136"
-                        used: "49254457344"
-                email: [email protected]
-                executable: /opt/conda/bin/python
-                gpu: NVIDIA H100 80GB HBM3
-                gpu_count: 1
-                gpu_nvidia:
-                    - architecture: Hopper
-                      cudaCores: 16896
-                      memoryTotal: "85520809984"
-                      name: NVIDIA H100 80GB HBM3
-                      uuid: GPU-4c999b2a-2578-9e62-0539-4b826d85fda8
-                host: serv-3342
-                memory:
-                    total: "2164176814080"
-                os: Linux-6.8.0-1043-nvidia-x86_64-with-glibc2.35
-                program: /netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/scripts/train.py
-                python: CPython 3.11.11
-                root: checkpoints/OLMo-1B-as_fm3_omi2/wandb
-                slurm:
-                    cluster_name: pegasus
-                    conf: /etc/slurm/slurm.conf
-                    cpu_bind: quiet,mask_cpu:0x000000000001FE00000000000000000000000001FE00000000000000
-                    cpu_bind_list: 0x000000000001FE00000000000000000000000001FE00000000000000
-                    cpu_bind_type: 'mask_cpu:'
-                    cpu_bind_verbose: quiet
-                    cpus_on_node: "16"
-                    cpus_per_task: "16"
-                    distribution: cyclic
-                    gpus: "1"
-                    gpus_on_node: "1"
-                    gtids: "0"
-                    job_cpus_per_node: "16"
-                    job_end_time: "1765734524"
-                    job_gid: "8000"
-                    job_group: iml
-                    job_id: "2383756"
-                    job_name: bash
-                    job_nodelist: serv-3342
-                    job_num_nodes: "1"
-                    job_partition: H100
-                    job_qos: normal
-                    job_start_time: "1765720124"
-                    job_uid: "13262"
-                    job_user: nguyen
-                    jobid: "2383756"
-                    launch_node_ipaddr: 192.168.33.114
-                    localid: "0"
-                    mem_per_cpu: "16384"
-                    mpi_type: pmix
-                    nnodes: "1"
-                    nodeid: "0"
-                    nodelist: serv-3342
-                    nprocs: "1"
-                    ntasks: "1"
-                    oom_kill_step: "0"
-                    pmix_mapping_serv: (vector,(0,1,1))
-                    pmixp_abort_agent_port: "33735"
-                    prio_process: "1"
-                    procid: "0"
-                    pty_port: "45219"
-                    pty_win_col: "156"
-                    pty_win_row: "41"
-                    srun_comm_host: 192.168.33.114
-                    srun_comm_port: "35153"
-                    step_gpus: "5"
-                    step_id: "0"
-                    step_launcher_port: "35153"
-                    step_nodelist: serv-3342
-                    step_num_nodes: "1"
-                    step_num_tasks: "1"
-                    step_tasks_per_node: "1"
-                    stepid: "0"
-                    submit_dir: /netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain
-                    submit_host: login1
-                    task_pid: "3684902"
-                    tasks_per_node: "1"
-                    topology_addr: serv-3342
-                    topology_addr_pattern: node
-                    tres_bind: gres/gpu:per_task:1
-                    tres_per_task: cpu=16,gres/gpu=1
-                    umask: "0022"
-                startedAt: "2025-12-14T14:37:31.123422Z"
-                writerId: mots0eu19y85h2hcucy8g5oum3x38ji3
-        m: []
-        python_version: 3.11.11
-        t:
-            "1":
-                - 1
-                - 5
-                - 11
-                - 41
-                - 49
-                - 51
-                - 53
-            "2":
-                - 1
-                - 5
-                - 11
-                - 41
-                - 49
-                - 51
-                - 53
-            "3":
-                - 2
-                - 13
-                - 15
-                - 16
-                - 61
-            "4": 3.11.11
-            "5": 0.23.1
-            "6": 4.57.3
-            "10":
-                - 19
-            "12": 0.23.1
-            "13": linux-x86_64
-activation_checkpointing:
-    value: null
-auxiliary_loss_multiplier:
-    value: 0.0001
-canceled_check_interval:
-    value: 6000
-compile:
-    value: null
-console_log_interval:
-    value: 1
-data:
-    value:
-        custom_dataset: null
-        datasets: null
-        drop_last: true
-        generate_attention_mask: false
-        generate_doc_lengths: false
-        instance_filter: null
-        label_mask_paths: null
-        memmap_dtype: uint16
-        num_workers: 32
-        pad_direction: right
-        paths:
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00000_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00001_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00002_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00003_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00004_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00005_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00006_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00007_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00008_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00009_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00010_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00011_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00012_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00016_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00019_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00020_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00021_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00022_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00023_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00024_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00025_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00026_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00027_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00028_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00029_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00030_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00031_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00032_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00033_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00034_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00035_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00036_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00037_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00038_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00039_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00040_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00041_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00042_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00043_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00044_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00045_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00046_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00047_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00048_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00049_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00050_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00051_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00052_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00053_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00054_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00055_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00056_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00057_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00058_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00059_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00060_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00061_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00062_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00063_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00064_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00065_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00066_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00067_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00068_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00069_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00070_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00071_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00072_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00073_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00074_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00075_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00076_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00077_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00078_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00000_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00001_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00002_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00003_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00004_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00005_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00006_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00007_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00008_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00009_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00010_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00011_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00012_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00013_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00014_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00015_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00016_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00017_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00018_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00019_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00020_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00021_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00022_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00023_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00024_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00025_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00026_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00027_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00028_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00029_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00030_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00031_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00032_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00033_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00034_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00035_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00036_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00037_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00038_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00039_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00040_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00041_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00042_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00043_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00044_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00045_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00046_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00047_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00048_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00049_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00050_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00051_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00052_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00053_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00054_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00055_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00056_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00057_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00058_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00059_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00060_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00061_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00062_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00063_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00064_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00065_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00066_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00067_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00068_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00069_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00070_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00071_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00072_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00073_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00074_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00075_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00076_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00077_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00078_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00079_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00080_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00081_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00082_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00083_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00084_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00085_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00086_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00087_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00088_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00089_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00090_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00091_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00092_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00093_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00094_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00095_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00096_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00097_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00098_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00099_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00000_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00001_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00002_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00003_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00004_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00005_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00006_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00007_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00008_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00009_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00010_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00011_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00012_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00013_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00014_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00015_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00016_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00017_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00018_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00019_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00020_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00021_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00022_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00023_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00024_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00025_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00026_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00027_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00028_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00029_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00030_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00031_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00032_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00033_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00034_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00035_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00036_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00037_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00038_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00039_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00040_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00041_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00042_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00043_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00044_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00045_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00046_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00047_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00048_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00049_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00050_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00051_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00052_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00053_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00054_00000_doc_shuffled.ds
-        persistent_workers: true
-        pin_memory: true
-        prefetch_factor: 8
-        seed: null
-        timeout: 0
-ddp:
-    value:
-        find_unused_params: false
-        grad_sync_mode: batch
-device_eval_batch_size:
-    value: 16
-device_train_batch_size:
-    value: 512
-device_train_grad_accum:
-    value: 32
-device_train_microbatch_size:
-    value: 16
-distributed_strategy:
-    value: fsdp
-dry_run:
-    value: false
-early_stopping_factor:
-    value: null
-epoch:
-    value: null
-eval_interval:
-    value: 5000
-eval_on_load:
-    value: false
-eval_subset_num_batches:
-    value: -1
-evaluators:
-    value: []
-extra_steps_after_cancel:
-    value: 10
-fast_forward_batches:
-    value: null
-force_save_unsharded:
-    value: false
-fsdp:
-    value:
-        hybrid_sharding_num_model_replicas: null
-        precision: mixed
-        sharding_strategy: FULL_SHARD
-        use_orig_params: true
-        wrapping_strategy: null
-fused_loss:
-    value: null
-gen1_gc_interval:
-    value: 1
-global_train_batch_size:
-    value: 512
-hf_datasets_cache_dir:
-    value: null
-load_path:
-    value: null
-load_path_sharded_checkpointer:
-    value: null
-max_duration:
-    value: 1ep
-max_grad_norm:
-    value: 1
-max_grad_norm_ratio:
-    value: null
-model:
-    value:
-        activation_type: swiglu
-        alibi: false
-        alibi_bias_max: 8
-        attention_dropout: 0
-        attention_layer_norm: false
-        attention_layer_norm_with_affine: false
-        bias_for_layer_norm: false
-        block_group_size: 1
-        block_type: sequential
-        clip_qkv: null
-        d_model: 2048
-        emb_init_std: null
-        embedding_dropout: 0
-        embedding_layer_norm: false
-        embedding_size: 32000
-        eos_token_id: 0
-        flash_attention: false
-        include_bias: false
-        init_cutoff_factor: 3
-        init_device: cuda
-        init_fn: normal
-        init_std: 0.02
-        layer_norm_eps: 1e-05
-        layer_norm_type: default
-        layer_norm_with_affine: false
-        max_sequence_length: 2048
-        mlp_hidden_size: null
-        mlp_ratio: 8
-        multi_query_attention: false
-        n_heads: 16
-        n_kv_heads: null
-        n_layers: 16
-        norm_after: false
-        pad_token_id: 1
-        precision: amp_bf16
-        residual_dropout: 0
-        rope: true
-        rope_full_precision: true
-        rope_theta: 10000
-        scale_emb_init: false
-        scale_logits: false
-        vocab_size: 32000
-        weight_tying: true
-module_outputs_save_steps:
-    value: null
-new_style_checkpoints:
-    value: null
-no_pre_train_checkpoint:
-    value: false
-optimizer:
-    value:
-        betas:
-            - 0.9
-            - 0.95
-        decay_embeddings: true
-        decay_norm_and_bias: true
-        eps: 1e-08
-        learning_rate: 0.0005
-        metrics_log_interval: 10
-        name: adamw
-        no_decay_norm_and_bias: null
-        record_update_metrics: false
-        selective_updates: false
-        weight_decay: 0.1
-precision:
-    value: amp_bf16
-python_profiling:
-    value: false
-remote_save_folder:
-    value: null
-reset_optimizer_state:
-    value: false
-reset_trainer_state:
-    value: false
-restore_dataloader:
-    value: true
-run_name:
-    value: OLMo-1B-as_fm3_omi2
-save_data_indices:
-    value: true
-save_folder:
-    value: checkpoints/OLMo-1B-as_fm3_omi2
-save_interval:
-    value: 3000
-save_interval_ephemeral:
-    value: null
-save_interval_unsharded:
-    value: 3000
-save_num_checkpoints_to_keep:
-    value: -1
-save_num_unsharded_checkpoints_to_keep:
-    value: -1
-save_overwrite:
-    value: true
-scheduler:
-    value:
-        alpha_f: 0.1
-        grad_clip_warmup_factor: null
-        grad_clip_warmup_steps: null
-        name: cosine_with_warmup
-        t_max: null
-        t_warmup: 2000
-        units: steps
-        warmup_min_lr: 0
-seed:
-    value: 6198
-sharded_checkpointer:
-    value: torch_legacy
-single:
-    value:
-        device: auto
-softmax_auxiliary_loss:
-    value: false
-speed_monitor:
-    value:
-        gpu_flops_available: null
-        window_size: 20
-stop_after:
-    value: null
-stop_at:
-    value: null
-time_limit:
-    value: null
-tokenizer:
-    value:
-        identifier: meta-llama/Llama-2-7b-hf
-        truncate_direction: right
-torch_profiling:
-    value: false
-try_load_latest_save:
-    value: false

models/OLMo-1B/wandb/wandb/run-20251214_153731-i0ugjt0v/files/output.log

@@ -1,71 +0,0 @@
-[2025-12-14 15:37:34] INFO     [olmo.data.iterable_dataset:79, rank=0] Saving global data order indices...
-[2025-12-14 15:37:35] INFO     [olmo.data.iterable_dataset:88, rank=0] Global data order indices saved to 'checkpoints/OLMo-1B-as_fm3_omi2/train_data/global_indices.npy'
-[2025-12-14 15:37:35] WARNING  [py.warnings:110, rank=0] /opt/conda/lib/python3.11/site-packages/torch/utils/data/dataloader.py:624: UserWarning: This DataLoader will create 32 worker processes in total. Our suggested max number of worker in current system is 16, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.
-  warnings.warn(
-tokenizer.json: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████| 1.84M/1.84M [00:00<00:00, 9.47MB/s]
-
-[2025-12-14 15:37:36] INFO     [train:139, rank=0] Building model...
-[2025-12-14 15:37:36] INFO     [olmo.model:1174, rank=0] Initializing model parameters...
-[2025-12-14 15:37:36] INFO     [train:141, rank=0] Total number of parameters: 1,139,277,824
-[2025-12-14 15:37:36] INFO     [train:142, rank=0] Number of non-embedding parameters: 1,073,741,824
-[2025-12-14 15:37:36] INFO     [train:143, rank=0] Peak GPU Memory (MB) before fsdp: 4559
-[2025-12-14 15:37:36] INFO     [train:172, rank=0] Wrapping model with FSDP...
-[2025-12-14 15:37:36] WARNING  [py.warnings:110, rank=0] /opt/conda/lib/python3.11/site-packages/torch/distributed/fsdp/_init_utils.py:444: UserWarning: FSDP is switching to use `NO_SHARD` instead of ShardingStrategy.FULL_SHARD since the world size is 1.
-  warnings.warn(
-
-[2025-12-14 15:37:36] INFO     [olmo.model:1174, rank=0] Initializing model parameters...
-[2025-12-14 15:37:36] INFO     [train:232, rank=0] Peak GPU Memory (MB) after fsdp: 9116
-[2025-12-14 15:37:36] INFO     [train:233, rank=0] Model:
-[2025-12-14 15:37:36] INFO     [train:234, rank=0] FullyShardedDataParallel(
-  (_fsdp_wrapped_module): OLMo(
-    (transformer): ModuleDict(
-      (wte): Embedding(32000, 2048)
-      (emb_drop): Dropout(p=0.0, inplace=False)
-      (ln_f): LayerNorm()
-      (blocks): ModuleList(
-        (0-15): 16 x OLMoSequentialBlock(
-          (dropout): Dropout(p=0.0, inplace=False)
-          (act): SwiGLU()
-          (attn_out): Linear(in_features=2048, out_features=2048, bias=False)
-          (ff_out): Linear(in_features=8192, out_features=2048, bias=False)
-          (rotary_emb): RotaryEmbedding()
-          (att_proj): Linear(in_features=2048, out_features=6144, bias=False)
-          (ff_proj): Linear(in_features=2048, out_features=16384, bias=False)
-          (attn_norm): LayerNorm()
-          (ff_norm): LayerNorm()
-        )
-      )
-    )
-  )
-)
-[2025-12-14 15:37:36] INFO     [olmo.optim:944, rank=0] Constructing optimizer with 1 param groups
-[2025-12-14 15:37:36] INFO     [train:335, rank=0] Saving pre-train checkpoint...
-[2025-12-14 15:37:36] WARNING  [py.warnings:110, rank=0] /opt/conda/lib/python3.11/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
-  warnings.warn(
-
-[2025-12-14 15:37:36] WARNING  [py.warnings:110, rank=0] /opt/conda/lib/python3.11/site-packages/torch/distributed/fsdp/_state_dict_utils.py:773: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned.
-  warnings.warn(
-
-[2025-12-14 15:37:36] WARNING  [py.warnings:110, rank=0] /opt/conda/lib/python3.11/site-packages/torch/distributed/fsdp/_state_dict_utils.py:711: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned.
-  warnings.warn(
-
-[2025-12-14 15:37:43] INFO     [olmo.checkpoint:607, rank=0] Saving config...
-[2025-12-14 15:37:44] INFO     [train:337, rank=0] Checkpoint saved to checkpoints/OLMo-1B-as_fm3_omi2/step0
-[2025-12-14 15:37:44] INFO     [train:340, rank=0] Attempting to load pre-train checkpoint...
-[2025-12-14 15:37:47] INFO     [olmo.checkpoint:1040, rank=0] Loading model state...
-[2025-12-14 15:37:47] WARNING  [py.warnings:110, rank=0] /opt/conda/lib/python3.11/site-packages/torch/distributed/fsdp/_state_dict_utils.py:827: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned.
-  warnings.warn(
-
-[2025-12-14 15:37:48] WARNING  [py.warnings:110, rank=0] /opt/conda/lib/python3.11/site-packages/torch/distributed/fsdp/_state_dict_utils.py:864: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned.
-  warnings.warn(
-
-[2025-12-14 15:37:48] INFO     [olmo.checkpoint:1044, rank=0] Loading optimizer state...
-[2025-12-14 15:37:48] INFO     [olmo.checkpoint:220, rank=0] Flattening sharded optimizer state...
-[2025-12-14 15:37:48] INFO     [olmo.checkpoint:234, rank=0] Loading flattened optimizer state...
-[2025-12-14 15:37:48] INFO     [olmo.train:409, rank=0] Resetting learning rate...
-[2025-12-14 15:37:48] INFO     [olmo.train:421, rank=0] Restoring RNG states...
-[2025-12-14 15:37:48] INFO     [train:344, rank=0] Checkpoint successfully loaded
-[2025-12-14 15:37:48] INFO     [train:375, rank=0] Starting training...
-[2025-12-14 15:37:48] INFO     [olmo.train:979, rank=0] Pre-train system metrics
-    System/Peak GPU Memory (MB)=9,116
-wandb: WARNING The `quiet` argument to `wandb.run.finish()` is deprecated, use `wandb.Settings(quiet=...)` to set this instead.

models/OLMo-1B/wandb/wandb/run-20251214_153731-i0ugjt0v/files/requirements.txt

@@ -1,271 +0,0 @@
-scikit-learn==1.8.0
-joblib==1.5.2
-threadpoolctl==3.6.0
-torchmetrics==1.8.2
-lightning-utilities==0.15.2
-wandb==0.23.1
-GitPython==3.1.45
-gitdb==4.0.12
-smmap==5.0.2
-datasets==4.4.1
-pandas==2.3.3
-multiprocess==0.70.18
-pyarrow==22.0.0
-tzdata==2025.3
-xxhash==3.6.0
-ai2-olmo==0.6.0
-ai2-olmo-core==2.4.0
-cached_path==1.8.0
-google-cloud-storage==2.19.0
-google-cloud-core==2.5.0
-boto3==1.42.9
-google-api-core==2.28.1
-google-auth==2.43.0
-s3transfer==0.16.0
-botocore==1.42.9
-google-resumable-media==2.8.0
-pyasn1_modules==0.4.2
-rich==13.9.4
-rsa==4.9.1
-bettermap==1.3.1
-google-crc32c==1.7.1
-jmespath==1.0.1
-numpy==1.26.4
-omegaconf==2.3.0
-proto-plus==1.26.1
-pyasn1==0.6.1
-python-dateutil==2.9.0.post0
-antlr4-python3-runtime==4.9.3
-zstandard==0.23.0
-zipp==3.21.0
-yarl==1.22.0
-xgrammar==0.1.18
-xformers==0.0.29.post2
-wrapt==2.0.1
-wheel==0.45.1
-websockets==15.0.1
-wcwidth==0.2.13
-watchfiles==1.1.1
-vllm==0.8.5.post1
-uvloop==0.22.1
-uvicorn==0.38.0
-urllib3==2.3.0
-typing-inspection==0.4.2
-typing_extensions==4.15.0
-types-dataclasses==0.6.6
-typer==0.20.0
-truststore==0.10.0
-triton==3.2.0
-transformers==4.57.3
-traitlets==5.14.3
-tqdm==4.67.1
-torchvision==0.21.0+cu124
-torchelastic==0.2.2
-torchaudio==2.6.0+cu124
-torch==2.6.0+cu124
-tokenizers==0.22.1
-tiktoken==0.12.0
-sympy==1.13.1
-starlette==0.50.0
-stack_data==0.6.3
-soupsieve==2.5
-sortedcontainers==2.4.0
-sniffio==1.3.1
-six==1.17.0
-shellingham==1.5.4
-setuptools==75.8.0
-sentry-sdk==2.47.0
-sentencepiece==0.2.1
-scipy==1.16.3
-safetensors==0.7.0
-ruamel.yaml.clib==0.2.8
-ruamel.yaml==0.18.10
-rpds-py==0.22.3
-rignore==0.7.6
-rich-toolkit==0.17.0
-requests==2.32.3
-regex==2025.11.3
-referencing==0.36.2
-ray==2.52.1
-pyzmq==27.1.0
-pytz==2024.2
-python-multipart==0.0.20
-python-json-logger==4.0.0
-python-etcd==0.4.5
-python-dotenv==1.2.1
-Pygments==2.19.1
-pydantic-extra-types==2.10.6
-pydantic_core==2.41.5
-pydantic==2.12.5
-pycparser==2.22
-pycountry==24.6.1
-pycosat==0.6.6
-py-cpuinfo==9.0.0
-pure_eval==0.2.3
-ptyprocess==0.7.0
-psutil==6.1.1
-protobuf==4.25.8
-propcache==0.4.1
-prompt_toolkit==3.0.50
-prometheus-fastapi-instrumentator==7.1.0
-prometheus_client==0.23.1
-pluggy==1.5.0
-platformdirs==4.3.6
-pkgutil_resolve_name==1.3.10
-pkginfo==1.12.0
-pip==24.3.1
-pillow==11.0.0
-pickleshare==0.7.5
-pexpect==4.9.0
-partial-json-parser==0.2.1.1.post7
-parso==0.8.4
-packaging==24.2
-outlines_core==0.1.26
-outlines==0.1.11
-optree==0.14.0
-opentelemetry-semantic-conventions-ai==0.4.13
-opentelemetry-semantic-conventions==0.47b0
-opentelemetry-sdk==1.26.0
-opentelemetry-proto==1.26.0
-opentelemetry-exporter-otlp-proto-http==1.26.0
-opentelemetry-exporter-otlp-proto-grpc==1.26.0
-opentelemetry-exporter-otlp-proto-common==1.26.0
-opentelemetry-exporter-otlp==1.26.0
-opentelemetry-api==1.26.0
-opencv-python-headless==4.12.0.88
-openai==2.11.0
-nvidia-nvtx-cu12==12.4.127
-nvidia-nvjitlink-cu12==12.4.127
-nvidia-nccl-cu12==2.21.5
-nvidia-cusparselt-cu12==0.6.2
-nvidia-cusparse-cu12==12.3.1.170
-nvidia-cusolver-cu12==11.6.1.9
-nvidia-curand-cu12==10.3.5.147
-nvidia-cufft-cu12==11.2.1.3
-nvidia-cudnn-cu12==9.1.0.70
-nvidia-cuda-runtime-cu12==12.4.127
-nvidia-cuda-nvrtc-cu12==12.4.127
-nvidia-cuda-cupti-cu12==12.4.127
-nvidia-cublas-cu12==12.4.5.8
-numba==0.61.2
-ninja==1.11.1.3
-networkx==3.4.2
-nest-asyncio==1.6.0
-multidict==6.7.0
-msgspec==0.20.0
-msgpack==1.1.2
-mpmath==1.3.0
-more-itertools==10.6.0
-mistral_common==1.8.6
-menuinst==2.2.0
-mdurl==0.1.2
-matplotlib-inline==0.1.7
-math-verify==0.8.0
-markdown-it-py==4.0.0
-lm-format-enforcer==0.10.12
-llvmlite==0.44.0
-llguidance==0.7.30
-lintrunner==0.12.7
-lief==0.14.1
-libmambapy==2.0.5
-libarchive-c==5.1
-latex2sympy2_extended==1.10.2
-lark==1.2.2
-jsonschema-specifications==2024.10.1
-jsonschema==4.23.0
-jsonpointer==3.0.0
-jsonpatch==1.33
-jiter==0.12.0
-Jinja2==3.1.5
-jedi==0.19.2
-ipython==8.31.0
-interegular==0.3.3
-importlib_resources==6.5.2
-importlib_metadata==8.0.0
-idna==3.10
-hypothesis==6.124.7
-hyperframe==6.0.1
-huggingface-hub==0.36.0
-httpx==0.28.1
-httptools==0.7.1
-httpcore==1.0.9
-hpack==4.0.0
-hf-xet==1.2.0
-h2==4.1.0
-h11==0.16.0
-grpcio==1.76.0
-googleapis-common-protos==1.72.0
-gguf==0.17.1
-fsspec==2024.12.0
-frozenlist==1.8.0
-frozendict==2.4.6
-filelock==3.17.0
-fastrlock==0.8.3
-fastar==0.8.0
-fastapi-cloud-cli==0.6.0
-fastapi-cli==0.0.16
-fastapi==0.124.4
-expecttest==0.3.0
-executing==2.1.0
-exceptiongroup==1.2.2
-email-validator==2.3.0
-einops==0.8.1
-dnspython==2.7.0
-distro==1.9.0
-diskcache==5.6.3
-dill==0.4.0
-depyf==0.18.0
-Deprecated==1.3.1
-decorator==5.1.1
-cupy-cuda12x==13.6.0
-conda_package_streaming==0.11.0
-conda-package-handling==2.4.0
-conda-libmamba-solver==25.1.1
-conda_index==0.5.0
-conda-build==25.1.1
-conda==25.1.0
-compressed-tensors==0.9.3
-colorama==0.4.6
-cmake==3.31.4
-cloudpickle==3.1.2
-click==8.1.8
-charset-normalizer==3.4.1
-chardet==5.2.0
-cffi==1.17.1
-certifi==2024.12.14
-cachetools==6.2.3
-boltons==24.0.0
-blake3==1.0.8
-beautifulsoup4==4.12.3
-attrs==25.1.0
-astunparse==1.6.3
-asttokens==3.0.0
-astor==0.8.1
-archspec==0.2.5
-anyio==4.12.0
-annotated-types==0.7.0
-annotated-doc==0.0.4
-airportsdata==20250909
-aiosignal==1.4.0
-aiohttp==3.13.2
-aiohappyeyeballs==2.6.1
-PyYAML==6.0.2
-PySocks==1.7.1
-MarkupSafe==3.0.2
-Brotli==1.1.0
-zipp==3.19.2
-wheel==0.43.0
-typing_extensions==4.12.2
-typeguard==4.3.0
-tomli==2.0.1
-platformdirs==4.2.2
-packaging==24.2
-more-itertools==10.3.0
-jaraco.text==3.12.1
-jaraco.functools==4.0.1
-jaraco.context==5.3.0
-jaraco.collections==5.1.0
-inflect==7.3.1
-importlib_metadata==8.0.0
-backports.tarfile==1.2.0
-autocommand==2.2.2

models/OLMo-1B/wandb/wandb/run-20251214_153731-i0ugjt0v/files/wandb-metadata.json

@@ -1,103 +0,0 @@
-{
-  "os":  "Linux-6.8.0-1043-nvidia-x86_64-with-glibc2.35",
-  "python":  "CPython 3.11.11",
-  "startedAt":  "2025-12-14T14:37:31.123422Z",
-  "args":  [
-    "pretraining/configs/RL-1B.yaml"
-  ],
-  "program":  "/netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/scripts/train.py",
-  "codePath":  "OLMo/scripts/train.py",
-  "codePathLocal":  "OLMo/scripts/train.py",
-  "email":  "[email protected]",
-  "root":  "checkpoints/OLMo-1B-as_fm3_omi2/wandb",
-  "host":  "serv-3342",
-  "executable":  "/opt/conda/bin/python",
-  "cpu_count":  112,
-  "cpu_count_logical":  224,
-  "gpu":  "NVIDIA H100 80GB HBM3",
-  "gpu_count":  1,
-  "disk":  {
-    "/":  {
-      "total":  "2055141851136",
-      "used":  "49254457344"
-    }
-  },
-  "memory":  {
-    "total":  "2164176814080"
-  },
-  "gpu_nvidia":  [
-    {
-      "name":  "NVIDIA H100 80GB HBM3",
-      "memoryTotal":  "85520809984",
-      "cudaCores":  16896,
-      "architecture":  "Hopper",
-      "uuid":  "GPU-4c999b2a-2578-9e62-0539-4b826d85fda8"
-    }
-  ],
-  "cudaVersion":  "13.0",
-  "slurm":  {
-    "cluster_name":  "pegasus",
-    "conf":  "/etc/slurm/slurm.conf",
-    "cpu_bind":  "quiet,mask_cpu:0x000000000001FE00000000000000000000000001FE00000000000000",
-    "cpu_bind_list":  "0x000000000001FE00000000000000000000000001FE00000000000000",
-    "cpu_bind_type":  "mask_cpu:",
-    "cpu_bind_verbose":  "quiet",
-    "cpus_on_node":  "16",
-    "cpus_per_task":  "16",
-    "distribution":  "cyclic",
-    "gpus":  "1",
-    "gpus_on_node":  "1",
-    "gtids":  "0",
-    "job_cpus_per_node":  "16",
-    "job_end_time":  "1765734524",
-    "job_gid":  "8000",
-    "job_group":  "iml",
-    "job_id":  "2383756",
-    "job_name":  "bash",
-    "job_nodelist":  "serv-3342",
-    "job_num_nodes":  "1",
-    "job_partition":  "H100",
-    "job_qos":  "normal",
-    "job_start_time":  "1765720124",
-    "job_uid":  "13262",
-    "job_user":  "nguyen",
-    "jobid":  "2383756",
-    "launch_node_ipaddr":  "192.168.33.114",
-    "localid":  "0",
-    "mem_per_cpu":  "16384",
-    "mpi_type":  "pmix",
-    "nnodes":  "1",
-    "nodeid":  "0",
-    "nodelist":  "serv-3342",
-    "nprocs":  "1",
-    "ntasks":  "1",
-    "oom_kill_step":  "0",
-    "pmix_mapping_serv":  "(vector,(0,1,1))",
-    "pmixp_abort_agent_port":  "33735",
-    "prio_process":  "1",
-    "procid":  "0",
-    "pty_port":  "45219",
-    "pty_win_col":  "156",
-    "pty_win_row":  "41",
-    "srun_comm_host":  "192.168.33.114",
-    "srun_comm_port":  "35153",
-    "step_gpus":  "5",
-    "step_id":  "0",
-    "step_launcher_port":  "35153",
-    "step_nodelist":  "serv-3342",
-    "step_num_nodes":  "1",
-    "step_num_tasks":  "1",
-    "step_tasks_per_node":  "1",
-    "stepid":  "0",
-    "submit_dir":  "/netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain",
-    "submit_host":  "login1",
-    "task_pid":  "3684902",
-    "tasks_per_node":  "1",
-    "topology_addr":  "serv-3342",
-    "topology_addr_pattern":  "node",
-    "tres_bind":  "gres/gpu:per_task:1",
-    "tres_per_task":  "cpu=16,gres/gpu=1",
-    "umask":  "0022"
-  },
-  "writerId":  "mots0eu19y85h2hcucy8g5oum3x38ji3"
-}

models/OLMo-1B/wandb/wandb/run-20251214_153731-i0ugjt0v/files/wandb-summary.json

@@ -1 +0,0 @@
-{"System/Peak GPU Memory (MB)":9116.3232421875,"_timestamp":1.765723068840134e+09,"_wandb":{"runtime":30},"_runtime":30.93714515,"_step":0}

models/OLMo-1B/wandb/wandb/run-20251214_153731-i0ugjt0v/logs/debug-internal.log

@@ -1,12 +0,0 @@
-{"time":"2025-12-14T15:37:31.412986217+01:00","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
-{"time":"2025-12-14T15:37:31.680606576+01:00","level":"INFO","msg":"stream: created new stream","id":"i0ugjt0v"}
-{"time":"2025-12-14T15:37:31.683856288+01:00","level":"INFO","msg":"handler: started","stream_id":"i0ugjt0v"}
-{"time":"2025-12-14T15:37:31.686578199+01:00","level":"INFO","msg":"stream: started","id":"i0ugjt0v"}
-{"time":"2025-12-14T15:37:31.686601008+01:00","level":"INFO","msg":"writer: started","stream_id":"i0ugjt0v"}
-{"time":"2025-12-14T15:37:31.686623633+01:00","level":"INFO","msg":"sender: started","stream_id":"i0ugjt0v"}
-{"time":"2025-12-14T15:38:03.558761809+01:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
-{"time":"2025-12-14T15:38:03.861133675+01:00","level":"INFO","msg":"handler: operation stats","stats":{}}
-{"time":"2025-12-14T15:38:03.86751239+01:00","level":"INFO","msg":"stream: closing","id":"i0ugjt0v"}
-{"time":"2025-12-14T15:38:03.870419678+01:00","level":"INFO","msg":"handler: closed","stream_id":"i0ugjt0v"}
-{"time":"2025-12-14T15:38:03.87246051+01:00","level":"INFO","msg":"sender: closed","stream_id":"i0ugjt0v"}
-{"time":"2025-12-14T15:38:03.873159055+01:00","level":"INFO","msg":"stream: closed","id":"i0ugjt0v"}

models/OLMo-1B/wandb/wandb/run-20251214_153731-i0ugjt0v/logs/debug.log

@@ -1,26 +0,0 @@
-2025-12-14 15:37:31,149 INFO    MainThread:3746193 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
-2025-12-14 15:37:31,151 INFO    MainThread:3746193 [wandb_setup.py:_flush():80] Configure stats pid to 3746193
-2025-12-14 15:37:31,151 INFO    MainThread:3746193 [wandb_setup.py:_flush():80] Loading settings from /root/.config/wandb/settings
-2025-12-14 15:37:31,152 INFO    MainThread:3746193 [wandb_setup.py:_flush():80] Loading settings from /netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/wandb/settings
-2025-12-14 15:37:31,153 INFO    MainThread:3746193 [wandb_setup.py:_flush():80] Loading settings from environment variables
-2025-12-14 15:37:31,154 INFO    MainThread:3746193 [wandb_init.py:setup_run_log_directory():714] Logging user logs to checkpoints/OLMo-1B-as_fm3_omi2/wandb/wandb/run-20251214_153731-i0ugjt0v/logs/debug.log
-2025-12-14 15:37:31,156 INFO    MainThread:3746193 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to checkpoints/OLMo-1B-as_fm3_omi2/wandb/wandb/run-20251214_153731-i0ugjt0v/logs/debug-internal.log
-2025-12-14 15:37:31,159 INFO    MainThread:3746193 [wandb_init.py:init():841] calling init triggers
-2025-12-14 15:37:31,159 INFO    MainThread:3746193 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
-config: {'run_name': 'OLMo-1B-as_fm3_omi2', 'seed': 6198, 'epoch': None, 'dry_run': False, 'model': {'d_model': 2048, 'n_heads': 16, 'n_kv_heads': None, 'clip_qkv': None, 'n_layers': 16, 'mlp_ratio': 8, 'mlp_hidden_size': None, 'activation_type': 'swiglu', 'block_type': 'sequential', 'block_group_size': 1, 'alibi': False, 'alibi_bias_max': 8.0, 'rope': True, 'rope_full_precision': True, 'rope_theta': 10000, 'flash_attention': False, 'attention_dropout': 0.0, 'multi_query_attention': False, 'attention_layer_norm': False, 'residual_dropout': 0.0, 'embedding_dropout': 0.0, 'embedding_layer_norm': False, 'layer_norm_type': 'default', 'layer_norm_with_affine': False, 'layer_norm_eps': 1e-05, 'attention_layer_norm_with_affine': False, 'max_sequence_length': 2048, 'include_bias': False, 'bias_for_layer_norm': False, 'scale_logits': False, 'vocab_size': 32000, 'embedding_size': 32000, 'weight_tying': True, 'eos_token_id': 0, 'pad_token_id': 1, 'init_device': 'cuda', 'init_fn': 'normal', 'init_std': 0.02, 'init_cutoff_factor': 3.0, 'precision': 'amp_bf16', 'scale_emb_init': False, 'emb_init_std': None, 'norm_after': False}, 'optimizer': {'name': 'adamw', 'learning_rate': 0.0005, 'weight_decay': 0.1, 'betas': (0.9, 0.95), 'eps': 1e-08, 'no_decay_norm_and_bias': None, 'selective_updates': False, 'decay_norm_and_bias': True, 'decay_embeddings': True, 'metrics_log_interval': 10, 'record_update_metrics': False}, 'scheduler': {'name': 'cosine_with_warmup', 'units': 'steps', 't_warmup': 2000, 't_max': None, 'alpha_f': 0.1, 'grad_clip_warmup_steps': None, 'grad_clip_warmup_factor': None, 'warmup_min_lr': 0.0}, 'data': {'paths': ['data_token/as_fm3_omi2/algebraic-stack-tokenized/00000_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00001_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00002_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00003_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00004_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00005_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00006_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00007_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00008_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00009_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00010_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00011_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00012_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00016_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00019_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00020_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00021_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00022_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00023_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00024_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00025_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00026_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00027_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00028_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00029_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00030_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00031_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00032_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00033_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00034_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00035_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00036_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00037_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00038_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00039_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00040_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00041_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00042_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00043_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00044_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00045_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00046_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00047_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00048_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00049_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00050_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00051_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00052_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00053_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00054_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00055_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00056_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00057_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00058_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00059_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00060_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00061_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00062_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00063_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00064_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00065_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00066_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00067_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00068_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00069_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00070_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00071_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00072_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00073_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00074_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00075_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00076_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00077_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00078_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00000_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00001_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00002_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00003_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00004_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00005_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00006_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00007_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00008_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00009_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00010_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00011_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00012_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00013_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00014_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00015_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00016_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00017_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00018_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00019_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00020_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00021_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00022_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00023_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00024_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00025_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00026_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00027_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00028_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00029_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00030_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00031_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00032_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00033_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00034_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00035_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00036_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00037_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00038_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00039_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00040_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00041_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00042_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00043_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00044_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00045_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00046_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00047_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00048_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00049_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00050_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00051_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00052_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00053_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00054_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00055_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00056_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00057_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00058_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00059_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00060_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00061_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00062_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00063_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00064_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00065_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00066_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00067_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00068_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00069_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00070_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00071_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00072_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00073_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00074_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00075_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00076_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00077_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00078_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00079_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00080_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00081_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00082_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00083_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00084_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00085_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00086_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00087_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00088_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00089_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00090_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00091_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00092_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00093_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00094_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00095_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00096_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00097_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00098_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00099_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00000_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00001_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00002_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00003_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00004_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00005_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00006_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00007_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00008_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00009_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00010_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00011_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00012_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00013_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00014_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00015_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00016_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00017_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00018_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00019_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00020_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00021_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00022_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00023_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00024_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00025_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00026_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00027_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00028_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00029_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00030_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00031_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00032_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00033_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00034_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00035_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00036_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00037_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00038_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00039_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00040_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00041_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00042_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00043_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00044_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00045_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00046_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00047_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00048_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00049_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00050_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00051_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00052_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00053_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00054_00000_doc_shuffled.ds'], 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 32, 'drop_last': True, 'pin_memory': True, 'prefetch_factor': 8, 'persistent_workers': True, 'timeout': 0, 'seed': None, 'instance_filter': None, 'custom_dataset': None}, 'restore_dataloader': True, 'fast_forward_batches': None, 'evaluators': [], 'eval_interval': 5000, 'tokenizer': {'identifier': 'meta-llama/Llama-2-7b-hf', 'truncate_direction': 'right'}, 'save_folder': 'checkpoints/OLMo-1B-as_fm3_omi2', 'remote_save_folder': None, 'canceled_check_interval': 6000, 'save_interval': 3000, 'save_interval_unsharded': 3000, 'save_interval_ephemeral': None, 'save_num_checkpoints_to_keep': -1, 'save_num_unsharded_checkpoints_to_keep': -1, 'save_overwrite': True, 'force_save_unsharded': False, 'no_pre_train_checkpoint': False, 'load_path': None, 'load_path_sharded_checkpointer': None, 'try_load_latest_save': False, 'reset_optimizer_state': False, 'reset_trainer_state': False, 'sharded_checkpointer': 'torch_legacy', 'new_style_checkpoints': None, 'max_duration': '1ep', 'global_train_batch_size': 512, 'device_train_batch_size': 512, 'device_train_microbatch_size': 16, 'device_eval_batch_size': 16, 'eval_subset_num_batches': -1, 'eval_on_load': False, 'device_train_grad_accum': 32, 'max_grad_norm': 1.0, 'max_grad_norm_ratio': None, 'precision': 'amp_bf16', 'speed_monitor': {'window_size': 20, 'gpu_flops_available': None}, 'console_log_interval': 1, 'gen1_gc_interval': 1, 'compile': None, 'distributed_strategy': 'fsdp', 'fsdp': {'use_orig_params': True, 'sharding_strategy': <ShardingStrategy.FULL_SHARD: 1>, 'wrapping_strategy': None, 'precision': 'mixed', 'hybrid_sharding_num_model_replicas': None}, 'ddp': {'grad_sync_mode': 'batch', 'find_unused_params': False}, 'single': {'device': 'auto'}, 'softmax_auxiliary_loss': False, 'auxiliary_loss_multiplier': 0.0001, 'time_limit': None, 'extra_steps_after_cancel': 10, 'early_stopping_factor': None, 'save_data_indices': True, 'python_profiling': False, 'torch_profiling': False, 'stop_at': None, 'stop_after': None, 'activation_checkpointing': None, 'fused_loss': None, 'hf_datasets_cache_dir': None, 'module_outputs_save_steps': None, '_wandb': {}}
-2025-12-14 15:37:31,160 INFO    MainThread:3746193 [wandb_init.py:init():889] starting backend
-2025-12-14 15:37:31,404 INFO    MainThread:3746193 [wandb_init.py:init():892] sending inform_init request
-2025-12-14 15:37:31,411 INFO    MainThread:3746193 [wandb_init.py:init():900] backend started and connected
-2025-12-14 15:37:31,414 INFO    MainThread:3746193 [wandb_init.py:init():970] updated telemetry
-2025-12-14 15:37:31,415 INFO    MainThread:3746193 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
-2025-12-14 15:37:31,929 INFO    MainThread:3746193 [wandb_init.py:init():1041] starting run threads in backend
-2025-12-14 15:37:32,020 INFO    MainThread:3746193 [wandb_run.py:_console_start():2521] atexit reg
-2025-12-14 15:37:32,021 INFO    MainThread:3746193 [wandb_run.py:_redirect():2369] redirect: wrap_raw
-2025-12-14 15:37:32,022 INFO    MainThread:3746193 [wandb_run.py:_redirect():2438] Wrapping output streams.
-2025-12-14 15:37:32,023 INFO    MainThread:3746193 [wandb_run.py:_redirect():2461] Redirects installed.
-2025-12-14 15:37:32,025 INFO    MainThread:3746193 [wandb_init.py:init():1081] run started, returning control to user process
-2025-12-14 15:38:02,853 INFO    MainThread:3746193 [wandb_run.py:_finish():2287] finishing run marksmans/olmo-debug/i0ugjt0v
-2025-12-14 15:38:02,861 INFO    MainThread:3746193 [wandb_run.py:_atexit_cleanup():2486] got exitcode: 1
-2025-12-14 15:38:02,862 INFO    MainThread:3746193 [wandb_run.py:_restore():2468] restore
-2025-12-14 15:38:02,863 INFO    MainThread:3746193 [wandb_run.py:_restore():2474] restore done
-2025-12-14 15:38:03,864 INFO    MainThread:3746193 [wandb_run.py:_footer_sync_info():3862] logging synced files

models/OLMo-1B/wandb/wandb/run-20251214_154039-z9xp3525/files/config.yaml

@@ -1,579 +0,0 @@
-_wandb:
-    value:
-        cli_version: 0.23.1
-        e:
-            hif91sfokkg4a3putu9qm5wzemlxp65c:
-                args:
-                    - pretraining/configs/RL-1B.yaml
-                codePath: OLMo/scripts/train.py
-                codePathLocal: OLMo/scripts/train.py
-                cpu_count: 112
-                cpu_count_logical: 224
-                cudaVersion: "13.0"
-                disk:
-                    /:
-                        total: "2055141851136"
-                        used: "49256366080"
-                email: [email protected]
-                executable: /opt/conda/bin/python
-                gpu: NVIDIA H100 80GB HBM3
-                gpu_count: 1
-                gpu_nvidia:
-                    - architecture: Hopper
-                      cudaCores: 16896
-                      memoryTotal: "85520809984"
-                      name: NVIDIA H100 80GB HBM3
-                      uuid: GPU-4c999b2a-2578-9e62-0539-4b826d85fda8
-                host: serv-3342
-                memory:
-                    total: "2164176814080"
-                os: Linux-6.8.0-1043-nvidia-x86_64-with-glibc2.35
-                program: /netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/scripts/train.py
-                python: CPython 3.11.11
-                root: checkpoints/OLMo-1B-as_fm3_omi2/wandb
-                slurm:
-                    cluster_name: pegasus
-                    conf: /etc/slurm/slurm.conf
-                    cpu_bind: quiet,mask_cpu:0x000000000001FE00000000000000000000000001FE00000000000000
-                    cpu_bind_list: 0x000000000001FE00000000000000000000000001FE00000000000000
-                    cpu_bind_type: 'mask_cpu:'
-                    cpu_bind_verbose: quiet
-                    cpus_on_node: "16"
-                    cpus_per_task: "16"
-                    distribution: cyclic
-                    gpus: "1"
-                    gpus_on_node: "1"
-                    gtids: "0"
-                    job_cpus_per_node: "16"
-                    job_end_time: "1765734524"
-                    job_gid: "8000"
-                    job_group: iml
-                    job_id: "2383756"
-                    job_name: bash
-                    job_nodelist: serv-3342
-                    job_num_nodes: "1"
-                    job_partition: H100
-                    job_qos: normal
-                    job_start_time: "1765720124"
-                    job_uid: "13262"
-                    job_user: nguyen
-                    jobid: "2383756"
-                    launch_node_ipaddr: 192.168.33.114
-                    localid: "0"
-                    mem_per_cpu: "16384"
-                    mpi_type: pmix
-                    nnodes: "1"
-                    nodeid: "0"
-                    nodelist: serv-3342
-                    nprocs: "1"
-                    ntasks: "1"
-                    oom_kill_step: "0"
-                    pmix_mapping_serv: (vector,(0,1,1))
-                    pmixp_abort_agent_port: "33735"
-                    prio_process: "1"
-                    procid: "0"
-                    pty_port: "45219"
-                    pty_win_col: "156"
-                    pty_win_row: "41"
-                    srun_comm_host: 192.168.33.114
-                    srun_comm_port: "35153"
-                    step_gpus: "5"
-                    step_id: "0"
-                    step_launcher_port: "35153"
-                    step_nodelist: serv-3342
-                    step_num_nodes: "1"
-                    step_num_tasks: "1"
-                    step_tasks_per_node: "1"
-                    stepid: "0"
-                    submit_dir: /netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain
-                    submit_host: login1
-                    task_pid: "3684902"
-                    tasks_per_node: "1"
-                    topology_addr: serv-3342
-                    topology_addr_pattern: node
-                    tres_bind: gres/gpu:per_task:1
-                    tres_per_task: cpu=16,gres/gpu=1
-                    umask: "0022"
-                startedAt: "2025-12-14T14:40:39.859989Z"
-                writerId: hif91sfokkg4a3putu9qm5wzemlxp65c
-        m: []
-        python_version: 3.11.11
-        t:
-            "1":
-                - 1
-                - 5
-                - 11
-                - 41
-                - 49
-                - 51
-                - 53
-            "2":
-                - 1
-                - 5
-                - 11
-                - 41
-                - 49
-                - 51
-                - 53
-            "3":
-                - 2
-                - 13
-                - 15
-                - 16
-                - 61
-            "4": 3.11.11
-            "5": 0.23.1
-            "6": 4.57.3
-            "10":
-                - 19
-            "12": 0.23.1
-            "13": linux-x86_64
-activation_checkpointing:
-    value: null
-auxiliary_loss_multiplier:
-    value: 0.0001
-canceled_check_interval:
-    value: 6000
-compile:
-    value: null
-console_log_interval:
-    value: 1
-data:
-    value:
-        custom_dataset: null
-        datasets: null
-        drop_last: true
-        generate_attention_mask: false
-        generate_doc_lengths: false
-        instance_filter: null
-        label_mask_paths: null
-        memmap_dtype: uint16
-        num_workers: 32
-        pad_direction: right
-        paths:
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00000_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00001_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00002_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00003_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00004_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00005_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00006_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00007_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00008_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00009_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00010_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00011_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00012_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00016_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00019_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00020_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00021_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00022_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00023_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00024_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00025_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00026_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00027_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00028_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00029_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00030_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00031_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00032_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00033_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00034_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00035_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00036_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00037_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00038_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00039_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00040_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00041_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00042_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00043_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00044_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00045_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00046_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00047_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00048_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00049_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00050_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00051_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00052_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00053_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00054_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00055_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00056_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00057_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00058_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00059_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00060_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00061_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00062_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00063_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00064_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00065_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00066_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00067_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00068_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00069_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00070_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00071_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00072_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00073_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00074_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00075_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00076_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00077_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00078_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00000_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00001_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00002_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00003_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00004_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00005_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00006_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00007_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00008_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00009_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00010_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00011_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00012_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00013_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00014_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00015_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00016_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00017_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00018_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00019_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00020_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00021_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00022_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00023_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00024_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00025_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00026_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00027_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00028_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00029_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00030_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00031_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00032_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00033_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00034_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00035_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00036_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00037_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00038_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00039_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00040_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00041_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00042_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00043_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00044_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00045_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00046_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00047_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00048_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00049_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00050_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00051_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00052_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00053_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00054_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00055_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00056_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00057_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00058_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00059_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00060_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00061_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00062_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00063_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00064_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00065_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00066_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00067_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00068_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00069_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00070_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00071_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00072_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00073_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00074_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00075_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00076_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00077_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00078_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00079_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00080_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00081_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00082_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00083_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00084_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00085_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00086_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00087_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00088_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00089_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00090_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00091_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00092_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00093_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00094_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00095_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00096_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00097_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00098_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00099_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00000_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00001_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00002_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00003_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00004_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00005_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00006_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00007_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00008_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00009_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00010_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00011_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00012_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00013_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00014_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00015_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00016_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00017_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00018_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00019_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00020_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00021_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00022_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00023_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00024_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00025_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00026_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00027_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00028_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00029_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00030_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00031_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00032_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00033_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00034_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00035_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00036_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00037_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00038_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00039_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00040_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00041_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00042_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00043_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00044_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00045_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00046_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00047_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00048_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00049_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00050_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00051_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00052_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00053_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00054_00000_doc_shuffled.ds
-        persistent_workers: true
-        pin_memory: true
-        prefetch_factor: 8
-        seed: null
-        timeout: 0
-ddp:
-    value:
-        find_unused_params: false
-        grad_sync_mode: batch
-device_eval_batch_size:
-    value: 16
-device_train_batch_size:
-    value: 512
-device_train_grad_accum:
-    value: 32
-device_train_microbatch_size:
-    value: 16
-distributed_strategy:
-    value: fsdp
-dry_run:
-    value: false
-early_stopping_factor:
-    value: null
-epoch:
-    value: null
-eval_interval:
-    value: 5000
-eval_on_load:
-    value: false
-eval_subset_num_batches:
-    value: -1
-evaluators:
-    value: []
-extra_steps_after_cancel:
-    value: 10
-fast_forward_batches:
-    value: null
-force_save_unsharded:
-    value: false
-fsdp:
-    value:
-        hybrid_sharding_num_model_replicas: null
-        precision: mixed
-        sharding_strategy: FULL_SHARD
-        use_orig_params: true
-        wrapping_strategy: null
-fused_loss:
-    value: null
-gen1_gc_interval:
-    value: 1
-global_train_batch_size:
-    value: 512
-hf_datasets_cache_dir:
-    value: null
-load_path:
-    value: null
-load_path_sharded_checkpointer:
-    value: null
-max_duration:
-    value: 1ep
-max_grad_norm:
-    value: 1
-max_grad_norm_ratio:
-    value: null
-model:
-    value:
-        activation_type: swiglu
-        alibi: false
-        alibi_bias_max: 8
-        attention_dropout: 0
-        attention_layer_norm: false
-        attention_layer_norm_with_affine: false
-        bias_for_layer_norm: false
-        block_group_size: 1
-        block_type: sequential
-        clip_qkv: null
-        d_model: 2048
-        emb_init_std: null
-        embedding_dropout: 0
-        embedding_layer_norm: false
-        embedding_size: 32000
-        eos_token_id: 0
-        flash_attention: false
-        include_bias: false
-        init_cutoff_factor: 3
-        init_device: cuda
-        init_fn: normal
-        init_std: 0.02
-        layer_norm_eps: 1e-05
-        layer_norm_type: default
-        layer_norm_with_affine: false
-        max_sequence_length: 2048
-        mlp_hidden_size: null
-        mlp_ratio: 8
-        multi_query_attention: false
-        n_heads: 16
-        n_kv_heads: null
-        n_layers: 16
-        norm_after: false
-        pad_token_id: 1
-        precision: amp_bf16
-        residual_dropout: 0
-        rope: true
-        rope_full_precision: true
-        rope_theta: 10000
-        scale_emb_init: false
-        scale_logits: false
-        vocab_size: 32000
-        weight_tying: true
-module_outputs_save_steps:
-    value: null
-new_style_checkpoints:
-    value: null
-no_pre_train_checkpoint:
-    value: false
-optimizer:
-    value:
-        betas:
-            - 0.9
-            - 0.95
-        decay_embeddings: true
-        decay_norm_and_bias: true
-        eps: 1e-08
-        learning_rate: 0.0005
-        metrics_log_interval: 10
-        name: adamw
-        no_decay_norm_and_bias: null
-        record_update_metrics: false
-        selective_updates: false
-        weight_decay: 0.1
-precision:
-    value: amp_bf16
-python_profiling:
-    value: false
-remote_save_folder:
-    value: null
-reset_optimizer_state:
-    value: false
-reset_trainer_state:
-    value: false
-restore_dataloader:
-    value: true
-run_name:
-    value: OLMo-1B-as_fm3_omi2
-save_data_indices:
-    value: true
-save_folder:
-    value: checkpoints/OLMo-1B-as_fm3_omi2
-save_interval:
-    value: 3000
-save_interval_ephemeral:
-    value: null
-save_interval_unsharded:
-    value: 3000
-save_num_checkpoints_to_keep:
-    value: -1
-save_num_unsharded_checkpoints_to_keep:
-    value: -1
-save_overwrite:
-    value: true
-scheduler:
-    value:
-        alpha_f: 0.1
-        grad_clip_warmup_factor: null
-        grad_clip_warmup_steps: null
-        name: cosine_with_warmup
-        t_max: null
-        t_warmup: 2000
-        units: steps
-        warmup_min_lr: 0
-seed:
-    value: 6198
-sharded_checkpointer:
-    value: torch_legacy
-single:
-    value:
-        device: auto
-softmax_auxiliary_loss:
-    value: false
-speed_monitor:
-    value:
-        gpu_flops_available: null
-        window_size: 20
-stop_after:
-    value: null
-stop_at:
-    value: null
-time_limit:
-    value: null
-tokenizer:
-    value:
-        identifier: meta-llama/Llama-2-7b-hf
-        truncate_direction: right
-torch_profiling:
-    value: false
-try_load_latest_save:
-    value: false

models/OLMo-1B/wandb/wandb/run-20251214_154039-z9xp3525/files/output.log

@@ -1,78 +0,0 @@
-[2025-12-14 15:40:42] INFO     [olmo.data.iterable_dataset:79, rank=0] Saving global data order indices...
-[2025-12-14 15:40:44] INFO     [olmo.data.iterable_dataset:88, rank=0] Global data order indices saved to 'checkpoints/OLMo-1B-as_fm3_omi2/train_data/global_indices.npy'
-[2025-12-14 15:40:44] WARNING  [py.warnings:110, rank=0] /opt/conda/lib/python3.11/site-packages/torch/utils/data/dataloader.py:624: UserWarning: This DataLoader will create 32 worker processes in total. Our suggested max number of worker in current system is 16, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.
-  warnings.warn(
-
-[2025-12-14 15:40:44] INFO     [train:139, rank=0] Building model...
-[2025-12-14 15:40:44] INFO     [olmo.model:1174, rank=0] Initializing model parameters...
-[2025-12-14 15:40:44] INFO     [train:141, rank=0] Total number of parameters: 1,139,277,824
-[2025-12-14 15:40:44] INFO     [train:142, rank=0] Number of non-embedding parameters: 1,073,741,824
-[2025-12-14 15:40:44] INFO     [train:143, rank=0] Peak GPU Memory (MB) before fsdp: 4559
-[2025-12-14 15:40:44] INFO     [train:172, rank=0] Wrapping model with FSDP...
-[2025-12-14 15:40:44] WARNING  [py.warnings:110, rank=0] /opt/conda/lib/python3.11/site-packages/torch/distributed/fsdp/_init_utils.py:444: UserWarning: FSDP is switching to use `NO_SHARD` instead of ShardingStrategy.FULL_SHARD since the world size is 1.
-  warnings.warn(
-
-[2025-12-14 15:40:44] INFO     [olmo.model:1174, rank=0] Initializing model parameters...
-[2025-12-14 15:40:44] INFO     [train:232, rank=0] Peak GPU Memory (MB) after fsdp: 9116
-[2025-12-14 15:40:44] INFO     [train:233, rank=0] Model:
-[2025-12-14 15:40:44] INFO     [train:234, rank=0] FullyShardedDataParallel(
-  (_fsdp_wrapped_module): OLMo(
-    (transformer): ModuleDict(
-      (wte): Embedding(32000, 2048)
-      (emb_drop): Dropout(p=0.0, inplace=False)
-      (ln_f): LayerNorm()
-      (blocks): ModuleList(
-        (0-15): 16 x OLMoSequentialBlock(
-          (dropout): Dropout(p=0.0, inplace=False)
-          (act): SwiGLU()
-          (attn_out): Linear(in_features=2048, out_features=2048, bias=False)
-          (ff_out): Linear(in_features=8192, out_features=2048, bias=False)
-          (rotary_emb): RotaryEmbedding()
-          (att_proj): Linear(in_features=2048, out_features=6144, bias=False)
-          (ff_proj): Linear(in_features=2048, out_features=16384, bias=False)
-          (attn_norm): LayerNorm()
-          (ff_norm): LayerNorm()
-        )
-      )
-    )
-  )
-)
-[2025-12-14 15:40:44] INFO     [olmo.optim:944, rank=0] Constructing optimizer with 1 param groups
-[2025-12-14 15:40:44] INFO     [train:335, rank=0] Saving pre-train checkpoint...
-[2025-12-14 15:40:44] WARNING  [py.warnings:110, rank=0] /opt/conda/lib/python3.11/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:690: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
-  warnings.warn(
-
-[2025-12-14 15:40:44] WARNING  [py.warnings:110, rank=0] /opt/conda/lib/python3.11/site-packages/torch/distributed/fsdp/_state_dict_utils.py:773: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned.
-  warnings.warn(
-
-[2025-12-14 15:40:44] WARNING  [py.warnings:110, rank=0] /opt/conda/lib/python3.11/site-packages/torch/distributed/fsdp/_state_dict_utils.py:711: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned.
-  warnings.warn(
-
-[2025-12-14 15:40:51] INFO     [olmo.checkpoint:607, rank=0] Saving config...
-[2025-12-14 15:40:51] INFO     [train:337, rank=0] Checkpoint saved to checkpoints/OLMo-1B-as_fm3_omi2/step0
-[2025-12-14 15:40:51] INFO     [train:340, rank=0] Attempting to load pre-train checkpoint...
-[2025-12-14 15:40:55] INFO     [olmo.checkpoint:1040, rank=0] Loading model state...
-[2025-12-14 15:40:55] WARNING  [py.warnings:110, rank=0] /opt/conda/lib/python3.11/site-packages/torch/distributed/fsdp/_state_dict_utils.py:827: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned.
-  warnings.warn(
-
-[2025-12-14 15:40:56] WARNING  [py.warnings:110, rank=0] /opt/conda/lib/python3.11/site-packages/torch/distributed/fsdp/_state_dict_utils.py:864: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned.
-  warnings.warn(
-
-[2025-12-14 15:40:56] INFO     [olmo.checkpoint:1044, rank=0] Loading optimizer state...
-[2025-12-14 15:40:56] INFO     [olmo.checkpoint:220, rank=0] Flattening sharded optimizer state...
-[2025-12-14 15:40:56] INFO     [olmo.checkpoint:234, rank=0] Loading flattened optimizer state...
-[2025-12-14 15:40:56] INFO     [olmo.train:409, rank=0] Resetting learning rate...
-[2025-12-14 15:40:56] INFO     [olmo.train:421, rank=0] Restoring RNG states...
-[2025-12-14 15:40:56] INFO     [train:344, rank=0] Checkpoint successfully loaded
-[2025-12-14 15:40:56] INFO     [train:375, rank=0] Starting training...
-[2025-12-14 15:40:56] INFO     [olmo.train:979, rank=0] Pre-train system metrics
-    System/Peak GPU Memory (MB)=9,116
-[2025-12-14 15:41:54] INFO     [olmo.train:979, rank=0] [step=1/59321,epoch=0]
-    optim/total_grad_norm=9.088
-    train/CrossEntropyLoss=10.76
-    train/Perplexity=47,163
-    throughput/total_tokens=1,048,576
-    throughput/total_training_Gflops=7,874,702
-    throughput/total_training_log_Gflops=15.88
-    System/Peak GPU Memory (MB)=80,250
-wandb: WARNING The `quiet` argument to `wandb.run.finish()` is deprecated, use `wandb.Settings(quiet=...)` to set this instead.

models/OLMo-1B/wandb/wandb/run-20251214_154039-z9xp3525/files/requirements.txt

@@ -1,271 +0,0 @@
-scikit-learn==1.8.0
-joblib==1.5.2
-threadpoolctl==3.6.0
-torchmetrics==1.8.2
-lightning-utilities==0.15.2
-wandb==0.23.1
-GitPython==3.1.45
-gitdb==4.0.12
-smmap==5.0.2
-datasets==4.4.1
-pandas==2.3.3
-multiprocess==0.70.18
-pyarrow==22.0.0
-tzdata==2025.3
-xxhash==3.6.0
-ai2-olmo==0.6.0
-ai2-olmo-core==2.4.0
-cached_path==1.8.0
-google-cloud-storage==2.19.0
-google-cloud-core==2.5.0
-boto3==1.42.9
-google-api-core==2.28.1
-google-auth==2.43.0
-s3transfer==0.16.0
-botocore==1.42.9
-google-resumable-media==2.8.0
-pyasn1_modules==0.4.2
-rich==13.9.4
-rsa==4.9.1
-bettermap==1.3.1
-google-crc32c==1.7.1
-jmespath==1.0.1
-numpy==1.26.4
-omegaconf==2.3.0
-proto-plus==1.26.1
-pyasn1==0.6.1
-python-dateutil==2.9.0.post0
-antlr4-python3-runtime==4.9.3
-zstandard==0.23.0
-zipp==3.21.0
-yarl==1.22.0
-xgrammar==0.1.18
-xformers==0.0.29.post2
-wrapt==2.0.1
-wheel==0.45.1
-websockets==15.0.1
-wcwidth==0.2.13
-watchfiles==1.1.1
-vllm==0.8.5.post1
-uvloop==0.22.1
-uvicorn==0.38.0
-urllib3==2.3.0
-typing-inspection==0.4.2
-typing_extensions==4.15.0
-types-dataclasses==0.6.6
-typer==0.20.0
-truststore==0.10.0
-triton==3.2.0
-transformers==4.57.3
-traitlets==5.14.3
-tqdm==4.67.1
-torchvision==0.21.0+cu124
-torchelastic==0.2.2
-torchaudio==2.6.0+cu124
-torch==2.6.0+cu124
-tokenizers==0.22.1
-tiktoken==0.12.0
-sympy==1.13.1
-starlette==0.50.0
-stack_data==0.6.3
-soupsieve==2.5
-sortedcontainers==2.4.0
-sniffio==1.3.1
-six==1.17.0
-shellingham==1.5.4
-setuptools==75.8.0
-sentry-sdk==2.47.0
-sentencepiece==0.2.1
-scipy==1.16.3
-safetensors==0.7.0
-ruamel.yaml.clib==0.2.8
-ruamel.yaml==0.18.10
-rpds-py==0.22.3
-rignore==0.7.6
-rich-toolkit==0.17.0
-requests==2.32.3
-regex==2025.11.3
-referencing==0.36.2
-ray==2.52.1
-pyzmq==27.1.0
-pytz==2024.2
-python-multipart==0.0.20
-python-json-logger==4.0.0
-python-etcd==0.4.5
-python-dotenv==1.2.1
-Pygments==2.19.1
-pydantic-extra-types==2.10.6
-pydantic_core==2.41.5
-pydantic==2.12.5
-pycparser==2.22
-pycountry==24.6.1
-pycosat==0.6.6
-py-cpuinfo==9.0.0
-pure_eval==0.2.3
-ptyprocess==0.7.0
-psutil==6.1.1
-protobuf==4.25.8
-propcache==0.4.1
-prompt_toolkit==3.0.50
-prometheus-fastapi-instrumentator==7.1.0
-prometheus_client==0.23.1
-pluggy==1.5.0
-platformdirs==4.3.6
-pkgutil_resolve_name==1.3.10
-pkginfo==1.12.0
-pip==24.3.1
-pillow==11.0.0
-pickleshare==0.7.5
-pexpect==4.9.0
-partial-json-parser==0.2.1.1.post7
-parso==0.8.4
-packaging==24.2
-outlines_core==0.1.26
-outlines==0.1.11
-optree==0.14.0
-opentelemetry-semantic-conventions-ai==0.4.13
-opentelemetry-semantic-conventions==0.47b0
-opentelemetry-sdk==1.26.0
-opentelemetry-proto==1.26.0
-opentelemetry-exporter-otlp-proto-http==1.26.0
-opentelemetry-exporter-otlp-proto-grpc==1.26.0
-opentelemetry-exporter-otlp-proto-common==1.26.0
-opentelemetry-exporter-otlp==1.26.0
-opentelemetry-api==1.26.0
-opencv-python-headless==4.12.0.88
-openai==2.11.0
-nvidia-nvtx-cu12==12.4.127
-nvidia-nvjitlink-cu12==12.4.127
-nvidia-nccl-cu12==2.21.5
-nvidia-cusparselt-cu12==0.6.2
-nvidia-cusparse-cu12==12.3.1.170
-nvidia-cusolver-cu12==11.6.1.9
-nvidia-curand-cu12==10.3.5.147
-nvidia-cufft-cu12==11.2.1.3
-nvidia-cudnn-cu12==9.1.0.70
-nvidia-cuda-runtime-cu12==12.4.127
-nvidia-cuda-nvrtc-cu12==12.4.127
-nvidia-cuda-cupti-cu12==12.4.127
-nvidia-cublas-cu12==12.4.5.8
-numba==0.61.2
-ninja==1.11.1.3
-networkx==3.4.2
-nest-asyncio==1.6.0
-multidict==6.7.0
-msgspec==0.20.0
-msgpack==1.1.2
-mpmath==1.3.0
-more-itertools==10.6.0
-mistral_common==1.8.6
-menuinst==2.2.0
-mdurl==0.1.2
-matplotlib-inline==0.1.7
-math-verify==0.8.0
-markdown-it-py==4.0.0
-lm-format-enforcer==0.10.12
-llvmlite==0.44.0
-llguidance==0.7.30
-lintrunner==0.12.7
-lief==0.14.1
-libmambapy==2.0.5
-libarchive-c==5.1
-latex2sympy2_extended==1.10.2
-lark==1.2.2
-jsonschema-specifications==2024.10.1
-jsonschema==4.23.0
-jsonpointer==3.0.0
-jsonpatch==1.33
-jiter==0.12.0
-Jinja2==3.1.5
-jedi==0.19.2
-ipython==8.31.0
-interegular==0.3.3
-importlib_resources==6.5.2
-importlib_metadata==8.0.0
-idna==3.10
-hypothesis==6.124.7
-hyperframe==6.0.1
-huggingface-hub==0.36.0
-httpx==0.28.1
-httptools==0.7.1
-httpcore==1.0.9
-hpack==4.0.0
-hf-xet==1.2.0
-h2==4.1.0
-h11==0.16.0
-grpcio==1.76.0
-googleapis-common-protos==1.72.0
-gguf==0.17.1
-fsspec==2024.12.0
-frozenlist==1.8.0
-frozendict==2.4.6
-filelock==3.17.0
-fastrlock==0.8.3
-fastar==0.8.0
-fastapi-cloud-cli==0.6.0
-fastapi-cli==0.0.16
-fastapi==0.124.4
-expecttest==0.3.0
-executing==2.1.0
-exceptiongroup==1.2.2
-email-validator==2.3.0
-einops==0.8.1
-dnspython==2.7.0
-distro==1.9.0
-diskcache==5.6.3
-dill==0.4.0
-depyf==0.18.0
-Deprecated==1.3.1
-decorator==5.1.1
-cupy-cuda12x==13.6.0
-conda_package_streaming==0.11.0
-conda-package-handling==2.4.0
-conda-libmamba-solver==25.1.1
-conda_index==0.5.0
-conda-build==25.1.1
-conda==25.1.0
-compressed-tensors==0.9.3
-colorama==0.4.6
-cmake==3.31.4
-cloudpickle==3.1.2
-click==8.1.8
-charset-normalizer==3.4.1
-chardet==5.2.0
-cffi==1.17.1
-certifi==2024.12.14
-cachetools==6.2.3
-boltons==24.0.0
-blake3==1.0.8
-beautifulsoup4==4.12.3
-attrs==25.1.0
-astunparse==1.6.3
-asttokens==3.0.0
-astor==0.8.1
-archspec==0.2.5
-anyio==4.12.0
-annotated-types==0.7.0
-annotated-doc==0.0.4
-airportsdata==20250909
-aiosignal==1.4.0
-aiohttp==3.13.2
-aiohappyeyeballs==2.6.1
-PyYAML==6.0.2
-PySocks==1.7.1
-MarkupSafe==3.0.2
-Brotli==1.1.0
-zipp==3.19.2
-wheel==0.43.0
-typing_extensions==4.12.2
-typeguard==4.3.0
-tomli==2.0.1
-platformdirs==4.2.2
-packaging==24.2
-more-itertools==10.3.0
-jaraco.text==3.12.1
-jaraco.functools==4.0.1
-jaraco.context==5.3.0
-jaraco.collections==5.1.0
-inflect==7.3.1
-importlib_metadata==8.0.0
-backports.tarfile==1.2.0
-autocommand==2.2.2

models/OLMo-1B/wandb/wandb/run-20251214_154039-z9xp3525/files/wandb-metadata.json

@@ -1,103 +0,0 @@
-{
-  "os":  "Linux-6.8.0-1043-nvidia-x86_64-with-glibc2.35",
-  "python":  "CPython 3.11.11",
-  "startedAt":  "2025-12-14T14:40:39.859989Z",
-  "args":  [
-    "pretraining/configs/RL-1B.yaml"
-  ],
-  "program":  "/netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/scripts/train.py",
-  "codePath":  "OLMo/scripts/train.py",
-  "codePathLocal":  "OLMo/scripts/train.py",
-  "email":  "[email protected]",
-  "root":  "checkpoints/OLMo-1B-as_fm3_omi2/wandb",
-  "host":  "serv-3342",
-  "executable":  "/opt/conda/bin/python",
-  "cpu_count":  112,
-  "cpu_count_logical":  224,
-  "gpu":  "NVIDIA H100 80GB HBM3",
-  "gpu_count":  1,
-  "disk":  {
-    "/":  {
-      "total":  "2055141851136",
-      "used":  "49256366080"
-    }
-  },
-  "memory":  {
-    "total":  "2164176814080"
-  },
-  "gpu_nvidia":  [
-    {
-      "name":  "NVIDIA H100 80GB HBM3",
-      "memoryTotal":  "85520809984",
-      "cudaCores":  16896,
-      "architecture":  "Hopper",
-      "uuid":  "GPU-4c999b2a-2578-9e62-0539-4b826d85fda8"
-    }
-  ],
-  "cudaVersion":  "13.0",
-  "slurm":  {
-    "cluster_name":  "pegasus",
-    "conf":  "/etc/slurm/slurm.conf",
-    "cpu_bind":  "quiet,mask_cpu:0x000000000001FE00000000000000000000000001FE00000000000000",
-    "cpu_bind_list":  "0x000000000001FE00000000000000000000000001FE00000000000000",
-    "cpu_bind_type":  "mask_cpu:",
-    "cpu_bind_verbose":  "quiet",
-    "cpus_on_node":  "16",
-    "cpus_per_task":  "16",
-    "distribution":  "cyclic",
-    "gpus":  "1",
-    "gpus_on_node":  "1",
-    "gtids":  "0",
-    "job_cpus_per_node":  "16",
-    "job_end_time":  "1765734524",
-    "job_gid":  "8000",
-    "job_group":  "iml",
-    "job_id":  "2383756",
-    "job_name":  "bash",
-    "job_nodelist":  "serv-3342",
-    "job_num_nodes":  "1",
-    "job_partition":  "H100",
-    "job_qos":  "normal",
-    "job_start_time":  "1765720124",
-    "job_uid":  "13262",
-    "job_user":  "nguyen",
-    "jobid":  "2383756",
-    "launch_node_ipaddr":  "192.168.33.114",
-    "localid":  "0",
-    "mem_per_cpu":  "16384",
-    "mpi_type":  "pmix",
-    "nnodes":  "1",
-    "nodeid":  "0",
-    "nodelist":  "serv-3342",
-    "nprocs":  "1",
-    "ntasks":  "1",
-    "oom_kill_step":  "0",
-    "pmix_mapping_serv":  "(vector,(0,1,1))",
-    "pmixp_abort_agent_port":  "33735",
-    "prio_process":  "1",
-    "procid":  "0",
-    "pty_port":  "45219",
-    "pty_win_col":  "156",
-    "pty_win_row":  "41",
-    "srun_comm_host":  "192.168.33.114",
-    "srun_comm_port":  "35153",
-    "step_gpus":  "5",
-    "step_id":  "0",
-    "step_launcher_port":  "35153",
-    "step_nodelist":  "serv-3342",
-    "step_num_nodes":  "1",
-    "step_num_tasks":  "1",
-    "step_tasks_per_node":  "1",
-    "stepid":  "0",
-    "submit_dir":  "/netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain",
-    "submit_host":  "login1",
-    "task_pid":  "3684902",
-    "tasks_per_node":  "1",
-    "topology_addr":  "serv-3342",
-    "topology_addr_pattern":  "node",
-    "tres_bind":  "gres/gpu:per_task:1",
-    "tres_per_task":  "cpu=16,gres/gpu=1",
-    "umask":  "0022"
-  },
-  "writerId":  "hif91sfokkg4a3putu9qm5wzemlxp65c"
-}

models/OLMo-1B/wandb/wandb/run-20251214_154039-z9xp3525/files/wandb-summary.json

@@ -1 +0,0 @@
-{"optim/grad/transformer.blocks.6.attn_out.weight.norm":0.5236513018608093,"optim/grad/transformer.blocks.1.attn_out.weight.norm":1.62814199924469,"_step":1,"optim/grad/transformer.blocks.10.ff_out.weight.norm":0.31942349672317505,"optim/grad/transformer.blocks.8.ff_out.weight.norm":0.3655010461807251,"optim/grad/transformer.blocks.0.attn_out.weight.norm":3.5133237838745117,"_runtime":73.879944699,"optim/grad/transformer.blocks.10.att_proj.weight.norm":0.3661356270313263,"optim/grad/transformer.blocks.5.ff_out.weight.norm":0.5184721946716309,"optim/grad/transformer.blocks.11.ff_out.weight.norm":0.29604294896125793,"optim/grad/transformer.blocks.5.att_proj.weight.norm":0.5883283019065857,"optim/grad/transformer.blocks.2.attn_out.weight.norm":1.118513584136963,"optim/grad/transformer.wte.weight.norm":3.908214569091797,"optim/grad/transformer.blocks.11.ff_proj.weight.norm":0.41096439957618713,"optim/grad/transformer.blocks.2.att_proj.weight.norm":1.1237092018127441,"optim/grad/transformer.blocks.6.att_proj.weight.norm":0.5209723114967346,"optim/grad/transformer.blocks.6.ff_proj.weight.norm":0.6300004720687866,"optim/grad/transformer.blocks.4.ff_out.weight.norm":0.5870352983474731,"optim/grad/transformer.blocks.13.ff_proj.weight.norm":0.37113988399505615,"throughput/total_tokens":1048576,"optim/total_grad_norm":9.087934494018555,"optim/grad/transformer.blocks.0.ff_out.weight.norm":2.177250623703003,"optim/grad/transformer.blocks.5.attn_out.weight.norm":0.6065305471420288,"optim/grad/transformer.blocks.3.ff_out.weight.norm":0.7264525294303894,"System/Peak GPU Memory (MB)":80250.1953125,"optim/grad/transformer.blocks.12.attn_out.weight.norm":0.3190648555755615,"optim/grad/transformer.blocks.1.att_proj.weight.norm":1.6082570552825928,"optim/grad/transformer.blocks.14.att_proj.weight.norm":0.28815507888793945,"optim/grad/transformer.blocks.15.attn_out.weight.norm":0.27553632855415344,"_wandb":{"runtime":73},"optim/grad/transformer.blocks.11.attn_out.weight.norm":0.3472404479980469,"optim/grad/transformer.blocks.10.ff_proj.weight.norm":0.45583638548851013,"optim/grad/transformer.blocks.14.attn_out.weight.norm":0.2960440516471863,"optim/grad/transformer.blocks.8.ff_proj.weight.norm":0.5061823129653931,"optim/grad/transformer.blocks.11.att_proj.weight.norm":0.33966541290283203,"optim/grad/transformer.blocks.15.att_proj.weight.norm":0.2798300087451935,"throughput/total_training_Gflops":7.874702278131712e+06,"optim/grad/transformer.blocks.7.attn_out.weight.norm":0.47163599729537964,"optim/grad/transformer.blocks.0.att_proj.weight.norm":3.6439361572265625,"optim/grad/transformer.blocks.8.attn_out.weight.norm":0.4244263768196106,"optim/grad/transformer.blocks.9.attn_out.weight.norm":0.3973117172718048,"optim/grad/transformer.blocks.14.ff_proj.weight.norm":0.3621259927749634,"train/Perplexity":47163.62832898086,"optim/grad/transformer.blocks.12.ff_proj.weight.norm":0.3901353180408478,"optim/grad/transformer.blocks.13.attn_out.weight.norm":0.3037296533584595,"optim/grad/transformer.blocks.1.ff_out.weight.norm":1.2330187559127808,"optim/grad/transformer.blocks.2.ff_out.weight.norm":0.8915591835975647,"optim/grad/transformer.blocks.3.attn_out.weight.norm":0.8428605794906616,"throughput/total_training_log_Gflops":15.879165936009779,"optim/grad/transformer.blocks.15.ff_proj.weight.norm":0.33649954199790955,"optim/grad/transformer.blocks.15.ff_out.weight.norm":0.243862122297287,"optim/grad/transformer.blocks.1.ff_proj.weight.norm":1.740834355354309,"optim/grad/transformer.blocks.12.ff_out.weight.norm":0.27772974967956543,"optim/grad/transformer.blocks.4.att_proj.weight.norm":0.7019151449203491,"train/CrossEntropyLoss":10.761378288269043,"optim/grad/transformer.blocks.8.att_proj.weight.norm":0.4309239089488983,"optim/learning_rate_group0":2.5e-07,"optim/grad/transformer.blocks.10.attn_out.weight.norm":0.3706686198711395,"optim/grad/transformer.blocks.12.att_proj.weight.norm":0.32427260279655457,"optim/grad/transformer.blocks.5.ff_proj.weight.norm":0.7240243554115295,"optim/grad/transformer.blocks.2.ff_proj.weight.norm":1.256575584411621,"optim/grad/transformer.blocks.13.att_proj.weight.norm":0.30371585488319397,"optim/grad/transformer.blocks.3.att_proj.weight.norm":0.8451918363571167,"_timestamp":1.7657233141800284e+09,"optim/grad/transformer.blocks.14.ff_out.weight.norm":0.25873398780822754,"optim/grad/transformer.blocks.0.ff_proj.weight.norm":3.110273599624634,"optim/grad/transformer.blocks.4.ff_proj.weight.norm":0.8388200998306274,"optim/grad/transformer.blocks.7.ff_proj.weight.norm":0.5779657363891602,"optim/grad/transformer.blocks.7.ff_out.weight.norm":0.40811124444007874,"optim/grad/transformer.blocks.7.att_proj.weight.norm":0.47150084376335144,"optim/grad/transformer.blocks.13.ff_out.weight.norm":0.2664201855659485,"optim/grad/transformer.blocks.9.att_proj.weight.norm":0.3878222703933716,"optim/grad/transformer.blocks.3.ff_proj.weight.norm":1.0055341720581055,"optim/grad/transformer.blocks.9.ff_proj.weight.norm":0.47650912404060364,"optim/grad/transformer.blocks.4.attn_out.weight.norm":0.7104970216751099,"optim/grad/transformer.blocks.6.ff_out.weight.norm":0.4432903230190277,"optim/grad/transformer.blocks.9.ff_out.weight.norm":0.3376409411430359}

models/OLMo-1B/wandb/wandb/run-20251214_154039-z9xp3525/logs/debug-internal.log

@@ -1,12 +0,0 @@
-{"time":"2025-12-14T15:40:40.168131806+01:00","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
-{"time":"2025-12-14T15:40:40.47528161+01:00","level":"INFO","msg":"stream: created new stream","id":"z9xp3525"}
-{"time":"2025-12-14T15:40:40.476452249+01:00","level":"INFO","msg":"handler: started","stream_id":"z9xp3525"}
-{"time":"2025-12-14T15:40:40.478203079+01:00","level":"INFO","msg":"stream: started","id":"z9xp3525"}
-{"time":"2025-12-14T15:40:40.47832097+01:00","level":"INFO","msg":"writer: started","stream_id":"z9xp3525"}
-{"time":"2025-12-14T15:40:40.478347155+01:00","level":"INFO","msg":"sender: started","stream_id":"z9xp3525"}
-{"time":"2025-12-14T15:41:55.381052285+01:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
-{"time":"2025-12-14T15:41:55.860048848+01:00","level":"INFO","msg":"handler: operation stats","stats":{}}
-{"time":"2025-12-14T15:41:55.865491898+01:00","level":"INFO","msg":"stream: closing","id":"z9xp3525"}
-{"time":"2025-12-14T15:41:55.866017166+01:00","level":"INFO","msg":"handler: closed","stream_id":"z9xp3525"}
-{"time":"2025-12-14T15:41:55.9059486+01:00","level":"INFO","msg":"sender: closed","stream_id":"z9xp3525"}
-{"time":"2025-12-14T15:41:55.906959791+01:00","level":"INFO","msg":"stream: closed","id":"z9xp3525"}

models/OLMo-1B/wandb/wandb/run-20251214_154039-z9xp3525/logs/debug.log

@@ -1,26 +0,0 @@
-2025-12-14 15:40:39,906 INFO    MainThread:3750944 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
-2025-12-14 15:40:39,907 INFO    MainThread:3750944 [wandb_setup.py:_flush():80] Configure stats pid to 3750944
-2025-12-14 15:40:39,908 INFO    MainThread:3750944 [wandb_setup.py:_flush():80] Loading settings from /root/.config/wandb/settings
-2025-12-14 15:40:39,909 INFO    MainThread:3750944 [wandb_setup.py:_flush():80] Loading settings from /netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/wandb/settings
-2025-12-14 15:40:39,910 INFO    MainThread:3750944 [wandb_setup.py:_flush():80] Loading settings from environment variables
-2025-12-14 15:40:39,911 INFO    MainThread:3750944 [wandb_init.py:setup_run_log_directory():714] Logging user logs to checkpoints/OLMo-1B-as_fm3_omi2/wandb/wandb/run-20251214_154039-z9xp3525/logs/debug.log
-2025-12-14 15:40:39,911 INFO    MainThread:3750944 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to checkpoints/OLMo-1B-as_fm3_omi2/wandb/wandb/run-20251214_154039-z9xp3525/logs/debug-internal.log
-2025-12-14 15:40:39,913 INFO    MainThread:3750944 [wandb_init.py:init():841] calling init triggers
-2025-12-14 15:40:39,914 INFO    MainThread:3750944 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
-config: {'run_name': 'OLMo-1B-as_fm3_omi2', 'seed': 6198, 'epoch': None, 'dry_run': False, 'model': {'d_model': 2048, 'n_heads': 16, 'n_kv_heads': None, 'clip_qkv': None, 'n_layers': 16, 'mlp_ratio': 8, 'mlp_hidden_size': None, 'activation_type': 'swiglu', 'block_type': 'sequential', 'block_group_size': 1, 'alibi': False, 'alibi_bias_max': 8.0, 'rope': True, 'rope_full_precision': True, 'rope_theta': 10000, 'flash_attention': False, 'attention_dropout': 0.0, 'multi_query_attention': False, 'attention_layer_norm': False, 'residual_dropout': 0.0, 'embedding_dropout': 0.0, 'embedding_layer_norm': False, 'layer_norm_type': 'default', 'layer_norm_with_affine': False, 'layer_norm_eps': 1e-05, 'attention_layer_norm_with_affine': False, 'max_sequence_length': 2048, 'include_bias': False, 'bias_for_layer_norm': False, 'scale_logits': False, 'vocab_size': 32000, 'embedding_size': 32000, 'weight_tying': True, 'eos_token_id': 0, 'pad_token_id': 1, 'init_device': 'cuda', 'init_fn': 'normal', 'init_std': 0.02, 'init_cutoff_factor': 3.0, 'precision': 'amp_bf16', 'scale_emb_init': False, 'emb_init_std': None, 'norm_after': False}, 'optimizer': {'name': 'adamw', 'learning_rate': 0.0005, 'weight_decay': 0.1, 'betas': (0.9, 0.95), 'eps': 1e-08, 'no_decay_norm_and_bias': None, 'selective_updates': False, 'decay_norm_and_bias': True, 'decay_embeddings': True, 'metrics_log_interval': 10, 'record_update_metrics': False}, 'scheduler': {'name': 'cosine_with_warmup', 'units': 'steps', 't_warmup': 2000, 't_max': None, 'alpha_f': 0.1, 'grad_clip_warmup_steps': None, 'grad_clip_warmup_factor': None, 'warmup_min_lr': 0.0}, 'data': {'paths': ['data_token/as_fm3_omi2/algebraic-stack-tokenized/00000_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00001_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00002_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00003_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00004_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00005_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00006_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00007_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00008_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00009_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00010_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00011_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00012_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00016_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00019_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00020_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00021_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00022_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00023_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00024_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00025_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00026_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00027_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00028_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00029_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00030_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00031_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00032_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00033_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00034_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00035_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00036_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00037_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00038_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00039_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00040_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00041_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00042_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00043_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00044_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00045_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00046_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00047_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00048_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00049_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00050_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00051_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00052_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00053_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00054_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00055_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00056_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00057_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00058_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00059_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00060_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00061_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00062_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00063_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00064_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00065_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00066_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00067_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00068_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00069_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00070_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00071_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00072_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00073_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00074_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00075_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00076_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00077_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00078_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00000_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00001_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00002_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00003_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00004_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00005_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00006_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00007_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00008_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00009_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00010_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00011_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00012_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00013_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00014_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00015_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00016_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00017_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00018_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00019_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00020_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00021_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00022_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00023_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00024_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00025_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00026_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00027_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00028_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00029_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00030_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00031_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00032_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00033_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00034_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00035_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00036_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00037_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00038_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00039_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00040_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00041_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00042_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00043_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00044_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00045_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00046_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00047_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00048_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00049_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00050_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00051_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00052_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00053_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00054_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00055_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00056_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00057_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00058_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00059_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00060_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00061_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00062_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00063_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00064_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00065_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00066_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00067_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00068_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00069_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00070_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00071_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00072_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00073_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00074_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00075_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00076_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00077_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00078_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00079_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00080_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00081_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00082_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00083_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00084_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00085_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00086_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00087_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00088_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00089_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00090_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00091_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00092_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00093_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00094_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00095_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00096_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00097_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00098_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00099_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00000_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00001_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00002_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00003_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00004_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00005_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00006_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00007_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00008_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00009_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00010_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00011_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00012_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00013_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00014_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00015_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00016_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00017_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00018_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00019_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00020_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00021_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00022_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00023_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00024_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00025_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00026_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00027_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00028_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00029_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00030_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00031_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00032_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00033_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00034_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00035_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00036_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00037_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00038_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00039_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00040_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00041_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00042_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00043_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00044_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00045_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00046_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00047_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00048_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00049_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00050_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00051_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00052_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00053_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00054_00000_doc_shuffled.ds'], 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 32, 'drop_last': True, 'pin_memory': True, 'prefetch_factor': 8, 'persistent_workers': True, 'timeout': 0, 'seed': None, 'instance_filter': None, 'custom_dataset': None}, 'restore_dataloader': True, 'fast_forward_batches': None, 'evaluators': [], 'eval_interval': 5000, 'tokenizer': {'identifier': 'meta-llama/Llama-2-7b-hf', 'truncate_direction': 'right'}, 'save_folder': 'checkpoints/OLMo-1B-as_fm3_omi2', 'remote_save_folder': None, 'canceled_check_interval': 6000, 'save_interval': 3000, 'save_interval_unsharded': 3000, 'save_interval_ephemeral': None, 'save_num_checkpoints_to_keep': -1, 'save_num_unsharded_checkpoints_to_keep': -1, 'save_overwrite': True, 'force_save_unsharded': False, 'no_pre_train_checkpoint': False, 'load_path': None, 'load_path_sharded_checkpointer': None, 'try_load_latest_save': False, 'reset_optimizer_state': False, 'reset_trainer_state': False, 'sharded_checkpointer': 'torch_legacy', 'new_style_checkpoints': None, 'max_duration': '1ep', 'global_train_batch_size': 512, 'device_train_batch_size': 512, 'device_train_microbatch_size': 16, 'device_eval_batch_size': 16, 'eval_subset_num_batches': -1, 'eval_on_load': False, 'device_train_grad_accum': 32, 'max_grad_norm': 1.0, 'max_grad_norm_ratio': None, 'precision': 'amp_bf16', 'speed_monitor': {'window_size': 20, 'gpu_flops_available': None}, 'console_log_interval': 1, 'gen1_gc_interval': 1, 'compile': None, 'distributed_strategy': 'fsdp', 'fsdp': {'use_orig_params': True, 'sharding_strategy': <ShardingStrategy.FULL_SHARD: 1>, 'wrapping_strategy': None, 'precision': 'mixed', 'hybrid_sharding_num_model_replicas': None}, 'ddp': {'grad_sync_mode': 'batch', 'find_unused_params': False}, 'single': {'device': 'auto'}, 'softmax_auxiliary_loss': False, 'auxiliary_loss_multiplier': 0.0001, 'time_limit': None, 'extra_steps_after_cancel': 10, 'early_stopping_factor': None, 'save_data_indices': True, 'python_profiling': False, 'torch_profiling': False, 'stop_at': None, 'stop_after': None, 'activation_checkpointing': None, 'fused_loss': None, 'hf_datasets_cache_dir': None, 'module_outputs_save_steps': None, '_wandb': {}}
-2025-12-14 15:40:39,915 INFO    MainThread:3750944 [wandb_init.py:init():889] starting backend
-2025-12-14 15:40:40,160 INFO    MainThread:3750944 [wandb_init.py:init():892] sending inform_init request
-2025-12-14 15:40:40,166 INFO    MainThread:3750944 [wandb_init.py:init():900] backend started and connected
-2025-12-14 15:40:40,169 INFO    MainThread:3750944 [wandb_init.py:init():970] updated telemetry
-2025-12-14 15:40:40,171 INFO    MainThread:3750944 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
-2025-12-14 15:40:40,795 INFO    MainThread:3750944 [wandb_init.py:init():1041] starting run threads in backend
-2025-12-14 15:40:40,887 INFO    MainThread:3750944 [wandb_run.py:_console_start():2521] atexit reg
-2025-12-14 15:40:40,888 INFO    MainThread:3750944 [wandb_run.py:_redirect():2369] redirect: wrap_raw
-2025-12-14 15:40:40,889 INFO    MainThread:3750944 [wandb_run.py:_redirect():2438] Wrapping output streams.
-2025-12-14 15:40:40,890 INFO    MainThread:3750944 [wandb_run.py:_redirect():2461] Redirects installed.
-2025-12-14 15:40:40,892 INFO    MainThread:3750944 [wandb_init.py:init():1081] run started, returning control to user process
-2025-12-14 15:41:54,674 INFO    MainThread:3750944 [wandb_run.py:_finish():2287] finishing run marksmans/olmo-debug/z9xp3525
-2025-12-14 15:41:54,675 INFO    MainThread:3750944 [wandb_run.py:_atexit_cleanup():2486] got exitcode: 1
-2025-12-14 15:41:54,676 INFO    MainThread:3750944 [wandb_run.py:_restore():2468] restore
-2025-12-14 15:41:54,676 INFO    MainThread:3750944 [wandb_run.py:_restore():2474] restore done
-2025-12-14 15:41:55,863 INFO    MainThread:3750944 [wandb_run.py:_footer_sync_info():3862] logging synced files

models/OLMo-1B/wandb/wandb/run-20251214_154707-2hopup81/files/config.yaml

@@ -1,575 +0,0 @@
-_wandb:
-    value:
-        cli_version: 0.23.1
-        e:
-            61ybci2w6pjjhx1rssgrci0sff4exdn7:
-                args:
-                    - pretraining/configs/RL-1B.yaml
-                codePath: OLMo/scripts/train.py
-                codePathLocal: OLMo/scripts/train.py
-                cpu_count: 112
-                cpu_count_logical: 224
-                cudaVersion: "13.0"
-                disk:
-                    /:
-                        total: "2055141851136"
-                        used: "49256501248"
-                email: [email protected]
-                executable: /opt/conda/bin/python
-                gpu: NVIDIA H100 80GB HBM3
-                gpu_count: 1
-                gpu_nvidia:
-                    - architecture: Hopper
-                      cudaCores: 16896
-                      memoryTotal: "85520809984"
-                      name: NVIDIA H100 80GB HBM3
-                      uuid: GPU-4c999b2a-2578-9e62-0539-4b826d85fda8
-                host: serv-3342
-                memory:
-                    total: "2164176814080"
-                os: Linux-6.8.0-1043-nvidia-x86_64-with-glibc2.35
-                program: /netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/scripts/train.py
-                python: CPython 3.11.11
-                root: checkpoints/OLMo-1B-as_fm3_omi2/wandb
-                slurm:
-                    cluster_name: pegasus
-                    conf: /etc/slurm/slurm.conf
-                    cpu_bind: quiet,mask_cpu:0x000000000001FE00000000000000000000000001FE00000000000000
-                    cpu_bind_list: 0x000000000001FE00000000000000000000000001FE00000000000000
-                    cpu_bind_type: 'mask_cpu:'
-                    cpu_bind_verbose: quiet
-                    cpus_on_node: "16"
-                    cpus_per_task: "16"
-                    distribution: cyclic
-                    gpus: "1"
-                    gpus_on_node: "1"
-                    gtids: "0"
-                    job_cpus_per_node: "16"
-                    job_end_time: "1765734524"
-                    job_gid: "8000"
-                    job_group: iml
-                    job_id: "2383756"
-                    job_name: bash
-                    job_nodelist: serv-3342
-                    job_num_nodes: "1"
-                    job_partition: H100
-                    job_qos: normal
-                    job_start_time: "1765720124"
-                    job_uid: "13262"
-                    job_user: nguyen
-                    jobid: "2383756"
-                    launch_node_ipaddr: 192.168.33.114
-                    localid: "0"
-                    mem_per_cpu: "16384"
-                    mpi_type: pmix
-                    nnodes: "1"
-                    nodeid: "0"
-                    nodelist: serv-3342
-                    nprocs: "1"
-                    ntasks: "1"
-                    oom_kill_step: "0"
-                    pmix_mapping_serv: (vector,(0,1,1))
-                    pmixp_abort_agent_port: "33735"
-                    prio_process: "1"
-                    procid: "0"
-                    pty_port: "45219"
-                    pty_win_col: "156"
-                    pty_win_row: "41"
-                    srun_comm_host: 192.168.33.114
-                    srun_comm_port: "35153"
-                    step_gpus: "5"
-                    step_id: "0"
-                    step_launcher_port: "35153"
-                    step_nodelist: serv-3342
-                    step_num_nodes: "1"
-                    step_num_tasks: "1"
-                    step_tasks_per_node: "1"
-                    stepid: "0"
-                    submit_dir: /netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain
-                    submit_host: login1
-                    task_pid: "3684902"
-                    tasks_per_node: "1"
-                    topology_addr: serv-3342
-                    topology_addr_pattern: node
-                    tres_bind: gres/gpu:per_task:1
-                    tres_per_task: cpu=16,gres/gpu=1
-                    umask: "0022"
-                startedAt: "2025-12-14T14:47:07.756230Z"
-                writerId: 61ybci2w6pjjhx1rssgrci0sff4exdn7
-        m: []
-        python_version: 3.11.11
-        t:
-            "1":
-                - 1
-                - 5
-                - 11
-                - 41
-                - 49
-                - 51
-                - 53
-            "2":
-                - 1
-                - 5
-                - 11
-                - 41
-                - 49
-                - 51
-                - 53
-            "3":
-                - 13
-                - 15
-                - 16
-            "4": 3.11.11
-            "5": 0.23.1
-            "6": 4.57.3
-            "12": 0.23.1
-            "13": linux-x86_64
-activation_checkpointing:
-    value: null
-auxiliary_loss_multiplier:
-    value: 0.0001
-canceled_check_interval:
-    value: 6000
-compile:
-    value: null
-console_log_interval:
-    value: 1
-data:
-    value:
-        custom_dataset: null
-        datasets: null
-        drop_last: true
-        generate_attention_mask: false
-        generate_doc_lengths: false
-        instance_filter: null
-        label_mask_paths: null
-        memmap_dtype: uint16
-        num_workers: 32
-        pad_direction: right
-        paths:
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00000_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00001_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00002_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00003_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00004_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00005_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00006_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00007_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00008_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00009_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00010_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00011_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00012_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00016_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00019_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00020_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00021_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00022_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00023_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00024_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00025_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00026_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00027_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00028_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00029_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00030_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00031_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00032_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00033_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00034_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00035_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00036_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00037_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00038_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00039_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00040_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00041_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00042_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00043_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00044_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00045_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00046_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00047_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00048_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00049_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00050_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00051_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00052_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00053_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00054_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00055_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00056_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00057_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00058_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00059_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00060_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00061_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00062_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00063_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00064_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00065_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00066_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00067_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00068_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00069_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00070_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00071_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00072_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00073_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00074_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00075_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00076_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00077_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/algebraic-stack-tokenized/00078_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00000_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00001_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00002_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00003_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00004_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00005_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00006_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00007_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00008_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00009_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00010_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00011_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00012_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00013_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00014_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00015_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00016_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00017_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00018_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00019_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00020_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00021_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00022_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00023_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00024_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00025_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00026_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00027_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00028_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00029_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00030_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00031_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00032_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00033_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00034_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00035_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00036_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00037_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00038_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00039_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00040_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00041_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00042_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00043_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00044_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00045_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00046_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00047_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00048_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00049_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00050_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00051_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00052_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00053_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00054_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00055_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00056_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00057_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00058_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00059_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00060_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00061_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00062_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00063_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00064_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00065_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00066_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00067_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00068_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00069_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00070_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00071_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00072_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00073_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00074_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00075_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00076_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00077_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00078_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00079_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00080_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00081_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00082_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00083_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00084_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00085_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00086_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00087_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00088_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00089_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00090_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00091_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00092_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00093_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00094_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00095_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00096_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00097_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00098_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/finemath3-tokenized/00099_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00000_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00001_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00002_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00003_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00004_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00005_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00006_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00007_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00008_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00009_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00010_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00011_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00012_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00013_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00014_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00015_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00016_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00017_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00018_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00019_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00020_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00021_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00022_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00023_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00024_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00025_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00026_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00027_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00028_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00029_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00030_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00031_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00032_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00033_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00034_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00035_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00036_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00037_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00038_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00039_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00040_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00041_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00042_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00043_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00044_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00045_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00046_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00047_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00048_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00049_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00050_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00051_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00052_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00053_00000_doc_shuffled.ds
-            - data_token/as_fm3_omi2/openmathinstruct2-tokenized/00054_00000_doc_shuffled.ds
-        persistent_workers: true
-        pin_memory: true
-        prefetch_factor: 8
-        seed: null
-        timeout: 0
-ddp:
-    value:
-        find_unused_params: false
-        grad_sync_mode: batch
-device_eval_batch_size:
-    value: 4
-device_train_batch_size:
-    value: 512
-device_train_grad_accum:
-    value: 128
-device_train_microbatch_size:
-    value: 4
-distributed_strategy:
-    value: fsdp
-dry_run:
-    value: false
-early_stopping_factor:
-    value: null
-epoch:
-    value: null
-eval_interval:
-    value: 5000
-eval_on_load:
-    value: false
-eval_subset_num_batches:
-    value: -1
-evaluators:
-    value: []
-extra_steps_after_cancel:
-    value: 10
-fast_forward_batches:
-    value: null
-force_save_unsharded:
-    value: false
-fsdp:
-    value:
-        hybrid_sharding_num_model_replicas: null
-        precision: mixed
-        sharding_strategy: FULL_SHARD
-        use_orig_params: true
-        wrapping_strategy: null
-fused_loss:
-    value: null
-gen1_gc_interval:
-    value: 1
-global_train_batch_size:
-    value: 512
-hf_datasets_cache_dir:
-    value: null
-load_path:
-    value: null
-load_path_sharded_checkpointer:
-    value: null
-max_duration:
-    value: 1ep
-max_grad_norm:
-    value: 1
-max_grad_norm_ratio:
-    value: null
-model:
-    value:
-        activation_type: swiglu
-        alibi: false
-        alibi_bias_max: 8
-        attention_dropout: 0
-        attention_layer_norm: false
-        attention_layer_norm_with_affine: false
-        bias_for_layer_norm: false
-        block_group_size: 1
-        block_type: sequential
-        clip_qkv: null
-        d_model: 2048
-        emb_init_std: null
-        embedding_dropout: 0
-        embedding_layer_norm: false
-        embedding_size: 32000
-        eos_token_id: 0
-        flash_attention: false
-        include_bias: false
-        init_cutoff_factor: 3
-        init_device: cuda
-        init_fn: normal
-        init_std: 0.02
-        layer_norm_eps: 1e-05
-        layer_norm_type: default
-        layer_norm_with_affine: false
-        max_sequence_length: 2048
-        mlp_hidden_size: null
-        mlp_ratio: 8
-        multi_query_attention: false
-        n_heads: 16
-        n_kv_heads: null
-        n_layers: 16
-        norm_after: false
-        pad_token_id: 1
-        precision: amp_bf16
-        residual_dropout: 0
-        rope: true
-        rope_full_precision: true
-        rope_theta: 10000
-        scale_emb_init: false
-        scale_logits: false
-        vocab_size: 32000
-        weight_tying: true
-module_outputs_save_steps:
-    value: null
-new_style_checkpoints:
-    value: null
-no_pre_train_checkpoint:
-    value: false
-optimizer:
-    value:
-        betas:
-            - 0.9
-            - 0.95
-        decay_embeddings: true
-        decay_norm_and_bias: true
-        eps: 1e-08
-        learning_rate: 0.0005
-        metrics_log_interval: 10
-        name: adamw
-        no_decay_norm_and_bias: null
-        record_update_metrics: false
-        selective_updates: false
-        weight_decay: 0.1
-precision:
-    value: amp_bf16
-python_profiling:
-    value: false
-remote_save_folder:
-    value: null
-reset_optimizer_state:
-    value: false
-reset_trainer_state:
-    value: false
-restore_dataloader:
-    value: true
-run_name:
-    value: OLMo-1B-as_fm3_omi2
-save_data_indices:
-    value: true
-save_folder:
-    value: checkpoints/OLMo-1B-as_fm3_omi2
-save_interval:
-    value: 3000
-save_interval_ephemeral:
-    value: null
-save_interval_unsharded:
-    value: 3000
-save_num_checkpoints_to_keep:
-    value: -1
-save_num_unsharded_checkpoints_to_keep:
-    value: -1
-save_overwrite:
-    value: true
-scheduler:
-    value:
-        alpha_f: 0.1
-        grad_clip_warmup_factor: null
-        grad_clip_warmup_steps: null
-        name: cosine_with_warmup
-        t_max: null
-        t_warmup: 2000
-        units: steps
-        warmup_min_lr: 0
-seed:
-    value: 6198
-sharded_checkpointer:
-    value: torch_legacy
-single:
-    value:
-        device: auto
-softmax_auxiliary_loss:
-    value: false
-speed_monitor:
-    value:
-        gpu_flops_available: null
-        window_size: 20
-stop_after:
-    value: null
-stop_at:
-    value: null
-time_limit:
-    value: null
-tokenizer:
-    value:
-        identifier: meta-llama/Llama-2-7b-hf
-        truncate_direction: right
-torch_profiling:
-    value: false
-try_load_latest_save:
-    value: false

models/OLMo-1B/wandb/wandb/run-20251214_154707-2hopup81/files/output.log

@@ -1,35 +0,0 @@
-[2025-12-14 15:47:11] INFO     [olmo.data.iterable_dataset:79, rank=0] Saving global data order indices...
-Traceback (most recent call last):
-  File "/netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/scripts/train.py", line 436, in <module>
-    main(cfg)
-  File "/netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/scripts/train.py", line 132, in main
-    train_loader = build_train_dataloader(cfg)
-                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/olmo/data/__init__.py", line 156, in build_train_dataloader
-    dataset = IterableDataset(
-              ^^^^^^^^^^^^^^^^
-  File "/netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/olmo/data/iterable_dataset.py", line 73, in __init__
-    self._build_and_save_global_indices()
-  File "/netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/olmo/data/iterable_dataset.py", line 86, in _build_and_save_global_indices
-    global_indices_mmap.flush()
-  File "/opt/conda/lib/python3.11/site-packages/numpy/core/memmap.py", line 301, in flush
-    def flush(self):
-
-KeyboardInterrupt
-Traceback (most recent call last):
-  File "/netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/scripts/train.py", line 436, in <module>
-    main(cfg)
-  File "/netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/scripts/train.py", line 132, in main
-    train_loader = build_train_dataloader(cfg)
-                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/olmo/data/__init__.py", line 156, in build_train_dataloader
-    dataset = IterableDataset(
-              ^^^^^^^^^^^^^^^^
-  File "/netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/olmo/data/iterable_dataset.py", line 73, in __init__
-    self._build_and_save_global_indices()
-  File "/netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/olmo/data/iterable_dataset.py", line 86, in _build_and_save_global_indices
-    global_indices_mmap.flush()
-  File "/opt/conda/lib/python3.11/site-packages/numpy/core/memmap.py", line 301, in flush
-    def flush(self):
-
-KeyboardInterrupt

models/OLMo-1B/wandb/wandb/run-20251214_154707-2hopup81/files/requirements.txt

@@ -1,271 +0,0 @@
-scikit-learn==1.8.0
-joblib==1.5.2
-threadpoolctl==3.6.0
-torchmetrics==1.8.2
-lightning-utilities==0.15.2
-wandb==0.23.1
-GitPython==3.1.45
-gitdb==4.0.12
-smmap==5.0.2
-datasets==4.4.1
-pandas==2.3.3
-multiprocess==0.70.18
-pyarrow==22.0.0
-tzdata==2025.3
-xxhash==3.6.0
-ai2-olmo==0.6.0
-ai2-olmo-core==2.4.0
-cached_path==1.8.0
-google-cloud-storage==2.19.0
-google-cloud-core==2.5.0
-boto3==1.42.9
-google-api-core==2.28.1
-google-auth==2.43.0
-s3transfer==0.16.0
-botocore==1.42.9
-google-resumable-media==2.8.0
-pyasn1_modules==0.4.2
-rich==13.9.4
-rsa==4.9.1
-bettermap==1.3.1
-google-crc32c==1.7.1
-jmespath==1.0.1
-numpy==1.26.4
-omegaconf==2.3.0
-proto-plus==1.26.1
-pyasn1==0.6.1
-python-dateutil==2.9.0.post0
-antlr4-python3-runtime==4.9.3
-zstandard==0.23.0
-zipp==3.21.0
-yarl==1.22.0
-xgrammar==0.1.18
-xformers==0.0.29.post2
-wrapt==2.0.1
-wheel==0.45.1
-websockets==15.0.1
-wcwidth==0.2.13
-watchfiles==1.1.1
-vllm==0.8.5.post1
-uvloop==0.22.1
-uvicorn==0.38.0
-urllib3==2.3.0
-typing-inspection==0.4.2
-typing_extensions==4.15.0
-types-dataclasses==0.6.6
-typer==0.20.0
-truststore==0.10.0
-triton==3.2.0
-transformers==4.57.3
-traitlets==5.14.3
-tqdm==4.67.1
-torchvision==0.21.0+cu124
-torchelastic==0.2.2
-torchaudio==2.6.0+cu124
-torch==2.6.0+cu124
-tokenizers==0.22.1
-tiktoken==0.12.0
-sympy==1.13.1
-starlette==0.50.0
-stack_data==0.6.3
-soupsieve==2.5
-sortedcontainers==2.4.0
-sniffio==1.3.1
-six==1.17.0
-shellingham==1.5.4
-setuptools==75.8.0
-sentry-sdk==2.47.0
-sentencepiece==0.2.1
-scipy==1.16.3
-safetensors==0.7.0
-ruamel.yaml.clib==0.2.8
-ruamel.yaml==0.18.10
-rpds-py==0.22.3
-rignore==0.7.6
-rich-toolkit==0.17.0
-requests==2.32.3
-regex==2025.11.3
-referencing==0.36.2
-ray==2.52.1
-pyzmq==27.1.0
-pytz==2024.2
-python-multipart==0.0.20
-python-json-logger==4.0.0
-python-etcd==0.4.5
-python-dotenv==1.2.1
-Pygments==2.19.1
-pydantic-extra-types==2.10.6
-pydantic_core==2.41.5
-pydantic==2.12.5
-pycparser==2.22
-pycountry==24.6.1
-pycosat==0.6.6
-py-cpuinfo==9.0.0
-pure_eval==0.2.3
-ptyprocess==0.7.0
-psutil==6.1.1
-protobuf==4.25.8
-propcache==0.4.1
-prompt_toolkit==3.0.50
-prometheus-fastapi-instrumentator==7.1.0
-prometheus_client==0.23.1
-pluggy==1.5.0
-platformdirs==4.3.6
-pkgutil_resolve_name==1.3.10
-pkginfo==1.12.0
-pip==24.3.1
-pillow==11.0.0
-pickleshare==0.7.5
-pexpect==4.9.0
-partial-json-parser==0.2.1.1.post7
-parso==0.8.4
-packaging==24.2
-outlines_core==0.1.26
-outlines==0.1.11
-optree==0.14.0
-opentelemetry-semantic-conventions-ai==0.4.13
-opentelemetry-semantic-conventions==0.47b0
-opentelemetry-sdk==1.26.0
-opentelemetry-proto==1.26.0
-opentelemetry-exporter-otlp-proto-http==1.26.0
-opentelemetry-exporter-otlp-proto-grpc==1.26.0
-opentelemetry-exporter-otlp-proto-common==1.26.0
-opentelemetry-exporter-otlp==1.26.0
-opentelemetry-api==1.26.0
-opencv-python-headless==4.12.0.88
-openai==2.11.0
-nvidia-nvtx-cu12==12.4.127
-nvidia-nvjitlink-cu12==12.4.127
-nvidia-nccl-cu12==2.21.5
-nvidia-cusparselt-cu12==0.6.2
-nvidia-cusparse-cu12==12.3.1.170
-nvidia-cusolver-cu12==11.6.1.9
-nvidia-curand-cu12==10.3.5.147
-nvidia-cufft-cu12==11.2.1.3
-nvidia-cudnn-cu12==9.1.0.70
-nvidia-cuda-runtime-cu12==12.4.127
-nvidia-cuda-nvrtc-cu12==12.4.127
-nvidia-cuda-cupti-cu12==12.4.127
-nvidia-cublas-cu12==12.4.5.8
-numba==0.61.2
-ninja==1.11.1.3
-networkx==3.4.2
-nest-asyncio==1.6.0
-multidict==6.7.0
-msgspec==0.20.0
-msgpack==1.1.2
-mpmath==1.3.0
-more-itertools==10.6.0
-mistral_common==1.8.6
-menuinst==2.2.0
-mdurl==0.1.2
-matplotlib-inline==0.1.7
-math-verify==0.8.0
-markdown-it-py==4.0.0
-lm-format-enforcer==0.10.12
-llvmlite==0.44.0
-llguidance==0.7.30
-lintrunner==0.12.7
-lief==0.14.1
-libmambapy==2.0.5
-libarchive-c==5.1
-latex2sympy2_extended==1.10.2
-lark==1.2.2
-jsonschema-specifications==2024.10.1
-jsonschema==4.23.0
-jsonpointer==3.0.0
-jsonpatch==1.33
-jiter==0.12.0
-Jinja2==3.1.5
-jedi==0.19.2
-ipython==8.31.0
-interegular==0.3.3
-importlib_resources==6.5.2
-importlib_metadata==8.0.0
-idna==3.10
-hypothesis==6.124.7
-hyperframe==6.0.1
-huggingface-hub==0.36.0
-httpx==0.28.1
-httptools==0.7.1
-httpcore==1.0.9
-hpack==4.0.0
-hf-xet==1.2.0
-h2==4.1.0
-h11==0.16.0
-grpcio==1.76.0
-googleapis-common-protos==1.72.0
-gguf==0.17.1
-fsspec==2024.12.0
-frozenlist==1.8.0
-frozendict==2.4.6
-filelock==3.17.0
-fastrlock==0.8.3
-fastar==0.8.0
-fastapi-cloud-cli==0.6.0
-fastapi-cli==0.0.16
-fastapi==0.124.4
-expecttest==0.3.0
-executing==2.1.0
-exceptiongroup==1.2.2
-email-validator==2.3.0
-einops==0.8.1
-dnspython==2.7.0
-distro==1.9.0
-diskcache==5.6.3
-dill==0.4.0
-depyf==0.18.0
-Deprecated==1.3.1
-decorator==5.1.1
-cupy-cuda12x==13.6.0
-conda_package_streaming==0.11.0
-conda-package-handling==2.4.0
-conda-libmamba-solver==25.1.1
-conda_index==0.5.0
-conda-build==25.1.1
-conda==25.1.0
-compressed-tensors==0.9.3
-colorama==0.4.6
-cmake==3.31.4
-cloudpickle==3.1.2
-click==8.1.8
-charset-normalizer==3.4.1
-chardet==5.2.0
-cffi==1.17.1
-certifi==2024.12.14
-cachetools==6.2.3
-boltons==24.0.0
-blake3==1.0.8
-beautifulsoup4==4.12.3
-attrs==25.1.0
-astunparse==1.6.3
-asttokens==3.0.0
-astor==0.8.1
-archspec==0.2.5
-anyio==4.12.0
-annotated-types==0.7.0
-annotated-doc==0.0.4
-airportsdata==20250909
-aiosignal==1.4.0
-aiohttp==3.13.2
-aiohappyeyeballs==2.6.1
-PyYAML==6.0.2
-PySocks==1.7.1
-MarkupSafe==3.0.2
-Brotli==1.1.0
-zipp==3.19.2
-wheel==0.43.0
-typing_extensions==4.12.2
-typeguard==4.3.0
-tomli==2.0.1
-platformdirs==4.2.2
-packaging==24.2
-more-itertools==10.3.0
-jaraco.text==3.12.1
-jaraco.functools==4.0.1
-jaraco.context==5.3.0
-jaraco.collections==5.1.0
-inflect==7.3.1
-importlib_metadata==8.0.0
-backports.tarfile==1.2.0
-autocommand==2.2.2

models/OLMo-1B/wandb/wandb/run-20251214_154707-2hopup81/files/wandb-metadata.json

@@ -1,103 +0,0 @@
-{
-  "os":  "Linux-6.8.0-1043-nvidia-x86_64-with-glibc2.35",
-  "python":  "CPython 3.11.11",
-  "startedAt":  "2025-12-14T14:47:07.756230Z",
-  "args":  [
-    "pretraining/configs/RL-1B.yaml"
-  ],
-  "program":  "/netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/OLMo/scripts/train.py",
-  "codePath":  "OLMo/scripts/train.py",
-  "codePathLocal":  "OLMo/scripts/train.py",
-  "email":  "[email protected]",
-  "root":  "checkpoints/OLMo-1B-as_fm3_omi2/wandb",
-  "host":  "serv-3342",
-  "executable":  "/opt/conda/bin/python",
-  "cpu_count":  112,
-  "cpu_count_logical":  224,
-  "gpu":  "NVIDIA H100 80GB HBM3",
-  "gpu_count":  1,
-  "disk":  {
-    "/":  {
-      "total":  "2055141851136",
-      "used":  "49256501248"
-    }
-  },
-  "memory":  {
-    "total":  "2164176814080"
-  },
-  "gpu_nvidia":  [
-    {
-      "name":  "NVIDIA H100 80GB HBM3",
-      "memoryTotal":  "85520809984",
-      "cudaCores":  16896,
-      "architecture":  "Hopper",
-      "uuid":  "GPU-4c999b2a-2578-9e62-0539-4b826d85fda8"
-    }
-  ],
-  "cudaVersion":  "13.0",
-  "slurm":  {
-    "cluster_name":  "pegasus",
-    "conf":  "/etc/slurm/slurm.conf",
-    "cpu_bind":  "quiet,mask_cpu:0x000000000001FE00000000000000000000000001FE00000000000000",
-    "cpu_bind_list":  "0x000000000001FE00000000000000000000000001FE00000000000000",
-    "cpu_bind_type":  "mask_cpu:",
-    "cpu_bind_verbose":  "quiet",
-    "cpus_on_node":  "16",
-    "cpus_per_task":  "16",
-    "distribution":  "cyclic",
-    "gpus":  "1",
-    "gpus_on_node":  "1",
-    "gtids":  "0",
-    "job_cpus_per_node":  "16",
-    "job_end_time":  "1765734524",
-    "job_gid":  "8000",
-    "job_group":  "iml",
-    "job_id":  "2383756",
-    "job_name":  "bash",
-    "job_nodelist":  "serv-3342",
-    "job_num_nodes":  "1",
-    "job_partition":  "H100",
-    "job_qos":  "normal",
-    "job_start_time":  "1765720124",
-    "job_uid":  "13262",
-    "job_user":  "nguyen",
-    "jobid":  "2383756",
-    "launch_node_ipaddr":  "192.168.33.114",
-    "localid":  "0",
-    "mem_per_cpu":  "16384",
-    "mpi_type":  "pmix",
-    "nnodes":  "1",
-    "nodeid":  "0",
-    "nodelist":  "serv-3342",
-    "nprocs":  "1",
-    "ntasks":  "1",
-    "oom_kill_step":  "0",
-    "pmix_mapping_serv":  "(vector,(0,1,1))",
-    "pmixp_abort_agent_port":  "33735",
-    "prio_process":  "1",
-    "procid":  "0",
-    "pty_port":  "45219",
-    "pty_win_col":  "156",
-    "pty_win_row":  "41",
-    "srun_comm_host":  "192.168.33.114",
-    "srun_comm_port":  "35153",
-    "step_gpus":  "5",
-    "step_id":  "0",
-    "step_launcher_port":  "35153",
-    "step_nodelist":  "serv-3342",
-    "step_num_nodes":  "1",
-    "step_num_tasks":  "1",
-    "step_tasks_per_node":  "1",
-    "stepid":  "0",
-    "submit_dir":  "/netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain",
-    "submit_host":  "login1",
-    "task_pid":  "3684902",
-    "tasks_per_node":  "1",
-    "topology_addr":  "serv-3342",
-    "topology_addr_pattern":  "node",
-    "tres_bind":  "gres/gpu:per_task:1",
-    "tres_per_task":  "cpu=16,gres/gpu=1",
-    "umask":  "0022"
-  },
-  "writerId":  "61ybci2w6pjjhx1rssgrci0sff4exdn7"
-}

models/OLMo-1B/wandb/wandb/run-20251214_154707-2hopup81/files/wandb-summary.json

@@ -1 +0,0 @@
-{"_wandb":{"runtime":3},"_runtime":3}

models/OLMo-1B/wandb/wandb/run-20251214_154707-2hopup81/logs/debug-internal.log

@@ -1,11 +0,0 @@
-{"time":"2025-12-14T15:47:08.105165256+01:00","level":"INFO","msg":"stream: starting","core version":"0.23.1"}
-{"time":"2025-12-14T15:47:08.37501868+01:00","level":"INFO","msg":"stream: created new stream","id":"2hopup81"}
-{"time":"2025-12-14T15:47:08.376220545+01:00","level":"INFO","msg":"handler: started","stream_id":"2hopup81"}
-{"time":"2025-12-14T15:47:08.491014958+01:00","level":"INFO","msg":"stream: started","id":"2hopup81"}
-{"time":"2025-12-14T15:47:08.491053676+01:00","level":"INFO","msg":"writer: started","stream_id":"2hopup81"}
-{"time":"2025-12-14T15:47:08.491071748+01:00","level":"INFO","msg":"sender: started","stream_id":"2hopup81"}
-{"time":"2025-12-14T15:47:12.190256481+01:00","level":"INFO","msg":"stream: closing","id":"2hopup81"}
-{"time":"2025-12-14T15:47:13.127841021+01:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
-{"time":"2025-12-14T15:47:13.326969802+01:00","level":"INFO","msg":"handler: closed","stream_id":"2hopup81"}
-{"time":"2025-12-14T15:47:13.330040788+01:00","level":"INFO","msg":"sender: closed","stream_id":"2hopup81"}
-{"time":"2025-12-14T15:47:13.330899511+01:00","level":"INFO","msg":"stream: closed","id":"2hopup81"}

models/OLMo-1B/wandb/wandb/run-20251214_154707-2hopup81/logs/debug.log

@@ -1,23 +0,0 @@
-2025-12-14 15:47:07,839 INFO    MainThread:3760501 [wandb_setup.py:_flush():80] Current SDK version is 0.23.1
-2025-12-14 15:47:07,840 INFO    MainThread:3760501 [wandb_setup.py:_flush():80] Configure stats pid to 3760501
-2025-12-14 15:47:07,841 INFO    MainThread:3760501 [wandb_setup.py:_flush():80] Loading settings from /root/.config/wandb/settings
-2025-12-14 15:47:07,842 INFO    MainThread:3760501 [wandb_setup.py:_flush():80] Loading settings from /netscratch/duynguyen/Research/gen_collapse/run_code/openrlhf-pretrain/wandb/settings
-2025-12-14 15:47:07,843 INFO    MainThread:3760501 [wandb_setup.py:_flush():80] Loading settings from environment variables
-2025-12-14 15:47:07,845 INFO    MainThread:3760501 [wandb_init.py:setup_run_log_directory():714] Logging user logs to checkpoints/OLMo-1B-as_fm3_omi2/wandb/wandb/run-20251214_154707-2hopup81/logs/debug.log
-2025-12-14 15:47:07,846 INFO    MainThread:3760501 [wandb_init.py:setup_run_log_directory():715] Logging internal logs to checkpoints/OLMo-1B-as_fm3_omi2/wandb/wandb/run-20251214_154707-2hopup81/logs/debug-internal.log
-2025-12-14 15:47:07,847 INFO    MainThread:3760501 [wandb_init.py:init():841] calling init triggers
-2025-12-14 15:47:07,848 INFO    MainThread:3760501 [wandb_init.py:init():846] wandb.init called with sweep_config: {}
-config: {'run_name': 'OLMo-1B-as_fm3_omi2', 'seed': 6198, 'epoch': None, 'dry_run': False, 'model': {'d_model': 2048, 'n_heads': 16, 'n_kv_heads': None, 'clip_qkv': None, 'n_layers': 16, 'mlp_ratio': 8, 'mlp_hidden_size': None, 'activation_type': 'swiglu', 'block_type': 'sequential', 'block_group_size': 1, 'alibi': False, 'alibi_bias_max': 8.0, 'rope': True, 'rope_full_precision': True, 'rope_theta': 10000, 'flash_attention': False, 'attention_dropout': 0.0, 'multi_query_attention': False, 'attention_layer_norm': False, 'residual_dropout': 0.0, 'embedding_dropout': 0.0, 'embedding_layer_norm': False, 'layer_norm_type': 'default', 'layer_norm_with_affine': False, 'layer_norm_eps': 1e-05, 'attention_layer_norm_with_affine': False, 'max_sequence_length': 2048, 'include_bias': False, 'bias_for_layer_norm': False, 'scale_logits': False, 'vocab_size': 32000, 'embedding_size': 32000, 'weight_tying': True, 'eos_token_id': 0, 'pad_token_id': 1, 'init_device': 'cuda', 'init_fn': 'normal', 'init_std': 0.02, 'init_cutoff_factor': 3.0, 'precision': 'amp_bf16', 'scale_emb_init': False, 'emb_init_std': None, 'norm_after': False}, 'optimizer': {'name': 'adamw', 'learning_rate': 0.0005, 'weight_decay': 0.1, 'betas': (0.9, 0.95), 'eps': 1e-08, 'no_decay_norm_and_bias': None, 'selective_updates': False, 'decay_norm_and_bias': True, 'decay_embeddings': True, 'metrics_log_interval': 10, 'record_update_metrics': False}, 'scheduler': {'name': 'cosine_with_warmup', 'units': 'steps', 't_warmup': 2000, 't_max': None, 'alpha_f': 0.1, 'grad_clip_warmup_steps': None, 'grad_clip_warmup_factor': None, 'warmup_min_lr': 0.0}, 'data': {'paths': ['data_token/as_fm3_omi2/algebraic-stack-tokenized/00000_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00001_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00002_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00003_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00004_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00005_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00006_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00007_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00008_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00009_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00010_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00011_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00012_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00016_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00019_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00020_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00021_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00022_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00023_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00024_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00025_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00026_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00027_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00028_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00029_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00030_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00031_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00032_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00033_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00034_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00035_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00036_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00037_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00038_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00039_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00040_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00041_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00042_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00043_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00044_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00045_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00046_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00047_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00048_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00049_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00050_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00051_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00052_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00053_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00054_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00055_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00056_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00057_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00058_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00059_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00060_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00061_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00062_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00063_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00064_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00065_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00066_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00067_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00068_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00069_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00070_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00071_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00072_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00073_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00074_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00075_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00076_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00077_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/algebraic-stack-tokenized/00078_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00000_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00001_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00002_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00003_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00004_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00005_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00006_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00007_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00008_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00009_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00010_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00011_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00012_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00013_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00014_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00015_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00016_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00017_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00018_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00019_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00020_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00021_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00022_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00023_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00024_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00025_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00026_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00027_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00028_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00029_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00030_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00031_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00032_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00033_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00034_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00035_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00036_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00037_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00038_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00039_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00040_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00041_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00042_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00043_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00044_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00045_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00046_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00047_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00048_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00049_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00050_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00051_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00052_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00053_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00054_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00055_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00056_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00057_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00058_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00059_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00060_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00061_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00062_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00063_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00064_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00065_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00066_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00067_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00068_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00069_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00070_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00071_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00072_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00073_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00074_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00075_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00076_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00077_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00078_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00079_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00080_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00081_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00082_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00083_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00084_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00085_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00086_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00087_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00088_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00089_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00090_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00091_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00092_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00093_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00094_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00095_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00096_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00097_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00098_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/finemath3-tokenized/00099_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00000_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00001_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00002_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00003_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00004_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00005_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00006_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00007_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00008_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00009_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00010_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00011_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00012_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00013_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00014_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00015_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00016_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00017_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00018_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00019_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00020_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00021_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00022_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00023_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00024_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00025_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00026_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00027_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00028_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00029_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00030_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00031_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00032_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00033_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00034_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00035_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00036_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00037_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00038_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00039_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00040_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00041_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00042_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00043_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00044_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00045_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00046_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00047_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00048_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00049_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00050_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00051_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00052_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00053_00000_doc_shuffled.ds', 'data_token/as_fm3_omi2/openmathinstruct2-tokenized/00054_00000_doc_shuffled.ds'], 'memmap_dtype': 'uint16', 'datasets': None, 'label_mask_paths': None, 'pad_direction': 'right', 'generate_attention_mask': False, 'generate_doc_lengths': False, 'num_workers': 32, 'drop_last': True, 'pin_memory': True, 'prefetch_factor': 8, 'persistent_workers': True, 'timeout': 0, 'seed': None, 'instance_filter': None, 'custom_dataset': None}, 'restore_dataloader': True, 'fast_forward_batches': None, 'evaluators': [], 'eval_interval': 5000, 'tokenizer': {'identifier': 'meta-llama/Llama-2-7b-hf', 'truncate_direction': 'right'}, 'save_folder': 'checkpoints/OLMo-1B-as_fm3_omi2', 'remote_save_folder': None, 'canceled_check_interval': 6000, 'save_interval': 3000, 'save_interval_unsharded': 3000, 'save_interval_ephemeral': None, 'save_num_checkpoints_to_keep': -1, 'save_num_unsharded_checkpoints_to_keep': -1, 'save_overwrite': True, 'force_save_unsharded': False, 'no_pre_train_checkpoint': False, 'load_path': None, 'load_path_sharded_checkpointer': None, 'try_load_latest_save': False, 'reset_optimizer_state': False, 'reset_trainer_state': False, 'sharded_checkpointer': 'torch_legacy', 'new_style_checkpoints': None, 'max_duration': '1ep', 'global_train_batch_size': 512, 'device_train_batch_size': 512, 'device_train_microbatch_size': 4, 'device_eval_batch_size': 4, 'eval_subset_num_batches': -1, 'eval_on_load': False, 'device_train_grad_accum': 128, 'max_grad_norm': 1.0, 'max_grad_norm_ratio': None, 'precision': 'amp_bf16', 'speed_monitor': {'window_size': 20, 'gpu_flops_available': None}, 'console_log_interval': 1, 'gen1_gc_interval': 1, 'compile': None, 'distributed_strategy': 'fsdp', 'fsdp': {'use_orig_params': True, 'sharding_strategy': <ShardingStrategy.FULL_SHARD: 1>, 'wrapping_strategy': None, 'precision': 'mixed', 'hybrid_sharding_num_model_replicas': None}, 'ddp': {'grad_sync_mode': 'batch', 'find_unused_params': False}, 'single': {'device': 'auto'}, 'softmax_auxiliary_loss': False, 'auxiliary_loss_multiplier': 0.0001, 'time_limit': None, 'extra_steps_after_cancel': 10, 'early_stopping_factor': None, 'save_data_indices': True, 'python_profiling': False, 'torch_profiling': False, 'stop_at': None, 'stop_after': None, 'activation_checkpointing': None, 'fused_loss': None, 'hf_datasets_cache_dir': None, 'module_outputs_save_steps': None, '_wandb': {}}
-2025-12-14 15:47:07,849 INFO    MainThread:3760501 [wandb_init.py:init():889] starting backend
-2025-12-14 15:47:08,094 INFO    MainThread:3760501 [wandb_init.py:init():892] sending inform_init request
-2025-12-14 15:47:08,103 INFO    MainThread:3760501 [wandb_init.py:init():900] backend started and connected
-2025-12-14 15:47:08,106 INFO    MainThread:3760501 [wandb_init.py:init():970] updated telemetry
-2025-12-14 15:47:08,107 INFO    MainThread:3760501 [wandb_init.py:init():994] communicating run to backend with 90.0 second timeout
-2025-12-14 15:47:08,863 INFO    MainThread:3760501 [wandb_init.py:init():1041] starting run threads in backend
-2025-12-14 15:47:08,953 INFO    MainThread:3760501 [wandb_run.py:_console_start():2521] atexit reg
-2025-12-14 15:47:08,954 INFO    MainThread:3760501 [wandb_run.py:_redirect():2369] redirect: wrap_raw
-2025-12-14 15:47:08,955 INFO    MainThread:3760501 [wandb_run.py:_redirect():2438] Wrapping output streams.
-2025-12-14 15:47:08,956 INFO    MainThread:3760501 [wandb_run.py:_redirect():2461] Redirects installed.
-2025-12-14 15:47:08,958 INFO    MainThread:3760501 [wandb_init.py:init():1081] run started, returning control to user process
-2025-12-14 15:47:12,190 INFO    wandb-AsyncioManager-main:3760501 [service_client.py:_forward_responses():80] Reached EOF.
-2025-12-14 15:47:12,191 INFO    wandb-AsyncioManager-main:3760501 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.