Update README.md

README.md

@@ -10,7 +10,120 @@ datasets:
 Training Dataset: Semant Search Summarization Dataset   
 Training configuration  
 ```yaml
-tbd.
+max_seq_len: 18000
+global_seed: 42
+
+# Run Name
+run_name: csmpt_summarization_d01 # If left blank, will be read from env var $COMPOSER_RUN_NAME
+
+# Model
+model:
+  name: hf_causal_lm
+  pretrained_model_name_or_path: BUT-FIT/csmpt7b
+  init_device: cpu
+  pretrained: true
+  trust_remote_code: true
+  config_overrides:
+    max_seq_len: ${max_seq_len}
+    attn_config:
+      attn_impl: flash
+      alibi: true
+    resid_pdrop: 0.1
+    tokenizer_name: BUT-FIT/csmpt7b
+
+dataset: &hf_dataset
+  hf_name: BUT-FIT/CzechRAGSummarization
+  decoder_only_format: true
+  max_seq_len: ${max_seq_len}
+
+# Tokenizer
+tokenizer:
+  name: BUT-FIT/csmpt7b
+  kwargs:
+    model_max_length: ${max_seq_len}
+
+# Dataloaders
+train_loader:
+  name: finetuning
+  dataset:
+    <<: *hf_dataset
+    split: train
+    shuffle: true
+    shuffle_seed: ${global_seed}
+  drop_last: true
+  num_workers: 8
+
+eval_loader:
+  name: finetuning
+  dataset:
+    <<: *hf_dataset
+    split: validation
+    shuffle: false
+    shuffle_seed: ${global_seed}
+  drop_last: false
+  num_workers: 2
+
+# Optimization
+scheduler:
+  name: cosine_with_warmup
+  t_warmup: 50ba
+  alpha_f: 0.1
+
+optimizer:
+  name: decoupled_lionw
+  lr: 1.0e-5
+  betas:
+    - 0.9
+    - 0.95
+  weight_decay: 0.0
+
+algorithms:
+  gradient_clipping:
+    clipping_type: norm
+    clipping_threshold: 5.0
+
+max_duration: 4500ba
+eval_interval: 200ba
+eval_first: false
+eval_subset_num_batches: -1
+global_train_batch_size: 64
+
+# System
+seed: ${global_seed}
+device_eval_batch_size: 2
+device_train_microbatch_size: 1
+# device_train_microbatch_size: auto
+precision: amp_bf16
+
+# FSDP
+fsdp_config:
+  sharding_strategy: FULL_SHARD
+  mixed_precision: PURE
+  activation_checkpointing: true
+  activation_checkpointing_reentrant: false
+  activation_cpu_offload: false
+  limit_all_gathers: true
+  verbose: false
+  backward_prefetch: BACKWARD_PRE
+
+# Logging
+progress_bar: true
+log_to_console: true
+console_log_interval: 1ba
+
+callbacks:
+  speed_monitor:
+    window_size: 10
+  lr_monitor: { }
+  memory_monitor: { }
+  runtime_estimator: { }
+
+
+save_num_checkpoints_to_keep: 10  # Important, this cleans up checkpoints saved to DISK
+save_interval: 200ba
+save_folder: ./{run_name}/checkpoints
+
+
 ```
 
 Example: