upload model weights and model card

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
+*.json filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,3 +1,114 @@
 ---
 license: llama3.1
+datasets:
+- OpenCoder-LLM/opc-sft-stage1
+- OpenCoder-LLM/opc-sft-stage2
+language:
+- en
+base_model:
+- meta-llama/Llama-3.1-8B-Instruct
+model-index:
+- name: Control-LLM-Llama3.1-8B-OpenCoder8
+  results:
+  - task:
+      type: code-evaluation
+    dataset:
+      type: mixed
+      name: Code Evaluation Dataset
+    metrics:
+    - name: pass_at_1,n=1 (code_instruct)
+      type: pass_at_1
+      value: 0.770508826583593
+      stderr: 0.013547264970313243
+      verified: false
+    - name: pass_at_1,n=1 (humaneval_greedy_instruct)
+      type: pass_at_1
+      value: 0.823170731707317
+      stderr: 0.029883277857485988
+      verified: false
+    - name: pass_at_1,n=1 (humaneval_plus_greedy_instruct)
+      type: pass_at_1
+      value: 0.7621951219512195
+      stderr: 0.033346454086653404
+      verified: false
+    - name: pass_at_1,n=1 (mbpp_plus_0shot_instruct)
+      type: pass_at_1
+      value: 0.7751322751322751
+      stderr: 0.02150209607822914
+      verified: false
+    - name: pass_at_1,n=1 (mbpp_sanitized_0shot_instruct)
+      type: pass_at_1
+      value: 0.7354085603112841
+      stderr: 0.027569713464529938
+      verified: false
+  - task:
+      type: original-capability
+    dataset:
+      type: meta/Llama-3.1-8B-Instruct-evals
+      name: Llama-3.1-8B-Instruct-evals Dataset
+      dataset_path: "meta-llama/llama-3.1-8_b-instruct-evals"
+      dataset_name: "Llama-3.1-8B-Instruct-evals__arc_challenge__details"
+    metrics:
+    - name: exact_match,strict-match (original_capability_instruct)
+      type: exact_match
+      value: 0.5599378769819771
+      stderr: 0.0028491774433443513
+      verified: false
+    - name: exact_match,strict-match (meta_arc_0shot_instruct)
+      type: exact_match
+      value: 0.8094420600858369
+      stderr: 0.011511446994122106
+      verified: false
+    - name: exact_match,strict-match (meta_gpqa_0shot_cot_instruct)
+      type: exact_match
+      value: 0.32589285714285715
+      stderr: 0.02216910313464341
+      verified: false
+    - name: exact_match,strict-match (meta_mmlu_0shot_instruct)
+      type: exact_match
+      value: 0.681241988320752
+      stderr: 0.003932622311434926
+      verified: false
+    - name: exact_match,strict-match (meta_mmlu_pro_5shot_instruct)
+      type: exact_match
+      value: 0.4029255319148936
+      stderr: 0.004471732136513382
+      verified: false
 ---
+# Control-LLM-Llama3.1-8B-OpenCoder8
+This is a fine-tuned model of Llama-3.1-8B-Instruct for coding tasks on OpenCoder SFT dataset.
+
+## Evaluation Results
+Here is an overview of the evaluation results and findings:
+
+### Hybrid Expansion on OpenCoder
+The following diagram illustrates how hybrid expansion works.
+
+![Catastrophic Forgetting](plots/control_llm_structure_analysis.png)
+
+### Benchmark Results Table
+The table below summarizes evaluation results across coding tasks and original capabilities.
+
+| **Model**          | **MB+** | **MS**  | **HE+** | **HE**  | **C-Avg** | **ARC** | **GP**  | **MLU** | **MLUP** | **O-Avg** | **Overall** |
+|--------------------|---------|---------|---------|---------|-----------|---------|---------|---------|----------|-----------|-------------|
+| Llama3.1-8B-Ins   | 70.4    | 67.7    | 66.5    | 70.7    | 69.1      | 83.4    | 29.9    | 72.4    | 46.7     | 60.5      | 64.8        |
+| OpenCoder-8B-Ins  | 81.2    | 76.3    | 78.0    | 82.3    | 79.5      | 8.2     | 25.4    | 37.4    | 11.3     | 24.6      | 52.1        |
+| Full Param Tune   | 75.1    | 69.6    | 71.3    | 76.8    | 73.3      | 24.4    | 21.9    | 43.0    | 19.2     | 31.5      | 52.4        |
+| Partial Param Tune | 75.7    | 71.6    | 74.4    | 79.3    | 75.0      | 70.2    | 28.1    | 60.7    | 32.4     | 48.3      | 61.7        |
+| Stack Expansion    | 77.2    | 72.8    | 73.2    | 78.7    | 75.6      | 80.0    | 26.3    | 66.6    | 38.2     | 54.2      | 64.9        |
+| **ControlLLM-Hybrid**  | 77.5    | 73.5    | **76.2**| **82.3**| 77.1      | 80.9    | **32.6**| 68.1    | 40.3     | 56.0      | 66.6        |
+
+---
+
+### Explanation:
+- **MB+**: MBPP Plus
+- **MS**: MBPP Sanitized
+- **HE+**: HumanEval Plus
+- **HE**: HumanEval
+- **C-Avg**: Coding - Size Weighted Average across MB+, MS, HE+, and HE
+- **ARC**: ARC benchmark
+- **GP**: GPQA benchmark
+- **MLU**: MMLU (Massive Multitask Language Understanding)
+- **MLUP**: MMLU Pro
+- **O-Avg**: Original Capability - Size Weighted Average across ARC, GPQA, MMLU, and MMLU Pro
+- **Overall**: Combined average across all tasks

benchmark_results_code_instruct.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd81d8a75dc67b64e07892ddef054a4f97b500a473eefef59fe9e199c2b62796
+size 18314

benchmark_results_original_capability_instruct.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88c9c5080306c367e72b0d15f4ecfb846397be12642931381f512373461bf94c
+size 17215

config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:191b75a990419ab5d7c37089f16c9a52e380e84a51a1507436e146164cb2bcdb
+size 1868

evaluation_results.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8c14f9cecfe02b33a127abf1ee2747d728353fa9b6f74c091c4397fa6b848e4
+size 434

generation_config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e3dea9cdfa4a3e0edf1b52d9b1e9609aa22efcddce630060b748284663c8e67
+size 177

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5bd557190a8a8aeb5f1eee0ee62d69efec6d0f88a951ee259cfc918e2f6d28d1
+size 4976715208

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97c6393935d4d382f706f57e22089724ef6b83c77cb235c355139dedf2d5d843
+size 4999836348

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:422f54136e61f1d2419c908bb8eb398ba87bc0c756d512ba405d6683b75e45b6
+size 4915933074

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27d2112bb1def37187f68c5c9cd18f6d4951a6846e2cf579ee56f22e67400e45
+size 4657972498

model.safetensors.index.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:749170718df030d8dc46588d46a1ba2d7f8a6ef945d36a3236b8fd7975924e17
+size 31691

special_tokens_map.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:453db79c09538b7953c4d9846d4bc0b46b86a296f285cdecc29f739f0b98f6a9
+size 572

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a200d62d1a1177908f4310d7e367f0194d474db0038dc1f2f2434a3ef74af7d9
+size 17210284

tokenizer_config.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e407a45cd60fbcdad88f1a22adab70157c47e858c0c94995de05e87b06205aa
+size 55820

train_params.yaml

@@ -0,0 +1,96 @@
+batch_size_training: '4'
+batching_strategy: padding
+checkpoint_type: StateDictType.SHARDED_STATE_DICT
+context_length: '8192'
+curriculum_learning: 'False'
+curriculum_phases: '3'
+dataset: '[''OpenCoderSFTStage2'']'
+ddp_timeout: '36000'
+debug: 'False'
+decay_steps: None
+dist_checkpoint_folder: fine-tuned
+drop_last: 'True'
+dynamic_batch_size: 'False'
+enable_deepspeed: 'False'
+enable_fsdp: 'True'
+enable_memory_profiling: 'False'
+enable_memory_trace: 'False'
+enable_mixed_precision: 'True'
+enable_tensorboard: 'True'
+eta_min: 1e-05
+eval_epoch: '1'
+eval_in_memory: 'False'
+eval_steps: '1000'
+evaluation_strategy: steps
+flop_counter: 'False'
+flop_counter_start: '3'
+fp16: 'False'
+freeze_layers: 'False'
+from_peft_checkpoint: ''
+fsdp_activation_checkpointing: 'True'
+fsdp_cpu_offload: 'False'
+fsdp_cpu_ram_efficient_loading: 'False'
+gamma: '0.85'
+gradient_accumulation_steps: '8'
+gradient_checkpointing: 'True'
+gradient_checkpointing_kwargs: '{''use_reentrant'': False}'
+gradient_clipping: 'False'
+gradient_clipping_threshold: '1.0'
+handle_long_sequences: 'True'
+hf_hub_metrics_cache_dir: /shared/public/data/controlllm/metrics/
+hsdp: 'True'
+learning_rate: 5e-05
+load_best_model_at_end: 'False'
+logging_steps: '500'
+long_sequence_threshold: '16384'
+low_cpu_fsdp: 'False'
+lr: '0.0001'
+lr_scheduler_per_iter: 'True'
+max_eval_step: '500'
+max_grad_norm: '1.0'
+max_step: '0'
+max_tokens_per_batch: '-1'
+max_train_step: '-1'
+memory_per_token: '-1'
+mixed_precision: 'True'
+model_name: PATH/to/Model
+no_cuda: 'False'
+num_epochs: '3'
+num_freeze_layers: '1'
+num_train_epochs: '20'
+num_unfrozen_layers: '8'
+num_workers_dataloader: '0'
+one_gpu: 'False'
+optimizer: AdamW
+overwrite_output_dir: 'False'
+peft_method: lora
+per_device_eval_batch_size: '1'
+per_device_train_batch_size: '12'
+precompute_batches: None
+pure_bf16: 'False'
+quantization: 'False'
+replica_group_size: '1'
+resume_checkpoint_folder: None
+resume_from_latest: 'True'
+run_validation: 'True'
+save_epoch: '1'
+save_metrics: 'False'
+save_model: 'True'
+save_optimizer: 'False'
+save_steps: '1000'
+seed: '42'
+sharding_group_size: '8'
+sharding_strategy: ShardingStrategy.HYBRID_SHARD
+step_size: '1'
+tokenizer_name: None
+trainer: native
+unfrozen_strategy: interweave
+use_fast_kernels: 'False'
+use_fp16: 'False'
+use_peft: 'False'
+use_profiler: 'False'
+use_wandb: 'False'
+val_batch_size: '1'
+warmup_steps: '1000'
+weight_decay: '0.01'
+weight_decay_ratio: '0.1'