Upload model

README.md

@@ -5,7 +5,7 @@ tags:
 - trl
 - sft
 - generated_from_trainer
-base_model: bigcode/starcoder2-15b
+base_model: bigcode/starcoder2-3b
 model-index:
 - name: finetune_starcoder2
   results: []
@@ -14,10 +14,9 @@ model-index:
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
 should probably proofread and complete it, then remove this comment. -->
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/vall-e/huggingface/runs/pflx7g43)
 # finetune_starcoder2
 
-This model is a fine-tuned version of [bigcode/starcoder2-15b](https://huggingface.co/bigcode/starcoder2-15b) on an unknown dataset.
+This model is a fine-tuned version of [bigcode/starcoder2-3b](https://huggingface.co/bigcode/starcoder2-3b) on an unknown dataset.
 
 ## Model description
 
@@ -55,7 +54,7 @@ The following hyperparameters were used during training:
 ### Framework versions
 
 - PEFT 0.11.1
-- Transformers 4.42.0.dev0
+- Transformers 4.41.2
 - Pytorch 2.3.0+cu121
-- Datasets 2.19.1
+- Datasets 2.20.0
 - Tokenizers 0.19.1

adapter_config.json

@@ -1,7 +1,7 @@
 {
   "alpha_pattern": {},
   "auto_mapping": null,
-  "base_model_name_or_path": "bigcode/starcoder2-15b",
+  "base_model_name_or_path": "bigcode/starcoder2-3b",
   "bias": "none",
   "fan_in_fan_out": false,
   "inference_mode": true,
@@ -21,12 +21,12 @@
   "revision": null,
   "target_modules": [
     "k_proj",
-    "v_proj",
     "o_proj",
     "up_proj",
-    "gate_proj",
+    "down_proj",
     "q_proj",
-    "down_proj"
+    "v_proj",
+    "gate_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9287a32a3396d0bd4d372e0c90796c32761ed4f7c2a9b59560f8f32e306b6fbf
-size 48539256
+oid sha256:893002930d3b445f48bec7cc1a2d144bc467dbd527211888e7bd53e2d0cc069b
+size 18218008

final_checkpoint/config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "bigcode/starcoder2-15b",
+  "_name_or_path": "bigcode/starcoder2-3b",
   "architectures": [
     "Starcoder2ForCausalLM"
   ],
@@ -8,38 +8,22 @@
   "embedding_dropout": 0.1,
   "eos_token_id": 0,
   "hidden_act": "gelu_pytorch_tanh",
-  "hidden_size": 6144,
-  "initializer_range": 0.01275,
-  "intermediate_size": 24576,
+  "hidden_size": 3072,
+  "initializer_range": 0.018042,
+  "intermediate_size": 12288,
   "max_position_embeddings": 16384,
   "mlp_type": "default",
   "model_type": "starcoder2",
   "norm_epsilon": 1e-05,
   "norm_type": "layer_norm",
-  "num_attention_heads": 48,
-  "num_hidden_layers": 40,
-  "num_key_value_heads": 4,
-  "quantization_config": {
-    "_load_in_4bit": true,
-    "_load_in_8bit": false,
-    "bnb_4bit_compute_dtype": "bfloat16",
-    "bnb_4bit_quant_storage": "uint8",
-    "bnb_4bit_quant_type": "nf4",
-    "bnb_4bit_use_double_quant": false,
-    "llm_int8_enable_fp32_cpu_offload": false,
-    "llm_int8_has_fp16_weight": false,
-    "llm_int8_skip_modules": null,
-    "llm_int8_threshold": 6.0,
-    "load_in_4bit": true,
-    "load_in_8bit": false,
-    "quant_method": "bitsandbytes"
-  },
+  "num_attention_heads": 24,
+  "num_hidden_layers": 30,
+  "num_key_value_heads": 2,
   "residual_dropout": 0.1,
-  "rope_theta": 100000,
+  "rope_theta": 999999.4420358813,
   "sliding_window": 4096,
-  "tie_word_embeddings": false,
   "torch_dtype": "float32",
-  "transformers_version": "4.42.0.dev0",
+  "transformers_version": "4.41.2",
   "use_bias": true,
   "use_cache": true,
   "vocab_size": 49152

final_checkpoint/generation_config.json

@@ -1,6 +1,6 @@
 {
   "_from_model_config": true,
-  "bos_token_id": 50256,
-  "eos_token_id": 50256,
-  "transformers_version": "4.42.0.dev0"
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "transformers_version": "4.41.2"
 }

final_checkpoint/model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:50f444c7f2952ebfc56ace73cce35b9422ca6c2deeac45776a897e571ff9e7de
-size 4950280112
+oid sha256:8a030b54c247c460e9fd49e0f0cf84a1899a4dc51cf0c05e3a9a7b86cf273f20
+size 4916180040

final_checkpoint/model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:81726e7af63cab3a8e8a7ec6d35e3216b7e38b19074bd121beaaab6014a3eee6
-size 4951978560
+oid sha256:2efe4f43f60f2d3a8098f198a34b38af8f7b8750fdca2a64b6dc2572140f274b
+size 4998826856

final_checkpoint/model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e8cf600df2701898434e15f495df429d54421618ab52daf150ca73b519223f0e
-size 1207959680
+oid sha256:c9a44c01e89bff09008dd19a6d09b4d0b274dbd5616b7cee45a979995b9a8795
+size 2224751784

tokenizer.json

@@ -1,11 +1,6 @@
 {
   "version": "1.0",
-  "truncation": {
-    "direction": "Right",
-    "max_length": 512,
-    "strategy": "LongestFirst",
-    "stride": 0
-  },
+  "truncation": null,
   "padding": null,
   "added_tokens": [
     {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:237fa6df610231b92c1bf8704ec441693524b8f0958486b43eb6dd5d2c63340c
+oid sha256:d3f299a5612bbad0e6e6568a00f74016f13f63ffa06effb3dc4082d7c90c912b
 size 5112