Around6827 commited on
Commit
e9800dc
·
1 Parent(s): 01ceb3e

Training in progress, epoch 1

Browse files
adapter_config.json CHANGED
@@ -16,13 +16,13 @@
16
  "rank_pattern": {},
17
  "revision": null,
18
  "target_modules": [
19
- "gate_proj",
20
  "down_proj",
 
 
21
  "v_proj",
 
22
  "k_proj",
23
- "up_proj",
24
- "q_proj",
25
- "o_proj"
26
  ],
27
  "task_type": "CAUSAL_LM"
28
  }
 
16
  "rank_pattern": {},
17
  "revision": null,
18
  "target_modules": [
 
19
  "down_proj",
20
+ "up_proj",
21
+ "gate_proj",
22
  "v_proj",
23
+ "o_proj",
24
  "k_proj",
25
+ "q_proj"
 
 
26
  ],
27
  "task_type": "CAUSAL_LM"
28
  }
adapter_model.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:370f35e8a15b92463a19e172f3d298bd495fbe604559302c85ef3f1a3d31ed00
3
  size 319977229
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5fd3ecb21dcf57bec80e101d7dc375946d20554d2e5176770f7a7657817a02d4
3
  size 319977229
config.json CHANGED
@@ -16,12 +16,24 @@
16
  "num_key_value_heads": 32,
17
  "pad_token_id": 0,
18
  "pretraining_tp": 1,
 
 
 
 
 
 
 
 
 
 
 
 
19
  "rms_norm_eps": 1e-05,
20
  "rope_scaling": null,
21
  "rope_theta": 10000.0,
22
  "tie_word_embeddings": false,
23
  "torch_dtype": "float16",
24
  "transformers_version": "4.34.0.dev0",
25
- "use_cache": true,
26
  "vocab_size": 32000
27
  }
 
16
  "num_key_value_heads": 32,
17
  "pad_token_id": 0,
18
  "pretraining_tp": 1,
19
+ "quantization_config": {
20
+ "bnb_4bit_compute_dtype": "float32",
21
+ "bnb_4bit_quant_type": "fp4",
22
+ "bnb_4bit_use_double_quant": false,
23
+ "llm_int8_enable_fp32_cpu_offload": false,
24
+ "llm_int8_has_fp16_weight": false,
25
+ "llm_int8_skip_modules": null,
26
+ "llm_int8_threshold": 6.0,
27
+ "load_in_4bit": false,
28
+ "load_in_8bit": true,
29
+ "quant_method": "bitsandbytes"
30
+ },
31
  "rms_norm_eps": 1e-05,
32
  "rope_scaling": null,
33
  "rope_theta": 10000.0,
34
  "tie_word_embeddings": false,
35
  "torch_dtype": "float16",
36
  "transformers_version": "4.34.0.dev0",
37
+ "use_cache": false,
38
  "vocab_size": 32000
39
  }
tokenizer_config.json CHANGED
@@ -1,26 +1,28 @@
1
  {
 
 
2
  "added_tokens_decoder": {
3
  "0": {
4
  "content": "<unk>",
5
- "lstrip": false,
6
- "normalized": true,
7
- "rstrip": false,
8
  "single_word": false,
9
  "special": true
10
  },
11
  "1": {
12
  "content": "<s>",
13
- "lstrip": false,
14
- "normalized": true,
15
- "rstrip": false,
16
  "single_word": false,
17
  "special": true
18
  },
19
  "2": {
20
  "content": "</s>",
21
- "lstrip": false,
22
- "normalized": true,
23
- "rstrip": false,
24
  "single_word": false,
25
  "special": true
26
  }
@@ -33,7 +35,11 @@
33
  "model_max_length": 1000000000000000019884624838656,
34
  "pad_token": "<unk>",
35
  "sp_model_kwargs": {},
 
36
  "tokenizer_class": "LlamaTokenizer",
 
 
37
  "unk_token": "<unk>",
38
- "use_default_system_prompt": true
 
39
  }
 
1
  {
2
+ "add_bos_token": true,
3
+ "add_eos_token": false,
4
  "added_tokens_decoder": {
5
  "0": {
6
  "content": "<unk>",
7
+ "lstrip": true,
8
+ "normalized": false,
9
+ "rstrip": true,
10
  "single_word": false,
11
  "special": true
12
  },
13
  "1": {
14
  "content": "<s>",
15
+ "lstrip": true,
16
+ "normalized": false,
17
+ "rstrip": true,
18
  "single_word": false,
19
  "special": true
20
  },
21
  "2": {
22
  "content": "</s>",
23
+ "lstrip": true,
24
+ "normalized": false,
25
+ "rstrip": true,
26
  "single_word": false,
27
  "special": true
28
  }
 
35
  "model_max_length": 1000000000000000019884624838656,
36
  "pad_token": "<unk>",
37
  "sp_model_kwargs": {},
38
+ "spaces_between_special_tokens": false,
39
  "tokenizer_class": "LlamaTokenizer",
40
+ "tokenizer_file": "/N/u/oleykin/BigRed200/SLATE/.cache/huggingface/hub/models--NousResearch--Llama-2-7b-hf/snapshots/dacdfcde31297e34b19ee0e7532f29586d2c17bc/tokenizer.json",
41
+ "trust_remote_code": false,
42
  "unk_token": "<unk>",
43
+ "use_default_system_prompt": true,
44
+ "use_fast": true
45
  }
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:17b8a61bed456f0e64f13280369796447b5a0eb70ef6f1bbaa8ff6c0edeefe45
3
  size 4475
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3a726b36a71a2000a2b99fd6770d79f9325b0522627d84d1ff16cdc48043749e
3
  size 4475