Update config.json

config.json

@@ -21,13 +21,17 @@
     "num_experts": 64,
     "num_shared_experts": 1,
     "norm_topk_prob": true,
-    "num_attention_heads": 42,
+    "num_attention_heads": 56,
     "num_experts_per_tok": 4,
     "num_hidden_layers": 88,
-    "num_key_value_heads": 6,
+    "num_key_value_heads": 8,
     "pretraining_tp": 1,
-    "rms_norm_eps": 1e-06,
-    "rope_scaling": null,
+    "rms_norm_eps": 1e-05,
+    "rope_scaling": {
+        "factor": 4.0,
+        "original_max_position_embeddings": 16384,
+        "type": "yarn"
+    },
     "rope_theta": 600000,
     "tie_word_embeddings": false,
     "torch_dtype": "bfloat16",
@@ -40,5 +44,6 @@
     "embedding_dropout": 0.0,
     "norm_head": true,
     "norm_softmax": false,
-    "output_dropout": 0.0
+    "output_dropout": 0.0,
+    "head_dim": 128
 }