modular components

config.json

@@ -2,7 +2,6 @@
     "LanguageModel": {
         "input": {
             "vocab_size": 51200,
-            "tie_embeddings": false,
             "pad_vocab_size_multiple": 8
         }
     },
@@ -15,21 +14,21 @@
     },
     "Block1": {
         "n_layers": 24,
-        "BlockType": "phi_block",
+        "BlockType": "modules.phi_block",
         "block_input": {
-            "residual_in_fp32": true,
             "resid_dropout": 0.0
         },
-        "CoreType": "discrete_mamba2",
+        "CoreType": "modules.mixers.discrete_mamba2",
         "core_input": {
             "d_state": 64,
-            "nheads": 32,
+            "n_v_heads": 32,
+            "n_qk_heads": 32,
+            "d_conv": 4,
+            "conv_bias": true,
             "expand": 1,
             "chunk_size": 128,
             "activation": "identity",
-            "use_ref_impl": false,
-            "bias": false,
-            "norm_cls": "none"
+            "bias": false
         }
     }
 }