| { | |
| "embed_dim": 1024, | |
| "num_heads": 4, | |
| "dropout": 0.05, | |
| "bias": true, | |
| "use_layernorm": true, | |
| "use_MLP": true, | |
| "MLP_h_size": 2048, | |
| "use_residual_mean": false, | |
| "use_residual_MLP": true, | |
| "ignore_cls_as_kv": true, | |
| "expand_emb_dim_to": 0, | |
| "compress_output_dim_to": 0 | |
| } |