RaymondLi
/

custom_gpt2_mqa

Text Generation

text-generation-inference

Model card Files Files and versions Community

RaymondLi commited on Dec 2, 2022

Commit

1e15ef3

·

1 Parent(s): a2873cd

Upload model

Files changed (1) hide show

config.json +1 -1

config.json CHANGED Viewed

@@ -22,7 +22,7 @@
   "n_positions": 2048,
   "reorder_and_upcast_attn": false,
   "resid_pdrop": 0.1,
-  "scale_attn_by_inverse_layer_idx": true,
   "scale_attn_weights": true,
   "summary_activation": null,
   "summary_first_dropout": 0.1,

   "n_positions": 2048,
   "reorder_and_upcast_attn": false,
   "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
   "scale_attn_weights": true,
   "summary_activation": null,
   "summary_first_dropout": 0.1,