DNA-LLM/synth_model_small_1.3

README.md

@@ -1,6 +1,7 @@
 ---
 library_name: transformers
-base_model: DNA-LLM/virus-pythia-1.2M-2048-ce_plus_energy
+license: apache-2.0
+base_model: DNA-LLM/virus-pythia-85M-2048-two_d_plus_ce
 tags:
 - generated_from_trainer
 model-index:
@@ -13,7 +14,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # virus-pythia-1.2M-2048-ce_synthetic
 
-This model is a fine-tuned version of [DNA-LLM/virus-pythia-1.2M-2048-ce_plus_energy](https://huggingface.co/DNA-LLM/virus-pythia-1.2M-2048-ce_plus_energy) on an unknown dataset.
+This model is a fine-tuned version of [DNA-LLM/virus-pythia-85M-2048-two_d_plus_ce](https://huggingface.co/DNA-LLM/virus-pythia-85M-2048-two_d_plus_ce) on an unknown dataset.
 
 ## Model description
 
@@ -33,11 +34,11 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 5e-05
-- train_batch_size: 112
+- train_batch_size: 16
 - eval_batch_size: 8
 - seed: 42
 - gradient_accumulation_steps: 4
-- total_train_batch_size: 448
+- total_train_batch_size: 64
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: linear
 - lr_scheduler_warmup_steps: 1

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "DNA-LLM/virus-pythia-1.2M-2048-ce_plus_energy",
+  "_name_or_path": "DNA-LLM/virus-pythia-85M-2048-two_d_plus_ce",
   "architectures": [
     "GPTNeoXForCausalLM"
   ],
@@ -10,14 +10,14 @@
   "eos_token_id": 0,
   "hidden_act": "gelu",
   "hidden_dropout": 0.0,
-  "hidden_size": 128,
+  "hidden_size": 768,
   "initializer_range": 0.02,
-  "intermediate_size": 512,
+  "intermediate_size": 3072,
   "layer_norm_eps": 1e-05,
   "max_position_embeddings": 2048,
   "model_type": "gpt_neox",
-  "num_attention_heads": 4,
-  "num_hidden_layers": 6,
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
   "partial_rotary_factor": 0.25,
   "rope_scaling": null,
   "rope_theta": 10000,

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e9d3ac4206e107d701514a28d8f1397dc05111731dbd3389586e4f313d809e7d
-size 6679704
+oid sha256:e7c731a8a5c96581d50752424002548eb6fa8fe232626f5595fcb598baef8a5a
+size 351711712

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:41bdf3c574c41652fe5a9dd08da7d3bc77e8a30a4ab0b1b1294e5702a77f0fb8
+oid sha256:2d38ac71a12b28c59d500c155cfbde41e0dbeae3a7a6c1fdb6731ddd928de37b
 size 5176