Training in progress, step 500, checkpoint

checkpoint-500/config.json

@@ -0,0 +1,184 @@
+{
+  "K": 4096,
+  "architectures": [
+    "MomentumDualEncoderModel"
+  ],
+  "assembly_config": {
+    "_name_or_path": "",
+    "add_cross_attention": false,
+    "architectures": [
+      "LongelmForMaskedLMWithEdgePrediction"
+    ],
+    "attention_probs_dropout_prob": 0.1,
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "block_size": 8,
+    "bos_token_id": 1,
+    "chunk_size_feed_forward": 0,
+    "classifier_dropout": null,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": 2,
+    "ep_add_linear_projection": true,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "global_memory_size": 1,
+    "hidden_act": "gelu",
+    "hidden_dropout_prob": 0.1,
+    "hidden_size": 768,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "layer_norm_eps": 1e-12,
+    "length_penalty": 1.0,
+    "max_blocks": 200,
+    "max_length": 20,
+    "max_position_embeddings": 1805,
+    "max_relative_position_embeddings": 8,
+    "min_length": 0,
+    "model_type": "longelm",
+    "no_repeat_ngram_size": 0,
+    "node_size": 1,
+    "num_attention_heads": 12,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_hidden_layers": 12,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": 3,
+    "position_embedding_type": "mixed",
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "sep_token_id": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": true,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": "float32",
+    "torchscript": false,
+    "type_vocab_size": 2,
+    "typical_p": 1.0,
+    "use_bfloat16": false,
+    "use_cache": true,
+    "vocab_size": 50265
+  },
+  "logit_scale_init_value": 2.6592,
+  "m": 0.999,
+  "model_type": "assembly-source-dual-encoder",
+  "projection_dim": 512,
+  "source_config": {
+    "_name_or_path": "Salesforce/codet5p-110m-embedding",
+    "add_cross_attention": false,
+    "architectures": [
+      "CodeT5p_Embedding"
+    ],
+    "auto_map": {
+      "AutoConfig": "Salesforce/codet5p-110m-embedding--configuration_codet5p_embedding.CodeT5pEmbeddingConfig",
+      "AutoModel": "Salesforce/codet5p-110m-embedding--modeling_codet5p_embedding.CodeT5pEmbeddingModel"
+    },
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": 1,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "d_ff": 3072,
+    "d_kv": 64,
+    "d_model": 768,
+    "decoder_start_token_id": 0,
+    "dense_act_fn": "relu",
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "dropout_rate": 0.1,
+    "early_stopping": false,
+    "embed_dim": 256,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": 2,
+    "exponential_decay_length_penalty": null,
+    "feed_forward_proj": "relu",
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "gradient_checkpointing": false,
+    "id2label": {
+      "0": "LABEL_0"
+    },
+    "initializer_factor": 1.0,
+    "is_decoder": false,
+    "is_encoder_decoder": true,
+    "is_gated_act": false,
+    "label2id": {
+      "LABEL_0": 0
+    },
+    "layer_norm_epsilon": 1e-06,
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "min_length": 0,
+    "model_type": "codet5p_embedding",
+    "n_positions": 512,
+    "no_repeat_ngram_size": 0,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_heads": 12,
+    "num_layers": 12,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_past": true,
+    "output_scores": false,
+    "pad_token_id": 0,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "relative_attention_max_distance": 128,
+    "relative_attention_num_buckets": 32,
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "sep_token_id": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": true,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": "float32",
+    "torchscript": false,
+    "typical_p": 1.0,
+    "use_bfloat16": false,
+    "use_cache": false,
+    "vocab_size": 32103
+  },
+  "torch_dtype": "float32",
+  "transformers_version": "4.39.3"
+}

checkpoint-500/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f54a28a63a304a0d289cd710ff04dc8ffa4f07875960a984748130ea72a415d8
+size 1901991436

checkpoint-500/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d941ef29656a61e809bb2f1c5e553c84249d926277471bee40250ac7e5acd1e
+size 1888540293

checkpoint-500/rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2550b0c0418fcb0451b08bbc745eaae4a2e5be3f00caf7efb9f01b99d9600cd2
+size 15607

checkpoint-500/rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3890e26cec6d5d419681617d2e80eafac5f45fe5792c537f7a2ad794add845b
+size 15607

checkpoint-500/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ac7b1e85d8b401c9451934d83baf0ea4ca610c19407d141306688eff409533d
+size 627

checkpoint-500/trainer_state.json

@@ -0,0 +1,379 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.01984126984126984,
+  "eval_steps": 500,
+  "global_step": 500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "grad_norm": 23.495065689086914,
+      "learning_rate": 4.0000000000000003e-07,
+      "loss": 8.6232,
+      "step": 10
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 24.87381935119629,
+      "learning_rate": 9e-07,
+      "loss": 8.4514,
+      "step": 20
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 22.63846778869629,
+      "learning_rate": 1.4000000000000001e-06,
+      "loss": 8.3152,
+      "step": 30
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 20.65507698059082,
+      "learning_rate": 1.9e-06,
+      "loss": 7.9732,
+      "step": 40
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 16.147356033325195,
+      "learning_rate": 2.4000000000000003e-06,
+      "loss": 7.6879,
+      "step": 50
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 14.252687454223633,
+      "learning_rate": 2.9e-06,
+      "loss": 7.4152,
+      "step": 60
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 12.011382102966309,
+      "learning_rate": 3.4000000000000005e-06,
+      "loss": 7.2472,
+      "step": 70
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 10.081536293029785,
+      "learning_rate": 3.9e-06,
+      "loss": 7.2433,
+      "step": 80
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 9.65556526184082,
+      "learning_rate": 4.4e-06,
+      "loss": 7.2298,
+      "step": 90
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 8.830799102783203,
+      "learning_rate": 4.9000000000000005e-06,
+      "loss": 7.19,
+      "step": 100
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 9.019427299499512,
+      "learning_rate": 5.4e-06,
+      "loss": 7.2362,
+      "step": 110
+    },
+    {
+      "epoch": 0.0,
+      "grad_norm": 9.13768482208252,
+      "learning_rate": 5.9e-06,
+      "loss": 7.2804,
+      "step": 120
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 9.02840805053711,
+      "learning_rate": 6.4000000000000006e-06,
+      "loss": 7.2345,
+      "step": 130
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 8.178731918334961,
+      "learning_rate": 6.900000000000001e-06,
+      "loss": 7.3014,
+      "step": 140
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 8.83882999420166,
+      "learning_rate": 7.4e-06,
+      "loss": 7.3875,
+      "step": 150
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 8.3687162399292,
+      "learning_rate": 7.9e-06,
+      "loss": 7.4546,
+      "step": 160
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 8.09121036529541,
+      "learning_rate": 8.400000000000001e-06,
+      "loss": 7.407,
+      "step": 170
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 12.731450080871582,
+      "learning_rate": 8.9e-06,
+      "loss": 7.4321,
+      "step": 180
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 7.601162910461426,
+      "learning_rate": 9.4e-06,
+      "loss": 7.4155,
+      "step": 190
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 8.006213188171387,
+      "learning_rate": 9.900000000000002e-06,
+      "loss": 7.3705,
+      "step": 200
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 7.79794454574585,
+      "learning_rate": 1.04e-05,
+      "loss": 7.3759,
+      "step": 210
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 7.281346321105957,
+      "learning_rate": 1.09e-05,
+      "loss": 7.3798,
+      "step": 220
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 7.605266094207764,
+      "learning_rate": 1.1400000000000001e-05,
+      "loss": 7.3878,
+      "step": 230
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 7.446412563323975,
+      "learning_rate": 1.19e-05,
+      "loss": 7.3098,
+      "step": 240
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 7.6324334144592285,
+      "learning_rate": 1.24e-05,
+      "loss": 7.2995,
+      "step": 250
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 8.330737113952637,
+      "learning_rate": 1.29e-05,
+      "loss": 7.1887,
+      "step": 260
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 7.283708572387695,
+      "learning_rate": 1.3400000000000002e-05,
+      "loss": 7.1184,
+      "step": 270
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 6.950069904327393,
+      "learning_rate": 1.3900000000000002e-05,
+      "loss": 6.9876,
+      "step": 280
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 6.844958782196045,
+      "learning_rate": 1.44e-05,
+      "loss": 6.9319,
+      "step": 290
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 7.499362468719482,
+      "learning_rate": 1.49e-05,
+      "loss": 6.9253,
+      "step": 300
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 8.45572280883789,
+      "learning_rate": 1.54e-05,
+      "loss": 6.7999,
+      "step": 310
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 8.108625411987305,
+      "learning_rate": 1.59e-05,
+      "loss": 6.7667,
+      "step": 320
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 8.131596565246582,
+      "learning_rate": 1.6400000000000002e-05,
+      "loss": 6.7599,
+      "step": 330
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 7.2159247398376465,
+      "learning_rate": 1.69e-05,
+      "loss": 6.7379,
+      "step": 340
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 7.335048675537109,
+      "learning_rate": 1.74e-05,
+      "loss": 6.6553,
+      "step": 350
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 7.361081600189209,
+      "learning_rate": 1.79e-05,
+      "loss": 6.6905,
+      "step": 360
+    },
+    {
+      "epoch": 0.01,
+      "grad_norm": 7.59442138671875,
+      "learning_rate": 1.84e-05,
+      "loss": 6.6356,
+      "step": 370
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 7.31695032119751,
+      "learning_rate": 1.8900000000000002e-05,
+      "loss": 6.6538,
+      "step": 380
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 7.541380882263184,
+      "learning_rate": 1.94e-05,
+      "loss": 6.6873,
+      "step": 390
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 8.056180000305176,
+      "learning_rate": 1.9900000000000003e-05,
+      "loss": 6.6332,
+      "step": 400
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 6.74859094619751,
+      "learning_rate": 2.04e-05,
+      "loss": 6.5488,
+      "step": 410
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 7.700525760650635,
+      "learning_rate": 2.09e-05,
+      "loss": 6.611,
+      "step": 420
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 7.347009181976318,
+      "learning_rate": 2.1400000000000002e-05,
+      "loss": 6.6621,
+      "step": 430
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 6.607337951660156,
+      "learning_rate": 2.19e-05,
+      "loss": 6.568,
+      "step": 440
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 6.816825866699219,
+      "learning_rate": 2.2400000000000002e-05,
+      "loss": 6.6293,
+      "step": 450
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 6.984443664550781,
+      "learning_rate": 2.29e-05,
+      "loss": 6.5667,
+      "step": 460
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 6.995021820068359,
+      "learning_rate": 2.3400000000000003e-05,
+      "loss": 6.5062,
+      "step": 470
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 6.876286029815674,
+      "learning_rate": 2.39e-05,
+      "loss": 6.5034,
+      "step": 480
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 6.953365802764893,
+      "learning_rate": 2.44e-05,
+      "loss": 6.459,
+      "step": 490
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 7.246520519256592,
+      "learning_rate": 2.4900000000000002e-05,
+      "loss": 6.4359,
+      "step": 500
+    },
+    {
+      "epoch": 0.02,
+      "eval_loss": 6.133457660675049,
+      "eval_runtime": 422.8798,
+      "eval_samples_per_second": 50.182,
+      "eval_steps_per_second": 0.393,
+      "step": 500
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 50400,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 500,
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-500/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37d08e731b46f2fc9ac1d4ac376bb688c70f2bf08a26eb4b0de280ef48858eb0
+size 4539