pylu5229's picture
Training in progress, epoch 25, checkpoint
7305512 verified
{
"best_metric": 0.923,
"best_model_checkpoint": "conditional-detr-resnet-50-uLED-obj-detect-test/checkpoint-902",
"epoch": 25.0,
"eval_steps": 500,
"global_step": 1025,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 1.0,
"eval_loss": 0.2459525614976883,
"eval_map": 0.7925,
"eval_map_50": 0.9619,
"eval_map_75": 0.9382,
"eval_map_large": -1.0,
"eval_map_medium": 0.7925,
"eval_map_small": -1.0,
"eval_map_trash": -1.0,
"eval_map_uLED": 0.7925,
"eval_mar_1": 0.0115,
"eval_mar_10": 0.1133,
"eval_mar_100": 0.8652,
"eval_mar_100_trash": -1.0,
"eval_mar_100_uLED": 0.8652,
"eval_mar_large": -1.0,
"eval_mar_medium": 0.8652,
"eval_mar_small": -1.0,
"eval_runtime": 49.0935,
"eval_samples_per_second": 4.705,
"eval_steps_per_second": 0.591,
"step": 41
},
{
"epoch": 2.0,
"eval_loss": 0.212324857711792,
"eval_map": 0.8121,
"eval_map_50": 0.9671,
"eval_map_75": 0.9527,
"eval_map_large": -1.0,
"eval_map_medium": 0.8121,
"eval_map_small": -1.0,
"eval_map_trash": -1.0,
"eval_map_uLED": 0.8121,
"eval_mar_1": 0.0111,
"eval_mar_10": 0.1125,
"eval_mar_100": 0.8797,
"eval_mar_100_trash": -1.0,
"eval_mar_100_uLED": 0.8797,
"eval_mar_large": -1.0,
"eval_mar_medium": 0.8797,
"eval_mar_small": -1.0,
"eval_runtime": 49.3207,
"eval_samples_per_second": 4.684,
"eval_steps_per_second": 0.588,
"step": 82
},
{
"epoch": 3.0,
"eval_loss": 0.15966945886611938,
"eval_map": 0.8576,
"eval_map_50": 0.9645,
"eval_map_75": 0.963,
"eval_map_large": -1.0,
"eval_map_medium": 0.8576,
"eval_map_small": -1.0,
"eval_map_trash": -1.0,
"eval_map_uLED": 0.8576,
"eval_mar_1": 0.0118,
"eval_mar_10": 0.1181,
"eval_mar_100": 0.9217,
"eval_mar_100_trash": -1.0,
"eval_mar_100_uLED": 0.9217,
"eval_mar_large": -1.0,
"eval_mar_medium": 0.9217,
"eval_mar_small": -1.0,
"eval_runtime": 48.3857,
"eval_samples_per_second": 4.774,
"eval_steps_per_second": 0.599,
"step": 123
},
{
"epoch": 4.0,
"eval_loss": 0.16448701918125153,
"eval_map": 0.8532,
"eval_map_50": 0.9644,
"eval_map_75": 0.9606,
"eval_map_large": -1.0,
"eval_map_medium": 0.8532,
"eval_map_small": -1.0,
"eval_map_trash": -1.0,
"eval_map_uLED": 0.8532,
"eval_mar_1": 0.0118,
"eval_mar_10": 0.1184,
"eval_mar_100": 0.9174,
"eval_mar_100_trash": -1.0,
"eval_mar_100_uLED": 0.9174,
"eval_mar_large": -1.0,
"eval_mar_medium": 0.9174,
"eval_mar_small": -1.0,
"eval_runtime": 49.5822,
"eval_samples_per_second": 4.659,
"eval_steps_per_second": 0.585,
"step": 164
},
{
"epoch": 5.0,
"eval_loss": 0.20374979078769684,
"eval_map": 0.824,
"eval_map_50": 0.9632,
"eval_map_75": 0.9614,
"eval_map_large": -1.0,
"eval_map_medium": 0.824,
"eval_map_small": -1.0,
"eval_map_trash": -1.0,
"eval_map_uLED": 0.824,
"eval_mar_1": 0.0115,
"eval_mar_10": 0.1142,
"eval_mar_100": 0.8826,
"eval_mar_100_trash": -1.0,
"eval_mar_100_uLED": 0.8826,
"eval_mar_large": -1.0,
"eval_mar_medium": 0.8826,
"eval_mar_small": -1.0,
"eval_runtime": 49.0006,
"eval_samples_per_second": 4.714,
"eval_steps_per_second": 0.592,
"step": 205
},
{
"epoch": 6.0,
"eval_loss": 0.1341749131679535,
"eval_map": 0.8864,
"eval_map_50": 0.9672,
"eval_map_75": 0.9665,
"eval_map_large": -1.0,
"eval_map_medium": 0.8864,
"eval_map_small": -1.0,
"eval_map_trash": -1.0,
"eval_map_uLED": 0.8864,
"eval_mar_1": 0.0119,
"eval_mar_10": 0.1213,
"eval_mar_100": 0.9429,
"eval_mar_100_trash": -1.0,
"eval_mar_100_uLED": 0.9429,
"eval_mar_large": -1.0,
"eval_mar_medium": 0.9429,
"eval_mar_small": -1.0,
"eval_runtime": 48.3343,
"eval_samples_per_second": 4.779,
"eval_steps_per_second": 0.6,
"step": 246
},
{
"epoch": 7.0,
"eval_loss": 0.1365080326795578,
"eval_map": 0.8821,
"eval_map_50": 0.9677,
"eval_map_75": 0.9672,
"eval_map_large": -1.0,
"eval_map_medium": 0.8821,
"eval_map_small": -1.0,
"eval_map_trash": -1.0,
"eval_map_uLED": 0.8821,
"eval_mar_1": 0.0121,
"eval_mar_10": 0.1218,
"eval_mar_100": 0.9362,
"eval_mar_100_trash": -1.0,
"eval_mar_100_uLED": 0.9362,
"eval_mar_large": -1.0,
"eval_mar_medium": 0.9362,
"eval_mar_small": -1.0,
"eval_runtime": 48.9561,
"eval_samples_per_second": 4.719,
"eval_steps_per_second": 0.592,
"step": 287
},
{
"epoch": 8.0,
"eval_loss": 0.1470160037279129,
"eval_map": 0.872,
"eval_map_50": 0.9666,
"eval_map_75": 0.9662,
"eval_map_large": -1.0,
"eval_map_medium": 0.872,
"eval_map_small": -1.0,
"eval_map_trash": -1.0,
"eval_map_uLED": 0.872,
"eval_mar_1": 0.0119,
"eval_mar_10": 0.12,
"eval_mar_100": 0.9326,
"eval_mar_100_trash": -1.0,
"eval_mar_100_uLED": 0.9326,
"eval_mar_large": -1.0,
"eval_mar_medium": 0.9326,
"eval_mar_small": -1.0,
"eval_runtime": 48.0042,
"eval_samples_per_second": 4.812,
"eval_steps_per_second": 0.604,
"step": 328
},
{
"epoch": 9.0,
"eval_loss": 0.17834806442260742,
"eval_map": 0.8495,
"eval_map_50": 0.9678,
"eval_map_75": 0.9673,
"eval_map_large": -1.0,
"eval_map_medium": 0.8495,
"eval_map_small": -1.0,
"eval_map_trash": -1.0,
"eval_map_uLED": 0.8495,
"eval_mar_1": 0.0118,
"eval_mar_10": 0.118,
"eval_mar_100": 0.9017,
"eval_mar_100_trash": -1.0,
"eval_mar_100_uLED": 0.9017,
"eval_mar_large": -1.0,
"eval_mar_medium": 0.9017,
"eval_mar_small": -1.0,
"eval_runtime": 49.1118,
"eval_samples_per_second": 4.704,
"eval_steps_per_second": 0.59,
"step": 369
},
{
"epoch": 10.0,
"eval_loss": 0.15633128583431244,
"eval_map": 0.8676,
"eval_map_50": 0.9662,
"eval_map_75": 0.9643,
"eval_map_large": -1.0,
"eval_map_medium": 0.8676,
"eval_map_small": -1.0,
"eval_map_trash": -1.0,
"eval_map_uLED": 0.8676,
"eval_mar_1": 0.012,
"eval_mar_10": 0.1203,
"eval_mar_100": 0.9225,
"eval_mar_100_trash": -1.0,
"eval_mar_100_uLED": 0.9225,
"eval_mar_large": -1.0,
"eval_mar_medium": 0.9225,
"eval_mar_small": -1.0,
"eval_runtime": 47.9908,
"eval_samples_per_second": 4.813,
"eval_steps_per_second": 0.604,
"step": 410
},
{
"epoch": 11.0,
"eval_loss": 0.14576929807662964,
"eval_map": 0.8783,
"eval_map_50": 0.966,
"eval_map_75": 0.9658,
"eval_map_large": -1.0,
"eval_map_medium": 0.8783,
"eval_map_small": -1.0,
"eval_map_trash": -1.0,
"eval_map_uLED": 0.8783,
"eval_mar_1": 0.012,
"eval_mar_10": 0.121,
"eval_mar_100": 0.9321,
"eval_mar_100_trash": -1.0,
"eval_mar_100_uLED": 0.9321,
"eval_mar_large": -1.0,
"eval_mar_medium": 0.9321,
"eval_mar_small": -1.0,
"eval_runtime": 49.3914,
"eval_samples_per_second": 4.677,
"eval_steps_per_second": 0.587,
"step": 451
},
{
"epoch": 12.0,
"eval_loss": 0.12726162374019623,
"eval_map": 0.8939,
"eval_map_50": 0.9669,
"eval_map_75": 0.9667,
"eval_map_large": -1.0,
"eval_map_medium": 0.8939,
"eval_map_small": -1.0,
"eval_map_trash": -1.0,
"eval_map_uLED": 0.8939,
"eval_mar_1": 0.0123,
"eval_mar_10": 0.1234,
"eval_mar_100": 0.9462,
"eval_mar_100_trash": -1.0,
"eval_mar_100_uLED": 0.9462,
"eval_mar_large": -1.0,
"eval_mar_medium": 0.9462,
"eval_mar_small": -1.0,
"eval_runtime": 48.8048,
"eval_samples_per_second": 4.733,
"eval_steps_per_second": 0.594,
"step": 492
},
{
"epoch": 12.195121951219512,
"grad_norm": 16.27830696105957,
"learning_rate": 3.223801962218372e-05,
"loss": 0.2348,
"step": 500
},
{
"epoch": 13.0,
"eval_loss": 0.13760322332382202,
"eval_map": 0.8862,
"eval_map_50": 0.9683,
"eval_map_75": 0.968,
"eval_map_large": -1.0,
"eval_map_medium": 0.8862,
"eval_map_small": -1.0,
"eval_map_trash": -1.0,
"eval_map_uLED": 0.8862,
"eval_mar_1": 0.0121,
"eval_mar_10": 0.1217,
"eval_mar_100": 0.9404,
"eval_mar_100_trash": -1.0,
"eval_mar_100_uLED": 0.9404,
"eval_mar_large": -1.0,
"eval_mar_medium": 0.9404,
"eval_mar_small": -1.0,
"eval_runtime": 48.1828,
"eval_samples_per_second": 4.794,
"eval_steps_per_second": 0.602,
"step": 533
},
{
"epoch": 14.0,
"eval_loss": 0.1337805837392807,
"eval_map": 0.8865,
"eval_map_50": 0.9669,
"eval_map_75": 0.9668,
"eval_map_large": -1.0,
"eval_map_medium": 0.8865,
"eval_map_small": -1.0,
"eval_map_trash": -1.0,
"eval_map_uLED": 0.8865,
"eval_mar_1": 0.0122,
"eval_mar_10": 0.1222,
"eval_mar_100": 0.9422,
"eval_mar_100_trash": -1.0,
"eval_mar_100_uLED": 0.9422,
"eval_mar_large": -1.0,
"eval_mar_medium": 0.9422,
"eval_mar_small": -1.0,
"eval_runtime": 48.3776,
"eval_samples_per_second": 4.775,
"eval_steps_per_second": 0.599,
"step": 574
},
{
"epoch": 15.0,
"eval_loss": 0.12577340006828308,
"eval_map": 0.8917,
"eval_map_50": 0.9685,
"eval_map_75": 0.9685,
"eval_map_large": -1.0,
"eval_map_medium": 0.8917,
"eval_map_small": -1.0,
"eval_map_trash": -1.0,
"eval_map_uLED": 0.8917,
"eval_mar_1": 0.012,
"eval_mar_10": 0.1221,
"eval_mar_100": 0.9454,
"eval_mar_100_trash": -1.0,
"eval_mar_100_uLED": 0.9454,
"eval_mar_large": -1.0,
"eval_mar_medium": 0.9454,
"eval_mar_small": -1.0,
"eval_runtime": 47.9608,
"eval_samples_per_second": 4.816,
"eval_steps_per_second": 0.605,
"step": 615
},
{
"epoch": 16.0,
"eval_loss": 0.12064919620752335,
"eval_map": 0.8998,
"eval_map_50": 0.9689,
"eval_map_75": 0.9689,
"eval_map_large": -1.0,
"eval_map_medium": 0.8998,
"eval_map_small": -1.0,
"eval_map_trash": -1.0,
"eval_map_uLED": 0.8998,
"eval_mar_1": 0.0123,
"eval_mar_10": 0.1233,
"eval_mar_100": 0.9524,
"eval_mar_100_trash": -1.0,
"eval_mar_100_uLED": 0.9524,
"eval_mar_large": -1.0,
"eval_mar_medium": 0.9524,
"eval_mar_small": -1.0,
"eval_runtime": 48.7655,
"eval_samples_per_second": 4.737,
"eval_steps_per_second": 0.595,
"step": 656
},
{
"epoch": 17.0,
"eval_loss": 0.10745943337678909,
"eval_map": 0.911,
"eval_map_50": 0.969,
"eval_map_75": 0.969,
"eval_map_large": -1.0,
"eval_map_medium": 0.911,
"eval_map_small": -1.0,
"eval_map_trash": -1.0,
"eval_map_uLED": 0.911,
"eval_mar_1": 0.0123,
"eval_mar_10": 0.1238,
"eval_mar_100": 0.9612,
"eval_mar_100_trash": -1.0,
"eval_mar_100_uLED": 0.9612,
"eval_mar_large": -1.0,
"eval_mar_medium": 0.9612,
"eval_mar_small": -1.0,
"eval_runtime": 47.426,
"eval_samples_per_second": 4.871,
"eval_steps_per_second": 0.611,
"step": 697
},
{
"epoch": 18.0,
"eval_loss": 0.10835644602775574,
"eval_map": 0.9113,
"eval_map_50": 0.9692,
"eval_map_75": 0.9691,
"eval_map_large": -1.0,
"eval_map_medium": 0.9113,
"eval_map_small": -1.0,
"eval_map_trash": -1.0,
"eval_map_uLED": 0.9113,
"eval_mar_1": 0.0123,
"eval_mar_10": 0.1237,
"eval_mar_100": 0.9628,
"eval_mar_100_trash": -1.0,
"eval_mar_100_uLED": 0.9628,
"eval_mar_large": -1.0,
"eval_mar_medium": 0.9628,
"eval_mar_small": -1.0,
"eval_runtime": 48.8868,
"eval_samples_per_second": 4.725,
"eval_steps_per_second": 0.593,
"step": 738
},
{
"epoch": 19.0,
"eval_loss": 0.11042997241020203,
"eval_map": 0.91,
"eval_map_50": 0.9688,
"eval_map_75": 0.9688,
"eval_map_large": -1.0,
"eval_map_medium": 0.91,
"eval_map_small": -1.0,
"eval_map_trash": -1.0,
"eval_map_uLED": 0.91,
"eval_mar_1": 0.0123,
"eval_mar_10": 0.1236,
"eval_mar_100": 0.9602,
"eval_mar_100_trash": -1.0,
"eval_mar_100_uLED": 0.9602,
"eval_mar_large": -1.0,
"eval_mar_medium": 0.9602,
"eval_mar_small": -1.0,
"eval_runtime": 47.701,
"eval_samples_per_second": 4.843,
"eval_steps_per_second": 0.608,
"step": 779
},
{
"epoch": 20.0,
"eval_loss": 0.10969330370426178,
"eval_map": 0.9103,
"eval_map_50": 0.9693,
"eval_map_75": 0.9693,
"eval_map_large": -1.0,
"eval_map_medium": 0.9103,
"eval_map_small": -1.0,
"eval_map_trash": -1.0,
"eval_map_uLED": 0.9103,
"eval_mar_1": 0.0123,
"eval_mar_10": 0.1241,
"eval_mar_100": 0.9616,
"eval_mar_100_trash": -1.0,
"eval_mar_100_uLED": 0.9616,
"eval_mar_large": -1.0,
"eval_mar_medium": 0.9616,
"eval_mar_small": -1.0,
"eval_runtime": 48.7585,
"eval_samples_per_second": 4.738,
"eval_steps_per_second": 0.595,
"step": 820
},
{
"epoch": 21.0,
"eval_loss": 0.11106648296117783,
"eval_map": 0.9106,
"eval_map_50": 0.9666,
"eval_map_75": 0.9665,
"eval_map_large": -1.0,
"eval_map_medium": 0.9106,
"eval_map_small": -1.0,
"eval_map_trash": -1.0,
"eval_map_uLED": 0.9106,
"eval_mar_1": 0.0123,
"eval_mar_10": 0.1242,
"eval_mar_100": 0.9624,
"eval_mar_100_trash": -1.0,
"eval_mar_100_uLED": 0.9624,
"eval_mar_large": -1.0,
"eval_mar_medium": 0.9624,
"eval_mar_small": -1.0,
"eval_runtime": 47.7102,
"eval_samples_per_second": 4.842,
"eval_steps_per_second": 0.608,
"step": 861
},
{
"epoch": 22.0,
"eval_loss": 0.10067534446716309,
"eval_map": 0.923,
"eval_map_50": 0.9667,
"eval_map_75": 0.9666,
"eval_map_large": -1.0,
"eval_map_medium": 0.923,
"eval_map_small": -1.0,
"eval_map_trash": -1.0,
"eval_map_uLED": 0.923,
"eval_mar_1": 0.0125,
"eval_mar_10": 0.1251,
"eval_mar_100": 0.972,
"eval_mar_100_trash": -1.0,
"eval_mar_100_uLED": 0.972,
"eval_mar_large": -1.0,
"eval_mar_medium": 0.972,
"eval_mar_small": -1.0,
"eval_runtime": 48.0814,
"eval_samples_per_second": 4.804,
"eval_steps_per_second": 0.603,
"step": 902
},
{
"epoch": 23.0,
"eval_loss": 0.1080288216471672,
"eval_map": 0.9103,
"eval_map_50": 0.9671,
"eval_map_75": 0.9671,
"eval_map_large": -1.0,
"eval_map_medium": 0.9103,
"eval_map_small": -1.0,
"eval_map_trash": -1.0,
"eval_map_uLED": 0.9103,
"eval_mar_1": 0.0123,
"eval_mar_10": 0.1242,
"eval_mar_100": 0.9612,
"eval_mar_100_trash": -1.0,
"eval_mar_100_uLED": 0.9612,
"eval_mar_large": -1.0,
"eval_mar_medium": 0.9612,
"eval_mar_small": -1.0,
"eval_runtime": 47.8899,
"eval_samples_per_second": 4.824,
"eval_steps_per_second": 0.606,
"step": 943
},
{
"epoch": 24.0,
"eval_loss": 0.09872538596391678,
"eval_map": 0.9197,
"eval_map_50": 0.967,
"eval_map_75": 0.967,
"eval_map_large": -1.0,
"eval_map_medium": 0.9197,
"eval_map_small": -1.0,
"eval_map_trash": -1.0,
"eval_map_uLED": 0.9197,
"eval_mar_1": 0.0124,
"eval_mar_10": 0.1253,
"eval_mar_100": 0.9697,
"eval_mar_100_trash": -1.0,
"eval_mar_100_uLED": 0.9697,
"eval_mar_large": -1.0,
"eval_mar_medium": 0.9697,
"eval_mar_small": -1.0,
"eval_runtime": 47.3564,
"eval_samples_per_second": 4.878,
"eval_steps_per_second": 0.612,
"step": 984
},
{
"epoch": 24.390243902439025,
"grad_norm": 20.484636306762695,
"learning_rate": 4.19111424408932e-06,
"loss": 0.1648,
"step": 1000
},
{
"epoch": 25.0,
"eval_loss": 0.09787507355213165,
"eval_map": 0.9226,
"eval_map_50": 0.9675,
"eval_map_75": 0.9675,
"eval_map_large": -1.0,
"eval_map_medium": 0.9226,
"eval_map_small": -1.0,
"eval_map_trash": -1.0,
"eval_map_uLED": 0.9226,
"eval_mar_1": 0.0125,
"eval_mar_10": 0.1253,
"eval_mar_100": 0.9715,
"eval_mar_100_trash": -1.0,
"eval_mar_100_uLED": 0.9715,
"eval_mar_large": -1.0,
"eval_mar_medium": 0.9715,
"eval_mar_small": -1.0,
"eval_runtime": 48.1314,
"eval_samples_per_second": 4.799,
"eval_steps_per_second": 0.603,
"step": 1025
}
],
"logging_steps": 500,
"max_steps": 1230,
"num_input_tokens_seen": 0,
"num_train_epochs": 30,
"save_steps": 500,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 5.8611278633472e+18,
"train_batch_size": 32,
"trial_name": null,
"trial_params": null
}