Training in progress, step 500

.gitattributes

@@ -39,4 +39,3 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 .venv/lib/python3.9/site-packages/torch/lib/libtorch_cpu.so filter=lfs diff=lfs merge=lfs -text
 .venv/lib/python3.9/site-packages/torch/lib/libtorch_cuda.so filter=lfs diff=lfs merge=lfs -text
 .venv/lib/python3.9/site-packages/torch/lib/libtorch_python.so filter=lfs diff=lfs merge=lfs -text
-wandb/run-20220126_105847-1xdiy2kf/run-1xdiy2kf.wandb filter=lfs diff=lfs merge=lfs -text

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5f0265c3190d4bad709792f032d753570867b48b4491c9f120d58b7516631525
+oid sha256:003e036bcfb179618e8f03a9aa7421a2853ee4bc931661f68ef1e17ba9894270
 size 1262083569

run-300M.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 export WANDB_PROJECT="xls-r-estonian"
-export CUDA_VISIBLE_DEVICES=2
+export CUDA_VISIBLE_DEVICES=1
 
 python src/run_speech_recognition_ctc_bnb.py \
        --dataset_name="mozilla-foundation/common_voice_7_0" \
@@ -10,9 +10,9 @@ python src/run_speech_recognition_ctc_bnb.py \
        --output_dir="./" \
        --overwrite_output_dir \
        --num_train_epochs=100 \
-       --per_device_train_batch_size=80 \
-       --per_device_eval_batch_size=80 \
-       --gradient_accumulation_steps=2 \
+       --per_device_train_batch_size=72 \
+       --per_device_eval_batch_size=72 \
+       --gradient_accumulation_steps=1 \
        --learning_rate=3e-4 \
        --save_total_limit=1 \
        --warmup_steps=500 \

special_tokens_map.json

@@ -1 +1 @@
-{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}
+{"bos_token": "<s>", "eos_token": "</s>", "unk_token": "[UNK]", "pad_token": "[PAD]", "additional_special_tokens": [{"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}, {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true}]}

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:020e62b69669cb5205d10821c92f57fc9ddfd0706b2f7dd41539b83bb4d40ffb
+oid sha256:4accf1cd31755d346bca292c83c6a2ec740ed76e30907abd55b9193f1ab81362
 size 3055

wandb/debug-internal.log

@@ -1 +1 @@
-run-20220129_111936-33z1cg4n/logs/debug-internal.log
+run-20220129_170809-3lal72pm/logs/debug-internal.log

wandb/debug.log

@@ -1 +1 @@
-run-20220129_111936-33z1cg4n/logs/debug.log
+run-20220129_170809-3lal72pm/logs/debug.log

wandb/latest-run

@@ -1 +1 @@
-run-20220129_111936-33z1cg4n
+run-20220129_170809-3lal72pm

wandb/run-20220126_105847-1xdiy2kf/files/wandb-metadata.json

@@ -1,76 +0,0 @@
-{
-    "os": "Linux-4.18.0-305.10.2.el8_4.x86_64-x86_64-with-glibc2.28",
-    "python": "3.9.6",
-    "heartbeatAt": "2022-01-26T15:58:48.495262",
-    "startedAt": "2022-01-26T15:58:47.536839",
-    "docker": null,
-    "gpu": "Tesla V100-PCIE-32GB",
-    "gpu_count": 3,
-    "cpu_count": 64,
-    "cuda": null,
-    "args": [
-        "--dataset_name=mozilla-foundation/common_voice_7_0",
-        "--model_name_or_path=facebook/wav2vec2-xls-r-300m",
-        "--dataset_config_name=et",
-        "--output_dir=./",
-        "--overwrite_output_dir",
-        "--num_train_epochs=500",
-        "--per_device_train_batch_size=80",
-        "--per_device_eval_batch_size=80",
-        "--gradient_accumulation_steps=2",
-        "--learning_rate=3e-4",
-        "--save_total_limit=1",
-        "--warmup_steps=500",
-        "--evaluation_strategy=steps",
-        "--text_column_name=sentence",
-        "--length_column_name=input_length",
-        "--save_steps=500",
-        "--eval_steps=500",
-        "--logging_steps=100",
-        "--layerdrop=0.0",
-        "--freeze_feature_encoder",
-        "--feat_proj_dropout=0.1",
-        "--chars_to_ignore",
-        ",",
-        "?",
-        ".",
-        "!",
-        "-",
-        ";",
-        ":",
-        "\"",
-        "\u201c",
-        "%",
-        "\u2018",
-        "\u201d",
-        "\ufffd",
-        "\u2014",
-        "\u2019",
-        "\u2026",
-        "\u2013",
-        "--gradient_checkpointing",
-        "--lr_scheduler_type=cosine",
-        "--fp16",
-        "--group_by_length",
-        "--mask_time_prob=0.1",
-        "--mask_time_length=10",
-        "--report_to=wandb",
-        "--run_name=cosine+drop_proj+low_specaugment-300M",
-        "--do_train",
-        "--do_eval",
-        "--use_auth_token",
-        "--push_to_hub"
-    ],
-    "state": "running",
-    "program": "/home/sagrilaft/Project/audio/xls-r-et/src/run_speech_recognition_ctc_bnb.py",
-    "codePath": "src/run_speech_recognition_ctc_bnb.py",
-    "git": {
-        "remote": "https://huggingface.co/shpotes/xls-r-et",
-        "commit": "2a437db2644553c5c9c2b9e81553ed9bdf92207e"
-    },
-    "email": "[email protected]",
-    "root": "/home/sagrilaft/Project/audio/xls-r-et",
-    "host": "ganymede.eafit.edu.co",
-    "username": "sagrilaft",
-    "executable": "/home/sagrilaft/Project/audio/xls-r-et/.venv/bin/python"
-}

wandb/run-20220126_105847-1xdiy2kf/logs/debug.log

@@ -1,186 +0,0 @@
-2022-01-26 10:58:47,539 INFO    MainThread:2220463 [wandb_setup.py:_flush():71] setting env: {'project': 'xls-r-estonian'}
-2022-01-26 10:58:47,540 INFO    MainThread:2220463 [wandb_setup.py:_flush():71] setting login settings: {}
-2022-01-26 10:58:47,540 INFO    MainThread:2220463 [wandb_init.py:_log_setup():371] Logging user logs to /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_105847-1xdiy2kf/logs/debug.log
-2022-01-26 10:58:47,540 INFO    MainThread:2220463 [wandb_init.py:_log_setup():372] Logging internal logs to /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_105847-1xdiy2kf/logs/debug-internal.log
-2022-01-26 10:58:47,540 INFO    MainThread:2220463 [wandb_init.py:init():404] calling init triggers
-2022-01-26 10:58:47,541 INFO    MainThread:2220463 [wandb_init.py:init():409] wandb.init called with sweep_config: {}
-config: {}
-2022-01-26 10:58:47,541 INFO    MainThread:2220463 [wandb_init.py:init():460] starting backend
-2022-01-26 10:58:47,541 INFO    MainThread:2220463 [backend.py:_multiprocessing_setup():99] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
-2022-01-26 10:58:47,575 INFO    MainThread:2220463 [backend.py:ensure_launched():216] starting backend process...
-2022-01-26 10:58:47,598 INFO    MainThread:2220463 [backend.py:ensure_launched():221] started backend process with pid: 2220828
-2022-01-26 10:58:47,600 INFO    MainThread:2220463 [wandb_init.py:init():469] backend started and connected
-2022-01-26 10:58:47,604 INFO    MainThread:2220463 [wandb_init.py:init():533] updated telemetry
-2022-01-26 10:58:47,664 INFO    MainThread:2220463 [wandb_init.py:init():563] communicating current version
-2022-01-26 10:58:48,298 INFO    MainThread:2220463 [wandb_init.py:init():568] got version response 
-2022-01-26 10:58:48,299 INFO    MainThread:2220463 [wandb_init.py:init():578] communicating run to backend with 30 second timeout
-2022-01-26 10:58:48,488 INFO    MainThread:2220463 [wandb_init.py:init():606] starting run threads in backend
-2022-01-26 10:58:48,534 INFO    MainThread:2220463 [wandb_run.py:_console_start():1810] atexit reg
-2022-01-26 10:58:48,535 INFO    MainThread:2220463 [wandb_run.py:_redirect():1684] redirect: SettingsConsole.REDIRECT
-2022-01-26 10:58:48,536 INFO    MainThread:2220463 [wandb_run.py:_redirect():1689] Redirecting console.
-2022-01-26 10:58:48,541 INFO    MainThread:2220463 [wandb_run.py:_redirect():1745] Redirects installed.
-2022-01-26 10:58:48,541 INFO    MainThread:2220463 [wandb_init.py:init():633] run started, returning control to user process
-2022-01-26 10:58:48,562 INFO    MainThread:2220463 [wandb_run.py:_config_callback():956] config_cb None None {'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'float32', 'use_bfloat16': False, 'pruned_heads': {}, 'tie_word_embeddings': True, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'chunk_size_feed_forward': 0, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'architectures': ['Wav2Vec2ForPreTraining'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 1, 'pad_token_id': 36, 'eos_token_id': 2, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'facebook/wav2vec2-xls-r-300m', 'transformers_version': '4.16.0.dev0', 'feat_extract_dropout': 0.0, 'model_type': 'wav2vec2', 'num_feat_extract_layers': 7, 'hidden_size': 1024, 'feat_extract_norm': 'layer', 'feat_extract_activation': 'gelu', 'conv_dim': [512, 512, 512, 512, 512, 512, 512], 'conv_stride': [5, 2, 2, 2, 2, 2, 2], 'conv_kernel': [10, 3, 3, 3, 3, 2, 2], 'conv_bias': True, 'num_conv_pos_embeddings': 128, 'num_conv_pos_embedding_groups': 16, 'num_hidden_layers': 24, 'intermediate_size': 4096, 'hidden_act': 'gelu', 'num_attention_heads': 16, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'activation_dropout': 0.0, 'feat_proj_dropout': 0.1, 'final_dropout': 0.0, 'layerdrop': 0.0, 'layer_norm_eps': 1e-05, 'initializer_range': 0.02, 'vocab_size': 39, 'do_stable_layer_norm': True, 'use_weighted_layer_sum': False, 'apply_spec_augment': True, 'mask_time_prob': 0.1, 'mask_time_length': 10, 'mask_time_min_masks': 2, 'mask_feature_prob': 0.0, 'mask_feature_length': 10, 'mask_feature_min_masks': 0, 'num_codevectors_per_group': 320, 'num_codevector_groups': 2, 'contrastive_logits_temperature': 0.1, 'feat_quantizer_dropout': 0.0, 'num_negatives': 100, 'codevector_dim': 768, 'proj_codevector_dim': 768, 'diversity_loss_weight': 0.1, 'ctc_loss_reduction': 'mean', 'ctc_zero_infinity': False, 'add_adapter': False, 'adapter_kernel_size': 3, 'adapter_stride': 2, 'num_adapter_layers': 3, 'output_hidden_size': 1024, 'classifier_proj_size': 256, 'tdnn_dim': [512, 512, 512, 512, 1500], 'tdnn_kernel': [5, 3, 3, 1, 1], 'tdnn_dilation': [1, 2, 3, 1, 1], 'xvector_output_dim': 512, 'output_dir': './', 'overwrite_output_dir': True, 'do_train': True, 'do_eval': True, 'do_predict': False, 'evaluation_strategy': 'steps', 'prediction_loss_only': False, 'per_device_train_batch_size': 80, 'per_device_eval_batch_size': 80, 'per_gpu_train_batch_size': 'None', 'per_gpu_eval_batch_size': 'None', 'gradient_accumulation_steps': 2, 'eval_accumulation_steps': 'None', 'learning_rate': 0.0003, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 500.0, 'max_steps': -1, 'lr_scheduler_type': 'cosine', 'warmup_ratio': 0.0, 'warmup_steps': 500, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jan26_10-56-56_ganymede.eafit.edu.co', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 100, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 500, 'save_total_limit': 1, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'bf16': False, 'fp16': True, 'fp16_opt_level': 'O1', 'half_precision_backend': 'amp', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': 'None', 'local_rank': -1, 'xpu_backend': 'None', 'tpu_num_cores': 'None', 'tpu_metrics_debug': False, 'debug': '[]', 'dataloader_drop_last': False, 'eval_steps': 500, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': 'cosine+drop_proj+low_specaugment-300M', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': 'None', 'load_best_model_at_end': False, 'metric_for_best_model': 'None', 'greater_is_better': 'None', 'ignore_data_skip': False, 'sharded_ddp': '[]', 'deepspeed': 'None', 'label_smoothing_factor': 0.0, 'optim': 'adamw_hf', 'adafactor': False, 'group_by_length': True, 'length_column_name': 'input_length', 'report_to': "['wandb']", 'ddp_find_unused_parameters': 'None', 'ddp_bucket_cap_mb': 'None', 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': 'None', 'hub_model_id': 'None', 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'gradient_checkpointing': True, 'fp16_backend': 'auto', 'push_to_hub_model_id': 'None', 'push_to_hub_organization': 'None', 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', '_n_gpu': 1, 'mp_parameters': '', 'train_batch_size': 80, 'eval_batch_size': 80}
-2022-01-26 10:58:48,565 INFO    MainThread:2220463 [wandb_watch.py:watch():43] Watching
-2022-01-28 11:57:53,891 INFO    MainThread:2220463 [wandb_run.py:_atexit_cleanup():1780] got exitcode: 1
-2022-01-28 11:57:53,896 INFO    MainThread:2220463 [wandb_run.py:_restore():1752] restore
-2022-01-28 11:57:56,278 INFO    MainThread:2220463 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts {
-  wandb_count: 1
-}
-pusher_stats {
-  uploaded_bytes: 2324
-  total_bytes: 2324
-}
-
-2022-01-28 11:57:56,435 INFO    MainThread:2220463 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts {
-  wandb_count: 1
-}
-pusher_stats {
-  uploaded_bytes: 2324
-  total_bytes: 2324
-}
-
-2022-01-28 11:57:57,061 INFO    MainThread:2220463 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts {
-  wandb_count: 1
-}
-pusher_stats {
-  uploaded_bytes: 2324
-  total_bytes: 2324
-}
-
-2022-01-28 11:57:58,002 INFO    MainThread:2220463 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts {
-  wandb_count: 5
-}
-pusher_stats {
-  uploaded_bytes: 2324
-  total_bytes: 3441922
-}
-
-2022-01-28 11:57:58,105 INFO    MainThread:2220463 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts {
-  wandb_count: 5
-}
-pusher_stats {
-  uploaded_bytes: 2324
-  total_bytes: 3441922
-}
-
-2022-01-28 11:57:58,208 INFO    MainThread:2220463 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts {
-  wandb_count: 5
-}
-pusher_stats {
-  uploaded_bytes: 2324
-  total_bytes: 3441922
-}
-
-2022-01-28 11:57:58,311 INFO    MainThread:2220463 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts {
-  wandb_count: 5
-}
-pusher_stats {
-  uploaded_bytes: 2246495
-  total_bytes: 3441922
-}
-
-2022-01-28 11:57:58,414 INFO    MainThread:2220463 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts {
-  wandb_count: 5
-}
-pusher_stats {
-  uploaded_bytes: 2246495
-  total_bytes: 3441922
-}
-
-2022-01-28 11:57:58,516 INFO    MainThread:2220463 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts {
-  wandb_count: 5
-}
-pusher_stats {
-  uploaded_bytes: 3441922
-  total_bytes: 3441922
-}
-
-2022-01-28 11:57:58,619 INFO    MainThread:2220463 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts {
-  wandb_count: 5
-}
-pusher_stats {
-  uploaded_bytes: 3441922
-  total_bytes: 3441922
-}
-
-2022-01-28 11:57:58,722 INFO    MainThread:2220463 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts {
-  wandb_count: 5
-}
-pusher_stats {
-  uploaded_bytes: 3441922
-  total_bytes: 3441922
-}
-
-2022-01-28 11:57:58,824 INFO    MainThread:2220463 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts {
-  wandb_count: 5
-}
-pusher_stats {
-  uploaded_bytes: 3441922
-  total_bytes: 3441922
-}
-
-2022-01-28 11:57:58,927 INFO    MainThread:2220463 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts {
-  wandb_count: 5
-}
-pusher_stats {
-  uploaded_bytes: 3441922
-  total_bytes: 3441922
-}
-
-2022-01-28 11:57:59,030 INFO    MainThread:2220463 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts {
-  wandb_count: 5
-}
-pusher_stats {
-  uploaded_bytes: 3441922
-  total_bytes: 3441922
-}
-
-2022-01-28 11:57:59,133 INFO    MainThread:2220463 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts {
-  wandb_count: 5
-}
-pusher_stats {
-  uploaded_bytes: 3441922
-  total_bytes: 3441922
-}
-
-2022-01-28 11:57:59,235 INFO    MainThread:2220463 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts {
-  wandb_count: 5
-}
-pusher_stats {
-  uploaded_bytes: 3441922
-  total_bytes: 3441922
-}
-
-2022-01-28 11:57:59,338 INFO    MainThread:2220463 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts {
-  wandb_count: 5
-}
-pusher_stats {
-  uploaded_bytes: 3441922
-  total_bytes: 3441922
-}
-
-2022-01-28 11:58:00,218 INFO    MainThread:2220463 [wandb_run.py:_wait_for_finish():1912] got exit ret: file_counts {
-  wandb_count: 5
-}
-pusher_stats {
-  uploaded_bytes: 3441922
-  total_bytes: 3441922
-}
-
-2022-01-28 11:58:00,597 INFO    MainThread:2220463 [wandb_run.py:_wait_for_finish():1912] got exit ret: done: true
-exit_result {
-}
-file_counts {
-  wandb_count: 5
-}
-pusher_stats {
-  uploaded_bytes: 3441922
-  total_bytes: 3441922
-}
-local_info {
-}
-
-2022-01-28 11:58:01,703 INFO    MainThread:2220463 [wandb_run.py:_append_history():2130] rendering history
-2022-01-28 11:58:01,705 INFO    MainThread:2220463 [wandb_run.py:_append_summary():2085] rendering summary
-2022-01-28 11:58:01,706 INFO    MainThread:2220463 [wandb_run.py:_append_files():2180] logging synced files

wandb/run-20220126_105847-1xdiy2kf/run-1xdiy2kf.wandb

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:aee0b40e3c968e6e44cf387e6b28e8eb547361393a2c20aa51d507601ce6dd4f
-size 125379374

wandb/run-20220126_121554-295enqix/files/config.yaml

@@ -1,649 +0,0 @@
-wandb_version: 1
-
-_n_gpu:
-  desc: null
-  value: 1
-_name_or_path:
-  desc: null
-  value: facebook/wav2vec2-xls-r-1b
-_wandb:
-  desc: null
-  value:
-    cli_version: 0.12.9
-    framework: huggingface
-    huggingface_version: 4.16.0.dev0
-    is_jupyter_run: false
-    is_kaggle_kernel: true
-    m:
-    - 1: train/global_step
-      6:
-      - 3
-    python_version: 3.9.6
-    start_time: 1643217354
-    t:
-      1:
-      - 1
-      - 5
-      - 11
-      3:
-      - 13
-      4: 3.9.6
-      5: 0.12.9
-      6: 4.16.0.dev0
-      8:
-      - 2
-      - 5
-activation_dropout:
-  desc: null
-  value: 0.0
-adafactor:
-  desc: null
-  value: false
-adam_beta1:
-  desc: null
-  value: 0.9
-adam_beta2:
-  desc: null
-  value: 0.999
-adam_epsilon:
-  desc: null
-  value: 1.0e-08
-adapter_kernel_size:
-  desc: null
-  value: 3
-adapter_stride:
-  desc: null
-  value: 2
-add_adapter:
-  desc: null
-  value: false
-add_cross_attention:
-  desc: null
-  value: false
-apply_spec_augment:
-  desc: null
-  value: true
-architectures:
-  desc: null
-  value:
-  - Wav2Vec2ForPreTraining
-attention_dropout:
-  desc: null
-  value: 0.0
-bad_words_ids:
-  desc: null
-  value: null
-bf16:
-  desc: null
-  value: false
-bf16_full_eval:
-  desc: null
-  value: false
-bos_token_id:
-  desc: null
-  value: 1
-chunk_size_feed_forward:
-  desc: null
-  value: 0
-classifier_proj_size:
-  desc: null
-  value: 256
-codevector_dim:
-  desc: null
-  value: 1024
-contrastive_logits_temperature:
-  desc: null
-  value: 0.1
-conv_bias:
-  desc: null
-  value: true
-conv_dim:
-  desc: null
-  value:
-  - 512
-  - 512
-  - 512
-  - 512
-  - 512
-  - 512
-  - 512
-conv_kernel:
-  desc: null
-  value:
-  - 10
-  - 3
-  - 3
-  - 3
-  - 3
-  - 2
-  - 2
-conv_stride:
-  desc: null
-  value:
-  - 5
-  - 2
-  - 2
-  - 2
-  - 2
-  - 2
-  - 2
-cross_attention_hidden_size:
-  desc: null
-  value: null
-ctc_loss_reduction:
-  desc: null
-  value: mean
-ctc_zero_infinity:
-  desc: null
-  value: false
-dataloader_drop_last:
-  desc: null
-  value: false
-dataloader_num_workers:
-  desc: null
-  value: 0
-dataloader_pin_memory:
-  desc: null
-  value: true
-ddp_bucket_cap_mb:
-  desc: null
-  value: None
-ddp_find_unused_parameters:
-  desc: null
-  value: None
-debug:
-  desc: null
-  value: '[]'
-decoder_start_token_id:
-  desc: null
-  value: null
-deepspeed:
-  desc: null
-  value: None
-disable_tqdm:
-  desc: null
-  value: false
-diversity_loss_weight:
-  desc: null
-  value: 0.1
-diversity_penalty:
-  desc: null
-  value: 0.0
-do_eval:
-  desc: null
-  value: true
-do_predict:
-  desc: null
-  value: false
-do_sample:
-  desc: null
-  value: false
-do_stable_layer_norm:
-  desc: null
-  value: true
-do_train:
-  desc: null
-  value: true
-early_stopping:
-  desc: null
-  value: false
-encoder_no_repeat_ngram_size:
-  desc: null
-  value: 0
-eos_token_id:
-  desc: null
-  value: 2
-eval_accumulation_steps:
-  desc: null
-  value: None
-eval_batch_size:
-  desc: null
-  value: 48
-eval_steps:
-  desc: null
-  value: 250
-evaluation_strategy:
-  desc: null
-  value: steps
-feat_extract_activation:
-  desc: null
-  value: gelu
-feat_extract_dropout:
-  desc: null
-  value: 0.0
-feat_extract_norm:
-  desc: null
-  value: layer
-feat_proj_dropout:
-  desc: null
-  value: 0.1
-feat_quantizer_dropout:
-  desc: null
-  value: 0.0
-final_dropout:
-  desc: null
-  value: 0.0
-finetuning_task:
-  desc: null
-  value: null
-forced_bos_token_id:
-  desc: null
-  value: null
-forced_eos_token_id:
-  desc: null
-  value: null
-fp16:
-  desc: null
-  value: true
-fp16_backend:
-  desc: null
-  value: auto
-fp16_full_eval:
-  desc: null
-  value: false
-fp16_opt_level:
-  desc: null
-  value: O1
-gradient_accumulation_steps:
-  desc: null
-  value: 2
-gradient_checkpointing:
-  desc: null
-  value: true
-greater_is_better:
-  desc: null
-  value: None
-group_by_length:
-  desc: null
-  value: true
-half_precision_backend:
-  desc: null
-  value: amp
-hidden_act:
-  desc: null
-  value: gelu
-hidden_dropout:
-  desc: null
-  value: 0.0
-hidden_size:
-  desc: null
-  value: 1280
-hub_model_id:
-  desc: null
-  value: None
-hub_strategy:
-  desc: null
-  value: every_save
-hub_token:
-  desc: null
-  value: <HUB_TOKEN>
-id2label:
-  desc: null
-  value:
-    '0': LABEL_0
-    '1': LABEL_1
-ignore_data_skip:
-  desc: null
-  value: false
-initializer_range:
-  desc: null
-  value: 0.02
-intermediate_size:
-  desc: null
-  value: 5120
-is_decoder:
-  desc: null
-  value: false
-is_encoder_decoder:
-  desc: null
-  value: false
-label2id:
-  desc: null
-  value:
-    LABEL_0: 0
-    LABEL_1: 1
-label_names:
-  desc: null
-  value: None
-label_smoothing_factor:
-  desc: null
-  value: 0.0
-layer_norm_eps:
-  desc: null
-  value: 1.0e-05
-layerdrop:
-  desc: null
-  value: 0.0
-learning_rate:
-  desc: null
-  value: 5.0e-05
-length_column_name:
-  desc: null
-  value: input_length
-length_penalty:
-  desc: null
-  value: 1.0
-load_best_model_at_end:
-  desc: null
-  value: false
-local_rank:
-  desc: null
-  value: -1
-log_level:
-  desc: null
-  value: -1
-log_level_replica:
-  desc: null
-  value: -1
-log_on_each_node:
-  desc: null
-  value: true
-logging_dir:
-  desc: null
-  value: ./1B/runs/Jan26_12-15-06_ganymede.eafit.edu.co
-logging_first_step:
-  desc: null
-  value: false
-logging_nan_inf_filter:
-  desc: null
-  value: true
-logging_steps:
-  desc: null
-  value: 100
-logging_strategy:
-  desc: null
-  value: steps
-lr_scheduler_type:
-  desc: null
-  value: cosine
-mask_feature_length:
-  desc: null
-  value: 10
-mask_feature_min_masks:
-  desc: null
-  value: 0
-mask_feature_prob:
-  desc: null
-  value: 0.0
-mask_time_length:
-  desc: null
-  value: 10
-mask_time_min_masks:
-  desc: null
-  value: 2
-mask_time_prob:
-  desc: null
-  value: 0.1
-max_grad_norm:
-  desc: null
-  value: 1.0
-max_length:
-  desc: null
-  value: 20
-max_steps:
-  desc: null
-  value: -1
-metric_for_best_model:
-  desc: null
-  value: None
-min_length:
-  desc: null
-  value: 0
-model_type:
-  desc: null
-  value: wav2vec2
-mp_parameters:
-  desc: null
-  value: ''
-no_cuda:
-  desc: null
-  value: false
-no_repeat_ngram_size:
-  desc: null
-  value: 0
-num_adapter_layers:
-  desc: null
-  value: 3
-num_attention_heads:
-  desc: null
-  value: 16
-num_beam_groups:
-  desc: null
-  value: 1
-num_beams:
-  desc: null
-  value: 1
-num_codevector_groups:
-  desc: null
-  value: 2
-num_codevectors_per_group:
-  desc: null
-  value: 320
-num_conv_pos_embedding_groups:
-  desc: null
-  value: 16
-num_conv_pos_embeddings:
-  desc: null
-  value: 128
-num_feat_extract_layers:
-  desc: null
-  value: 7
-num_hidden_layers:
-  desc: null
-  value: 48
-num_negatives:
-  desc: null
-  value: 100
-num_return_sequences:
-  desc: null
-  value: 1
-num_train_epochs:
-  desc: null
-  value: 100.0
-optim:
-  desc: null
-  value: adamw_hf
-output_attentions:
-  desc: null
-  value: false
-output_dir:
-  desc: null
-  value: ./1B
-output_hidden_size:
-  desc: null
-  value: 1280
-output_hidden_states:
-  desc: null
-  value: false
-output_scores:
-  desc: null
-  value: false
-overwrite_output_dir:
-  desc: null
-  value: true
-pad_token_id:
-  desc: null
-  value: 36
-past_index:
-  desc: null
-  value: -1
-per_device_eval_batch_size:
-  desc: null
-  value: 48
-per_device_train_batch_size:
-  desc: null
-  value: 48
-per_gpu_eval_batch_size:
-  desc: null
-  value: None
-per_gpu_train_batch_size:
-  desc: null
-  value: None
-prediction_loss_only:
-  desc: null
-  value: false
-prefix:
-  desc: null
-  value: null
-problem_type:
-  desc: null
-  value: null
-proj_codevector_dim:
-  desc: null
-  value: 1024
-pruned_heads:
-  desc: null
-  value: {}
-push_to_hub:
-  desc: null
-  value: false
-push_to_hub_model_id:
-  desc: null
-  value: None
-push_to_hub_organization:
-  desc: null
-  value: None
-push_to_hub_token:
-  desc: null
-  value: <PUSH_TO_HUB_TOKEN>
-remove_invalid_values:
-  desc: null
-  value: false
-remove_unused_columns:
-  desc: null
-  value: true
-repetition_penalty:
-  desc: null
-  value: 1.0
-report_to:
-  desc: null
-  value: '[''wandb'']'
-resume_from_checkpoint:
-  desc: null
-  value: None
-return_dict:
-  desc: null
-  value: true
-return_dict_in_generate:
-  desc: null
-  value: false
-run_name:
-  desc: null
-  value: cosine+drop_proj+low_specaugment-1b
-save_on_each_node:
-  desc: null
-  value: false
-save_steps:
-  desc: null
-  value: 1000
-save_strategy:
-  desc: null
-  value: steps
-save_total_limit:
-  desc: null
-  value: 1
-seed:
-  desc: null
-  value: 42
-sep_token_id:
-  desc: null
-  value: null
-sharded_ddp:
-  desc: null
-  value: '[]'
-skip_memory_metrics:
-  desc: null
-  value: true
-task_specific_params:
-  desc: null
-  value: null
-tdnn_dilation:
-  desc: null
-  value:
-  - 1
-  - 2
-  - 3
-  - 1
-  - 1
-tdnn_dim:
-  desc: null
-  value:
-  - 512
-  - 512
-  - 512
-  - 512
-  - 1500
-tdnn_kernel:
-  desc: null
-  value:
-  - 5
-  - 3
-  - 3
-  - 1
-  - 1
-temperature:
-  desc: null
-  value: 1.0
-tf32:
-  desc: null
-  value: None
-tie_encoder_decoder:
-  desc: null
-  value: false
-tie_word_embeddings:
-  desc: null
-  value: true
-tokenizer_class:
-  desc: null
-  value: null
-top_k:
-  desc: null
-  value: 50
-top_p:
-  desc: null
-  value: 1.0
-torch_dtype:
-  desc: null
-  value: float32
-torchscript:
-  desc: null
-  value: false
-tpu_metrics_debug:
-  desc: null
-  value: false
-tpu_num_cores:
-  desc: null
-  value: None
-train_batch_size:
-  desc: null
-  value: 48
-transformers_version:
-  desc: null
-  value: 4.16.0.dev0
-use_bfloat16:
-  desc: null
-  value: false
-use_legacy_prediction_loop:
-  desc: null
-  value: false
-use_weighted_layer_sum:
-  desc: null
-  value: false
-vocab_size:
-  desc: null
-  value: 39
-warmup_ratio:
-  desc: null
-  value: 0.0
-warmup_steps:
-  desc: null
-  value: 500
-weight_decay:
-  desc: null
-  value: 0.0
-xpu_backend:
-  desc: null
-  value: None
-xvector_output_dim:
-  desc: null
-  value: 512

wandb/run-20220126_121554-295enqix/files/output.log

@@ -1,32 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-

wandb/run-20220126_121554-295enqix/files/requirements.txt

@@ -1,87 +0,0 @@
-aiohttp==3.8.1
-aiosignal==1.2.0
-appdirs==1.4.4
-async-timeout==4.0.2
-attrs==21.4.0
-audioread==2.1.9
-bitsandbytes-cuda113==0.26.0
-certifi==2021.10.8
-cffi==1.15.0
-charset-normalizer==2.0.10
-click==8.0.3
-clldutils==3.10.1
-colorlog==6.6.0
-configparser==5.2.0
-csvw==1.11.0
-datasets==1.18.1.dev0
-decorator==5.1.1
-dill==0.3.4
-dlinfo==1.2.1
-docker-pycreds==0.4.0
-filelock==3.4.2
-frozenlist==1.3.0
-fsspec==2022.1.0
-gitdb==4.0.9
-gitpython==3.1.26
-huggingface-hub==0.4.0
-hypothesis==6.36.0
-idna==3.3
-isodate==0.6.1
-jiwer==2.3.0
-joblib==1.1.0
-librosa==0.8.1
-llvmlite==0.38.0
-multidict==6.0.2
-multiprocess==0.70.12.2
-numba==0.55.0
-numpy==1.21.5
-packaging==21.3
-pandas==1.4.0
-pathtools==0.1.2
-phonemizer==3.0.1
-pip==21.3.1
-pooch==1.6.0
-promise==2.3
-protobuf==3.19.3
-psutil==5.9.0
-pyarrow==6.0.1
-pycparser==2.21
-pyctcdecode==0.3.0
-pygtrie==2.4.2
-pyparsing==3.0.7
-python-dateutil==2.8.2
-python-levenshtein==0.12.2
-pytz==2021.3
-pyyaml==6.0
-regex==2022.1.18
-requests==2.27.1
-resampy==0.2.2
-rfc3986==2.0.0
-sacremoses==0.0.47
-scikit-learn==1.0.2
-scipy==1.7.3
-segments==2.2.0
-sentry-sdk==1.5.4
-setuptools==60.2.0
-shortuuid==1.0.8
-six==1.16.0
-smmap==5.0.0
-sortedcontainers==2.4.0
-soundfile==0.10.3.post1
-subprocess32==3.5.4
-tabulate==0.8.9
-termcolor==1.1.0
-threadpoolctl==3.0.0
-tokenizers==0.11.4
-torch==1.10.1
-torchaudio==0.10.1
-tqdm==4.62.3
-transformers==4.16.0.dev0
-typing-extensions==4.0.1
-uritemplate==4.1.1
-urllib3==1.26.8
-wandb==0.12.9
-wheel==0.37.1
-xxhash==2.0.2
-yarl==1.7.2
-yaspin==2.1.0

wandb/run-20220126_121554-295enqix/files/wandb-metadata.json

@@ -1,74 +0,0 @@
-{
-    "os": "Linux-4.18.0-305.10.2.el8_4.x86_64-x86_64-with-glibc2.28",
-    "python": "3.9.6",
-    "heartbeatAt": "2022-01-26T17:15:55.285233",
-    "startedAt": "2022-01-26T17:15:54.341820",
-    "docker": null,
-    "gpu": "Tesla V100-PCIE-32GB",
-    "gpu_count": 3,
-    "cpu_count": 64,
-    "cuda": null,
-    "args": [
-        "--dataset_name=mozilla-foundation/common_voice_7_0",
-        "--model_name_or_path=facebook/wav2vec2-xls-r-1b",
-        "--dataset_config_name=et",
-        "--output_dir=./1B",
-        "--overwrite_output_dir",
-        "--num_train_epochs=100",
-        "--per_device_train_batch_size=48",
-        "--per_device_eval_batch_size=48",
-        "--gradient_accumulation_steps=2",
-        "--learning_rate=5e-5",
-        "--save_total_limit=1",
-        "--warmup_steps=500",
-        "--evaluation_strategy=steps",
-        "--text_column_name=sentence",
-        "--length_column_name=input_length",
-        "--save_steps=1000",
-        "--eval_steps=250",
-        "--logging_steps=100",
-        "--layerdrop=0.0",
-        "--freeze_feature_encoder",
-        "--feat_proj_dropout=0.1",
-        "--chars_to_ignore",
-        ",",
-        "?",
-        ".",
-        "!",
-        "-",
-        ";",
-        ":",
-        "\"",
-        "\u201c",
-        "%",
-        "\u2018",
-        "\u201d",
-        "\ufffd",
-        "\u2014",
-        "\u2019",
-        "\u2026",
-        "\u2013",
-        "--gradient_checkpointing",
-        "--lr_scheduler_type=cosine",
-        "--fp16",
-        "--group_by_length",
-        "--mask_time_prob=0.1",
-        "--mask_time_length=10",
-        "--report_to=wandb",
-        "--run_name=cosine+drop_proj+low_specaugment-1b",
-        "--do_train",
-        "--do_eval"
-    ],
-    "state": "running",
-    "program": "/home/sagrilaft/Project/audio/xls-r-et/src/run_speech_recognition_ctc_bnb.py",
-    "codePath": "src/run_speech_recognition_ctc_bnb.py",
-    "git": {
-        "remote": "https://huggingface.co/shpotes/xls-r-et",
-        "commit": "2a437db2644553c5c9c2b9e81553ed9bdf92207e"
-    },
-    "email": "[email protected]",
-    "root": "/home/sagrilaft/Project/audio/xls-r-et",
-    "host": "ganymede.eafit.edu.co",
-    "username": "sagrilaft",
-    "executable": "/home/sagrilaft/Project/audio/xls-r-et/.venv/bin/python"
-}

wandb/run-20220126_121554-295enqix/files/wandb-summary.json

@@ -1 +0,0 @@
-{}

wandb/run-20220126_121554-295enqix/logs/debug-internal.log

@@ -1,136 +0,0 @@
-2022-01-26 12:15:54,828 INFO    MainThread:2226380 [internal.py:wandb_internal():87] W&B internal server running at pid: 2226380, started at: 2022-01-26 12:15:54.828545
-2022-01-26 12:15:54,831 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: check_version
-2022-01-26 12:15:54,831 INFO    WriterThread:2226380 [datastore.py:open_for_write():77] open: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/run-295enqix.wandb
-2022-01-26 12:15:54,833 DEBUG   SenderThread:2226380 [sender.py:send():234] send: header
-2022-01-26 12:15:54,834 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: check_version
-2022-01-26 12:15:55,103 DEBUG   SenderThread:2226380 [sender.py:send():234] send: run
-2022-01-26 12:15:55,276 INFO    SenderThread:2226380 [dir_watcher.py:__init__():169] watching files in: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files
-2022-01-26 12:15:55,276 INFO    SenderThread:2226380 [sender.py:_start_run_threads():804] run started: 295enqix with start time 1643217354
-2022-01-26 12:15:55,276 DEBUG   SenderThread:2226380 [sender.py:send():234] send: summary
-2022-01-26 12:15:55,276 INFO    SenderThread:2226380 [sender.py:_save_file():939] saving file wandb-summary.json with policy end
-2022-01-26 12:15:55,278 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: run_start
-2022-01-26 12:15:55,284 DEBUG   HandlerThread:2226380 [meta.py:__init__():40] meta init
-2022-01-26 12:15:55,285 DEBUG   HandlerThread:2226380 [meta.py:__init__():54] meta init done
-2022-01-26 12:15:55,285 DEBUG   HandlerThread:2226380 [meta.py:probe():214] probe
-2022-01-26 12:15:55,291 DEBUG   HandlerThread:2226380 [meta.py:_setup_git():204] setup git
-2022-01-26 12:15:55,309 DEBUG   HandlerThread:2226380 [meta.py:_setup_git():211] setup git done
-2022-01-26 12:15:55,309 DEBUG   HandlerThread:2226380 [meta.py:_save_pip():58] save pip
-2022-01-26 12:15:55,309 DEBUG   HandlerThread:2226380 [meta.py:_save_pip():72] save pip done
-2022-01-26 12:15:55,309 DEBUG   HandlerThread:2226380 [meta.py:probe():252] probe done
-2022-01-26 12:15:55,313 DEBUG   SenderThread:2226380 [sender.py:send():234] send: files
-2022-01-26 12:15:55,313 INFO    SenderThread:2226380 [sender.py:_save_file():939] saving file wandb-metadata.json with policy now
-2022-01-26 12:15:55,322 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
-2022-01-26 12:15:55,323 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
-2022-01-26 12:15:55,460 DEBUG   SenderThread:2226380 [sender.py:send():234] send: config
-2022-01-26 12:15:55,461 DEBUG   SenderThread:2226380 [sender.py:send():234] send: metric
-2022-01-26 12:15:55,462 DEBUG   SenderThread:2226380 [sender.py:send():234] send: metric
-2022-01-26 12:15:55,462 WARNING SenderThread:2226380 [sender.py:send_metric():897] Seen metric with glob (shouldnt happen)
-2022-01-26 12:15:55,937 INFO    Thread-11 :2226380 [upload_job.py:push():137] Uploaded file /tmp/tmp0kjyvd1nwandb/sa83kpbt-wandb-metadata.json
-2022-01-26 12:15:56,277 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_created():217] file/dir created: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/wandb-metadata.json
-2022-01-26 12:15:56,278 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_created():217] file/dir created: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/requirements.txt
-2022-01-26 12:15:56,278 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_created():217] file/dir created: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
-2022-01-26 12:15:56,278 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_created():217] file/dir created: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/wandb-summary.json
-2022-01-26 12:15:58,277 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
-2022-01-26 12:16:10,577 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
-2022-01-26 12:16:10,577 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
-2022-01-26 12:16:16,285 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
-2022-01-26 12:16:24,276 DEBUG   SenderThread:2226380 [sender.py:send():234] send: stats
-2022-01-26 12:16:25,718 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
-2022-01-26 12:16:25,719 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
-2022-01-26 12:16:26,289 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/config.yaml
-2022-01-26 12:16:30,291 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
-2022-01-26 12:16:40,861 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
-2022-01-26 12:16:40,862 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
-2022-01-26 12:16:44,298 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
-2022-01-26 12:16:55,255 DEBUG   SenderThread:2226380 [sender.py:send():234] send: stats
-2022-01-26 12:16:56,119 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
-2022-01-26 12:16:56,119 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
-2022-01-26 12:16:56,303 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
-2022-01-26 12:17:06,308 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
-2022-01-26 12:17:11,395 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
-2022-01-26 12:17:11,395 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
-2022-01-26 12:17:16,312 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
-2022-01-26 12:17:26,266 DEBUG   SenderThread:2226380 [sender.py:send():234] send: stats
-2022-01-26 12:17:26,317 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
-2022-01-26 12:17:26,631 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
-2022-01-26 12:17:26,632 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
-2022-01-26 12:17:34,321 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
-2022-01-26 12:17:41,857 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
-2022-01-26 12:17:41,858 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
-2022-01-26 12:17:42,325 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
-2022-01-26 12:17:50,329 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
-2022-01-26 12:17:56,331 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
-2022-01-26 12:17:57,090 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
-2022-01-26 12:17:57,090 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
-2022-01-26 12:17:57,233 DEBUG   SenderThread:2226380 [sender.py:send():234] send: stats
-2022-01-26 12:18:02,334 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
-2022-01-26 12:18:08,337 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
-2022-01-26 12:18:12,358 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
-2022-01-26 12:18:12,359 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
-2022-01-26 12:18:13,339 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
-2022-01-26 12:18:27,555 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
-2022-01-26 12:18:27,555 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
-2022-01-26 12:18:28,215 DEBUG   SenderThread:2226380 [sender.py:send():234] send: stats
-2022-01-26 12:18:31,347 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
-2022-01-26 12:18:42,695 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
-2022-01-26 12:18:42,695 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
-2022-01-26 12:18:45,354 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
-2022-01-26 12:18:57,952 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
-2022-01-26 12:18:57,952 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
-2022-01-26 12:18:59,199 DEBUG   SenderThread:2226380 [sender.py:send():234] send: stats
-2022-01-26 12:18:59,360 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
-2022-01-26 12:19:11,366 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
-2022-01-26 12:19:13,205 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
-2022-01-26 12:19:13,206 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
-2022-01-26 12:19:23,373 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
-2022-01-26 12:19:28,370 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
-2022-01-26 12:19:28,371 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
-2022-01-26 12:19:30,192 DEBUG   SenderThread:2226380 [sender.py:send():234] send: stats
-2022-01-26 12:19:33,379 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
-2022-01-26 12:19:41,382 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
-2022-01-26 12:19:43,543 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
-2022-01-26 12:19:43,544 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
-2022-01-26 12:19:51,386 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
-2022-01-26 12:19:58,761 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
-2022-01-26 12:19:58,761 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
-2022-01-26 12:19:59,389 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
-2022-01-26 12:20:01,184 DEBUG   SenderThread:2226380 [sender.py:send():234] send: stats
-2022-01-26 12:20:07,393 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
-2022-01-26 12:20:13,395 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
-2022-01-26 12:20:13,950 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
-2022-01-26 12:20:13,950 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
-2022-01-26 12:20:19,398 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
-2022-01-26 12:20:25,401 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
-2022-01-26 12:20:29,257 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
-2022-01-26 12:20:29,257 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
-2022-01-26 12:20:29,402 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
-2022-01-26 12:20:32,140 DEBUG   SenderThread:2226380 [sender.py:send():234] send: stats
-2022-01-26 12:20:44,546 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
-2022-01-26 12:20:44,547 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
-2022-01-26 12:20:47,411 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
-2022-01-26 12:20:59,689 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
-2022-01-26 12:20:59,690 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
-2022-01-26 12:21:03,070 DEBUG   SenderThread:2226380 [sender.py:send():234] send: stats
-2022-01-26 12:21:04,418 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
-2022-01-26 12:21:14,840 DEBUG   HandlerThread:2226380 [handler.py:handle_request():130] handle_request: stop_status
-2022-01-26 12:21:14,840 DEBUG   SenderThread:2226380 [sender.py:send_request():248] send_request: stop_status
-2022-01-26 12:21:18,425 INFO    Thread-8  :2226380 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
-2022-01-26 12:21:32,200 WARNING MainThread:2226380 [internal.py:is_dead():366] Internal process exiting, parent pid 2226158 disappeared
-2022-01-26 12:21:32,200 ERROR   MainThread:2226380 [internal.py:wandb_internal():142] Internal process shutdown.
-2022-01-26 12:21:32,483 INFO    SenderThread:2226380 [sender.py:finish():1070] shutting down sender
-2022-01-26 12:21:32,484 INFO    SenderThread:2226380 [dir_watcher.py:finish():283] shutting down directory watcher
-2022-01-26 12:21:32,492 INFO    HandlerThread:2226380 [handler.py:finish():731] shutting down handler
-2022-01-26 12:21:32,503 INFO    WriterThread:2226380 [datastore.py:close():281] close: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/run-295enqix.wandb
-2022-01-26 12:21:33,428 INFO    SenderThread:2226380 [dir_watcher.py:finish():313] scan: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files
-2022-01-26 12:21:33,429 INFO    SenderThread:2226380 [dir_watcher.py:finish():327] scan save: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/config.yaml config.yaml
-2022-01-26 12:21:33,429 INFO    SenderThread:2226380 [dir_watcher.py:finish():327] scan save: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/wandb-summary.json wandb-summary.json
-2022-01-26 12:21:33,429 INFO    SenderThread:2226380 [dir_watcher.py:finish():327] scan save: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/requirements.txt requirements.txt
-2022-01-26 12:21:33,429 INFO    SenderThread:2226380 [dir_watcher.py:finish():327] scan save: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/wandb-metadata.json wandb-metadata.json
-2022-01-26 12:21:33,430 INFO    SenderThread:2226380 [dir_watcher.py:finish():327] scan save: /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log output.log
-2022-01-26 12:21:33,430 INFO    SenderThread:2226380 [file_pusher.py:finish():177] shutting down file pusher
-2022-01-26 12:21:33,430 INFO    SenderThread:2226380 [file_pusher.py:join():182] waiting for file pusher
-2022-01-26 12:21:34,064 INFO    Thread-14 :2226380 [upload_job.py:push():137] Uploaded file /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/requirements.txt
-2022-01-26 12:21:34,092 INFO    Thread-13 :2226380 [upload_job.py:push():137] Uploaded file /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/wandb-summary.json
-2022-01-26 12:21:34,134 INFO    Thread-12 :2226380 [upload_job.py:push():137] Uploaded file /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/config.yaml
-2022-01-26 12:21:34,181 INFO    Thread-15 :2226380 [upload_job.py:push():137] Uploaded file /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/files/output.log
-2022-01-26 12:21:34,621 INFO    MainThread:2226380 [internal.py:handle_exit():77] Internal process exited

wandb/run-20220126_121554-295enqix/logs/debug.log

@@ -1,24 +0,0 @@
-2022-01-26 12:15:54,344 INFO    MainThread:2226158 [wandb_setup.py:_flush():71] setting env: {'project': 'xls-r-estonian'}
-2022-01-26 12:15:54,344 INFO    MainThread:2226158 [wandb_setup.py:_flush():71] setting login settings: {}
-2022-01-26 12:15:54,345 INFO    MainThread:2226158 [wandb_init.py:_log_setup():371] Logging user logs to /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/logs/debug.log
-2022-01-26 12:15:54,345 INFO    MainThread:2226158 [wandb_init.py:_log_setup():372] Logging internal logs to /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220126_121554-295enqix/logs/debug-internal.log
-2022-01-26 12:15:54,345 INFO    MainThread:2226158 [wandb_init.py:init():404] calling init triggers
-2022-01-26 12:15:54,346 INFO    MainThread:2226158 [wandb_init.py:init():409] wandb.init called with sweep_config: {}
-config: {}
-2022-01-26 12:15:54,346 INFO    MainThread:2226158 [wandb_init.py:init():460] starting backend
-2022-01-26 12:15:54,346 INFO    MainThread:2226158 [backend.py:_multiprocessing_setup():99] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
-2022-01-26 12:15:54,373 INFO    MainThread:2226158 [backend.py:ensure_launched():216] starting backend process...
-2022-01-26 12:15:54,395 INFO    MainThread:2226158 [backend.py:ensure_launched():221] started backend process with pid: 2226380
-2022-01-26 12:15:54,396 INFO    MainThread:2226158 [wandb_init.py:init():469] backend started and connected
-2022-01-26 12:15:54,401 INFO    MainThread:2226158 [wandb_init.py:init():533] updated telemetry
-2022-01-26 12:15:54,459 INFO    MainThread:2226158 [wandb_init.py:init():563] communicating current version
-2022-01-26 12:15:55,101 INFO    MainThread:2226158 [wandb_init.py:init():568] got version response 
-2022-01-26 12:15:55,101 INFO    MainThread:2226158 [wandb_init.py:init():578] communicating run to backend with 30 second timeout
-2022-01-26 12:15:55,277 INFO    MainThread:2226158 [wandb_init.py:init():606] starting run threads in backend
-2022-01-26 12:15:55,320 INFO    MainThread:2226158 [wandb_run.py:_console_start():1810] atexit reg
-2022-01-26 12:15:55,321 INFO    MainThread:2226158 [wandb_run.py:_redirect():1684] redirect: SettingsConsole.REDIRECT
-2022-01-26 12:15:55,322 INFO    MainThread:2226158 [wandb_run.py:_redirect():1689] Redirecting console.
-2022-01-26 12:15:55,325 INFO    MainThread:2226158 [wandb_run.py:_redirect():1745] Redirects installed.
-2022-01-26 12:15:55,325 INFO    MainThread:2226158 [wandb_init.py:init():633] run started, returning control to user process
-2022-01-26 12:15:55,341 INFO    MainThread:2226158 [wandb_run.py:_config_callback():956] config_cb None None {'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'float32', 'use_bfloat16': False, 'pruned_heads': {}, 'tie_word_embeddings': True, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'chunk_size_feed_forward': 0, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'architectures': ['Wav2Vec2ForPreTraining'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 1, 'pad_token_id': 36, 'eos_token_id': 2, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'facebook/wav2vec2-xls-r-1b', 'transformers_version': '4.16.0.dev0', 'feat_extract_dropout': 0.0, 'model_type': 'wav2vec2', 'num_feat_extract_layers': 7, 'hidden_size': 1280, 'feat_extract_norm': 'layer', 'feat_extract_activation': 'gelu', 'conv_dim': [512, 512, 512, 512, 512, 512, 512], 'conv_stride': [5, 2, 2, 2, 2, 2, 2], 'conv_kernel': [10, 3, 3, 3, 3, 2, 2], 'conv_bias': True, 'num_conv_pos_embeddings': 128, 'num_conv_pos_embedding_groups': 16, 'num_hidden_layers': 48, 'intermediate_size': 5120, 'hidden_act': 'gelu', 'num_attention_heads': 16, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'activation_dropout': 0.0, 'feat_proj_dropout': 0.1, 'final_dropout': 0.0, 'layerdrop': 0.0, 'layer_norm_eps': 1e-05, 'initializer_range': 0.02, 'vocab_size': 39, 'do_stable_layer_norm': True, 'use_weighted_layer_sum': False, 'apply_spec_augment': True, 'mask_time_prob': 0.1, 'mask_time_length': 10, 'mask_time_min_masks': 2, 'mask_feature_prob': 0.0, 'mask_feature_length': 10, 'mask_feature_min_masks': 0, 'num_codevectors_per_group': 320, 'num_codevector_groups': 2, 'contrastive_logits_temperature': 0.1, 'feat_quantizer_dropout': 0.0, 'num_negatives': 100, 'codevector_dim': 1024, 'proj_codevector_dim': 1024, 'diversity_loss_weight': 0.1, 'ctc_loss_reduction': 'mean', 'ctc_zero_infinity': False, 'add_adapter': False, 'adapter_kernel_size': 3, 'adapter_stride': 2, 'num_adapter_layers': 3, 'output_hidden_size': 1280, 'classifier_proj_size': 256, 'tdnn_dim': [512, 512, 512, 512, 1500], 'tdnn_kernel': [5, 3, 3, 1, 1], 'tdnn_dilation': [1, 2, 3, 1, 1], 'xvector_output_dim': 512, 'output_dir': './1B', 'overwrite_output_dir': True, 'do_train': True, 'do_eval': True, 'do_predict': False, 'evaluation_strategy': 'steps', 'prediction_loss_only': False, 'per_device_train_batch_size': 48, 'per_device_eval_batch_size': 48, 'per_gpu_train_batch_size': 'None', 'per_gpu_eval_batch_size': 'None', 'gradient_accumulation_steps': 2, 'eval_accumulation_steps': 'None', 'learning_rate': 5e-05, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 100.0, 'max_steps': -1, 'lr_scheduler_type': 'cosine', 'warmup_ratio': 0.0, 'warmup_steps': 500, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './1B/runs/Jan26_12-15-06_ganymede.eafit.edu.co', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 100, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 1000, 'save_total_limit': 1, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'bf16': False, 'fp16': True, 'fp16_opt_level': 'O1', 'half_precision_backend': 'amp', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': 'None', 'local_rank': -1, 'xpu_backend': 'None', 'tpu_num_cores': 'None', 'tpu_metrics_debug': False, 'debug': '[]', 'dataloader_drop_last': False, 'eval_steps': 250, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': 'cosine+drop_proj+low_specaugment-1b', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': 'None', 'load_best_model_at_end': False, 'metric_for_best_model': 'None', 'greater_is_better': 'None', 'ignore_data_skip': False, 'sharded_ddp': '[]', 'deepspeed': 'None', 'label_smoothing_factor': 0.0, 'optim': 'adamw_hf', 'adafactor': False, 'group_by_length': True, 'length_column_name': 'input_length', 'report_to': "['wandb']", 'ddp_find_unused_parameters': 'None', 'ddp_bucket_cap_mb': 'None', 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': 'None', 'hub_model_id': 'None', 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'gradient_checkpointing': True, 'fp16_backend': 'auto', 'push_to_hub_model_id': 'None', 'push_to_hub_organization': 'None', 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', '_n_gpu': 1, 'mp_parameters': '', 'train_batch_size': 48, 'eval_batch_size': 48}
-2022-01-26 12:15:55,344 INFO    MainThread:2226158 [wandb_watch.py:watch():43] Watching

wandb/run-20220129_111936-33z1cg4n/files/requirements.txt

@@ -1,87 +0,0 @@
-aiohttp==3.8.1
-aiosignal==1.2.0
-appdirs==1.4.4
-async-timeout==4.0.2
-attrs==21.4.0
-audioread==2.1.9
-bitsandbytes-cuda113==0.26.0
-certifi==2021.10.8
-cffi==1.15.0
-charset-normalizer==2.0.10
-click==8.0.3
-clldutils==3.10.1
-colorlog==6.6.0
-configparser==5.2.0
-csvw==1.11.0
-datasets==1.18.1.dev0
-decorator==5.1.1
-dill==0.3.4
-dlinfo==1.2.1
-docker-pycreds==0.4.0
-filelock==3.4.2
-frozenlist==1.3.0
-fsspec==2022.1.0
-gitdb==4.0.9
-gitpython==3.1.26
-huggingface-hub==0.4.0
-hypothesis==6.36.0
-idna==3.3
-isodate==0.6.1
-jiwer==2.3.0
-joblib==1.1.0
-librosa==0.8.1
-llvmlite==0.38.0
-multidict==6.0.2
-multiprocess==0.70.12.2
-numba==0.55.0
-numpy==1.21.5
-packaging==21.3
-pandas==1.4.0
-pathtools==0.1.2
-phonemizer==3.0.1
-pip==21.3.1
-pooch==1.6.0
-promise==2.3
-protobuf==3.19.3
-psutil==5.9.0
-pyarrow==6.0.1
-pycparser==2.21
-pyctcdecode==0.3.0
-pygtrie==2.4.2
-pyparsing==3.0.7
-python-dateutil==2.8.2
-python-levenshtein==0.12.2
-pytz==2021.3
-pyyaml==6.0
-regex==2022.1.18
-requests==2.27.1
-resampy==0.2.2
-rfc3986==2.0.0
-sacremoses==0.0.47
-scikit-learn==1.0.2
-scipy==1.7.3
-segments==2.2.0
-sentry-sdk==1.5.4
-setuptools==60.2.0
-shortuuid==1.0.8
-six==1.16.0
-smmap==5.0.0
-sortedcontainers==2.4.0
-soundfile==0.10.3.post1
-subprocess32==3.5.4
-tabulate==0.8.9
-termcolor==1.1.0
-threadpoolctl==3.0.0
-tokenizers==0.11.4
-torch==1.10.1
-torchaudio==0.10.1
-tqdm==4.62.3
-transformers==4.16.0.dev0
-typing-extensions==4.0.1
-uritemplate==4.1.1
-urllib3==1.26.8
-wandb==0.12.9
-wheel==0.37.1
-xxhash==2.0.2
-yarl==1.7.2
-yaspin==2.1.0

wandb/{run-20220129_111936-33z1cg4n → run-20220129_170809-3lal72pm}/files/config.yaml

@@ -4803,7 +4803,7 @@ _wandb:
       6:
       - 1
     python_version: 3.9.6
-    start_time: 1643473176
+    start_time: 1643494089
     t:
       1:
       - 1
@@ -4982,7 +4982,7 @@ eval_accumulation_steps:
   value: None
 eval_batch_size:
   desc: null
-  value: 80
+  value: 72
 eval_steps:
   desc: null
   value: 500
@@ -5030,7 +5030,7 @@ fp16_opt_level:
   value: O1
 gradient_accumulation_steps:
   desc: null
-  value: 2
+  value: 1
 gradient_checkpointing:
   desc: null
   value: true
@@ -5124,7 +5124,7 @@ log_on_each_node:
   value: true
 logging_dir:
   desc: null
-  value: ./runs/Jan29_11-17-08_ganymede.eafit.edu.co
+  value: ./runs/Jan29_17-06-16_ganymede.eafit.edu.co
 logging_first_step:
   desc: null
   value: false
@@ -5253,10 +5253,10 @@ past_index:
   value: -1
 per_device_eval_batch_size:
   desc: null
-  value: 80
+  value: 72
 per_device_train_batch_size:
   desc: null
-  value: 80
+  value: 72
 per_gpu_eval_batch_size:
   desc: null
   value: None
@@ -5400,7 +5400,7 @@ tpu_num_cores:
   value: None
 train_batch_size:
   desc: null
-  value: 80
+  value: 72
 transformers_version:
   desc: null
   value: 4.16.0.dev0

wandb/{run-20220129_111936-33z1cg4n → run-20220129_170809-3lal72pm}/files/output.log

@@ -98,8 +98,8 @@
 
 
 
-  3%|█▋                                                            | 99/3600 [15:46<10:30:00, 10.80s/it]
 
+  1%|██                                                                                                                                                                      | 100/8000 [07:12<9:10:12,  4.18s/it]
 
 
 
@@ -199,8 +199,8 @@
 
 
 
+  2%|████▏                                                                                                                                                                  | 200/8000 [14:20<10:55:34,  5.04s/it]
 
-  6%|███▍                                                         | 200/3600 [31:33<10:03:22, 10.65s/it]
 
 
 
@@ -298,9 +298,9 @@
 
 
 
+  4%|██████▎                                                                                                                                                                | 300/8000 [21:26<11:53:08,  5.56s/it]
 
 
-  8%|█████▏                                                        | 299/3600 [47:03<9:53:18, 10.78s/it]
 
 
 
@@ -396,14 +396,7 @@
 
 
 
-
-
-
-
-
- 11%|██████▋                                                     | 399/3600 [1:02:40<9:47:13, 11.01s/it]
-
-
+  5%|████████▍                                                                                                                                                               | 400/8000 [28:21<5:51:55,  2.78s/it]
 
 
 
@@ -499,14 +492,14 @@
 
 
 
+  6%|██████████▍                                                                                                                                                             | 499/8000 [35:20<4:57:32,  2.38s/it]
+  6%|██████████▌                                                                                                                                                             | 500/8000 [35:28<8:36:37,  4.13s/it]The following columns in the evaluation set  don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.
+***** Running Evaluation *****
+  Num examples = 2609
+  Batch size = 72
 
 
 
- 14%|████████▎                                                   | 499/3600 [1:18:15<6:47:48,  7.89s/it]
- 14%|████████▎                                                   | 500/3600 [1:18:20<5:57:31,  6.92s/it]The following columns in the evaluation set  don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.
-***** Running Evaluation *****
-  Num examples = 2609
-  Batch size = 80
 
 
 
@@ -539,8 +532,8 @@
 
 
 
-100%|███████████████████████████████████████████████████████████████████| 33/33 [02:06<00:00,  2.59s/it]
 
 Configuration saved in ./checkpoint-500/config.json
+{'eval_loss': 0.4678382873535156, 'eval_wer': 0.5768055997454661, 'eval_runtime': 131.6765, 'eval_samples_per_second': 19.814, 'eval_steps_per_second': 0.281, 'epoch': 6.25}
 Model weights saved in ./checkpoint-500/pytorch_model.bin
 Configuration saved in ./checkpoint-500/preprocessor_config.json

wandb/{run-20220129_111936-33z1cg4n → run-20220129_170809-3lal72pm}/files/wandb-metadata.json

@@ -1,8 +1,8 @@
 {
     "os": "Linux-4.18.0-305.10.2.el8_4.x86_64-x86_64-with-glibc2.28",
     "python": "3.9.6",
-    "heartbeatAt": "2022-01-29T16:19:37.625823",
-    "startedAt": "2022-01-29T16:19:36.629047",
+    "heartbeatAt": "2022-01-29T22:08:10.210586",
+    "startedAt": "2022-01-29T22:08:09.026413",
     "docker": null,
     "gpu": "Tesla V100-PCIE-32GB",
     "gpu_count": 3,
@@ -15,9 +15,9 @@
         "--output_dir=./",
         "--overwrite_output_dir",
         "--num_train_epochs=100",
-        "--per_device_train_batch_size=80",
-        "--per_device_eval_batch_size=80",
-        "--gradient_accumulation_steps=2",
+        "--per_device_train_batch_size=72",
+        "--per_device_eval_batch_size=72",
+        "--gradient_accumulation_steps=1",
         "--learning_rate=3e-4",
         "--save_total_limit=1",
         "--warmup_steps=500",
@@ -66,7 +66,7 @@
     "codePath": "src/run_speech_recognition_ctc_bnb.py",
     "git": {
         "remote": "https://huggingface.co/shpotes/xls-r-et",
-        "commit": "fa2a9d24f0bae1b5002fb327d3cfa727ef5b4caf"
+        "commit": "1f68524110998ba851e20ef822076294e0945a07"
     },
     "email": "[email protected]",
     "root": "/home/sagrilaft/Project/audio/xls-r-et",

wandb/{run-20220129_111936-33z1cg4n → run-20220129_170809-3lal72pm}/logs/debug.log

@@ -1,24 +1,24 @@
-2022-01-29 11:19:36,632 INFO    MainThread:2523114 [wandb_setup.py:_flush():71] setting env: {'project': 'xls-r-estonian'}
-2022-01-29 11:19:36,632 INFO    MainThread:2523114 [wandb_setup.py:_flush():71] setting login settings: {}
-2022-01-29 11:19:36,632 INFO    MainThread:2523114 [wandb_init.py:_log_setup():371] Logging user logs to /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220129_111936-33z1cg4n/logs/debug.log
-2022-01-29 11:19:36,632 INFO    MainThread:2523114 [wandb_init.py:_log_setup():372] Logging internal logs to /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220129_111936-33z1cg4n/logs/debug-internal.log
-2022-01-29 11:19:36,633 INFO    MainThread:2523114 [wandb_init.py:init():404] calling init triggers
-2022-01-29 11:19:36,633 INFO    MainThread:2523114 [wandb_init.py:init():409] wandb.init called with sweep_config: {}
+2022-01-29 17:08:09,029 INFO    MainThread:2547831 [wandb_setup.py:_flush():71] setting env: {'project': 'xls-r-estonian'}
+2022-01-29 17:08:09,029 INFO    MainThread:2547831 [wandb_setup.py:_flush():71] setting login settings: {}
+2022-01-29 17:08:09,030 INFO    MainThread:2547831 [wandb_init.py:_log_setup():371] Logging user logs to /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220129_170809-3lal72pm/logs/debug.log
+2022-01-29 17:08:09,030 INFO    MainThread:2547831 [wandb_init.py:_log_setup():372] Logging internal logs to /home/sagrilaft/Project/audio/xls-r-et/wandb/run-20220129_170809-3lal72pm/logs/debug-internal.log
+2022-01-29 17:08:09,031 INFO    MainThread:2547831 [wandb_init.py:init():404] calling init triggers
+2022-01-29 17:08:09,031 INFO    MainThread:2547831 [wandb_init.py:init():409] wandb.init called with sweep_config: {}
 config: {}
-2022-01-29 11:19:36,633 INFO    MainThread:2523114 [wandb_init.py:init():460] starting backend
-2022-01-29 11:19:36,633 INFO    MainThread:2523114 [backend.py:_multiprocessing_setup():99] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
-2022-01-29 11:19:36,666 INFO    MainThread:2523114 [backend.py:ensure_launched():216] starting backend process...
-2022-01-29 11:19:36,693 INFO    MainThread:2523114 [backend.py:ensure_launched():221] started backend process with pid: 2523574
-2022-01-29 11:19:36,694 INFO    MainThread:2523114 [wandb_init.py:init():469] backend started and connected
-2022-01-29 11:19:36,701 INFO    MainThread:2523114 [wandb_init.py:init():533] updated telemetry
-2022-01-29 11:19:36,765 INFO    MainThread:2523114 [wandb_init.py:init():563] communicating current version
-2022-01-29 11:19:37,425 INFO    MainThread:2523114 [wandb_init.py:init():568] got version response 
-2022-01-29 11:19:37,426 INFO    MainThread:2523114 [wandb_init.py:init():578] communicating run to backend with 30 second timeout
-2022-01-29 11:19:37,617 INFO    MainThread:2523114 [wandb_init.py:init():606] starting run threads in backend
-2022-01-29 11:19:37,676 INFO    MainThread:2523114 [wandb_run.py:_console_start():1810] atexit reg
-2022-01-29 11:19:37,677 INFO    MainThread:2523114 [wandb_run.py:_redirect():1684] redirect: SettingsConsole.REDIRECT
-2022-01-29 11:19:37,677 INFO    MainThread:2523114 [wandb_run.py:_redirect():1689] Redirecting console.
-2022-01-29 11:19:37,680 INFO    MainThread:2523114 [wandb_run.py:_redirect():1745] Redirects installed.
-2022-01-29 11:19:37,680 INFO    MainThread:2523114 [wandb_init.py:init():633] run started, returning control to user process
-2022-01-29 11:19:37,694 INFO    MainThread:2523114 [wandb_run.py:_config_callback():956] config_cb None None {'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'float32', 'use_bfloat16': False, 'pruned_heads': {}, 'tie_word_embeddings': True, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'chunk_size_feed_forward': 0, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'architectures': ['Wav2Vec2ForPreTraining'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 1, 'pad_token_id': 36, 'eos_token_id': 2, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'facebook/wav2vec2-xls-r-300m', 'transformers_version': '4.16.0.dev0', 'feat_extract_dropout': 0.0, 'model_type': 'wav2vec2', 'num_feat_extract_layers': 7, 'hidden_size': 1024, 'feat_extract_norm': 'layer', 'feat_extract_activation': 'gelu', 'conv_dim': [512, 512, 512, 512, 512, 512, 512], 'conv_stride': [5, 2, 2, 2, 2, 2, 2], 'conv_kernel': [10, 3, 3, 3, 3, 2, 2], 'conv_bias': True, 'num_conv_pos_embeddings': 128, 'num_conv_pos_embedding_groups': 16, 'num_hidden_layers': 24, 'intermediate_size': 4096, 'hidden_act': 'gelu', 'num_attention_heads': 16, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'activation_dropout': 0.0, 'feat_proj_dropout': 0.1, 'final_dropout': 0.0, 'layerdrop': 0.0, 'layer_norm_eps': 1e-05, 'initializer_range': 0.02, 'vocab_size': 39, 'do_stable_layer_norm': True, 'use_weighted_layer_sum': False, 'apply_spec_augment': True, 'mask_time_prob': 0.1, 'mask_time_length': 10, 'mask_time_min_masks': 2, 'mask_feature_prob': 0.0, 'mask_feature_length': 10, 'mask_feature_min_masks': 0, 'num_codevectors_per_group': 320, 'num_codevector_groups': 2, 'contrastive_logits_temperature': 0.1, 'feat_quantizer_dropout': 0.0, 'num_negatives': 100, 'codevector_dim': 768, 'proj_codevector_dim': 768, 'diversity_loss_weight': 0.1, 'ctc_loss_reduction': 'mean', 'ctc_zero_infinity': False, 'add_adapter': False, 'adapter_kernel_size': 3, 'adapter_stride': 2, 'num_adapter_layers': 3, 'output_hidden_size': 1024, 'classifier_proj_size': 256, 'tdnn_dim': [512, 512, 512, 512, 1500], 'tdnn_kernel': [5, 3, 3, 1, 1], 'tdnn_dilation': [1, 2, 3, 1, 1], 'xvector_output_dim': 512, 'output_dir': './', 'overwrite_output_dir': True, 'do_train': True, 'do_eval': True, 'do_predict': False, 'evaluation_strategy': 'steps', 'prediction_loss_only': False, 'per_device_train_batch_size': 80, 'per_device_eval_batch_size': 80, 'per_gpu_train_batch_size': 'None', 'per_gpu_eval_batch_size': 'None', 'gradient_accumulation_steps': 2, 'eval_accumulation_steps': 'None', 'learning_rate': 0.0003, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 100.0, 'max_steps': -1, 'lr_scheduler_type': 'cosine', 'warmup_ratio': 0.0, 'warmup_steps': 500, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jan29_11-17-08_ganymede.eafit.edu.co', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 100, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 500, 'save_total_limit': 1, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'bf16': False, 'fp16': True, 'fp16_opt_level': 'O1', 'half_precision_backend': 'amp', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': 'None', 'local_rank': -1, 'xpu_backend': 'None', 'tpu_num_cores': 'None', 'tpu_metrics_debug': False, 'debug': '[]', 'dataloader_drop_last': False, 'eval_steps': 500, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': 'cosine+drop_proj+low_specaugment-300M', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': 'None', 'load_best_model_at_end': False, 'metric_for_best_model': 'None', 'greater_is_better': 'None', 'ignore_data_skip': False, 'sharded_ddp': '[]', 'deepspeed': 'None', 'label_smoothing_factor': 0.0, 'optim': 'adamw_hf', 'adafactor': False, 'group_by_length': True, 'length_column_name': 'input_length', 'report_to': "['wandb']", 'ddp_find_unused_parameters': 'None', 'ddp_bucket_cap_mb': 'None', 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': 'None', 'hub_model_id': 'None', 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'gradient_checkpointing': True, 'fp16_backend': 'auto', 'push_to_hub_model_id': 'None', 'push_to_hub_organization': 'None', 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', '_n_gpu': 1, 'mp_parameters': '', 'train_batch_size': 80, 'eval_batch_size': 80}
-2022-01-29 11:19:37,697 INFO    MainThread:2523114 [wandb_watch.py:watch():43] Watching
+2022-01-29 17:08:09,031 INFO    MainThread:2547831 [wandb_init.py:init():460] starting backend
+2022-01-29 17:08:09,031 INFO    MainThread:2547831 [backend.py:_multiprocessing_setup():99] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2022-01-29 17:08:09,069 INFO    MainThread:2547831 [backend.py:ensure_launched():216] starting backend process...
+2022-01-29 17:08:09,101 INFO    MainThread:2547831 [backend.py:ensure_launched():221] started backend process with pid: 2548178
+2022-01-29 17:08:09,103 INFO    MainThread:2547831 [wandb_init.py:init():469] backend started and connected
+2022-01-29 17:08:09,109 INFO    MainThread:2547831 [wandb_init.py:init():533] updated telemetry
+2022-01-29 17:08:09,183 INFO    MainThread:2547831 [wandb_init.py:init():563] communicating current version
+2022-01-29 17:08:09,991 INFO    MainThread:2547831 [wandb_init.py:init():568] got version response 
+2022-01-29 17:08:09,991 INFO    MainThread:2547831 [wandb_init.py:init():578] communicating run to backend with 30 second timeout
+2022-01-29 17:08:10,201 INFO    MainThread:2547831 [wandb_init.py:init():606] starting run threads in backend
+2022-01-29 17:08:10,250 INFO    MainThread:2547831 [wandb_run.py:_console_start():1810] atexit reg
+2022-01-29 17:08:10,251 INFO    MainThread:2547831 [wandb_run.py:_redirect():1684] redirect: SettingsConsole.REDIRECT
+2022-01-29 17:08:10,252 INFO    MainThread:2547831 [wandb_run.py:_redirect():1689] Redirecting console.
+2022-01-29 17:08:10,257 INFO    MainThread:2547831 [wandb_run.py:_redirect():1745] Redirects installed.
+2022-01-29 17:08:10,257 INFO    MainThread:2547831 [wandb_init.py:init():633] run started, returning control to user process
+2022-01-29 17:08:10,281 INFO    MainThread:2547831 [wandb_run.py:_config_callback():956] config_cb None None {'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'float32', 'use_bfloat16': False, 'pruned_heads': {}, 'tie_word_embeddings': True, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'chunk_size_feed_forward': 0, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'architectures': ['Wav2Vec2ForPreTraining'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 1, 'pad_token_id': 36, 'eos_token_id': 2, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'facebook/wav2vec2-xls-r-300m', 'transformers_version': '4.16.0.dev0', 'feat_extract_dropout': 0.0, 'model_type': 'wav2vec2', 'num_feat_extract_layers': 7, 'hidden_size': 1024, 'feat_extract_norm': 'layer', 'feat_extract_activation': 'gelu', 'conv_dim': [512, 512, 512, 512, 512, 512, 512], 'conv_stride': [5, 2, 2, 2, 2, 2, 2], 'conv_kernel': [10, 3, 3, 3, 3, 2, 2], 'conv_bias': True, 'num_conv_pos_embeddings': 128, 'num_conv_pos_embedding_groups': 16, 'num_hidden_layers': 24, 'intermediate_size': 4096, 'hidden_act': 'gelu', 'num_attention_heads': 16, 'hidden_dropout': 0.0, 'attention_dropout': 0.0, 'activation_dropout': 0.0, 'feat_proj_dropout': 0.1, 'final_dropout': 0.0, 'layerdrop': 0.0, 'layer_norm_eps': 1e-05, 'initializer_range': 0.02, 'vocab_size': 39, 'do_stable_layer_norm': True, 'use_weighted_layer_sum': False, 'apply_spec_augment': True, 'mask_time_prob': 0.1, 'mask_time_length': 10, 'mask_time_min_masks': 2, 'mask_feature_prob': 0.0, 'mask_feature_length': 10, 'mask_feature_min_masks': 0, 'num_codevectors_per_group': 320, 'num_codevector_groups': 2, 'contrastive_logits_temperature': 0.1, 'feat_quantizer_dropout': 0.0, 'num_negatives': 100, 'codevector_dim': 768, 'proj_codevector_dim': 768, 'diversity_loss_weight': 0.1, 'ctc_loss_reduction': 'mean', 'ctc_zero_infinity': False, 'add_adapter': False, 'adapter_kernel_size': 3, 'adapter_stride': 2, 'num_adapter_layers': 3, 'output_hidden_size': 1024, 'classifier_proj_size': 256, 'tdnn_dim': [512, 512, 512, 512, 1500], 'tdnn_kernel': [5, 3, 3, 1, 1], 'tdnn_dilation': [1, 2, 3, 1, 1], 'xvector_output_dim': 512, 'output_dir': './', 'overwrite_output_dir': True, 'do_train': True, 'do_eval': True, 'do_predict': False, 'evaluation_strategy': 'steps', 'prediction_loss_only': False, 'per_device_train_batch_size': 72, 'per_device_eval_batch_size': 72, 'per_gpu_train_batch_size': 'None', 'per_gpu_eval_batch_size': 'None', 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': 'None', 'learning_rate': 0.0003, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 100.0, 'max_steps': -1, 'lr_scheduler_type': 'cosine', 'warmup_ratio': 0.0, 'warmup_steps': 500, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jan29_17-06-16_ganymede.eafit.edu.co', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 100, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 500, 'save_total_limit': 1, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'bf16': False, 'fp16': True, 'fp16_opt_level': 'O1', 'half_precision_backend': 'amp', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': 'None', 'local_rank': -1, 'xpu_backend': 'None', 'tpu_num_cores': 'None', 'tpu_metrics_debug': False, 'debug': '[]', 'dataloader_drop_last': False, 'eval_steps': 500, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': 'cosine+drop_proj+low_specaugment-300M', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': 'None', 'load_best_model_at_end': False, 'metric_for_best_model': 'None', 'greater_is_better': 'None', 'ignore_data_skip': False, 'sharded_ddp': '[]', 'deepspeed': 'None', 'label_smoothing_factor': 0.0, 'optim': 'adamw_hf', 'adafactor': False, 'group_by_length': True, 'length_column_name': 'input_length', 'report_to': "['wandb']", 'ddp_find_unused_parameters': 'None', 'ddp_bucket_cap_mb': 'None', 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': 'None', 'hub_model_id': 'None', 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'gradient_checkpointing': True, 'fp16_backend': 'auto', 'push_to_hub_model_id': 'None', 'push_to_hub_organization': 'None', 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', '_n_gpu': 1, 'mp_parameters': '', 'train_batch_size': 72, 'eval_batch_size': 72}
+2022-01-29 17:08:10,284 INFO    MainThread:2547831 [wandb_watch.py:watch():43] Watching