





import inspect
import os
import unittest

import torch

import accelerate
from accelerate import debug_launcher
from accelerate.test_utils import (
    execute_subprocess_async,
    require_cpu,
    require_huggingface_suite,
    require_multi_gpu,
    require_single_gpu,
)
from accelerate.utils import patch_environment



@require_huggingface_suite
class        _SCREAMING_SNAKE_CASE						(     unittest.TestCase   ):







 '''simple docstring'''

 def 							SCREAMING_SNAKE_CASE_ (self    :       Optional[int])	->Tuple:



   '''simple docstring'''



   lowerCamelCase__:				List[str]        =inspect.getfile(accelerate.test_utils)
   lowerCamelCase__:				str        =os.path.sep.join(
       mod_file.split(os.path.sep)[:-1] + ["scripts", "external_deps", "test_metrics.py"])

   from accelerate.test_utils.scripts.external_deps import test_metrics  # noqa: F401

   lowerCamelCase__:				str        =test_metrics

 @require_cpu
 def 							SCREAMING_SNAKE_CASE_ (self    :       int)	->Any:



   '''simple docstring'''



   debug_launcher(self.test_metrics.main	,			num_processes=1)

 @require_cpu
 def 							SCREAMING_SNAKE_CASE_ (self    :       int)	->int:



   '''simple docstring'''



   debug_launcher(self.test_metrics.main)

 @require_single_gpu
 def 							SCREAMING_SNAKE_CASE_ (self    :       Any)	->int:



   '''simple docstring'''



   self.test_metrics.main()







 @require_multi_gpu
 def 							SCREAMING_SNAKE_CASE_ (self    :       Tuple)	->Dict:



   '''simple docstring'''



   print(F"""Found {torch.cuda.device_count()} devices.""")
   lowerCamelCase__:				Optional[Any]        =["torchrun", F"""--nproc_per_node={torch.cuda.device_count()}""", self.test_file_path]
   with patch_environment(omp_num_threads=1):
     execute_subprocess_async(UpperCAmelCase_	,			env=os.environ.copy())








import argparse
import logging
import pickle
from collections import Counter


logging.basicConfig(
    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO
)
__A     =    logging.getLogger(__name__)

if __name__ == "__main__":
     __A     =    argparse.ArgumentParser(
         description="Token Counts for smoothing the masking probabilities in MLM (cf XLM/word2vec)"
     )
     parser.add_argument(
         "--data_file", type=str, default="data/dump.bert-base-uncased.pickle", help="The binarized dataset."
     )
     parser.add_argument(
         "--token_counts_dump", type=str, default="data/token_counts.bert-base-uncased.pickle", help="The dump file."
     )
     parser.add_argument("--vocab_size", default=3_0522, type=int)
     __A     =    parser.parse_args()

     logger.info(f'Loading data from {args.data_file}')
     with open(args.data_file, "rb") as fp:
          __A     =    pickle.load(fp)

     logger.info("Counting occurrences for MLM.")
     __A     =    Counter()
     for tk_ids in data:
          counter.update(tk_ids)
     __A     =    [0] * args.vocab_size
     for k, v in counter.items():
          __A     =    v

     logger.info(f'Dump to {args.token_counts_dump}')
     with open(args.token_counts_dump, "wb") as handle:
          pickle.dump(counts, handle, protocol=pickle.HIGHEST_PROTOCOL)

