






'''simple docstring'''



import argparse
import logging
import pickle
from collections import Counter


logging.basicConfig(
    format='%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt='%m/%d/%Y %H:%M:%S', level=logging.INFO
)
SCREAMING_SNAKE_CASE							=						logging.getLogger(__name__)

if __name__ == "__main__":
      SCREAMING_SNAKE_CASE							=						argparse.ArgumentParser(
          description='Token Counts for smoothing the masking probabilities in MLM (cf XLM/word2vec)'
      )
      parser.add_argument(
          '--data_file', type=str, default='data/dump.bert-base-uncased.pickle', help='The binarized dataset.'
      )
      parser.add_argument(
          '--token_counts_dump', type=str, default='data/token_counts.bert-base-uncased.pickle', help='The dump file.'
      )
      parser.add_argument('--vocab_size', default=30_522, type=int)
      SCREAMING_SNAKE_CASE							=						parser.parse_args()

      logger.info(f"""Loading data from {args.data_file}""")
      with open(args.data_file, 'rb') as fp:
            SCREAMING_SNAKE_CASE							=						pickle.load(fp)

      logger.info('Counting occurrences for MLM.')
      SCREAMING_SNAKE_CASE							=						Counter()
      for tk_ids in data:
            counter.update(tk_ids)
      SCREAMING_SNAKE_CASE							=						[0] * args.vocab_size
      for k, v in counter.items():
            SCREAMING_SNAKE_CASE							=						v

      logger.info(f"""Dump to {args.token_counts_dump}""")
      with open(args.token_counts_dump, 'wb') as handle:
            pickle.dump(counts, handle, protocol=pickle.HIGHEST_PROTOCOL)











'''simple docstring'''



import unittest
from dataclasses import dataclass

import pytest

from accelerate.commands.config.config_args import SageMakerConfig
from accelerate.utils import ComputeEnvironment
from accelerate.utils.launch import _convert_nargs_to_dict
@dataclass
class UpperCAmelCase_       (     __A					):




      """simple docstring"""




      UpperCamelCase_             =      ComputeEnvironment.AMAZON_SAGEMAKER
      UpperCamelCase_             =      True
      UpperCamelCase_             =      '''ml.p3.2xlarge'''
      UpperCamelCase_             =      '''accelerate_sagemaker_execution_role'''
      UpperCamelCase_             =      '''hf-sm'''
      UpperCamelCase_             =      '''us-east-1'''
      UpperCamelCase_             =      1
      UpperCamelCase_             =      '''accelerate-sagemaker-1'''
      UpperCamelCase_             =      '''1.6'''
      UpperCamelCase_             =      '''4.4'''
      UpperCamelCase_             =      '''train.py'''
      UpperCamelCase_             =      [
          '''--model_name_or_path''',
          '''bert''',
          '''--do_train''',
          '''False''',
          '''--epochs''',
          '''3''',
          '''--learning_rate''',
          '''5e-5''',
          '''--max_steps''',
          '''50.5''',
      ]
      UpperCamelCase_             =      [
          '''--model_name_or_path''',
          '''bert''',
          '''--do_train''',
          '''--do_test''',
          '''False''',
          '''--do_predict''',
          '''--epochs''',
          '''3''',
          '''--learning_rate''',
          '''5e-5''',
          '''--max_steps''',
          '''50.5''',
      ]





class UpperCAmelCase_       (     unittest.TestCase					):




      """simple docstring"""




      def 					A__   (			self      :		int		)	->	Dict:


       '''simple docstring'''

       lowercase       : str						=_convert_nargs_to_dict(MockLaunchConfig.success_training_script_args		)
       assert isinstance(converted_args['''model_name_or_path''']  ,   UpperCAmelCase		)
       assert isinstance(converted_args['''do_train''']  ,   UpperCAmelCase		)
       assert isinstance(converted_args['''epochs''']  ,   UpperCAmelCase		)
       assert isinstance(converted_args['''learning_rate''']  ,   UpperCAmelCase		)
       assert isinstance(converted_args['''max_steps''']  ,   UpperCAmelCase		)

       with pytest.raises(UpperCAmelCase		):
        _convert_nargs_to_dict(MockLaunchConfig.fail_training_script_args		)



