



"""simple docstring"""

# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os

from accelerate.test_utils import execute_subprocess_async





def    __magic_name__     ( _lowerCamelCase					:			Tuple=None       ):
							if subparsers is not None:
														__a		:		str						=							subparsers.add_parser("""test"""       )
							else:
														__a		:		int						=							argparse.ArgumentParser("""Accelerate test command"""       )

							parser.add_argument(
							    """--config_file"""	,							default=UpperCamelCase__	,							help=(
							        """The path to use to store the config file. Will default to a file named default_config.yaml in the cache """
							        """location, which is the content of the environment `HF_HOME` suffixed with 'accelerate', or if you don't have """
							        """such an environment variable, your cache directory ('~/.cache' or the content of `XDG_CACHE_HOME`) suffixed """
							        """with 'huggingface'."""
							    )	,							)

							if subparsers is not None:
														parser.set_defaults(func=UpperCamelCase__       )
							return parser





def    __magic_name__     ( _lowerCamelCase					:			Optional[Any]       ):
							__a		:		Optional[Any]						=							os.path.sep.join(__file__.split(os.path.sep       )[:-2] + ["""test_utils""", """scripts""", """test_script.py"""]       )

							if args.config_file is None:
														__a		:		List[str]						=							script_name
							else:
														__a		:		Union[str, Any]						=							F'''--config_file={args.config_file} {script_name}'''

							__a		:		Dict						=							["""accelerate-launch"""] + test_args.split()
							__a		:		Tuple						=							execute_subprocess_async(UpperCamelCase__	,							env=os.environ.copy()       )
							if result.returncode == 0:
														print("""Test is a success! You are ready for your distributed training!"""       )





def    __magic_name__     ( ):
							__a		:		Optional[Any]						=							test_command_parser()
							__a		:		Any						=							parser.parse_args()
							test_command(UpperCamelCase__       )


if __name__ == "__main__":
	main()








import argparse
import logging
import pickle
from collections import Counter


logging.basicConfig(
    format='%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt='%m/%d/%Y %H:%M:%S', level=logging.INFO
)
_lowerCamelCase 					=      logging.getLogger(__name__)

if __name__ == "__main__":
	_lowerCamelCase 					=      argparse.ArgumentParser(
	    description='Token Counts for smoothing the masking probabilities in MLM (cf XLM/word2vec)'
	)
	parser.add_argument(
	    '--data_file', type=str, default='data/dump.bert-base-uncased.pickle', help='The binarized dataset.'
	)
	parser.add_argument(
	    '--token_counts_dump', type=str, default='data/token_counts.bert-base-uncased.pickle', help='The dump file.'
	)
	parser.add_argument('--vocab_size', default=30522, type=int)
	_lowerCamelCase 					=      parser.parse_args()

	logger.info(F'''Loading data from {args.data_file}''')
	with open(args.data_file, 'rb') as fp:
		_lowerCamelCase 					=      pickle.load(fp)

	logger.info('Counting occurrences for MLM.')
	_lowerCamelCase 					=      Counter()
	for tk_ids in data:
		counter.update(tk_ids)
	_lowerCamelCase 					=      [0] * args.vocab_size
	for k, v in counter.items():
		_lowerCamelCase 					=      v

	logger.info(F'''Dump to {args.token_counts_dump}''')
	with open(args.token_counts_dump, 'wb') as handle:
		pickle.dump(counts, handle, protocol=pickle.HIGHEST_PROTOCOL)