

"""simple docstring"""




# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import os

from accelerate.utils import ComputeEnvironment

from .cluster import get_cluster_input
from .config_args import cache_dir, default_config_file, default_yaml_config_file, load_config_from_file  # noqa: F401
from .config_utils import _ask_field, _ask_options, _convert_compute_environment  # noqa: F401
from .sagemaker import get_sagemaker_input


A_				:      Optional[Any]           =			"Launches a series of prompts to create and save a `default_config.yaml` configuration file for your training system. Should always be ran first on your machine"

def 						A							(  ):




			'''simple docstring'''


			SCREAMING_SNAKE_CASE__										=   _ask_options(
			    """In which compute environment are you running?"""		,					["""This machine""", """AWS (Amazon SageMaker)"""]		,					_convert_compute_environment		,					)
			if compute_environment == ComputeEnvironment.AMAZON_SAGEMAKER:
						SCREAMING_SNAKE_CASE__										=   get_sagemaker_input()
			else:
						SCREAMING_SNAKE_CASE__										=   get_cluster_input()
			return config

def 						A							(  snake_case__=None     ):




			'''simple docstring'''


			if subparsers is not None:
						SCREAMING_SNAKE_CASE__										=   subparsers.add_parser("""config"""		,					description=snake_case__     )
			else:
						SCREAMING_SNAKE_CASE__										=   argparse.ArgumentParser("""Accelerate config command"""		,					description=snake_case__     )

			parser.add_argument(
			    """--config_file"""		,					default=snake_case__		,					help=(
			        """The path to use to store the config file. Will default to a file named default_config.yaml in the cache """
			        """location, which is the content of the environment `HF_HOME` suffixed with 'accelerate', or if you don't have """
			        """such an environment variable, your cache directory ('~/.cache' or the content of `XDG_CACHE_HOME`) suffixed """
			        """with 'huggingface'."""
			    )		,					)

			if subparsers is not None:
						parser.set_defaults(func=snake_case__     )
			return parser

def 						A							(  snake_case__     ):




			'''simple docstring'''


			SCREAMING_SNAKE_CASE__										=   get_user_input()
			if args.config_file is not None:
						SCREAMING_SNAKE_CASE__										=   args.config_file
			else:
						if not os.path.isdir(snake_case__     ):
									os.makedirs(snake_case__     )
						SCREAMING_SNAKE_CASE__										=   default_yaml_config_file

			if config_file.endswith(""".json"""     ):
						config.to_json_file(snake_case__     )
			else:
						config.to_yaml_file(snake_case__     )
			print(f"""accelerate configuration saved at {config_file}"""     )

def 						A							(  ):




			'''simple docstring'''


			SCREAMING_SNAKE_CASE__										=   config_command_parser()
			SCREAMING_SNAKE_CASE__										=   parser.parse_args()
			config_command(snake_case__     )


if __name__ == "__main__":
						main()





"""simple docstring"""




import os
import unittest

from transformers.models.phobert.tokenization_phobert import VOCAB_FILES_NAMES, PhobertTokenizer

from ...test_tokenization_common import TokenizerTesterMixin





class  lowerCamelCase    (A__      ,unittest.TestCase      ):
			lowerCamelCase__ :		List[str]       		=				PhobertTokenizer
			lowerCamelCase__ :		Union[str, Any]       		=				False

			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     Any:
						super().setUp()

						# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
						SCREAMING_SNAKE_CASE__										=   ["""T@@""", """i""", """I""", """R@@""", """r""", """e@@"""]
						SCREAMING_SNAKE_CASE__										=   dict(zip(__UpperCAmelCase     ,					range(len(__UpperCAmelCase							)							)							)							)
						SCREAMING_SNAKE_CASE__										=   ["""#version: 0.2""", """l à</w>"""]
						SCREAMING_SNAKE_CASE__										=   {"""unk_token""": """<unk>"""}

						SCREAMING_SNAKE_CASE__										=   os.path.join(self.tmpdirname     ,					VOCAB_FILES_NAMES["""vocab_file"""]							)
						SCREAMING_SNAKE_CASE__										=   os.path.join(self.tmpdirname     ,					VOCAB_FILES_NAMES["""merges_file"""]							)

						with open(self.vocab_file     ,					"""w"""     ,					encoding="""utf-8"""							) as fp:
									for token in vocab_tokens:
												fp.write(F"""{token} {vocab_tokens[token]}\n"""							)
						with open(self.merges_file     ,					"""w"""     ,					encoding="""utf-8"""							) as fp:
									fp.write("""\n""".join(__UpperCAmelCase							)							)

			def        SCREAMING_SNAKE_CASE					(       self					:    List[Any]     ,					**__UpperCAmelCase					:    int							) ->     Optional[int]:
						kwargs.update(self.special_tokens_map							)
						return PhobertTokenizer.from_pretrained(self.tmpdirname     ,					**__UpperCAmelCase							)

			def        SCREAMING_SNAKE_CASE					(       self					:    Dict     ,					__UpperCAmelCase					:    int							) ->     Optional[Any]:
						SCREAMING_SNAKE_CASE__										=   """Tôi là VinAI Research"""
						SCREAMING_SNAKE_CASE__										=   """T<unk> i <unk> <unk> <unk> <unk> <unk> <unk> I Re<unk> e<unk> <unk> <unk> <unk>"""
						return input_text, output_text





			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[int]							) ->     int:
						SCREAMING_SNAKE_CASE__										=   PhobertTokenizer(self.vocab_file     ,					self.merges_file     ,					**self.special_tokens_map							)
						SCREAMING_SNAKE_CASE__										=   """Tôi là VinAI Research"""
						SCREAMING_SNAKE_CASE__										=   """T@@ ô@@ i l@@ à V@@ i@@ n@@ A@@ I R@@ e@@ s@@ e@@ a@@ r@@ c@@ h""".split()
						SCREAMING_SNAKE_CASE__										=   tokenizer.tokenize(__UpperCAmelCase							)
						print(__UpperCAmelCase							)
						self.assertListEqual(__UpperCAmelCase     ,					__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   tokens + [tokenizer.unk_token]

						SCREAMING_SNAKE_CASE__										=   [4, 3, 5, 3, 3, 3, 3, 3, 3, 6, 7, 9, 3, 9, 3, 3, 3, 3, 3]
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase							)     ,					__UpperCAmelCase							)


