

import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class 							lowerCAmelCase__  (	UpperCamelCase					,							unittest.TestCase						):
					__A     :    Optional[int]           =						BertTokenizer
					__A     :    str           =						BertTokenizerFast
					__A     :    Optional[Any]           =						True
					__A     :    Union[str, Any]           =						True
					__A     :    Union[str, Any]           =						filter_non_english







					def  _lowercase			(       self      :     str):
											super().setUp()

											A__	:							List[Any]						    = [
											    "[UNK]",
											    "[CLS]",
											    "[SEP]",
											    "[PAD]",
											    "[MASK]",
											    "want",
											    "##want",
											    "##ed",
											    "wa",
											    "un",
											    "runn",
											    "##ing",
											    ",",
											    "low",
											    "lowest",
											]
											A__	:							Any						    = os.path.join(self.tmpdirname			,		VOCAB_FILES_NAMES["vocab_file"])
											with open(self.vocab_file			,		"w"			,		encoding="utf-8") as vocab_writer:
																	vocab_writer.write("".join([x + "\n" for x in vocab_tokens]))







					def  _lowercase			(       self      :     Any			,		_A      :     int):
											A__	:							List[str]						    = "UNwant\u00E9d,running"
											A__	:							List[Any]						    = "unwanted, running"
											return input_text, output_text







					def  _lowercase			(       self      :     Any):
											A__	:							Tuple						    = self.tokenizer_class(self.vocab_file)

											A__	:							str						    = tokenizer.tokenize("UNwant\u00E9d,running")
											self.assertListEqual(_A			,		["un", "##want", "##ed", ",", "runn", "##ing"])
											self.assertListEqual(tokenizer.convert_tokens_to_ids(_A)			,		[9, 6, 7, 12, 10, 11])







					def  _lowercase			(       self      :     Dict):
											if not self.test_rust_tokenizer:
																	return

											A__	:							Union[str, Any]						    = self.get_tokenizer()
											A__	:							List[str]						    = self.get_rust_tokenizer()

											A__	:							List[Any]						    = "UNwant\u00E9d,running"

											A__	:							Union[str, Any]						    = tokenizer.tokenize(_A)
											A__	:							List[str]						    = rust_tokenizer.tokenize(_A)
											self.assertListEqual(_A			,		_A)

											A__	:							Tuple						    = tokenizer.encode(_A			,		add_special_tokens=_A)
											A__	:							str						    = rust_tokenizer.encode(_A			,		add_special_tokens=_A)
											self.assertListEqual(_A			,		_A)

											A__	:							List[Any]						    = self.get_rust_tokenizer()
											A__	:							int						    = tokenizer.encode(_A)
											A__	:							Tuple						    = rust_tokenizer.encode(_A)
											self.assertListEqual(_A			,		_A)

											# With lower casing
											A__	:							Any						    = self.get_tokenizer(do_lower_case=_A)
											A__	:							List[str]						    = self.get_rust_tokenizer(do_lower_case=_A)

											A__	:							str						    = "UNwant\u00E9d,running"

											A__	:							str						    = tokenizer.tokenize(_A)
											A__	:							List[str]						    = rust_tokenizer.tokenize(_A)
											self.assertListEqual(_A			,		_A)

											A__	:							List[Any]						    = tokenizer.encode(_A			,		add_special_tokens=_A)
											A__	:							Optional[int]						    = rust_tokenizer.encode(_A			,		add_special_tokens=_A)
											self.assertListEqual(_A			,		_A)

											A__	:							List[Any]						    = self.get_rust_tokenizer()
											A__	:							int						    = tokenizer.encode(_A)
											A__	:							List[str]						    = rust_tokenizer.encode(_A)
											self.assertListEqual(_A			,		_A)







					def  _lowercase			(       self      :     Dict):
											A__	:							Union[str, Any]						    = BasicTokenizer()

											self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz")			,		["ah", "\u535A", "\u63A8", "zz"])







					def  _lowercase			(       self      :     Optional[Any]):
											A__	:							str						    = BasicTokenizer(do_lower_case=_A)

											self.assertListEqual(
											    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  ")			,		["hello", "!", "how", "are", "you", "?"])
											self.assertListEqual(tokenizer.tokenize("H\u00E9llo")			,		["hello"])







					def  _lowercase			(       self      :     List[str]):
											A__	:							Union[str, Any]						    = BasicTokenizer(do_lower_case=_A			,		strip_accents=_A)

											self.assertListEqual(
											    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")			,		["hällo", "!", "how", "are", "you", "?"])
											self.assertListEqual(tokenizer.tokenize("H\u00E9llo")			,		["h\u00E9llo"])







					def  _lowercase			(       self      :     Optional[Any]):
											A__	:							Union[str, Any]						    = BasicTokenizer(do_lower_case=_A			,		strip_accents=_A)

											self.assertListEqual(
											    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")			,		["hallo", "!", "how", "are", "you", "?"])
											self.assertListEqual(tokenizer.tokenize("H\u00E9llo")			,		["hello"])







					def  _lowercase			(       self      :     Dict):
											A__	:							List[Any]						    = BasicTokenizer(do_lower_case=_A)

											self.assertListEqual(
											    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")			,		["hallo", "!", "how", "are", "you", "?"])
											self.assertListEqual(tokenizer.tokenize("H\u00E9llo")			,		["hello"])







					def  _lowercase			(       self      :     Dict):
											A__	:							Optional[int]						    = BasicTokenizer(do_lower_case=_A)

											self.assertListEqual(
											    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  ")			,		["HeLLo", "!", "how", "Are", "yoU", "?"])







					def  _lowercase			(       self      :     Tuple):
											A__	:							Tuple						    = BasicTokenizer(do_lower_case=_A			,		strip_accents=_A)

											self.assertListEqual(
											    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")			,		["HäLLo", "!", "how", "Are", "yoU", "?"])







					def  _lowercase			(       self      :     Union[str, Any]):
											A__	:							Dict						    = BasicTokenizer(do_lower_case=_A			,		strip_accents=_A)

											self.assertListEqual(
											    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  ")			,		["HaLLo", "!", "how", "Are", "yoU", "?"])







					def  _lowercase			(       self      :     Union[str, Any]):
											A__	:							List[str]						    = BasicTokenizer(do_lower_case=_A			,		never_split=["[UNK]"])

											self.assertListEqual(
											    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]")			,		["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"])







					def  _lowercase			(       self      :     List[Any]):
											A__	:							List[Any]						    = BasicTokenizer()
											A__	:							Optional[int]						    = "a\n'll !!to?'d of, can't."
											A__	:							Tuple						    = ["a", "'", "ll", "!", "!", "to", "?", "'", "d", "of", ",", "can", "'", "t", "."]
											self.assertListEqual(tokenizer.tokenize(_A)			,		_A)







					def  _lowercase			(       self      :     Tuple):
											A__	:							List[str]						    = ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

											A__	:							Dict						    = {}
											for i, token in enumerate(_A):
																	A__	:							Tuple						    = i
											A__	:							Dict						    = WordpieceTokenizer(vocab=_A			,		unk_token="[UNK]")

											self.assertListEqual(tokenizer.tokenize("")			,		[])

											self.assertListEqual(tokenizer.tokenize("unwanted running")			,		["un", "##want", "##ed", "runn", "##ing"])

											self.assertListEqual(tokenizer.tokenize("unwantedX running")			,		["[UNK]", "runn", "##ing"])







					def  _lowercase			(       self      :     Dict):
											self.assertTrue(_is_whitespace(" "))
											self.assertTrue(_is_whitespace("\t"))
											self.assertTrue(_is_whitespace("\r"))
											self.assertTrue(_is_whitespace("\n"))
											self.assertTrue(_is_whitespace("\u00A0"))

											self.assertFalse(_is_whitespace("A"))
											self.assertFalse(_is_whitespace("-"))







					def  _lowercase			(       self      :     List[str]):
											self.assertTrue(_is_control("\u0005"))

											self.assertFalse(_is_control("A"))
											self.assertFalse(_is_control(" "))
											self.assertFalse(_is_control("\t"))
											self.assertFalse(_is_control("\r"))







					def  _lowercase			(       self      :     str):
											self.assertTrue(_is_punctuation("-"))
											self.assertTrue(_is_punctuation("$"))
											self.assertTrue(_is_punctuation("`"))
											self.assertTrue(_is_punctuation("."))

											self.assertFalse(_is_punctuation("A"))
											self.assertFalse(_is_punctuation(" "))







					def  _lowercase			(       self      :     Dict):
											A__	:							Optional[Any]						    = self.get_tokenizer()
											A__	:							Dict						    = self.get_rust_tokenizer()

											# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
											self.assertListEqual([tokenizer.tokenize(_A) for t in ["Test", "\xad", "test"]]			,		[["[UNK]"], [], ["[UNK]"]])

											self.assertListEqual(
											    [rust_tokenizer.tokenize(_A) for t in ["Test", "\xad", "test"]]			,		[["[UNK]"], [], ["[UNK]"]])







					@slow
					def  _lowercase			(       self      :     List[str]):
											A__	:							int						    = self.tokenizer_class.from_pretrained("bert-base-uncased")

											A__	:							str						    = tokenizer.encode("sequence builders"			,		add_special_tokens=_A)
											A__	:							Dict						    = tokenizer.encode("multi-sequence build"			,		add_special_tokens=_A)

											A__	:							Optional[Any]						    = tokenizer.build_inputs_with_special_tokens(_A)
											A__	:							Union[str, Any]						    = tokenizer.build_inputs_with_special_tokens(_A			,		_A)

											assert encoded_sentence == [101] + text + [102]
											assert encoded_pair == [101] + text + [102] + text_a + [102]







					def  _lowercase			(       self      :     int):
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																	with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'):
																							A__	:							Union[str, Any]						    = self.rust_tokenizer_class.from_pretrained(_A			,		**_A)

																							A__	:							List[str]						    = F'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
																							A__	:							Optional[int]						    = tokenizer_r.encode_plus(
																							    _A			,		return_attention_mask=_A			,		return_token_type_ids=_A			,		return_offsets_mapping=_A			,		add_special_tokens=_A			,		)

																							A__	:							Tuple						    = tokenizer_r.do_lower_case if hasattr(_A			,		"do_lower_case") else False
																							A__	:							Any						    = (
																							    [
																							        ((0, 0), tokenizer_r.cls_token),
																							        ((0, 1), "A"),
																							        ((1, 2), ","),
																							        ((3, 5), "na"),
																							        ((5, 6), "##ï"),
																							        ((6, 8), "##ve"),
																							        ((9, 15), tokenizer_r.mask_token),
																							        ((16, 21), "Allen"),
																							        ((21, 23), "##NL"),
																							        ((23, 24), "##P"),
																							        ((25, 33), "sentence"),
																							        ((33, 34), "."),
																							        ((0, 0), tokenizer_r.sep_token),
																							    ]
																							    if not do_lower_case
																							    else [
																							        ((0, 0), tokenizer_r.cls_token),
																							        ((0, 1), "a"),
																							        ((1, 2), ","),
																							        ((3, 8), "naive"),
																							        ((9, 15), tokenizer_r.mask_token),
																							        ((16, 21), "allen"),
																							        ((21, 23), "##nl"),
																							        ((23, 24), "##p"),
																							        ((25, 33), "sentence"),
																							        ((33, 34), "."),
																							        ((0, 0), tokenizer_r.sep_token),
																							    ]
																							)

																							self.assertEqual(
																							    [e[1] for e in expected_results]			,		tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]))
																							self.assertEqual([e[0] for e in expected_results]			,		tokens["offset_mapping"])







					def  _lowercase			(       self      :     Optional[Any]):
											A__	:							str						    = ["的", "人", "有"]
											A__	:							List[Any]						    = "".join(_A)
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																	with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'):
																							A__	:							str						    = True
																							A__	:							Optional[Any]						    = self.tokenizer_class.from_pretrained(_A			,		**_A)
																							A__	:							List[str]						    = self.rust_tokenizer_class.from_pretrained(_A			,		**_A)

																							A__	:							List[Any]						    = tokenizer_p.encode(_A			,		add_special_tokens=_A)
																							A__	:							str						    = tokenizer_r.encode(_A			,		add_special_tokens=_A)

																							A__	:							Tuple						    = tokenizer_r.convert_ids_to_tokens(_A)
																							A__	:							List[str]						    = tokenizer_p.convert_ids_to_tokens(_A)

																							# it is expected that each Chinese character is not preceded by "##"
																							self.assertListEqual(_A			,		_A)
																							self.assertListEqual(_A			,		_A)

																							A__	:							Optional[Any]						    = False
																							A__	:							List[Any]						    = self.rust_tokenizer_class.from_pretrained(_A			,		**_A)
																							A__	:							Dict						    = self.tokenizer_class.from_pretrained(_A			,		**_A)

																							A__	:							Optional[int]						    = tokenizer_r.encode(_A			,		add_special_tokens=_A)
																							A__	:							Any						    = tokenizer_p.encode(_A			,		add_special_tokens=_A)

																							A__	:							Union[str, Any]						    = tokenizer_r.convert_ids_to_tokens(_A)
																							A__	:							Union[str, Any]						    = tokenizer_p.convert_ids_to_tokens(_A)

																							# it is expected that only the first Chinese character is not preceded by "##".
																							A__	:							str						    = [
																							    F'##{token}' if idx != 0 else token for idx, token in enumerate(_A)
																							]
																							self.assertListEqual(_A			,		_A)
																							self.assertListEqual(_A			,		_A)




from argparse import ArgumentParser

from .add_new_model import AddNewModelCommand
from .add_new_model_like import AddNewModelLikeCommand
from .convert import ConvertCommand
from .download import DownloadCommand
from .env import EnvironmentCommand
from .lfs import LfsCommands
from .pt_to_tf import PTtoTFCommand
from .run import RunCommand
from .serving import ServeCommand
from .user import UserCommands
def 	snake_case__       (					)		->		List[Any]:







						"""simple docstring"""

						A__	:							List[Any]						    = ArgumentParser("Transformers CLI tool"							,     usage="transformers-cli <command> [<args>]"       )
						A__	:							int						    = parser.add_subparsers(help="transformers-cli command helpers"       )

						# Register commands
						ConvertCommand.register_subcommand(__lowercase       )
						DownloadCommand.register_subcommand(__lowercase       )
						EnvironmentCommand.register_subcommand(__lowercase       )
						RunCommand.register_subcommand(__lowercase       )
						ServeCommand.register_subcommand(__lowercase       )
						UserCommands.register_subcommand(__lowercase       )
						AddNewModelCommand.register_subcommand(__lowercase       )
						AddNewModelLikeCommand.register_subcommand(__lowercase       )
						LfsCommands.register_subcommand(__lowercase       )
						PTtoTFCommand.register_subcommand(__lowercase       )

						# Let's go
						A__	:							int						    = parser.parse_args()

						if not hasattr(__lowercase							,     "func"       ):
												parser.print_help()
												exit(1       )

						# Run
						A__	:							Any						    = args.func(__lowercase       )
						service.run()


if __name__ == "__main__":
		main()