




import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class        UpperCamelCase_					(				UpperCAmelCase__	,      unittest.TestCase  ):

    '''simple docstring'''






    UpperCAmelCase__													=   BertTokenizer
    UpperCAmelCase__													=   BertTokenizerFast
    UpperCAmelCase__													=   True
    UpperCAmelCase__													=   True
    UpperCAmelCase__													=   filter_non_english


    def  SCREAMING_SNAKE_CASE							(    self   :			Union[str, Any])		->Dict:





           '''simple docstring'''

           super().setUp()

           A__   				=					[
               '''[UNK]''',
               '''[CLS]''',
               '''[SEP]''',
               '''[PAD]''',
               '''[MASK]''',
               '''want''',
               '''##want''',
               '''##ed''',
               '''wa''',
               '''un''',
               '''runn''',
               '''##ing''',
               ''',''',
               '''low''',
               '''lowest''',
           ]
           A__   				=					os.path.join(self.tmpdirname      ,				VOCAB_FILES_NAMES['''vocab_file'''])
           with open(self.vocab_file      ,				'''w'''      ,				encoding='''utf-8''') as vocab_writer:
                  vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]))


    def  SCREAMING_SNAKE_CASE							(    self   :			Union[str, Any]      ,				UpperCAmelCase__   :			Tuple)		->List[str]:





           '''simple docstring'''

           A__   				=					'''UNwant\u00E9d,running'''
           A__   				=					'''unwanted, running'''
           return input_text, output_text


    def  SCREAMING_SNAKE_CASE							(    self   :			List[str])		->Tuple:





           '''simple docstring'''

           A__   				=					self.tokenizer_class(self.vocab_file)

           A__   				=					tokenizer.tokenize('''UNwant\u00E9d,running''')
           self.assertListEqual(UpperCAmelCase__      ,				['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing'''])
           self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase__)      ,				[9, 6, 7, 12, 10, 11])


    def  SCREAMING_SNAKE_CASE							(    self   :			Any)		->int:





           '''simple docstring'''

           if not self.test_rust_tokenizer:
                  return

           A__   				=					self.get_tokenizer()
           A__   				=					self.get_rust_tokenizer()

           A__   				=					'''UNwant\u00E9d,running'''

           A__   				=					tokenizer.tokenize(UpperCAmelCase__)
           A__   				=					rust_tokenizer.tokenize(UpperCAmelCase__)
           self.assertListEqual(UpperCAmelCase__      ,				UpperCAmelCase__)

           A__   				=					tokenizer.encode(UpperCAmelCase__      ,				add_special_tokens=UpperCAmelCase__)
           A__   				=					rust_tokenizer.encode(UpperCAmelCase__      ,				add_special_tokens=UpperCAmelCase__)
           self.assertListEqual(UpperCAmelCase__      ,				UpperCAmelCase__)

           A__   				=					self.get_rust_tokenizer()
           A__   				=					tokenizer.encode(UpperCAmelCase__)
           A__   				=					rust_tokenizer.encode(UpperCAmelCase__)
           self.assertListEqual(UpperCAmelCase__      ,				UpperCAmelCase__)

           # With lower casing
           A__   				=					self.get_tokenizer(do_lower_case=UpperCAmelCase__)
           A__   				=					self.get_rust_tokenizer(do_lower_case=UpperCAmelCase__)

           A__   				=					'''UNwant\u00E9d,running'''

           A__   				=					tokenizer.tokenize(UpperCAmelCase__)
           A__   				=					rust_tokenizer.tokenize(UpperCAmelCase__)
           self.assertListEqual(UpperCAmelCase__      ,				UpperCAmelCase__)

           A__   				=					tokenizer.encode(UpperCAmelCase__      ,				add_special_tokens=UpperCAmelCase__)
           A__   				=					rust_tokenizer.encode(UpperCAmelCase__      ,				add_special_tokens=UpperCAmelCase__)
           self.assertListEqual(UpperCAmelCase__      ,				UpperCAmelCase__)

           A__   				=					self.get_rust_tokenizer()
           A__   				=					tokenizer.encode(UpperCAmelCase__)
           A__   				=					rust_tokenizer.encode(UpperCAmelCase__)
           self.assertListEqual(UpperCAmelCase__      ,				UpperCAmelCase__)


    def  SCREAMING_SNAKE_CASE							(    self   :			Union[str, Any])		->Optional[int]:





           '''simple docstring'''

           A__   				=					BasicTokenizer()

           self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz''')      ,				['''ah''', '''\u535A''', '''\u63A8''', '''zz'''])


    def  SCREAMING_SNAKE_CASE							(    self   :			str)		->str:





           '''simple docstring'''

           A__   				=					BasicTokenizer(do_lower_case=UpperCAmelCase__)

           self.assertListEqual(
               tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  ''')      ,				['''hello''', '''!''', '''how''', '''are''', '''you''', '''?'''])
           self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''')      ,				['''hello'''])


    def  SCREAMING_SNAKE_CASE							(    self   :			Tuple)		->Optional[Any]:





           '''simple docstring'''

           A__   				=					BasicTokenizer(do_lower_case=UpperCAmelCase__      ,				strip_accents=UpperCAmelCase__)

           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''')      ,				['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?'''])
           self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''')      ,				['''h\u00E9llo'''])


    def  SCREAMING_SNAKE_CASE							(    self   :			Optional[Any])		->Dict:





           '''simple docstring'''

           A__   				=					BasicTokenizer(do_lower_case=UpperCAmelCase__      ,				strip_accents=UpperCAmelCase__)

           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''')      ,				['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?'''])
           self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''')      ,				['''hello'''])


    def  SCREAMING_SNAKE_CASE							(    self   :			Tuple)		->Dict:





           '''simple docstring'''

           A__   				=					BasicTokenizer(do_lower_case=UpperCAmelCase__)

           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''')      ,				['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?'''])
           self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''')      ,				['''hello'''])


    def  SCREAMING_SNAKE_CASE							(    self   :			int)		->Tuple:





           '''simple docstring'''

           A__   				=					BasicTokenizer(do_lower_case=UpperCAmelCase__)

           self.assertListEqual(
               tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  ''')      ,				['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''])


    def  SCREAMING_SNAKE_CASE							(    self   :			List[str])		->int:





           '''simple docstring'''

           A__   				=					BasicTokenizer(do_lower_case=UpperCAmelCase__      ,				strip_accents=UpperCAmelCase__)

           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''')      ,				['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''])


    def  SCREAMING_SNAKE_CASE							(    self   :			Optional[Any])		->Any:





           '''simple docstring'''

           A__   				=					BasicTokenizer(do_lower_case=UpperCAmelCase__      ,				strip_accents=UpperCAmelCase__)

           self.assertListEqual(
               tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''')      ,				['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''])


    def  SCREAMING_SNAKE_CASE							(    self   :			List[str])		->List[Any]:





           '''simple docstring'''

           A__   				=					BasicTokenizer(do_lower_case=UpperCAmelCase__      ,				never_split=['''[UNK]'''])

           self.assertListEqual(
               tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]''')      ,				['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]'''])


    def  SCREAMING_SNAKE_CASE							(    self   :			Optional[int])		->Tuple:





           '''simple docstring'''

           A__   				=					BasicTokenizer()
           A__   				=					'''a\n\'ll !!to?\'d of, can\'t.'''
           A__   				=					['''a''', '''\'''', '''ll''', '''!''', '''!''', '''to''', '''?''', '''\'''', '''d''', '''of''', ''',''', '''can''', '''\'''', '''t''', '''.''']
           self.assertListEqual(tokenizer.tokenize(UpperCAmelCase__)      ,				UpperCAmelCase__)


    def  SCREAMING_SNAKE_CASE							(    self   :			List[str])		->Optional[int]:





           '''simple docstring'''

           A__   				=					['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

           A__   				=					{}
           for i, token in enumerate(UpperCAmelCase__):
                  A__   				=					i
           A__   				=					WordpieceTokenizer(vocab=UpperCAmelCase__      ,				unk_token='''[UNK]''')

           self.assertListEqual(tokenizer.tokenize('''''')      ,				[])

           self.assertListEqual(tokenizer.tokenize('''unwanted running''')      ,				['''un''', '''##want''', '''##ed''', '''runn''', '''##ing'''])

           self.assertListEqual(tokenizer.tokenize('''unwantedX running''')      ,				['''[UNK]''', '''runn''', '''##ing'''])


    def  SCREAMING_SNAKE_CASE							(    self   :			Dict)		->List[Any]:





           '''simple docstring'''

           self.assertTrue(_is_whitespace(''' '''))
           self.assertTrue(_is_whitespace('''\t'''))
           self.assertTrue(_is_whitespace('''\r'''))
           self.assertTrue(_is_whitespace('''\n'''))
           self.assertTrue(_is_whitespace('''\u00A0'''))

           self.assertFalse(_is_whitespace('''A'''))
           self.assertFalse(_is_whitespace('''-'''))


    def  SCREAMING_SNAKE_CASE							(    self   :			List[Any])		->List[str]:





           '''simple docstring'''

           self.assertTrue(_is_control('''\u0005'''))

           self.assertFalse(_is_control('''A'''))
           self.assertFalse(_is_control(''' '''))
           self.assertFalse(_is_control('''\t'''))
           self.assertFalse(_is_control('''\r'''))


    def  SCREAMING_SNAKE_CASE							(    self   :			Dict)		->Optional[int]:





           '''simple docstring'''

           self.assertTrue(_is_punctuation('''-'''))
           self.assertTrue(_is_punctuation('''$'''))
           self.assertTrue(_is_punctuation('''`'''))
           self.assertTrue(_is_punctuation('''.'''))

           self.assertFalse(_is_punctuation('''A'''))
           self.assertFalse(_is_punctuation(''' '''))


    def  SCREAMING_SNAKE_CASE							(    self   :			List[str])		->Tuple:





           '''simple docstring'''

           A__   				=					self.get_tokenizer()
           A__   				=					self.get_rust_tokenizer()

           # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
           self.assertListEqual([tokenizer.tokenize(UpperCAmelCase__) for t in ['''Test''', '''\xad''', '''test''']]      ,				[['''[UNK]'''], [], ['''[UNK]''']])

           self.assertListEqual(
               [rust_tokenizer.tokenize(UpperCAmelCase__) for t in ['''Test''', '''\xad''', '''test''']]      ,				[['''[UNK]'''], [], ['''[UNK]''']])


    @slow
    def  SCREAMING_SNAKE_CASE							(    self   :			List[str])		->Optional[int]:





           '''simple docstring'''

           A__   				=					self.tokenizer_class.from_pretrained('''bert-base-uncased''')

           A__   				=					tokenizer.encode('''sequence builders'''      ,				add_special_tokens=UpperCAmelCase__)
           A__   				=					tokenizer.encode('''multi-sequence build'''      ,				add_special_tokens=UpperCAmelCase__)

           A__   				=					tokenizer.build_inputs_with_special_tokens(UpperCAmelCase__)
           A__   				=					tokenizer.build_inputs_with_special_tokens(UpperCAmelCase__      ,				UpperCAmelCase__)

           assert encoded_sentence == [101] + text + [102]
           assert encoded_pair == [101] + text + [102] + text_a + [102]


    def  SCREAMING_SNAKE_CASE							(    self   :			List[str])		->Union[str, Any]:





           '''simple docstring'''

           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                  with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
                         A__   				=					self.rust_tokenizer_class.from_pretrained(UpperCAmelCase__      ,				**UpperCAmelCase__)

                         A__   				=					f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
                         A__   				=					tokenizer_r.encode_plus(
                             UpperCAmelCase__      ,				return_attention_mask=UpperCAmelCase__      ,				return_token_type_ids=UpperCAmelCase__      ,				return_offsets_mapping=UpperCAmelCase__      ,				add_special_tokens=UpperCAmelCase__      ,				)

                         A__   				=					tokenizer_r.do_lower_case if hasattr(UpperCAmelCase__      ,				'''do_lower_case''') else False
                         A__   				=					(
                             [
                                 ((0, 0), tokenizer_r.cls_token),
                                 ((0, 1), '''A'''),
                                 ((1, 2), ''','''),
                                 ((3, 5), '''na'''),
                                 ((5, 6), '''##ï'''),
                                 ((6, 8), '''##ve'''),
                                 ((9, 15), tokenizer_r.mask_token),
                                 ((16, 21), '''Allen'''),
                                 ((21, 23), '''##NL'''),
                                 ((23, 24), '''##P'''),
                                 ((25, 33), '''sentence'''),
                                 ((33, 34), '''.'''),
                                 ((0, 0), tokenizer_r.sep_token),
                             ]
                             if not do_lower_case
                             else [
                                 ((0, 0), tokenizer_r.cls_token),
                                 ((0, 1), '''a'''),
                                 ((1, 2), ''','''),
                                 ((3, 8), '''naive'''),
                                 ((9, 15), tokenizer_r.mask_token),
                                 ((16, 21), '''allen'''),
                                 ((21, 23), '''##nl'''),
                                 ((23, 24), '''##p'''),
                                 ((25, 33), '''sentence'''),
                                 ((33, 34), '''.'''),
                                 ((0, 0), tokenizer_r.sep_token),
                             ]
                         )

                         self.assertEqual(
                             [e[1] for e in expected_results]      ,				tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']))
                         self.assertEqual([e[0] for e in expected_results]      ,				tokens['''offset_mapping'''])





    def  SCREAMING_SNAKE_CASE							(    self   :			Tuple)		->List[Any]:





           '''simple docstring'''

           A__   				=					['''的''', '''人''', '''有''']
           A__   				=					''''''.join(UpperCAmelCase__)
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                  with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
                         A__   				=					True
                         A__   				=					self.tokenizer_class.from_pretrained(UpperCAmelCase__      ,				**UpperCAmelCase__)
                         A__   				=					self.rust_tokenizer_class.from_pretrained(UpperCAmelCase__      ,				**UpperCAmelCase__)

                         A__   				=					tokenizer_p.encode(UpperCAmelCase__      ,				add_special_tokens=UpperCAmelCase__)
                         A__   				=					tokenizer_r.encode(UpperCAmelCase__      ,				add_special_tokens=UpperCAmelCase__)

                         A__   				=					tokenizer_r.convert_ids_to_tokens(UpperCAmelCase__)
                         A__   				=					tokenizer_p.convert_ids_to_tokens(UpperCAmelCase__)

                         # it is expected that each Chinese character is not preceded by "##"
                         self.assertListEqual(UpperCAmelCase__      ,				UpperCAmelCase__)
                         self.assertListEqual(UpperCAmelCase__      ,				UpperCAmelCase__)

                         A__   				=					False
                         A__   				=					self.rust_tokenizer_class.from_pretrained(UpperCAmelCase__      ,				**UpperCAmelCase__)
                         A__   				=					self.tokenizer_class.from_pretrained(UpperCAmelCase__      ,				**UpperCAmelCase__)

                         A__   				=					tokenizer_r.encode(UpperCAmelCase__      ,				add_special_tokens=UpperCAmelCase__)
                         A__   				=					tokenizer_p.encode(UpperCAmelCase__      ,				add_special_tokens=UpperCAmelCase__)

                         A__   				=					tokenizer_r.convert_ids_to_tokens(UpperCAmelCase__)
                         A__   				=					tokenizer_p.convert_ids_to_tokens(UpperCAmelCase__)

                         # it is expected that only the first Chinese character is not preceded by "##".
                         A__   				=					[
                             f"""##{token}""" if idx != 0 else token for idx, token in enumerate(UpperCAmelCase__)
                         ]
                         self.assertListEqual(UpperCAmelCase__      ,				UpperCAmelCase__)
                         self.assertListEqual(UpperCAmelCase__      ,				UpperCAmelCase__)








from argparse import ArgumentParser

from . import BaseTransformersCLICommand







def SCREAMING_SNAKE_CASE       ( lowercase_		)     -> str:



       """simple docstring"""

       return DownloadCommand(args.model				,       args.cache_dir				,       args.force				,       args.trust_remote_code		)


class        UpperCamelCase_					(				UpperCAmelCase__  ):

    '''simple docstring'''






    @staticmethod
    def  SCREAMING_SNAKE_CASE							(    UpperCAmelCase__   :			ArgumentParser)		->Any:





           '''simple docstring'''

           A__   				=					parser.add_parser('''download''')
           download_parser.add_argument(
               '''--cache-dir'''      ,				type=UpperCAmelCase__      ,				default=UpperCAmelCase__      ,				help='''Path to location to store the models''')
           download_parser.add_argument(
               '''--force'''      ,				action='''store_true'''      ,				help='''Force the model to be download even if already in cache-dir''')
           download_parser.add_argument(
               '''--trust-remote-code'''      ,				action='''store_true'''      ,				help='''Whether or not to allow for custom models defined on the Hub in their own modeling files. Use only if you\'ve reviewed the code as it will execute on your local machine'''      ,				)
           download_parser.add_argument('''model'''      ,				type=UpperCAmelCase__      ,				help='''Name of the model to download''')
           download_parser.set_defaults(func=UpperCAmelCase__)


    def __init__(    self   :			Dict      ,				UpperCAmelCase__   :			str      ,				UpperCAmelCase__   :			str      ,				UpperCAmelCase__   :			bool      ,				UpperCAmelCase__   :			bool)		->int:





           '''simple docstring'''

           A__   				=					model
           A__   				=					cache
           A__   				=					force
           A__   				=					trust_remote_code





    def  SCREAMING_SNAKE_CASE							(    self   :			str)		->Union[str, Any]:





           '''simple docstring'''

           from ..models.auto import AutoModel, AutoTokenizer

           AutoModel.from_pretrained(
               self._model      ,				cache_dir=self._cache      ,				force_download=self._force      ,				trust_remote_code=self._trust_remote_code)
           AutoTokenizer.from_pretrained(
               self._model      ,				cache_dir=self._cache      ,				force_download=self._force      ,				trust_remote_code=self._trust_remote_code)


