






import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english





@require_tokenizers
class 					A__     (     __snake_case   ,    unittest.TestCase ):

     '''simple docstring'''






     snake_case__					=					RoCBertTokenizer
     snake_case__					=					None
     snake_case__					=					False
     snake_case__					=					True
     snake_case__					=					filter_non_english







     def   _SCREAMING_SNAKE_CASE     ( self      :    Union[str, Any] ):






           """simple docstring"""




           super().setUp()

           UpperCamelCase					       =  ['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', '你', '好', '是', '谁', 'a', 'b', 'c', 'd']
           UpperCamelCase					       =  {}
           UpperCamelCase					       =  {}
           for i, value in enumerate(_SCREAMING_SNAKE_CASE ):
                 UpperCamelCase					       =  i
                 UpperCamelCase					       =  i
           UpperCamelCase					       =  os.path.join(self.tmpdirname   ,						VOCAB_FILES_NAMES['vocab_file'] )
           UpperCamelCase					       =  os.path.join(self.tmpdirname   ,						VOCAB_FILES_NAMES['word_shape_file'] )
           UpperCamelCase					       =  os.path.join(self.tmpdirname   ,						VOCAB_FILES_NAMES['word_pronunciation_file'] )
           with open(self.vocab_file   ,						'w'   ,						encoding='utf-8' ) as vocab_writer:
                 vocab_writer.write(''.join([x + '\n' for x in vocab_tokens] ) )
           with open(self.word_shape_file   ,						'w'   ,						encoding='utf-8' ) as word_shape_writer:
                 json.dump(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE   ,						ensure_ascii=_SCREAMING_SNAKE_CASE )
           with open(self.word_pronunciation_file   ,						'w'   ,						encoding='utf-8' ) as word_pronunciation_writer:
                 json.dump(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE   ,						ensure_ascii=_SCREAMING_SNAKE_CASE )







     def   _SCREAMING_SNAKE_CASE     ( self      :    str ):






           """simple docstring"""




           UpperCamelCase					       =  self.tokenizer_class(self.vocab_file   ,						self.word_shape_file   ,						self.word_pronunciation_file )

           UpperCamelCase					       =  tokenizer.tokenize('你好[SEP]你是谁' )
           self.assertListEqual(_SCREAMING_SNAKE_CASE   ,						['你', '好', '[SEP]', '你', '是', '谁'] )
           self.assertListEqual(tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE )   ,						[5, 6, 2, 5, 7, 8] )
           self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(_SCREAMING_SNAKE_CASE )   ,						[5, 6, 2, 5, 7, 8] )
           self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(_SCREAMING_SNAKE_CASE )   ,						[5, 6, 2, 5, 7, 8] )







     def   _SCREAMING_SNAKE_CASE     ( self      :    Any ):






           """simple docstring"""




           UpperCamelCase					       =  RoCBertBasicTokenizer()

           self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz' )   ,						['ah', '\u535A', '\u63A8', 'zz'] )







     def   _SCREAMING_SNAKE_CASE     ( self      :    str ):






           """simple docstring"""




           UpperCamelCase					       =  RoCBertBasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE )

           self.assertListEqual(
               tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ' )   ,						['hello', '!', 'how', 'are', 'you', '?'] )
           self.assertListEqual(tokenizer.tokenize('H\u00E9llo' )   ,						['hello'] )







     def   _SCREAMING_SNAKE_CASE     ( self      :    List[Any] ):






           """simple docstring"""




           UpperCamelCase					       =  RoCBertBasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE   ,						strip_accents=_SCREAMING_SNAKE_CASE )

           self.assertListEqual(
               tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ' )   ,						['hällo', '!', 'how', 'are', 'you', '?'] )
           self.assertListEqual(tokenizer.tokenize('H\u00E9llo' )   ,						['h\u00E9llo'] )







     def   _SCREAMING_SNAKE_CASE     ( self      :    List[str] ):






           """simple docstring"""




           UpperCamelCase					       =  RoCBertBasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE   ,						strip_accents=_SCREAMING_SNAKE_CASE )

           self.assertListEqual(
               tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ' )   ,						['hallo', '!', 'how', 'are', 'you', '?'] )
           self.assertListEqual(tokenizer.tokenize('H\u00E9llo' )   ,						['hello'] )







     def   _SCREAMING_SNAKE_CASE     ( self      :    Union[str, Any] ):






           """simple docstring"""




           UpperCamelCase					       =  RoCBertBasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE )

           self.assertListEqual(
               tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ' )   ,						['hallo', '!', 'how', 'are', 'you', '?'] )
           self.assertListEqual(tokenizer.tokenize('H\u00E9llo' )   ,						['hello'] )







     def   _SCREAMING_SNAKE_CASE     ( self      :    Dict ):






           """simple docstring"""




           UpperCamelCase					       =  RoCBertBasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE )

           self.assertListEqual(
               tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ' )   ,						['HeLLo', '!', 'how', 'Are', 'yoU', '?'] )







     def   _SCREAMING_SNAKE_CASE     ( self      :    Optional[Any] ):






           """simple docstring"""




           UpperCamelCase					       =  RoCBertBasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE   ,						strip_accents=_SCREAMING_SNAKE_CASE )

           self.assertListEqual(
               tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ' )   ,						['HäLLo', '!', 'how', 'Are', 'yoU', '?'] )







     def   _SCREAMING_SNAKE_CASE     ( self      :    int ):






           """simple docstring"""




           UpperCamelCase					       =  RoCBertBasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE   ,						strip_accents=_SCREAMING_SNAKE_CASE )

           self.assertListEqual(
               tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ' )   ,						['HaLLo', '!', 'how', 'Are', 'yoU', '?'] )







     def   _SCREAMING_SNAKE_CASE     ( self      :    str ):






           """simple docstring"""




           UpperCamelCase					       =  RoCBertBasicTokenizer(do_lower_case=_SCREAMING_SNAKE_CASE   ,						never_split=['[UNK]'] )

           self.assertListEqual(
               tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]' )   ,						['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]'] )







     def   _SCREAMING_SNAKE_CASE     ( self      :    str ):






           """simple docstring"""




           UpperCamelCase					       =  ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

           UpperCamelCase					       =  {}
           for i, token in enumerate(_SCREAMING_SNAKE_CASE ):
                 UpperCamelCase					       =  i
           UpperCamelCase					       =  RoCBertWordpieceTokenizer(vocab=_SCREAMING_SNAKE_CASE   ,						unk_token='[UNK]' )

           self.assertListEqual(tokenizer.tokenize('' )   ,						[] )

           self.assertListEqual(tokenizer.tokenize('unwanted running' )   ,						['un', '##want', '##ed', 'runn', '##ing'] )

           self.assertListEqual(tokenizer.tokenize('unwantedX running' )   ,						['[UNK]', 'runn', '##ing'] )







     def   _SCREAMING_SNAKE_CASE     ( self      :    List[Any] ):






           """simple docstring"""




           self.assertTrue(_is_whitespace(' ' ) )
           self.assertTrue(_is_whitespace('\t' ) )
           self.assertTrue(_is_whitespace('\r' ) )
           self.assertTrue(_is_whitespace('\n' ) )
           self.assertTrue(_is_whitespace('\u00A0' ) )

           self.assertFalse(_is_whitespace('A' ) )
           self.assertFalse(_is_whitespace('-' ) )







     def   _SCREAMING_SNAKE_CASE     ( self      :    List[Any] ):






           """simple docstring"""




           self.assertTrue(_is_control('\u0005' ) )

           self.assertFalse(_is_control('A' ) )
           self.assertFalse(_is_control(' ' ) )
           self.assertFalse(_is_control('\t' ) )
           self.assertFalse(_is_control('\r' ) )







     def   _SCREAMING_SNAKE_CASE     ( self      :    Optional[int] ):






           """simple docstring"""




           self.assertTrue(_is_punctuation('-' ) )
           self.assertTrue(_is_punctuation('$' ) )
           self.assertTrue(_is_punctuation('`' ) )
           self.assertTrue(_is_punctuation('.' ) )

           self.assertFalse(_is_punctuation('A' ) )
           self.assertFalse(_is_punctuation(' ' ) )







     def   _SCREAMING_SNAKE_CASE     ( self      :    Optional[int] ):






           """simple docstring"""




           UpperCamelCase					       =  self.get_tokenizer()

           # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
           self.assertListEqual([tokenizer.tokenize(_SCREAMING_SNAKE_CASE ) for t in ['Test', '\xad', 'test']]   ,						[['[UNK]'], [], ['[UNK]']] )

           if self.test_rust_tokenizer:
                 UpperCamelCase					       =  self.get_rust_tokenizer()
                 self.assertListEqual(
                     [rust_tokenizer.tokenize(_SCREAMING_SNAKE_CASE ) for t in ['Test', '\xad', 'test']]   ,						[['[UNK]'], [], ['[UNK]']] )







     def   _SCREAMING_SNAKE_CASE     ( self      :    Any ):






           """simple docstring"""




           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})' ):
                       UpperCamelCase					       =  self.rust_tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE   ,						**_SCREAMING_SNAKE_CASE )

                       UpperCamelCase					       =  f'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
                       UpperCamelCase					       =  tokenizer_r.encode_plus(
                           _SCREAMING_SNAKE_CASE   ,						return_attention_mask=_SCREAMING_SNAKE_CASE   ,						return_token_type_ids=_SCREAMING_SNAKE_CASE   ,						return_offsets_mapping=_SCREAMING_SNAKE_CASE   ,						add_special_tokens=_SCREAMING_SNAKE_CASE   ,						)

                       UpperCamelCase					       =  tokenizer_r.do_lower_case if hasattr(_SCREAMING_SNAKE_CASE   ,						'do_lower_case' ) else False
                       UpperCamelCase					       =  (
                           [
                               ((0, 0), tokenizer_r.cls_token),
                               ((0, 1), 'A'),
                               ((1, 2), ','),
                               ((3, 5), 'na'),
                               ((5, 6), '##ï'),
                               ((6, 8), '##ve'),
                               ((9, 15), tokenizer_r.mask_token),
                               ((16, 21), 'Allen'),
                               ((21, 23), '##NL'),
                               ((23, 24), '##P'),
                               ((25, 33), 'sentence'),
                               ((33, 34), '.'),
                               ((0, 0), tokenizer_r.sep_token),
                           ]
                           if not do_lower_case
                           else [
                               ((0, 0), tokenizer_r.cls_token),
                               ((0, 1), 'a'),
                               ((1, 2), ','),
                               ((3, 8), 'naive'),
                               ((9, 15), tokenizer_r.mask_token),
                               ((16, 21), 'allen'),
                               ((21, 23), '##nl'),
                               ((23, 24), '##p'),
                               ((25, 33), 'sentence'),
                               ((33, 34), '.'),
                               ((0, 0), tokenizer_r.sep_token),
                           ]
                       )

                       self.assertEqual(
                           [e[1] for e in expected_results]   ,						tokenizer_r.convert_ids_to_tokens(tokens['input_ids'] ) )
                       self.assertEqual([e[0] for e in expected_results]   ,						tokens['offset_mapping'] )







     def   _SCREAMING_SNAKE_CASE     ( self      :    Any ):






           """simple docstring"""




           UpperCamelCase					       =  ['的', '人', '有']
           UpperCamelCase					       =  ''.join(_SCREAMING_SNAKE_CASE )
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})' ):
                       UpperCamelCase					       =  True
                       UpperCamelCase					       =  self.tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE   ,						**_SCREAMING_SNAKE_CASE )
                       UpperCamelCase					       =  self.rust_tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE   ,						**_SCREAMING_SNAKE_CASE )

                       UpperCamelCase					       =  tokenizer_p.encode(_SCREAMING_SNAKE_CASE   ,						add_special_tokens=_SCREAMING_SNAKE_CASE )
                       UpperCamelCase					       =  tokenizer_r.encode(_SCREAMING_SNAKE_CASE   ,						add_special_tokens=_SCREAMING_SNAKE_CASE )

                       UpperCamelCase					       =  tokenizer_r.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE )
                       UpperCamelCase					       =  tokenizer_p.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE )

                       # it is expected that each Chinese character is not preceded by "##"
                       self.assertListEqual(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )
                       self.assertListEqual(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )

                       UpperCamelCase					       =  False
                       UpperCamelCase					       =  self.rust_tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE   ,						**_SCREAMING_SNAKE_CASE )
                       UpperCamelCase					       =  self.tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE   ,						**_SCREAMING_SNAKE_CASE )

                       UpperCamelCase					       =  tokenizer_r.encode(_SCREAMING_SNAKE_CASE   ,						add_special_tokens=_SCREAMING_SNAKE_CASE )
                       UpperCamelCase					       =  tokenizer_p.encode(_SCREAMING_SNAKE_CASE   ,						add_special_tokens=_SCREAMING_SNAKE_CASE )

                       UpperCamelCase					       =  tokenizer_r.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE )
                       UpperCamelCase					       =  tokenizer_p.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE )

                       # it is expected that only the first Chinese character is not preceded by "##".
                       UpperCamelCase					       =  [
                           f'##{token}' if idx != 0 else token for idx, token in enumerate(_SCREAMING_SNAKE_CASE )
                       ]
                       self.assertListEqual(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )
                       self.assertListEqual(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )







     @slow
     def   _SCREAMING_SNAKE_CASE     ( self      :    Any ):






           """simple docstring"""




           UpperCamelCase					       =  self.tokenizer_class(self.vocab_file   ,						self.word_shape_file   ,						self.word_pronunciation_file )

           UpperCamelCase					       =  tokenizer.encode('你好'   ,						add_special_tokens=_SCREAMING_SNAKE_CASE )
           UpperCamelCase					       =  tokenizer.encode('你是谁'   ,						add_special_tokens=_SCREAMING_SNAKE_CASE )

           UpperCamelCase					       =  tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE )
           UpperCamelCase					       =  tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )

           assert encoded_sentence == [1] + text + [2]
           assert encoded_pair == [1] + text + [2] + text_a + [2]







     def   _SCREAMING_SNAKE_CASE     ( self      :    Optional[Any] ):






           """simple docstring"""




           UpperCamelCase					       =  self.get_tokenizers(do_lower_case=_SCREAMING_SNAKE_CASE )
           for tokenizer in tokenizers:
                 with self.subTest(f'{tokenizer.__class__.__name__}' ):
                       UpperCamelCase					       =  '你好，你是谁'
                       UpperCamelCase					       =  tokenizer.tokenize(_SCREAMING_SNAKE_CASE )
                       UpperCamelCase					       =  tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE )
                       UpperCamelCase					       =  tokenizer.convert_tokens_to_shape_ids(_SCREAMING_SNAKE_CASE )
                       UpperCamelCase					       =  tokenizer.convert_tokens_to_pronunciation_ids(_SCREAMING_SNAKE_CASE )
                       UpperCamelCase					       =  tokenizer.prepare_for_model(
                           _SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE   ,						add_special_tokens=_SCREAMING_SNAKE_CASE )

                       UpperCamelCase					       =  tokenizer.encode_plus(_SCREAMING_SNAKE_CASE   ,						add_special_tokens=_SCREAMING_SNAKE_CASE )

                       self.assertEqual(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )











from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tokenizers_available,
    is_torch_available,
)


__magic_name__   :  Tuple        =							{'''configuration_plbart''': ['''PLBART_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''PLBartConfig''']}

try:
       if not is_sentencepiece_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       __magic_name__   :  Optional[Any]        =							['''PLBartTokenizer''']

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       __magic_name__   :  Tuple        =							[
           '''PLBART_PRETRAINED_MODEL_ARCHIVE_LIST''',
           '''PLBartForCausalLM''',
           '''PLBartForConditionalGeneration''',
           '''PLBartForSequenceClassification''',
           '''PLBartModel''',
           '''PLBartPreTrainedModel''',
       ]


if TYPE_CHECKING:
       from .configuration_plbart import PLBART_PRETRAINED_CONFIG_ARCHIVE_MAP, PLBartConfig

       try:
              if not is_sentencepiece_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .tokenization_plbart import PLBartTokenizer

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_plbart import (
                  PLBART_PRETRAINED_MODEL_ARCHIVE_LIST,
                  PLBartForCausalLM,
                  PLBartForConditionalGeneration,
                  PLBartForSequenceClassification,
                  PLBartModel,
                  PLBartPreTrainedModel,
              )


else:
       import sys

       __magic_name__   :  Optional[int]        =							_LazyModule(__name__, globals()['''__file__'''], _import_structure)



