

"""simple docstring"""




import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english







@require_tokenizers
class      __lowercase			(    __lowerCamelCase	,			unittest.TestCase					):
  snake_case_				       =       RoCBertTokenizer
  snake_case_				       =       None
  snake_case_				       =       False
  snake_case_				       =       True
  snake_case_				       =       filter_non_english

  def    __lowercase       (  self		:			Union[str, Any]    ):






   '''simple docstring'''
   super().setUp()

   UpperCAmelCase__					:  str					   =					["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """你""", """好""", """是""", """谁""", """a""", """b""", """c""", """d"""]
   UpperCAmelCase__					:  str					   =					{}
   UpperCAmelCase__					:  str					   =					{}
   for i, value in enumerate(A    ):
    UpperCAmelCase__					:  List[str]					   =					i
    UpperCAmelCase__					:  Any					   =					i
   UpperCAmelCase__					:  Optional[int]					   =					os.path.join(self.tmpdirname					,VOCAB_FILES_NAMES["""vocab_file"""]    )
   UpperCAmelCase__					:  Optional[int]					   =					os.path.join(self.tmpdirname					,VOCAB_FILES_NAMES["""word_shape_file"""]    )
   UpperCAmelCase__					:  Optional[Any]					   =					os.path.join(self.tmpdirname					,VOCAB_FILES_NAMES["""word_pronunciation_file"""]    )
   with open(self.vocab_file					,"""w"""					,encoding="""utf-8"""    ) as vocab_writer:
    vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]    )    )
   with open(self.word_shape_file					,"""w"""					,encoding="""utf-8"""    ) as word_shape_writer:
    json.dump(A					,A					,ensure_ascii=A    )
   with open(self.word_pronunciation_file					,"""w"""					,encoding="""utf-8"""    ) as word_pronunciation_writer:
    json.dump(A					,A					,ensure_ascii=A    )

  def    __lowercase       (  self		:			str    ):






   '''simple docstring'''
   UpperCAmelCase__					:  str					   =					self.tokenizer_class(self.vocab_file					,self.word_shape_file					,self.word_pronunciation_file    )

   UpperCAmelCase__					:  str					   =					tokenizer.tokenize("""你好[SEP]你是谁"""    )
   self.assertListEqual(A					,["""你""", """好""", """[SEP]""", """你""", """是""", """谁"""]    )
   self.assertListEqual(tokenizer.convert_tokens_to_ids(A    )					,[5, 6, 2, 5, 7, 8]    )
   self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(A    )					,[5, 6, 2, 5, 7, 8]    )
   self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(A    )					,[5, 6, 2, 5, 7, 8]    )

  def    __lowercase       (  self		:			List[str]    ):






   '''simple docstring'''
   UpperCAmelCase__					:  str					   =					RoCBertBasicTokenizer()

   self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""    )					,["""ah""", """\u535A""", """\u63A8""", """zz"""]    )

  def    __lowercase       (  self		:			Dict    ):






   '''simple docstring'''
   UpperCAmelCase__					:  int					   =					RoCBertBasicTokenizer(do_lower_case=A    )

   self.assertListEqual(
       tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """    )					,["""hello""", """!""", """how""", """are""", """you""", """?"""]    )
   self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""    )					,["""hello"""]    )

  def    __lowercase       (  self		:			Optional[int]    ):






   '''simple docstring'''
   UpperCAmelCase__					:  int					   =					RoCBertBasicTokenizer(do_lower_case=A					,strip_accents=A    )

   self.assertListEqual(
       tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )					,["""hällo""", """!""", """how""", """are""", """you""", """?"""]    )
   self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""    )					,["""h\u00E9llo"""]    )

  def    __lowercase       (  self		:			Dict    ):






   '''simple docstring'''
   UpperCAmelCase__					:  List[Any]					   =					RoCBertBasicTokenizer(do_lower_case=A					,strip_accents=A    )

   self.assertListEqual(
       tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )					,["""hallo""", """!""", """how""", """are""", """you""", """?"""]    )
   self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""    )					,["""hello"""]    )

  def    __lowercase       (  self		:			int    ):






   '''simple docstring'''
   UpperCAmelCase__					:  str					   =					RoCBertBasicTokenizer(do_lower_case=A    )

   self.assertListEqual(
       tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )					,["""hallo""", """!""", """how""", """are""", """you""", """?"""]    )
   self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""    )					,["""hello"""]    )

  def    __lowercase       (  self		:			Union[str, Any]    ):






   '''simple docstring'''
   UpperCAmelCase__					:  int					   =					RoCBertBasicTokenizer(do_lower_case=A    )

   self.assertListEqual(
       tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """    )					,["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]    )

  def    __lowercase       (  self		:			List[Any]    ):






   '''simple docstring'''
   UpperCAmelCase__					:  Dict					   =					RoCBertBasicTokenizer(do_lower_case=A					,strip_accents=A    )

   self.assertListEqual(
       tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )					,["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]    )

  def    __lowercase       (  self		:			List[str]    ):






   '''simple docstring'''
   UpperCAmelCase__					:  int					   =					RoCBertBasicTokenizer(do_lower_case=A					,strip_accents=A    )

   self.assertListEqual(
       tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """    )					,["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]    )

  def    __lowercase       (  self		:			Dict    ):






   '''simple docstring'''
   UpperCAmelCase__					:  int					   =					RoCBertBasicTokenizer(do_lower_case=A					,never_split=["""[UNK]"""]    )

   self.assertListEqual(
       tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""    )					,["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]    )

  def    __lowercase       (  self		:			Tuple    ):






   '''simple docstring'''
   UpperCAmelCase__					:  Union[str, Any]					   =					["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

   UpperCAmelCase__					:  Tuple					   =					{}
   for i, token in enumerate(A    ):
    UpperCAmelCase__					:  Dict					   =					i
   UpperCAmelCase__					:  Optional[int]					   =					RoCBertWordpieceTokenizer(vocab=A					,unk_token="""[UNK]"""    )

   self.assertListEqual(tokenizer.tokenize(""""""    )					,[]    )

   self.assertListEqual(tokenizer.tokenize("""unwanted running"""    )					,["""un""", """##want""", """##ed""", """runn""", """##ing"""]    )

   self.assertListEqual(tokenizer.tokenize("""unwantedX running"""    )					,["""[UNK]""", """runn""", """##ing"""]    )

  def    __lowercase       (  self		:			Optional[int]    ):






   '''simple docstring'''
   self.assertTrue(_is_whitespace(""" """    )    )
   self.assertTrue(_is_whitespace("""\t"""    )    )
   self.assertTrue(_is_whitespace("""\r"""    )    )
   self.assertTrue(_is_whitespace("""\n"""    )    )
   self.assertTrue(_is_whitespace("""\u00A0"""    )    )

   self.assertFalse(_is_whitespace("""A"""    )    )
   self.assertFalse(_is_whitespace("""-"""    )    )

  def    __lowercase       (  self		:			int    ):






   '''simple docstring'''
   self.assertTrue(_is_control("""\u0005"""    )    )

   self.assertFalse(_is_control("""A"""    )    )
   self.assertFalse(_is_control(""" """    )    )
   self.assertFalse(_is_control("""\t"""    )    )
   self.assertFalse(_is_control("""\r"""    )    )

  def    __lowercase       (  self		:			List[Any]    ):






   '''simple docstring'''
   self.assertTrue(_is_punctuation("""-"""    )    )
   self.assertTrue(_is_punctuation("""$"""    )    )
   self.assertTrue(_is_punctuation("""`"""    )    )
   self.assertTrue(_is_punctuation("""."""    )    )

   self.assertFalse(_is_punctuation("""A"""    )    )
   self.assertFalse(_is_punctuation(""" """    )    )

  def    __lowercase       (  self		:			Dict    ):






   '''simple docstring'''
   UpperCAmelCase__					:  List[str]					   =					self.get_tokenizer()

   # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
   self.assertListEqual([tokenizer.tokenize(A    ) for t in ["""Test""", """\xad""", """test"""]]					,[["""[UNK]"""], [], ["""[UNK]"""]]    )

   if self.test_rust_tokenizer:
    UpperCAmelCase__					:  Optional[int]					   =					self.get_rust_tokenizer()
    self.assertListEqual(
        [rust_tokenizer.tokenize(A    ) for t in ["""Test""", """\xad""", """test"""]]					,[["""[UNK]"""], [], ["""[UNK]"""]]    )

  def    __lowercase       (  self		:			Dict    ):






   '''simple docstring'''
   for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
    with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"    ):
     UpperCAmelCase__					:  Any					   =					self.rust_tokenizer_class.from_pretrained(A					,**A    )

     UpperCAmelCase__					:  Tuple					   =					f"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
     UpperCAmelCase__					:  Tuple					   =					tokenizer_r.encode_plus(
         A					,return_attention_mask=A					,return_token_type_ids=A					,return_offsets_mapping=A					,add_special_tokens=A					,)

     UpperCAmelCase__					:  Tuple					   =					tokenizer_r.do_lower_case if hasattr(A					,"""do_lower_case"""    ) else False
     UpperCAmelCase__					:  List[str]					   =					(
         [
             ((0, 0), tokenizer_r.cls_token),
             ((0, 1), """A"""),
             ((1, 2), ""","""),
             ((3, 5), """na"""),
             ((5, 6), """##ï"""),
             ((6, 8), """##ve"""),
             ((9, 15), tokenizer_r.mask_token),
             ((16, 21), """Allen"""),
             ((21, 23), """##NL"""),
             ((23, 24), """##P"""),
             ((25, 33), """sentence"""),
             ((33, 34), """."""),
             ((0, 0), tokenizer_r.sep_token),
         ]
         if not do_lower_case
         else [
             ((0, 0), tokenizer_r.cls_token),
             ((0, 1), """a"""),
             ((1, 2), ""","""),
             ((3, 8), """naive"""),
             ((9, 15), tokenizer_r.mask_token),
             ((16, 21), """allen"""),
             ((21, 23), """##nl"""),
             ((23, 24), """##p"""),
             ((25, 33), """sentence"""),
             ((33, 34), """."""),
             ((0, 0), tokenizer_r.sep_token),
         ]
     )

     self.assertEqual(
         [e[1] for e in expected_results]					,tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]    )    )
     self.assertEqual([e[0] for e in expected_results]					,tokens["""offset_mapping"""]    )

  def    __lowercase       (  self		:			Any    ):






   '''simple docstring'''
   UpperCAmelCase__					:  str					   =					["""的""", """人""", """有"""]
   UpperCAmelCase__					:  int					   =					"""""".join(A    )
   for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
    with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"    ):
     UpperCAmelCase__					:  List[Any]					   =					True
     UpperCAmelCase__					:  List[str]					   =					self.tokenizer_class.from_pretrained(A					,**A    )
     UpperCAmelCase__					:  Dict					   =					self.rust_tokenizer_class.from_pretrained(A					,**A    )

     UpperCAmelCase__					:  List[str]					   =					tokenizer_p.encode(A					,add_special_tokens=A    )
     UpperCAmelCase__					:  Any					   =					tokenizer_r.encode(A					,add_special_tokens=A    )

     UpperCAmelCase__					:  List[str]					   =					tokenizer_r.convert_ids_to_tokens(A    )
     UpperCAmelCase__					:  Any					   =					tokenizer_p.convert_ids_to_tokens(A    )

     # it is expected that each Chinese character is not preceded by "##"
     self.assertListEqual(A					,A    )
     self.assertListEqual(A					,A    )

     UpperCAmelCase__					:  List[str]					   =					False
     UpperCAmelCase__					:  int					   =					self.rust_tokenizer_class.from_pretrained(A					,**A    )
     UpperCAmelCase__					:  int					   =					self.tokenizer_class.from_pretrained(A					,**A    )

     UpperCAmelCase__					:  Optional[Any]					   =					tokenizer_r.encode(A					,add_special_tokens=A    )
     UpperCAmelCase__					:  Dict					   =					tokenizer_p.encode(A					,add_special_tokens=A    )

     UpperCAmelCase__					:  Optional[Any]					   =					tokenizer_r.convert_ids_to_tokens(A    )
     UpperCAmelCase__					:  Union[str, Any]					   =					tokenizer_p.convert_ids_to_tokens(A    )

     # it is expected that only the first Chinese character is not preceded by "##".
     UpperCAmelCase__					:  str					   =					[
         f"##{token}" if idx != 0 else token for idx, token in enumerate(A    )
     ]
     self.assertListEqual(A					,A    )
     self.assertListEqual(A					,A    )

  @slow
  def    __lowercase       (  self		:			Dict    ):






   '''simple docstring'''
   UpperCAmelCase__					:  str					   =					self.tokenizer_class(self.vocab_file					,self.word_shape_file					,self.word_pronunciation_file    )

   UpperCAmelCase__					:  Optional[Any]					   =					tokenizer.encode("""你好"""					,add_special_tokens=A    )
   UpperCAmelCase__					:  str					   =					tokenizer.encode("""你是谁"""					,add_special_tokens=A    )

   UpperCAmelCase__					:  Optional[int]					   =					tokenizer.build_inputs_with_special_tokens(A    )
   UpperCAmelCase__					:  int					   =					tokenizer.build_inputs_with_special_tokens(A					,A    )

   assert encoded_sentence == [1] + text + [2]
   assert encoded_pair == [1] + text + [2] + text_a + [2]




  def    __lowercase       (  self		:			str    ):






   '''simple docstring'''
   UpperCAmelCase__					:  List[str]					   =					self.get_tokenizers(do_lower_case=A    )
   for tokenizer in tokenizers:
    with self.subTest(f"{tokenizer.__class__.__name__}"    ):
     UpperCAmelCase__					:  Optional[Any]					   =					"""你好，你是谁"""
     UpperCAmelCase__					:  List[Any]					   =					tokenizer.tokenize(A    )
     UpperCAmelCase__					:  Optional[int]					   =					tokenizer.convert_tokens_to_ids(A    )
     UpperCAmelCase__					:  Union[str, Any]					   =					tokenizer.convert_tokens_to_shape_ids(A    )
     UpperCAmelCase__					:  List[str]					   =					tokenizer.convert_tokens_to_pronunciation_ids(A    )
     UpperCAmelCase__					:  List[str]					   =					tokenizer.prepare_for_model(
         A					,A					,A					,add_special_tokens=A    )

     UpperCAmelCase__					:  Tuple					   =					tokenizer.encode_plus(A					,add_special_tokens=A    )

     self.assertEqual(A					,A    )








"""simple docstring"""




def   lowerCAmelCase						(					__UpperCamelCase      ):
 '''simple docstring'''





 UpperCAmelCase__					:  str					   =					[]
 if len(__UpperCamelCase      ) == 1:
  return [nums.copy()]
 for _ in range(len(__UpperCamelCase      )      ):
  UpperCAmelCase__					:  Tuple					   =					nums.pop(0      )
  UpperCAmelCase__					:  List[str]					   =					permute(__UpperCamelCase      )
  for perm in permutations:
   perm.append(__UpperCamelCase      )
  result.extend(__UpperCamelCase      )
  nums.append(__UpperCamelCase      )
 return result
def   lowerCAmelCase						(					__UpperCamelCase      ):
 '''simple docstring'''





 def backtrack(__UpperCamelCase      ):
  if start == len(__UpperCamelCase      ) - 1:
   output.append(nums[:]      )
  else:
   for i in range(__UpperCamelCase							,    len(__UpperCamelCase      )      ):
    UpperCAmelCase__   ,      UpperCAmelCase__					:  Union[str, Any]					   =					nums[i], nums[start]
    backtrack(start + 1      )
    UpperCAmelCase__   ,      UpperCAmelCase__					:  Union[str, Any]					   =					nums[i], nums[start]  # backtrack

 UpperCAmelCase__					:  List[str]					   =					[]
 backtrack(0      )
 return output


if __name__ == "__main__":
      import doctest

      # use res to print the data in permute2 function
      __UpperCAmelCase =				permutea([1, 2, 3])
      print(res)
      doctest.testmod()





