
# Imports
import numpy as np



class    _snake_case :


 def __init__(       self       ,	SCREAMING_SNAKE_CASE_=None       ,	SCREAMING_SNAKE_CASE_=None       ,	SCREAMING_SNAKE_CASE_=None       ,	SCREAMING_SNAKE_CASE_=None       ,	SCREAMING_SNAKE_CASE_=None):


   '''simple docstring'''




   self.set_matricies(red=SCREAMING_SNAKE_CASE_       ,	green=SCREAMING_SNAKE_CASE_       ,	blue=SCREAMING_SNAKE_CASE_       ,	red_edge=SCREAMING_SNAKE_CASE_       ,	nir=SCREAMING_SNAKE_CASE_)


 def   lowercase__  (       self       ,	SCREAMING_SNAKE_CASE_=None       ,	SCREAMING_SNAKE_CASE_=None       ,	SCREAMING_SNAKE_CASE_=None       ,	SCREAMING_SNAKE_CASE_=None       ,	SCREAMING_SNAKE_CASE_=None):


   '''simple docstring'''




   if red is not None:
     lowercase__	: List[str]     		=   red
   if green is not None:
     lowercase__	: Union[str, Any]     		=   green
   if blue is not None:
     lowercase__	: int     		=   blue
   if red_edge is not None:
     lowercase__	: Optional[Any]     		=   red_edge
   if nir is not None:
     lowercase__	: Optional[Any]     		=   nir
   return True


 def   lowercase__  (       self       ,	SCREAMING_SNAKE_CASE_=""       ,	SCREAMING_SNAKE_CASE_=None       ,	SCREAMING_SNAKE_CASE_=None       ,	SCREAMING_SNAKE_CASE_=None       ,	SCREAMING_SNAKE_CASE_=None       ,	SCREAMING_SNAKE_CASE_=None):


   '''simple docstring'''




   self.set_matricies(red=SCREAMING_SNAKE_CASE_       ,	green=SCREAMING_SNAKE_CASE_       ,	blue=SCREAMING_SNAKE_CASE_       ,	red_edge=SCREAMING_SNAKE_CASE_       ,	nir=SCREAMING_SNAKE_CASE_)
   lowercase__	: Dict     		=   {
       """ARVI2""": self.arvaa,
       """CCCI""": self.ccci,
       """CVI""": self.cvi,
       """GLI""": self.gli,
       """NDVI""": self.ndvi,
       """BNDVI""": self.bndvi,
       """redEdgeNDVI""": self.red_edge_ndvi,
       """GNDVI""": self.gndvi,
       """GBNDVI""": self.gbndvi,
       """GRNDVI""": self.grndvi,
       """RBNDVI""": self.rbndvi,
       """PNDVI""": self.pndvi,
       """ATSAVI""": self.atsavi,
       """BWDRVI""": self.bwdrvi,
       """CIgreen""": self.ci_green,
       """CIrededge""": self.ci_rededge,
       """CI""": self.ci,
       """CTVI""": self.ctvi,
       """GDVI""": self.gdvi,
       """EVI""": self.evi,
       """GEMI""": self.gemi,
       """GOSAVI""": self.gosavi,
       """GSAVI""": self.gsavi,
       """Hue""": self.hue,
       """IVI""": self.ivi,
       """IPVI""": self.ipvi,
       """I""": self.i,
       """RVI""": self.rvi,
       """MRVI""": self.mrvi,
       """MSAVI""": self.m_savi,
       """NormG""": self.norm_g,
       """NormNIR""": self.norm_nir,
       """NormR""": self.norm_r,
       """NGRDI""": self.ngrdi,
       """RI""": self.ri,
       """S""": self.s,
       """IF""": self._if,
       """DVI""": self.dvi,
       """TVI""": self.tvi,
       """NDRE""": self.ndre,
   }

   try:
     return funcs[index]()
   except KeyError:
     print("""Index not in the list!""")
     return False


 def   lowercase__  (       self):


   '''simple docstring'''




   return -0.1_8 + (1.1_7 * ((self.nir - self.red) / (self.nir + self.red)))


 def   lowercase__  (       self):


   '''simple docstring'''




   return ((self.nir - self.redEdge) / (self.nir + self.redEdge)) / (
       (self.nir - self.red) / (self.nir + self.red)
   )


 def   lowercase__  (       self):


   '''simple docstring'''




   return self.nir * (self.red / (self.green**2))


 def   lowercase__  (       self):


   '''simple docstring'''




   return (2 * self.green - self.red - self.blue) / (
       2 * self.green + self.red + self.blue
   )


 def   lowercase__  (       self):


   '''simple docstring'''




   return (self.nir - self.red) / (self.nir + self.red)


 def   lowercase__  (       self):


   '''simple docstring'''




   return (self.nir - self.blue) / (self.nir + self.blue)


 def   lowercase__  (       self):


   '''simple docstring'''




   return (self.redEdge - self.red) / (self.redEdge + self.red)


 def   lowercase__  (       self):


   '''simple docstring'''




   return (self.nir - self.green) / (self.nir + self.green)


 def   lowercase__  (       self):


   '''simple docstring'''




   return (self.nir - (self.green + self.blue)) / (
       self.nir + (self.green + self.blue)
   )


 def   lowercase__  (       self):


   '''simple docstring'''




   return (self.nir - (self.green + self.red)) / (
       self.nir + (self.green + self.red)
   )


 def   lowercase__  (       self):


   '''simple docstring'''




   return (self.nir - (self.blue + self.red)) / (self.nir + (self.blue + self.red))


 def   lowercase__  (       self):


   '''simple docstring'''




   return (self.nir - (self.green + self.red + self.blue)) / (
       self.nir + (self.green + self.red + self.blue)
   )


 def   lowercase__  (       self       ,	SCREAMING_SNAKE_CASE_=0.0_8       ,	SCREAMING_SNAKE_CASE_=1.2_2       ,	SCREAMING_SNAKE_CASE_=0.0_3):


   '''simple docstring'''




   return a * (
       (self.nir - a * self.red - b)
       / (a * self.nir + self.red - a * b + x * (1 + a**2))
   )


 def   lowercase__  (       self):


   '''simple docstring'''




   return (0.1 * self.nir - self.blue) / (0.1 * self.nir + self.blue)


 def   lowercase__  (       self):


   '''simple docstring'''




   return (self.nir / self.green) - 1


 def   lowercase__  (       self):


   '''simple docstring'''




   return (self.nir / self.redEdge) - 1


 def   lowercase__  (       self):


   '''simple docstring'''




   return (self.red - self.blue) / self.red


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: int     		=   self.ndvi()
   return ((ndvi + 0.5) / (abs(ndvi + 0.5))) * (abs(ndvi + 0.5) ** (1 / 2))


 def   lowercase__  (       self):


   '''simple docstring'''




   return self.nir - self.green


 def   lowercase__  (       self):


   '''simple docstring'''




   return 2.5 * (
       (self.nir - self.red) / (self.nir + 6 * self.red - 7.5 * self.blue + 1)
   )


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Union[str, Any]     		=   (2 * (self.nir**2 - self.red**2) + 1.5 * self.nir + 0.5 * self.red) / (
       self.nir + self.red + 0.5
   )
   return n * (1 - 0.2_5 * n) - (self.red - 0.1_2_5) / (1 - self.red)


 def   lowercase__  (       self       ,	SCREAMING_SNAKE_CASE_=0.1_6):


   '''simple docstring'''




   return (self.nir - self.green) / (self.nir + self.green + y)


 def   lowercase__  (       self       ,	SCREAMING_SNAKE_CASE_=0.5):


   '''simple docstring'''




   return ((self.nir - self.green) / (self.nir + self.green + n)) * (1 + n)


 def   lowercase__  (       self):


   '''simple docstring'''




   return np.arctan(
       ((2 * self.red - self.green - self.blue) / 3_0.5) * (self.green - self.blue))


 def   lowercase__  (       self       ,	SCREAMING_SNAKE_CASE_=None       ,	SCREAMING_SNAKE_CASE_=None):


   '''simple docstring'''




   return (self.nir - b) / (a * self.red)


 def   lowercase__  (       self):


   '''simple docstring'''




   return (self.nir / ((self.nir + self.red) / 2)) * (self.ndvi() + 1)


 def   lowercase__  (       self):


   '''simple docstring'''




   return (self.red + self.green + self.blue) / 3_0.5


 def   lowercase__  (       self):


   '''simple docstring'''




   return self.nir / self.red


 def   lowercase__  (       self):


   '''simple docstring'''




   return (self.rvi() - 1) / (self.rvi() + 1)


 def   lowercase__  (       self):


   '''simple docstring'''




   return (
       (2 * self.nir + 1)
       - ((2 * self.nir + 1) ** 2 - 8 * (self.nir - self.red)) ** (1 / 2)
   ) / 2


 def   lowercase__  (       self):


   '''simple docstring'''




   return self.green / (self.nir + self.red + self.green)


 def   lowercase__  (       self):


   '''simple docstring'''




   return self.nir / (self.nir + self.red + self.green)


 def   lowercase__  (       self):


   '''simple docstring'''




   return self.red / (self.nir + self.red + self.green)


 def   lowercase__  (       self):


   '''simple docstring'''




   return (self.green - self.red) / (self.green + self.red)


 def   lowercase__  (       self):


   '''simple docstring'''




   return (self.red - self.green) / (self.red + self.green)


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Tuple     		=   np.max([np.max(self.red), np.max(self.green), np.max(self.blue)])
   lowercase__	: Dict     		=   np.min([np.min(self.red), np.min(self.green), np.min(self.blue)])
   return (max_value - min_value) / max_value


 def   lowercase__  (       self):


   '''simple docstring'''




   return (2 * self.red - self.green - self.blue) / (self.green - self.blue)


 def   lowercase__  (       self):


   '''simple docstring'''




   return self.nir / self.red


 def   lowercase__  (       self):


   '''simple docstring'''




   return (self.ndvi() + 0.5) ** (1 / 2)



 def   lowercase__  (       self):


   '''simple docstring'''




   return (self.nir - self.redEdge) / (self.nir + self.redEdge)








import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english



@require_tokenizers
class    _snake_case (						UpperCAmelCase_	,    unittest.TestCase				):
 __lowerCAmelCase	:  Union[str, Any]	   =      RoCBertTokenizer
 __lowerCAmelCase	:  Union[str, Any]	   =      None
 __lowerCAmelCase	:  str	   =      False
 __lowerCAmelCase	:  List[Any]	   =      True
 __lowerCAmelCase	:  Optional[int]	   =      filter_non_english


 def   lowercase__  (       self):


   '''simple docstring'''




   super().setUp()

   lowercase__	: Optional[int]     		=   ["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """你""", """好""", """是""", """谁""", """a""", """b""", """c""", """d"""]
   lowercase__	: Dict     		=   {}
   lowercase__	: Tuple     		=   {}
   for i, value in enumerate(SCREAMING_SNAKE_CASE_):
     lowercase__	: Tuple     		=   i
     lowercase__	: Any     		=   i
   lowercase__	: str     		=   os.path.join(self.tmpdirname       ,	VOCAB_FILES_NAMES["""vocab_file"""])
   lowercase__	: Union[str, Any]     		=   os.path.join(self.tmpdirname       ,	VOCAB_FILES_NAMES["""word_shape_file"""])
   lowercase__	: Tuple     		=   os.path.join(self.tmpdirname       ,	VOCAB_FILES_NAMES["""word_pronunciation_file"""])
   with open(self.vocab_file       ,	"""w"""       ,	encoding="""utf-8""") as vocab_writer:
     vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]))
   with open(self.word_shape_file       ,	"""w"""       ,	encoding="""utf-8""") as word_shape_writer:
     json.dump(SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_       ,	ensure_ascii=SCREAMING_SNAKE_CASE_)
   with open(self.word_pronunciation_file       ,	"""w"""       ,	encoding="""utf-8""") as word_pronunciation_writer:
     json.dump(SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_       ,	ensure_ascii=SCREAMING_SNAKE_CASE_)


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Dict     		=   self.tokenizer_class(self.vocab_file       ,	self.word_shape_file       ,	self.word_pronunciation_file)

   lowercase__	: Optional[int]     		=   tokenizer.tokenize("""你好[SEP]你是谁""")
   self.assertListEqual(SCREAMING_SNAKE_CASE_       ,	["""你""", """好""", """[SEP]""", """你""", """是""", """谁"""])
   self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_)       ,	[5, 6, 2, 5, 7, 8])
   self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(SCREAMING_SNAKE_CASE_)       ,	[5, 6, 2, 5, 7, 8])
   self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(SCREAMING_SNAKE_CASE_)       ,	[5, 6, 2, 5, 7, 8])


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: int     		=   RoCBertBasicTokenizer()

   self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz""")       ,	["""ah""", """\u535A""", """\u63A8""", """zz"""])


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Dict     		=   RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_)

   self.assertListEqual(
       tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """)       ,	["""hello""", """!""", """how""", """are""", """you""", """?"""])
   self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""")       ,	["""hello"""])


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Any     		=   RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_       ,	strip_accents=SCREAMING_SNAKE_CASE_)

   self.assertListEqual(
       tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """)       ,	["""hällo""", """!""", """how""", """are""", """you""", """?"""])
   self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""")       ,	["""h\u00E9llo"""])


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Optional[Any]     		=   RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_       ,	strip_accents=SCREAMING_SNAKE_CASE_)

   self.assertListEqual(
       tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """)       ,	["""hallo""", """!""", """how""", """are""", """you""", """?"""])
   self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""")       ,	["""hello"""])


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Optional[Any]     		=   RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_)

   self.assertListEqual(
       tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """)       ,	["""hallo""", """!""", """how""", """are""", """you""", """?"""])
   self.assertListEqual(tokenizer.tokenize("""H\u00E9llo""")       ,	["""hello"""])


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Union[str, Any]     		=   RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_)

   self.assertListEqual(
       tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """)       ,	["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""])


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: str     		=   RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_       ,	strip_accents=SCREAMING_SNAKE_CASE_)

   self.assertListEqual(
       tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """)       ,	["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""])


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Tuple     		=   RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_       ,	strip_accents=SCREAMING_SNAKE_CASE_)

   self.assertListEqual(
       tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """)       ,	["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""])


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Dict     		=   RoCBertBasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_       ,	never_split=["""[UNK]"""])

   self.assertListEqual(
       tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]""")       ,	["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""])


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Optional[int]     		=   ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

   lowercase__	: Optional[int]     		=   {}
   for i, token in enumerate(SCREAMING_SNAKE_CASE_):
     lowercase__	: Optional[Any]     		=   i
   lowercase__	: Union[str, Any]     		=   RoCBertWordpieceTokenizer(vocab=SCREAMING_SNAKE_CASE_       ,	unk_token="""[UNK]""")

   self.assertListEqual(tokenizer.tokenize("""""")       ,	[])

   self.assertListEqual(tokenizer.tokenize("""unwanted running""")       ,	["""un""", """##want""", """##ed""", """runn""", """##ing"""])

   self.assertListEqual(tokenizer.tokenize("""unwantedX running""")       ,	["""[UNK]""", """runn""", """##ing"""])


 def   lowercase__  (       self):


   '''simple docstring'''




   self.assertTrue(_is_whitespace(""" """))
   self.assertTrue(_is_whitespace("""\t"""))
   self.assertTrue(_is_whitespace("""\r"""))
   self.assertTrue(_is_whitespace("""\n"""))
   self.assertTrue(_is_whitespace("""\u00A0"""))

   self.assertFalse(_is_whitespace("""A"""))
   self.assertFalse(_is_whitespace("""-"""))


 def   lowercase__  (       self):


   '''simple docstring'''




   self.assertTrue(_is_control("""\u0005"""))

   self.assertFalse(_is_control("""A"""))
   self.assertFalse(_is_control(""" """))
   self.assertFalse(_is_control("""\t"""))
   self.assertFalse(_is_control("""\r"""))


 def   lowercase__  (       self):


   '''simple docstring'''




   self.assertTrue(_is_punctuation("""-"""))
   self.assertTrue(_is_punctuation("""$"""))
   self.assertTrue(_is_punctuation("""`"""))
   self.assertTrue(_is_punctuation("""."""))

   self.assertFalse(_is_punctuation("""A"""))
   self.assertFalse(_is_punctuation(""" """))


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Union[str, Any]     		=   self.get_tokenizer()

   # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
   self.assertListEqual([tokenizer.tokenize(SCREAMING_SNAKE_CASE_) for t in ["""Test""", """\xad""", """test"""]]       ,	[["""[UNK]"""], [], ["""[UNK]"""]])

   if self.test_rust_tokenizer:
     lowercase__	: int     		=   self.get_rust_tokenizer()
     self.assertListEqual(
         [rust_tokenizer.tokenize(SCREAMING_SNAKE_CASE_) for t in ["""Test""", """\xad""", """test"""]]       ,	[["""[UNK]"""], [], ["""[UNK]"""]])


 def   lowercase__  (       self):


   '''simple docstring'''




   for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
     with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'):
       lowercase__	: str     		=   self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_       ,	**SCREAMING_SNAKE_CASE_)

       lowercase__	: Optional[int]     		=   f'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
       lowercase__	: List[str]     		=   tokenizer_r.encode_plus(
           SCREAMING_SNAKE_CASE_       ,	return_attention_mask=SCREAMING_SNAKE_CASE_       ,	return_token_type_ids=SCREAMING_SNAKE_CASE_       ,	return_offsets_mapping=SCREAMING_SNAKE_CASE_       ,	add_special_tokens=SCREAMING_SNAKE_CASE_       ,	)

       lowercase__	: str     		=   tokenizer_r.do_lower_case if hasattr(SCREAMING_SNAKE_CASE_       ,	"""do_lower_case""") else False
       lowercase__	: Optional[Any]     		=   (
           [
               ((0, 0), tokenizer_r.cls_token),
               ((0, 1), """A"""),
               ((1, 2), ""","""),
               ((3, 5), """na"""),
               ((5, 6), """##ï"""),
               ((6, 8), """##ve"""),
               ((9, 15), tokenizer_r.mask_token),
               ((16, 21), """Allen"""),
               ((21, 23), """##NL"""),
               ((23, 24), """##P"""),
               ((25, 33), """sentence"""),
               ((33, 34), """."""),
               ((0, 0), tokenizer_r.sep_token),
           ]
           if not do_lower_case
           else [
               ((0, 0), tokenizer_r.cls_token),
               ((0, 1), """a"""),
               ((1, 2), ""","""),
               ((3, 8), """naive"""),
               ((9, 15), tokenizer_r.mask_token),
               ((16, 21), """allen"""),
               ((21, 23), """##nl"""),
               ((23, 24), """##p"""),
               ((25, 33), """sentence"""),
               ((33, 34), """."""),
               ((0, 0), tokenizer_r.sep_token),
           ]
       )

       self.assertEqual(
           [e[1] for e in expected_results]       ,	tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]))
       self.assertEqual([e[0] for e in expected_results]       ,	tokens["""offset_mapping"""])


 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Any     		=   ["""的""", """人""", """有"""]
   lowercase__	: List[str]     		=   """""".join(SCREAMING_SNAKE_CASE_)
   for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
     with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'):
       lowercase__	: Union[str, Any]     		=   True
       lowercase__	: Tuple     		=   self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_       ,	**SCREAMING_SNAKE_CASE_)
       lowercase__	: List[Any]     		=   self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_       ,	**SCREAMING_SNAKE_CASE_)

       lowercase__	: Optional[Any]     		=   tokenizer_p.encode(SCREAMING_SNAKE_CASE_       ,	add_special_tokens=SCREAMING_SNAKE_CASE_)
       lowercase__	: str     		=   tokenizer_r.encode(SCREAMING_SNAKE_CASE_       ,	add_special_tokens=SCREAMING_SNAKE_CASE_)

       lowercase__	: List[str]     		=   tokenizer_r.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_)
       lowercase__	: List[str]     		=   tokenizer_p.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_)

       # it is expected that each Chinese character is not preceded by "##"
       self.assertListEqual(SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_)
       self.assertListEqual(SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_)

       lowercase__	: Any     		=   False
       lowercase__	: Optional[Any]     		=   self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_       ,	**SCREAMING_SNAKE_CASE_)
       lowercase__	: Optional[Any]     		=   self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_       ,	**SCREAMING_SNAKE_CASE_)

       lowercase__	: Optional[int]     		=   tokenizer_r.encode(SCREAMING_SNAKE_CASE_       ,	add_special_tokens=SCREAMING_SNAKE_CASE_)
       lowercase__	: Tuple     		=   tokenizer_p.encode(SCREAMING_SNAKE_CASE_       ,	add_special_tokens=SCREAMING_SNAKE_CASE_)

       lowercase__	: Tuple     		=   tokenizer_r.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_)
       lowercase__	: Optional[Any]     		=   tokenizer_p.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_)

       # it is expected that only the first Chinese character is not preceded by "##".
       lowercase__	: Any     		=   [
           f'##{token}' if idx != 0 else token for idx, token in enumerate(SCREAMING_SNAKE_CASE_)
       ]
       self.assertListEqual(SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_)
       self.assertListEqual(SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_)


 @slow
 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Dict     		=   self.tokenizer_class(self.vocab_file       ,	self.word_shape_file       ,	self.word_pronunciation_file)

   lowercase__	: Optional[Any]     		=   tokenizer.encode("""你好"""       ,	add_special_tokens=SCREAMING_SNAKE_CASE_)
   lowercase__	: Any     		=   tokenizer.encode("""你是谁"""       ,	add_special_tokens=SCREAMING_SNAKE_CASE_)

   lowercase__	: Optional[Any]     		=   tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE_)
   lowercase__	: Tuple     		=   tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_)

   assert encoded_sentence == [1] + text + [2]
   assert encoded_pair == [1] + text + [2] + text_a + [2]



 def   lowercase__  (       self):


   '''simple docstring'''




   lowercase__	: Optional[int]     		=   self.get_tokenizers(do_lower_case=SCREAMING_SNAKE_CASE_)
   for tokenizer in tokenizers:
     with self.subTest(f'{tokenizer.__class__.__name__}'):
       lowercase__	: Optional[int]     		=   """你好，你是谁"""
       lowercase__	: List[Any]     		=   tokenizer.tokenize(SCREAMING_SNAKE_CASE_)
       lowercase__	: Union[str, Any]     		=   tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_)
       lowercase__	: Tuple     		=   tokenizer.convert_tokens_to_shape_ids(SCREAMING_SNAKE_CASE_)
       lowercase__	: List[str]     		=   tokenizer.convert_tokens_to_pronunciation_ids(SCREAMING_SNAKE_CASE_)
       lowercase__	: Any     		=   tokenizer.prepare_for_model(
           SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_       ,	add_special_tokens=SCREAMING_SNAKE_CASE_)

       lowercase__	: Dict     		=   tokenizer.encode_plus(SCREAMING_SNAKE_CASE_       ,	add_special_tokens=SCREAMING_SNAKE_CASE_)

       self.assertEqual(SCREAMING_SNAKE_CASE_       ,	SCREAMING_SNAKE_CASE_)






