

import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
     __lowercase     : str       =						'''pt'''
elif is_tf_available():
     __lowercase     : str       =						'''tf'''
else:
     __lowercase     : int       =						'''jax'''



class 						_A     (				snake_case						,    unittest.TestCase				):





  '''simple docstring'''



  __lowerCamelCase			:      Tuple					=   ByTaTokenizer
  __lowerCamelCase			:      Union[str, Any]					=   False





  def    snake_case_       (  self    ):

      '''simple docstring'''
      super().setUp()
      snake_case							:		List[Any]        =		ByTaTokenizer()
      tokenizer.save_pretrained(self.tmpdirname    )





  @cached_property
  def    snake_case_       (  self    ):

      '''simple docstring'''
      return ByTaTokenizer.from_pretrained("""google/byt5-small"""    )





  def    snake_case_       (  self   ,**SCREAMING_SNAKE_CASE_    ):

      '''simple docstring'''
      return self.tokenizer_class.from_pretrained(self.tmpdirname   ,**SCREAMING_SNAKE_CASE_    )





  def    snake_case_       (  self   ,SCREAMING_SNAKE_CASE_   ,SCREAMING_SNAKE_CASE_=False   ,SCREAMING_SNAKE_CASE_=20   ,SCREAMING_SNAKE_CASE_=5    ):

      '''simple docstring'''
      # XXX The default common tokenizer tests assume that every ID is decodable on its own.
      # This assumption is invalid for ByT5 because single bytes might not be
      # valid utf-8 (byte 128 for instance).
      # Here we're overriding the smallest possible method to provide
      # a clean sequence without making the same assumption.

      snake_case							:		str        =		[]
      for i in range(len(SCREAMING_SNAKE_CASE_    )    ):
          try:
              snake_case							:		Optional[Any]        =		tokenizer.decode([i]   ,clean_up_tokenization_spaces=SCREAMING_SNAKE_CASE_    )
          except UnicodeDecodeError:
              pass
          toks.append((i, tok)    )

      snake_case							:		List[str]        =		list(filter(lambda SCREAMING_SNAKE_CASE_    : re.match(R"""^[ a-zA-Z]+$"""   ,t[1]    )   ,SCREAMING_SNAKE_CASE_    )    )
      snake_case							:		Tuple        =		list(filter(lambda SCREAMING_SNAKE_CASE_    : [t[0]] == tokenizer.encode(t[1]   ,add_special_tokens=SCREAMING_SNAKE_CASE_    )   ,SCREAMING_SNAKE_CASE_    )    )
      if max_length is not None and len(SCREAMING_SNAKE_CASE_    ) > max_length:
          snake_case							:		List[str]        =		toks[:max_length]
      if min_length is not None and len(SCREAMING_SNAKE_CASE_    ) < min_length and len(SCREAMING_SNAKE_CASE_    ) > 0:
          while len(SCREAMING_SNAKE_CASE_    ) < min_length:
              snake_case							:		Tuple        =		toks + toks
        # toks_str = [t[1] for t in toks]
      snake_case							:		Optional[Any]        =		[t[0] for t in toks]

      # Ensure consistency
      snake_case							:		Optional[Any]        =		tokenizer.decode(SCREAMING_SNAKE_CASE_   ,clean_up_tokenization_spaces=SCREAMING_SNAKE_CASE_    )
      if " " not in output_txt and len(SCREAMING_SNAKE_CASE_    ) > 1:
          snake_case							:		List[Any]        =		(
              tokenizer.decode([toks_ids[0]]   ,clean_up_tokenization_spaces=SCREAMING_SNAKE_CASE_    )
              + """ """
              + tokenizer.decode(toks_ids[1:]   ,clean_up_tokenization_spaces=SCREAMING_SNAKE_CASE_    )
          )
      if with_prefix_space:
          snake_case							:		Tuple        =		""" """ + output_txt
      snake_case							:		List[str]        =		tokenizer.encode(SCREAMING_SNAKE_CASE_   ,add_special_tokens=SCREAMING_SNAKE_CASE_    )
      return output_txt, output_ids





  def    snake_case_       (  self    ):

      '''simple docstring'''
      snake_case							:		int        =		self.ta_base_tokenizer
      snake_case							:		Any        =		tokenizer(["""hi</s>""", """I went to the gym</s>""", """</s>"""]    )
      snake_case							:		Tuple        =		tokenizer(["""hi""", """I went to the gym""", """"""]    )
      self.assertListEqual(batch_with_eos_added["""input_ids"""]   ,batch_without_eos_added["""input_ids"""]    )





  def    snake_case_       (  self    ):

      '''simple docstring'''
      snake_case							:		Any        =		self.ta_base_tokenizer
      snake_case							:		Union[str, Any]        =		"""Unicode €."""
      snake_case							:		Any        =		tokenizer(SCREAMING_SNAKE_CASE_    )
      snake_case							:		Optional[Any]        =		[88, 113, 108, 102, 114, 103, 104, 35, 229, 133, 175, 49, 1]
      self.assertEqual(encoded["""input_ids"""]   ,SCREAMING_SNAKE_CASE_    )

      # decoding
      snake_case							:		Optional[Any]        =		tokenizer.decode(SCREAMING_SNAKE_CASE_    )
      self.assertEqual(SCREAMING_SNAKE_CASE_   ,"""Unicode €.</s>"""    )

      snake_case							:		str        =		tokenizer("""e è é ê ë"""    )
      snake_case							:		Dict        =		[104, 35, 198, 171, 35, 198, 172, 35, 198, 173, 35, 198, 174, 1]
      self.assertEqual(encoded["""input_ids"""]   ,SCREAMING_SNAKE_CASE_    )
      # decoding
      snake_case							:		Dict        =		tokenizer.decode(SCREAMING_SNAKE_CASE_    )
      self.assertEqual(SCREAMING_SNAKE_CASE_   ,"""e è é ê ë</s>"""    )

      # encode/decode, but with `encode` instead of `__call__`
      self.assertEqual(tokenizer.decode(tokenizer.encode("""e è é ê ë"""    )    )   ,"""e è é ê ë</s>"""    )





  def    snake_case_       (  self    ):

      '''simple docstring'''
      snake_case							:		Tuple        =		self.ta_base_tokenizer
      snake_case							:		List[str]        =		["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
      # fmt: off
      snake_case							:		Tuple        =		[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 1, 0]
      # fmt: on
      snake_case							:		Optional[Any]        =		tokenizer(SCREAMING_SNAKE_CASE_   ,padding=SCREAMING_SNAKE_CASE_   ,return_tensors=SCREAMING_SNAKE_CASE_    )
      self.assertIsInstance(SCREAMING_SNAKE_CASE_   ,SCREAMING_SNAKE_CASE_    )

      if FRAMEWORK != "jax":
          snake_case							:		int        =		list(batch.input_ids.numpy()[0]    )
      else:
          snake_case							:		Any        =		list(batch.input_ids.tolist()[0]    )

      self.assertListEqual(SCREAMING_SNAKE_CASE_   ,SCREAMING_SNAKE_CASE_    )

      self.assertEqual((2, 37)   ,batch.input_ids.shape    )
      self.assertEqual((2, 37)   ,batch.attention_mask.shape    )





  def    snake_case_       (  self    ):

      '''simple docstring'''
      snake_case							:		Optional[Any]        =		self.ta_base_tokenizer
      snake_case							:		Optional[Any]        =		["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
      snake_case							:		Dict        =		tokenizer(SCREAMING_SNAKE_CASE_   ,padding=SCREAMING_SNAKE_CASE_   ,return_tensors=SCREAMING_SNAKE_CASE_    )
      # check if input_ids are returned and no decoder_input_ids
      self.assertIn("""input_ids"""   ,SCREAMING_SNAKE_CASE_    )
      self.assertIn("""attention_mask"""   ,SCREAMING_SNAKE_CASE_    )
      self.assertNotIn("""decoder_input_ids"""   ,SCREAMING_SNAKE_CASE_    )
      self.assertNotIn("""decoder_attention_mask"""   ,SCREAMING_SNAKE_CASE_    )





  def    snake_case_       (  self    ):

      '''simple docstring'''
      snake_case							:		Union[str, Any]        =		self.ta_base_tokenizer
      snake_case							:		Dict        =		[
          """Summary of the text.""",
          """Another summary.""",
      ]
      snake_case							:		Optional[Any]        =		tokenizer(
          text_target=SCREAMING_SNAKE_CASE_   ,max_length=32   ,padding="""max_length"""   ,truncation=SCREAMING_SNAKE_CASE_   ,return_tensors=SCREAMING_SNAKE_CASE_    )
      self.assertEqual(32   ,targets["""input_ids"""].shape[1]    )





  def    snake_case_       (  self    ):

      '''simple docstring'''
      snake_case							:		List[Any]        =		self.ta_base_tokenizer
      snake_case							:		Dict        =		["""A long paragraph for summarization. </s>"""]
      snake_case							:		int        =		["""Summary of the text. </s>"""]
      # fmt: off
      snake_case							:		Optional[int]        =		[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 35, 1]
      snake_case							:		int        =		[86, 120, 112, 112, 100, 117, 124, 35, 114, 105, 35, 119, 107, 104, 35, 119, 104, 123, 119, 49, 35, 1]
      # fmt: on

      snake_case							:		Dict        =		tokenizer(SCREAMING_SNAKE_CASE_   ,text_target=SCREAMING_SNAKE_CASE_    )

      self.assertEqual(SCREAMING_SNAKE_CASE_   ,batch["""input_ids"""][0]    )
      self.assertEqual(SCREAMING_SNAKE_CASE_   ,batch["""labels"""][0]    )





  def    snake_case_       (  self    ):

      '''simple docstring'''
      # safety check on max_len default value so we are sure the test works
      snake_case							:		List[str]        =		self.get_tokenizers()
      for tokenizer in tokenizers:
          with self.subTest(F"""{tokenizer.__class__.__name__}"""    ):
              self.assertNotEqual(tokenizer.model_max_length   ,42    )

        # Now let's start the test
      snake_case							:		str        =		self.get_tokenizers()
      for tokenizer in tokenizers:
          with self.subTest(F"""{tokenizer.__class__.__name__}"""    ):
              # Isolate this from the other tests because we save additional tokens/etc
              snake_case							:		Union[str, Any]        =		tempfile.mkdtemp()

              snake_case							:		Optional[Any]        =		""" He is very happy, UNwant\u00E9d,running"""
              snake_case							:		Any        =		tokenizer.encode(SCREAMING_SNAKE_CASE_   ,add_special_tokens=SCREAMING_SNAKE_CASE_    )
              tokenizer.save_pretrained(SCREAMING_SNAKE_CASE_    )

              snake_case							:		int        =		tokenizer.__class__.from_pretrained(SCREAMING_SNAKE_CASE_    )
              snake_case							:		int        =		after_tokenizer.encode(SCREAMING_SNAKE_CASE_   ,add_special_tokens=SCREAMING_SNAKE_CASE_    )
              self.assertListEqual(SCREAMING_SNAKE_CASE_   ,SCREAMING_SNAKE_CASE_    )

              shutil.rmtree(SCREAMING_SNAKE_CASE_    )

      snake_case							:		str        =		self.get_tokenizers(model_max_length=42    )
      for tokenizer in tokenizers:
          with self.subTest(F"""{tokenizer.__class__.__name__}"""    ):
              # Isolate this from the other tests because we save additional tokens/etc
              snake_case							:		Any        =		tempfile.mkdtemp()

              snake_case							:		Optional[Any]        =		""" He is very happy, UNwant\u00E9d,running"""
              tokenizer.add_tokens(["""bim""", """bambam"""]    )
              snake_case							:		List[Any]        =		tokenizer.additional_special_tokens
              additional_special_tokens.append("""new_additional_special_token"""    )
              tokenizer.add_special_tokens({"""additional_special_tokens""": additional_special_tokens}    )
              snake_case							:		Optional[int]        =		tokenizer.encode(SCREAMING_SNAKE_CASE_   ,add_special_tokens=SCREAMING_SNAKE_CASE_    )
              tokenizer.save_pretrained(SCREAMING_SNAKE_CASE_    )

              snake_case							:		List[Any]        =		tokenizer.__class__.from_pretrained(SCREAMING_SNAKE_CASE_    )
              snake_case							:		Union[str, Any]        =		after_tokenizer.encode(SCREAMING_SNAKE_CASE_   ,add_special_tokens=SCREAMING_SNAKE_CASE_    )
              self.assertListEqual(SCREAMING_SNAKE_CASE_   ,SCREAMING_SNAKE_CASE_    )
              self.assertIn("""new_additional_special_token"""   ,after_tokenizer.additional_special_tokens    )
              self.assertEqual(after_tokenizer.model_max_length   ,42    )

              snake_case							:		Dict        =		tokenizer.__class__.from_pretrained(SCREAMING_SNAKE_CASE_   ,model_max_length=43    )
              self.assertEqual(tokenizer.model_max_length   ,43    )

              shutil.rmtree(SCREAMING_SNAKE_CASE_    )





  def    snake_case_       (  self    ):

      '''simple docstring'''
      snake_case							:		Optional[int]        =		[]
      if self.test_slow_tokenizer:
          tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())    )

      if self.test_rust_tokenizer:
          tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())    )

      for tokenizer_class, tokenizer_utils in tokenizer_list:
          with tempfile.TemporaryDirectory() as tmp_dir:
              tokenizer_utils.save_pretrained(SCREAMING_SNAKE_CASE_    )

              with open(os.path.join(SCREAMING_SNAKE_CASE_   ,"""special_tokens_map.json"""    )   ,encoding="""utf-8"""    ) as json_file:
                  snake_case							:		int        =		json.load(SCREAMING_SNAKE_CASE_    )

              with open(os.path.join(SCREAMING_SNAKE_CASE_   ,"""tokenizer_config.json"""    )   ,encoding="""utf-8"""    ) as json_file:
                  snake_case							:		int        =		json.load(SCREAMING_SNAKE_CASE_    )

              snake_case							:		Union[str, Any]        =		[F"""<extra_id_{i}>""" for i in range(125    )]

              snake_case							:		List[Any]        =		added_tokens_extra_ids + [
                  """an_additional_special_token"""
              ]
              snake_case							:		Optional[int]        =		added_tokens_extra_ids + [
                  """an_additional_special_token"""
              ]

              with open(os.path.join(SCREAMING_SNAKE_CASE_   ,"""special_tokens_map.json"""    )   ,"""w"""   ,encoding="""utf-8"""    ) as outfile:
                  json.dump(SCREAMING_SNAKE_CASE_   ,SCREAMING_SNAKE_CASE_    )
              with open(os.path.join(SCREAMING_SNAKE_CASE_   ,"""tokenizer_config.json"""    )   ,"""w"""   ,encoding="""utf-8"""    ) as outfile:
                  json.dump(SCREAMING_SNAKE_CASE_   ,SCREAMING_SNAKE_CASE_    )

              # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
              # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
              # "special_tokens_map.json" files
              snake_case							:		List[Any]        =		tokenizer_class.from_pretrained(
                  SCREAMING_SNAKE_CASE_   ,)
              self.assertIn(
                  """an_additional_special_token"""   ,tokenizer_without_change_in_init.additional_special_tokens    )
              # self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
              self.assertEqual(
                  ["""an_additional_special_token"""]   ,tokenizer_without_change_in_init.convert_ids_to_tokens(
                      tokenizer_without_change_in_init.convert_tokens_to_ids(["""an_additional_special_token"""]    )    )   ,)

              # Now we test that we can change the value of additional_special_tokens in the from_pretrained
              snake_case							:		Any        =		added_tokens_extra_ids + [AddedToken("""a_new_additional_special_token"""   ,lstrip=SCREAMING_SNAKE_CASE_    )]
              snake_case							:		int        =		tokenizer_class.from_pretrained(
                  SCREAMING_SNAKE_CASE_   ,additional_special_tokens=SCREAMING_SNAKE_CASE_   ,)

              self.assertIn("""a_new_additional_special_token"""   ,tokenizer.additional_special_tokens    )
              self.assertEqual(
                  ["""a_new_additional_special_token"""]   ,tokenizer.convert_ids_to_tokens(
                      tokenizer.convert_tokens_to_ids(["""a_new_additional_special_token"""]    )    )   ,)





  def    snake_case_       (  self    ):

      '''simple docstring'''
      snake_case							:		Optional[int]        =		[]
      if self.test_slow_tokenizer:
          tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())    )

      if self.test_rust_tokenizer:
          tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())    )

      for tokenizer_class, tokenizer_utils in tokenizer_list:
          with tempfile.TemporaryDirectory() as tmp_dir:
              tokenizer_utils.save_pretrained(SCREAMING_SNAKE_CASE_    )

              snake_case							:		Any        =		tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_    )

              self.assertTrue(tokenizer.decode([255]    ) == """"""    )





  def    snake_case_       (  self    ):

      '''simple docstring'''
      pass





  def    snake_case_       (  self    ):

      '''simple docstring'''
      pass





  def    snake_case_       (  self    ):

      '''simple docstring'''
      pass





  def    snake_case_       (  self    ):

      '''simple docstring'''
      pass





  def    snake_case_       (  self    ):

      '''simple docstring'''
      # The default common tokenizer tests uses invalid tokens for ByT5 that can only accept one-character strings
      # and special added tokens as tokens
      snake_case							:		Any        =		self.get_tokenizers(fast=SCREAMING_SNAKE_CASE_   ,do_lower_case=SCREAMING_SNAKE_CASE_    )
      for tokenizer in tokenizers:
          with self.subTest(F"""{tokenizer.__class__.__name__}"""    ):
              snake_case							:		Tuple        =		["""t""", """h""", """i""", """s""", """ """, """i""", """s""", """ """, """a""", """ """, """t""", """e""", """x""", """t""", """</s>"""]
              snake_case							:		Union[str, Any]        =		tokenizer.convert_tokens_to_string(SCREAMING_SNAKE_CASE_    )

              self.assertIsInstance(SCREAMING_SNAKE_CASE_   ,SCREAMING_SNAKE_CASE_    )





  def    snake_case_       (  self    ):

      '''simple docstring'''
      snake_case							:		Optional[int]        =		self.get_tokenizers()
      for tokenizer in tokenizers:
          with self.subTest(F"""{tokenizer.__class__.__name__}"""    ):
              snake_case							:		List[str]        =		[
                  """bos_token""",
                  """eos_token""",
                  """unk_token""",
                  """sep_token""",
                  """pad_token""",
                  """cls_token""",
                  """mask_token""",
              ]

              snake_case							:		Dict        =		0
              snake_case							:		Optional[int]        =		tokenizer.convert_ids_to_tokens(
                  SCREAMING_SNAKE_CASE_   ,skip_special_tokens=SCREAMING_SNAKE_CASE_    )

              for attr in attributes_list:
                  setattr(SCREAMING_SNAKE_CASE_   ,attr + """_id"""   ,SCREAMING_SNAKE_CASE_    )
                  self.assertEqual(getattr(SCREAMING_SNAKE_CASE_   ,SCREAMING_SNAKE_CASE_    )   ,SCREAMING_SNAKE_CASE_    )
                  self.assertEqual(getattr(SCREAMING_SNAKE_CASE_   ,attr + """_id"""    )   ,SCREAMING_SNAKE_CASE_    )

                  setattr(SCREAMING_SNAKE_CASE_   ,attr + """_id"""   ,SCREAMING_SNAKE_CASE_    )
                  self.assertEqual(getattr(SCREAMING_SNAKE_CASE_   ,SCREAMING_SNAKE_CASE_    )   ,SCREAMING_SNAKE_CASE_    )
                  self.assertEqual(getattr(SCREAMING_SNAKE_CASE_   ,attr + """_id"""    )   ,SCREAMING_SNAKE_CASE_    )

              setattr(SCREAMING_SNAKE_CASE_   ,"""additional_special_tokens_ids"""   ,[]    )
              self.assertListEqual(getattr(SCREAMING_SNAKE_CASE_   ,"""additional_special_tokens"""    )   ,[]    )
              self.assertListEqual(getattr(SCREAMING_SNAKE_CASE_   ,"""additional_special_tokens_ids"""    )   ,[]    )

              setattr(SCREAMING_SNAKE_CASE_   ,"""additional_special_tokens_ids"""   ,[token_id_to_test_setters]    )
              self.assertListEqual(getattr(SCREAMING_SNAKE_CASE_   ,"""additional_special_tokens"""    )   ,[token_to_test_setters]    )
              self.assertListEqual(getattr(SCREAMING_SNAKE_CASE_   ,"""additional_special_tokens_ids"""    )   ,[token_id_to_test_setters]    )






import unittest

from transformers import load_tool

from .test_tools_common import ToolTesterMixin


lowercase_   							=  '\nHugging Face was founded in 2016 by French entrepreneurs Clément Delangue, Julien Chaumond, and Thomas Wolf originally as a company that developed a chatbot app targeted at teenagers.[2] After open-sourcing the model behind the chatbot, the company pivoted to focus on being a platform for machine learning.\n\nIn March 2021, Hugging Face raised $40 million in a Series B funding round.[3]\n\nOn April 28, 2021, the company launched the BigScience Research Workshop in collaboration with several other research groups to release an open large language model.[4] In 2022, the workshop concluded with the announcement of BLOOM, a multilingual large language model with 176 billion parameters.[5]\n'





class 					__lowerCAmelCase	(   unittest.TestCase     ,     SCREAMING_SNAKE_CASE				):




  def     A__	(							self   )    ->					Union[str, Any]:
        '''simple docstring'''






        _lowercase      			=load_tool('text-question-answering'   )
        self.tool.setup()
        _lowercase      			=load_tool('text-question-answering'				,  remote=lowerCAmelCase   )




  def     A__	(							self   )    ->					Dict:
        '''simple docstring'''






        _lowercase      			=self.tool(lowerCAmelCase				,  'What did Hugging Face do in April 2021?'   )
        self.assertEqual(lowerCAmelCase				,  'launched the BigScience Research Workshop'   )




  def     A__	(							self   )    ->					Optional[Any]:
        '''simple docstring'''






        _lowercase      			=self.remote_tool(lowerCAmelCase				,  'What did Hugging Face do in April 2021?'   )
        self.assertEqual(lowerCAmelCase				,  'launched the BigScience Research Workshop'   )




  def     A__	(							self   )    ->					Any:
        '''simple docstring'''






        _lowercase      			=self.tool(text=lowerCAmelCase				,  question='What did Hugging Face do in April 2021?'   )
        self.assertEqual(lowerCAmelCase				,  'launched the BigScience Research Workshop'   )




  def     A__	(							self   )    ->					Optional[int]:
        '''simple docstring'''






        _lowercase      			=self.remote_tool(text=lowerCAmelCase				,  question='What did Hugging Face do in April 2021?'   )
        self.assertEqual(lowerCAmelCase				,  'launched the BigScience Research Workshop'   )


