




'''simple docstring'''



import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
      UpperCamelCase_    	=				"pt"
elif is_tf_available():
      UpperCamelCase_    	=				"tf"
else:
      UpperCamelCase_    	=				"jax"






class 							_a				(			SCREAMING_SNAKE_CASE ,      unittest.TestCase ):





       '''simple docstring'''






       A						:				Optional[int]        =						ByTaTokenizer
       A						:				Tuple        =						False







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             super().setUp()
             SCREAMING_SNAKE_CASE   :      List[Any]    					=						ByTaTokenizer()
             tokenizer.save_pretrained(self.tmpdirname  )







       @cached_property
       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             return ByTaTokenizer.from_pretrained('google/byt5-small'  )







       def        UpperCamelCase_							(      self,							**A  ):







             '''simple docstring'''







             return self.tokenizer_class.from_pretrained(self.tmpdirname,							**A  )







       def        UpperCamelCase_							(      self,							A,							A=False,							A=20,							A=5  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						[]
             for i in range(len(A  )  ):
                   try:
                         SCREAMING_SNAKE_CASE   :      int    					=						tokenizer.decode([i],							clean_up_tokenization_spaces=A  )
                   except UnicodeDecodeError:
                         pass
                   toks.append((i, tok)  )

             SCREAMING_SNAKE_CASE   :      Tuple    					=						list(filter(lambda A  : re.match(r'^[ a-zA-Z]+$',							t[1]  ),							A  )  )
             SCREAMING_SNAKE_CASE   :      Any    					=						list(filter(lambda A  : [t[0]] == tokenizer.encode(t[1],							add_special_tokens=A  ),							A  )  )
             if max_length is not None and len(A  ) > max_length:
                   SCREAMING_SNAKE_CASE   :      List[Any]    					=						toks[:max_length]
             if min_length is not None and len(A  ) < min_length and len(A  ) > 0:
                   while len(A  ) < min_length:
                         SCREAMING_SNAKE_CASE   :      str    					=						toks + toks
        # toks_str = [t[1] for t in toks]
             SCREAMING_SNAKE_CASE   :      str    					=						[t[0] for t in toks]

             # Ensure consistency
             SCREAMING_SNAKE_CASE   :      Optional[int]    					=						tokenizer.decode(A,							clean_up_tokenization_spaces=A  )
             if " " not in output_txt and len(A  ) > 1:
                   SCREAMING_SNAKE_CASE   :      Optional[int]    					=						(
                       tokenizer.decode([toks_ids[0]],							clean_up_tokenization_spaces=A  )
                       + ' '
                       + tokenizer.decode(toks_ids[1:],							clean_up_tokenization_spaces=A  )
                   )
             if with_prefix_space:
                   SCREAMING_SNAKE_CASE   :      Dict    					=						' ' + output_txt
             SCREAMING_SNAKE_CASE   :      Any    					=						tokenizer.encode(A,							add_special_tokens=A  )
             return output_txt, output_ids







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      str    					=						self.ta_base_tokenizer
             SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						tokenizer(['hi</s>', 'I went to the gym</s>', '</s>']  )
             SCREAMING_SNAKE_CASE   :      str    					=						tokenizer(['hi', 'I went to the gym', '']  )
             self.assertListEqual(batch_with_eos_added['input_ids'],							batch_without_eos_added['input_ids']  )







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      Dict    					=						self.ta_base_tokenizer
             SCREAMING_SNAKE_CASE   :      Dict    					=						'Unicode €.'
             SCREAMING_SNAKE_CASE   :      Any    					=						tokenizer(A  )
             SCREAMING_SNAKE_CASE   :      List[Any]    					=						[88, 113, 108, 102, 114, 103, 104, 35, 229, 133, 175, 49, 1]
             self.assertEqual(encoded['input_ids'],							A  )

             # decoding
             SCREAMING_SNAKE_CASE   :      Dict    					=						tokenizer.decode(A  )
             self.assertEqual(A,							'Unicode €.</s>'  )

             SCREAMING_SNAKE_CASE   :      int    					=						tokenizer('e è é ê ë'  )
             SCREAMING_SNAKE_CASE   :      Optional[int]    					=						[104, 35, 198, 171, 35, 198, 172, 35, 198, 173, 35, 198, 174, 1]
             self.assertEqual(encoded['input_ids'],							A  )
             # decoding
             SCREAMING_SNAKE_CASE   :      str    					=						tokenizer.decode(A  )
             self.assertEqual(A,							'e è é ê ë</s>'  )

             # encode/decode, but with `encode` instead of `__call__`
             self.assertEqual(tokenizer.decode(tokenizer.encode('e è é ê ë'  )  ),							'e è é ê ë</s>'  )







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						self.ta_base_tokenizer
             SCREAMING_SNAKE_CASE   :      Tuple    					=						['A long paragraph for summarization.', 'Another paragraph for summarization.']
             # fmt: off
             SCREAMING_SNAKE_CASE   :      Tuple    					=						[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 1, 0]
             # fmt: on
             SCREAMING_SNAKE_CASE   :      Any    					=						tokenizer(A,							padding=A,							return_tensors=A  )
             self.assertIsInstance(A,							A  )

             if FRAMEWORK != "jax":
                   SCREAMING_SNAKE_CASE   :      List[str]    					=						list(batch.input_ids.numpy()[0]  )
             else:
                   SCREAMING_SNAKE_CASE   :      List[Any]    					=						list(batch.input_ids.tolist()[0]  )

             self.assertListEqual(A,							A  )

             self.assertEqual((2, 37),							batch.input_ids.shape  )
             self.assertEqual((2, 37),							batch.attention_mask.shape  )







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						self.ta_base_tokenizer
             SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						['A long paragraph for summarization.', 'Another paragraph for summarization.']
             SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						tokenizer(A,							padding=A,							return_tensors=A  )
             # check if input_ids are returned and no decoder_input_ids
             self.assertIn('input_ids',							A  )
             self.assertIn('attention_mask',							A  )
             self.assertNotIn('decoder_input_ids',							A  )
             self.assertNotIn('decoder_attention_mask',							A  )







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						self.ta_base_tokenizer
             SCREAMING_SNAKE_CASE   :      List[Any]    					=						[
                 'Summary of the text.',
                 'Another summary.',
             ]
             SCREAMING_SNAKE_CASE   :      str    					=						tokenizer(
                 text_target=A,							max_length=32,							padding='max_length',							truncation=A,							return_tensors=A  )
             self.assertEqual(32,							targets['input_ids'].shape[1]  )







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      List[Any]    					=						self.ta_base_tokenizer
             SCREAMING_SNAKE_CASE   :      List[str]    					=						['A long paragraph for summarization. </s>']
             SCREAMING_SNAKE_CASE   :      Tuple    					=						['Summary of the text. </s>']
             # fmt: off
             SCREAMING_SNAKE_CASE   :      Dict    					=						[68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 35, 1]
             SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						[86, 120, 112, 112, 100, 117, 124, 35, 114, 105, 35, 119, 107, 104, 35, 119, 104, 123, 119, 49, 35, 1]
             # fmt: on

             SCREAMING_SNAKE_CASE   :      int    					=						tokenizer(A,							text_target=A  )

             self.assertEqual(A,							batch['input_ids'][0]  )
             self.assertEqual(A,							batch['labels'][0]  )







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      Tuple    					=						self.get_tokenizers()
             for tokenizer in tokenizers:
                   with self.subTest(F"{tokenizer.__class__.__name__}"  ):
                         self.assertNotEqual(tokenizer.model_max_length,							42  )

        # Now let's start the test
             SCREAMING_SNAKE_CASE   :      Optional[int]    					=						self.get_tokenizers()
             for tokenizer in tokenizers:
                   with self.subTest(F"{tokenizer.__class__.__name__}"  ):
                         # Isolate this from the other tests because we save additional tokens/etc
                         SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						tempfile.mkdtemp()

                         SCREAMING_SNAKE_CASE   :      int    					=						' He is very happy, UNwant\u00E9d,running'
                         SCREAMING_SNAKE_CASE   :      Any    					=						tokenizer.encode(A,							add_special_tokens=A  )
                         tokenizer.save_pretrained(A  )

                         SCREAMING_SNAKE_CASE   :      Optional[int]    					=						tokenizer.__class__.from_pretrained(A  )
                         SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						after_tokenizer.encode(A,							add_special_tokens=A  )
                         self.assertListEqual(A,							A  )

                         shutil.rmtree(A  )

             SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						self.get_tokenizers(model_max_length=42  )
             for tokenizer in tokenizers:
                   with self.subTest(F"{tokenizer.__class__.__name__}"  ):
                         # Isolate this from the other tests because we save additional tokens/etc
                         SCREAMING_SNAKE_CASE   :      str    					=						tempfile.mkdtemp()

                         SCREAMING_SNAKE_CASE   :      int    					=						' He is very happy, UNwant\u00E9d,running'
                         tokenizer.add_tokens(['bim', 'bambam']  )
                         SCREAMING_SNAKE_CASE   :      Tuple    					=						tokenizer.additional_special_tokens
                         additional_special_tokens.append('new_additional_special_token'  )
                         tokenizer.add_special_tokens({'additional_special_tokens': additional_special_tokens}  )
                         SCREAMING_SNAKE_CASE   :      str    					=						tokenizer.encode(A,							add_special_tokens=A  )
                         tokenizer.save_pretrained(A  )

                         SCREAMING_SNAKE_CASE   :      Tuple    					=						tokenizer.__class__.from_pretrained(A  )
                         SCREAMING_SNAKE_CASE   :      List[Any]    					=						after_tokenizer.encode(A,							add_special_tokens=A  )
                         self.assertListEqual(A,							A  )
                         self.assertIn('new_additional_special_token',							after_tokenizer.additional_special_tokens  )
                         self.assertEqual(after_tokenizer.model_max_length,							42  )

                         SCREAMING_SNAKE_CASE   :      Dict    					=						tokenizer.__class__.from_pretrained(A,							model_max_length=43  )
                         self.assertEqual(tokenizer.model_max_length,							43  )

                         shutil.rmtree(A  )







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      Optional[int]    					=						[]
             if self.test_slow_tokenizer:
                   tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())  )

             if self.test_rust_tokenizer:
                   tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())  )

             for tokenizer_class, tokenizer_utils in tokenizer_list:
                   with tempfile.TemporaryDirectory() as tmp_dir:
                         tokenizer_utils.save_pretrained(A  )

                         with open(os.path.join(A,							'special_tokens_map.json'  ),							encoding='utf-8'  ) as json_file:
                               SCREAMING_SNAKE_CASE   :      List[Any]    					=						json.load(A  )

                         with open(os.path.join(A,							'tokenizer_config.json'  ),							encoding='utf-8'  ) as json_file:
                               SCREAMING_SNAKE_CASE   :      Any    					=						json.load(A  )

                         SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						[F"<extra_id_{i}>" for i in range(125  )]

                         SCREAMING_SNAKE_CASE   :      List[Any]    					=						added_tokens_extra_ids + [
                             'an_additional_special_token'
                         ]
                         SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						added_tokens_extra_ids + [
                             'an_additional_special_token'
                         ]

                         with open(os.path.join(A,							'special_tokens_map.json'  ),							'w',							encoding='utf-8'  ) as outfile:
                               json.dump(A,							A  )
                         with open(os.path.join(A,							'tokenizer_config.json'  ),							'w',							encoding='utf-8'  ) as outfile:
                               json.dump(A,							A  )

                         # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
                         # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
                         # "special_tokens_map.json" files
                         SCREAMING_SNAKE_CASE   :      Dict    					=						tokenizer_class.from_pretrained(
                             A,							)
                         self.assertIn(
                             'an_additional_special_token',							tokenizer_without_change_in_init.additional_special_tokens  )
                         # self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
                         self.assertEqual(
                             ['an_additional_special_token'],							tokenizer_without_change_in_init.convert_ids_to_tokens(
                                 tokenizer_without_change_in_init.convert_tokens_to_ids(['an_additional_special_token']  )  ),							)

                         # Now we test that we can change the value of additional_special_tokens in the from_pretrained
                         SCREAMING_SNAKE_CASE   :      Any    					=						added_tokens_extra_ids + [AddedToken('a_new_additional_special_token',							lstrip=A  )]
                         SCREAMING_SNAKE_CASE   :      Optional[int]    					=						tokenizer_class.from_pretrained(
                             A,							additional_special_tokens=A,							)

                         self.assertIn('a_new_additional_special_token',							tokenizer.additional_special_tokens  )
                         self.assertEqual(
                             ['a_new_additional_special_token'],							tokenizer.convert_ids_to_tokens(
                                 tokenizer.convert_tokens_to_ids(['a_new_additional_special_token']  )  ),							)







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      int    					=						[]
             if self.test_slow_tokenizer:
                   tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())  )

             if self.test_rust_tokenizer:
                   tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())  )

             for tokenizer_class, tokenizer_utils in tokenizer_list:
                   with tempfile.TemporaryDirectory() as tmp_dir:
                         tokenizer_utils.save_pretrained(A  )

                         SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						tokenizer_class.from_pretrained(A  )

                         self.assertTrue(tokenizer.decode([255]  ) == ''  )







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             pass







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             pass







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             pass







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             pass







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      Tuple    					=						self.get_tokenizers(fast=A,							do_lower_case=A  )
             for tokenizer in tokenizers:
                   with self.subTest(F"{tokenizer.__class__.__name__}"  ):
                         SCREAMING_SNAKE_CASE   :      Optional[Any]    					=						['t', 'h', 'i', 's', ' ', 'i', 's', ' ', 'a', ' ', 't', 'e', 'x', 't', '</s>']
                         SCREAMING_SNAKE_CASE   :      List[str]    					=						tokenizer.convert_tokens_to_string(A  )

                         self.assertIsInstance(A,							A  )







       def        UpperCamelCase_							(      self  ):







             '''simple docstring'''







             SCREAMING_SNAKE_CASE   :      Tuple    					=						self.get_tokenizers()
             for tokenizer in tokenizers:
                   with self.subTest(F"{tokenizer.__class__.__name__}"  ):
                         SCREAMING_SNAKE_CASE   :      Union[str, Any]    					=						[
                             'bos_token',
                             'eos_token',
                             'unk_token',
                             'sep_token',
                             'pad_token',
                             'cls_token',
                             'mask_token',
                         ]

                         SCREAMING_SNAKE_CASE   :      Any    					=						0
                         SCREAMING_SNAKE_CASE   :      Tuple    					=						tokenizer.convert_ids_to_tokens(
                             A,							skip_special_tokens=A  )

                         for attr in attributes_list:
                               setattr(A,							attr + '_id',							A  )
                               self.assertEqual(getattr(A,							A  ),							A  )
                               self.assertEqual(getattr(A,							attr + '_id'  ),							A  )

                               setattr(A,							attr + '_id',							A  )
                               self.assertEqual(getattr(A,							A  ),							A  )
                               self.assertEqual(getattr(A,							attr + '_id'  ),							A  )

                         setattr(A,							'additional_special_tokens_ids',							[]  )
                         self.assertListEqual(getattr(A,							'additional_special_tokens'  ),							[]  )
                         self.assertListEqual(getattr(A,							'additional_special_tokens_ids'  ),							[]  )

                         setattr(A,							'additional_special_tokens_ids',							[token_id_to_test_setters]  )
                         self.assertListEqual(getattr(A,							'additional_special_tokens'  ),							[token_to_test_setters]  )
                         self.assertListEqual(getattr(A,							'additional_special_tokens_ids'  ),							[token_id_to_test_setters]  )






'''simple docstring'''






from typing import List, Union

from ..utils import (
    add_end_docstrings,
    is_tf_available,
    is_torch_available,
    is_vision_available,
    logging,
    requires_backends,
)
from .base import PIPELINE_INIT_ARGS, Pipeline


if is_vision_available():
 from PIL import Image

 from ..image_utils import load_image

if is_tf_available():
 import tensorflow as tf

 from ..models.auto.modeling_tf_auto import TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING
 from ..tf_utils import stable_softmax

if is_torch_available():
 from ..models.auto.modeling_auto import MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING

lowercase				=logging.get_logger(__name__)






@add_end_docstrings(lowerCAmelCase							)
class    __magic_name__				(						lowerCAmelCase							):






       def __init__(     self    ,				*snake_case    ,				**snake_case)		->    Dict:




          '''simple docstring'''
          super().__init__(*snake_case    ,				**snake_case)
          requires_backends(self    ,				'vision')
          self.check_model_type(
              TF_MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING
              if self.framework == 'tf'
              else MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING)






       def      lowerCAmelCase						(     self    ,				snake_case=None)		->    str:




          '''simple docstring'''
          _UpperCAmelCase  :  Any				       ={}
          if top_k is not None:
             _UpperCAmelCase  :  Optional[int]				       =top_k
          return {}, {}, postprocess_params






       def __call__(     self    ,				snake_case    ,				**snake_case)		->    List[str]:




          '''simple docstring'''
          return super().__call__(snake_case    ,				**snake_case)






       def      lowerCAmelCase						(     self    ,				snake_case)		->    List[str]:




          '''simple docstring'''
          _UpperCAmelCase  :  int				       =load_image(snake_case)
          _UpperCAmelCase  :  Tuple				       =self.image_processor(images=snake_case    ,				return_tensors=self.framework)
          return model_inputs






       def      lowerCAmelCase						(     self    ,				snake_case)		->    Optional[Any]:




          '''simple docstring'''
          _UpperCAmelCase  :  List[str]				       =self.model(**snake_case)
          return model_outputs






       def      lowerCAmelCase						(     self    ,				snake_case    ,				snake_case=5)		->    Any:




          '''simple docstring'''
          if top_k > self.model.config.num_labels:
             _UpperCAmelCase  :  Optional[Any]				       =self.model.config.num_labels

          if self.framework == "pt":
             _UpperCAmelCase  :  List[Any]				       =model_outputs.logits.softmax(-1)[0]
             _UpperCAmelCase       ,			_UpperCAmelCase  :  Optional[Any]				       =probs.topk(snake_case)
          elif self.framework == "tf":
             _UpperCAmelCase  :  int				       =stable_softmax(model_outputs.logits    ,				axis=-1)[0]
             _UpperCAmelCase  :  List[Any]				       =tf.math.top_k(snake_case    ,				k=snake_case)
             _UpperCAmelCase       ,			_UpperCAmelCase  :  List[str]				       =topk.values.numpy(), topk.indices.numpy()
          else:
             raise ValueError(f"Unsupported framework: {self.framework}")

          _UpperCAmelCase  :  int				       =scores.tolist()
          _UpperCAmelCase  :  Tuple				       =ids.tolist()
          return [{"score": score, "label": self.model.config.idalabel[_id]} for score, _id in zip(snake_case    ,				snake_case)]

