
'''simple docstring'''


import re
import warnings
from contextlib import contextmanager

from ...processing_utils import ProcessorMixin


class 						lowercase__	(  snake_case_   ):






   '''simple docstring'''







   _snake_case		   =							['''image_processor''', '''tokenizer''']
   _snake_case		   =							'''AutoImageProcessor'''
   _snake_case		   =							'''AutoTokenizer'''







   def __init__(			self ,   lowerCamelCase__=None ,   lowerCamelCase__=None ,   **lowerCamelCase__   ):

      '''simple docstring'''







      UpperCamelCase       =  None
      if "feature_extractor" in kwargs:
         warnings.warn(
             '''The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`'''
             ''' instead.''' ,   lowerCamelCase__ ,   )
         UpperCamelCase       =  kwargs.pop('''feature_extractor'''   )

      UpperCamelCase       =  image_processor if image_processor is not None else feature_extractor
      if image_processor is None:
         raise ValueError('''You need to specify an `image_processor`.'''   )
      if tokenizer is None:
         raise ValueError('''You need to specify a `tokenizer`.'''   )

      super().__init__(lowerCamelCase__ ,   lowerCamelCase__   )
      UpperCamelCase       =  self.image_processor
      UpperCamelCase       =  False







   def __call__(			self ,   *lowerCamelCase__ ,   **lowerCamelCase__   ):

      '''simple docstring'''







      if self._in_target_context_manager:
         return self.current_processor(*lowerCamelCase__ ,   **lowerCamelCase__   )

      UpperCamelCase       =  kwargs.pop('''images''' ,   lowerCamelCase__   )
      UpperCamelCase       =  kwargs.pop('''text''' ,   lowerCamelCase__   )
      if len(lowerCamelCase__   ) > 0:
         UpperCamelCase       =  args[0]
         UpperCamelCase       =  args[1:]

      if images is None and text is None:
         raise ValueError('''You need to specify either an `images` or `text` input to process.'''   )

      if images is not None:
         UpperCamelCase       =  self.image_processor(lowerCamelCase__ ,   *lowerCamelCase__ ,   **lowerCamelCase__   )
      if text is not None:
         UpperCamelCase       =  self.tokenizer(lowerCamelCase__ ,   **lowerCamelCase__   )

      if text is None:
         return inputs
      elif images is None:
         return encodings
      else:
         UpperCamelCase       =  encodings['''input_ids''']
         return inputs







   def       UpperCAmelCase				(			self ,   *lowerCamelCase__ ,   **lowerCamelCase__   ):

      '''simple docstring'''







      return self.tokenizer.batch_decode(*lowerCamelCase__ ,   **lowerCamelCase__   )







   def       UpperCAmelCase				(			self ,   *lowerCamelCase__ ,   **lowerCamelCase__   ):

      '''simple docstring'''







      return self.tokenizer.decode(*lowerCamelCase__ ,   **lowerCamelCase__   )







   @contextmanager
   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      warnings.warn(
          '''`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your '''
          '''labels by using the argument `text` of the regular `__call__` method (either in the same call as '''
          '''your images inputs, or in a separate call.'''   )
      UpperCamelCase       =  True
      UpperCamelCase       =  self.tokenizer
      yield
      UpperCamelCase       =  self.image_processor
      UpperCamelCase       =  False







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__=False ,   lowerCamelCase__=None   ):

      '''simple docstring'''







      if added_vocab is None:
         UpperCamelCase       =  self.tokenizer.get_added_vocab()

      UpperCamelCase       =  {}

      while tokens:
         UpperCamelCase       =  re.search(R'''<s_(.*?)>''' ,   lowerCamelCase__ ,   re.IGNORECASE   )
         if start_token is None:
            break
         UpperCamelCase       =  start_token.group(1   )
         UpperCamelCase       =  re.search(Rf'</s_{key}>' ,   lowerCamelCase__ ,   re.IGNORECASE   )
         UpperCamelCase       =  start_token.group()
         if end_token is None:
            UpperCamelCase       =  tokens.replace(lowerCamelCase__ ,   ''''''   )
         else:
            UpperCamelCase       =  end_token.group()
            UpperCamelCase       =  re.escape(lowerCamelCase__   )
            UpperCamelCase       =  re.escape(lowerCamelCase__   )
            UpperCamelCase       =  re.search(f'{start_token_escaped}(.*?){end_token_escaped}' ,   lowerCamelCase__ ,   re.IGNORECASE   )
            if content is not None:
               UpperCamelCase       =  content.group(1   ).strip()
               if r"<s_" in content and r"</s_" in content:  # non-leaf node
                  UpperCamelCase       =  self.tokenajson(lowerCamelCase__ ,   is_inner_value=lowerCamelCase__ ,   added_vocab=lowerCamelCase__   )
                  if value:
                     if len(lowerCamelCase__   ) == 1:
                        UpperCamelCase       =  value[0]
                     UpperCamelCase       =  value
               else:  # leaf nodes
                  UpperCamelCase       =  []
                  for leaf in content.split(R'''<sep/>'''   ):
                     UpperCamelCase       =  leaf.strip()
                     if leaf in added_vocab and leaf[0] == "<" and leaf[-2:] == "/>":
                        UpperCamelCase       =  leaf[1:-2]  # for categorical special tokens
                     output[key].append(lowerCamelCase__   )
                  if len(output[key]   ) == 1:
                     UpperCamelCase       =  output[key][0]

            UpperCamelCase       =  tokens[tokens.find(lowerCamelCase__   ) + len(lowerCamelCase__   ) :].strip()
            if tokens[:6] == r"<sep/>":  # non-leaf nodes
               return [output] + self.tokenajson(tokens[6:] ,   is_inner_value=lowerCamelCase__ ,   added_vocab=lowerCamelCase__   )

      if len(lowerCamelCase__   ):
         return [output] if is_inner_value else output
      else:
         return [] if is_inner_value else {"text_sequence": tokens}







   @property
   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      warnings.warn(
          '''`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead.''' ,   lowerCamelCase__ ,   )
      return self.image_processor_class







   @property
   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      warnings.warn(
          '''`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead.''' ,   lowerCamelCase__ ,   )
      return self.image_processor







'''simple docstring'''


import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
   from .tokenization_pegasus import PegasusTokenizer
else:
   snake_case_ :  Dict     =					None


snake_case_ :  int     =					logging.get_logger(__name__)


snake_case_ :  int     =					'▁'

snake_case_ :  Optional[int]     =					{'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

snake_case_ :  List[Any]     =					{
    'vocab_file': {'google/pegasus-xsum': 'https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model'},
    'tokenizer_file': {
        'google/pegasus-xsum': 'https://huggingface.co/google/pegasus-xsum/resolve/main/tokenizer.json'
    },
}

snake_case_ :  Tuple     =					{
    'google/pegasus-xsum': 512,
}


class 						lowercase__	(  snake_case_   ):






   '''simple docstring'''







   _snake_case		   =							VOCAB_FILES_NAMES
   _snake_case		   =							PRETRAINED_VOCAB_FILES_MAP
   _snake_case		   =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _snake_case		   =							PegasusTokenizer
   _snake_case		   =							['''input_ids''', '''attention_mask''']







   def __init__(			self ,   lowerCamelCase__=None ,   lowerCamelCase__=None ,   lowerCamelCase__="<pad>" ,   lowerCamelCase__="</s>" ,   lowerCamelCase__="<unk>" ,   lowerCamelCase__="<mask_2>" ,   lowerCamelCase__="<mask_1>" ,   lowerCamelCase__=None ,   lowerCamelCase__=1_0_3 ,   **lowerCamelCase__ ,   ):

      '''simple docstring'''







      UpperCamelCase       =  offset

      if additional_special_tokens is not None:
         if not isinstance(lowerCamelCase__ ,   lowerCamelCase__   ):
            raise TypeError(
                f'additional_special_tokens should be of type {type(lowerCamelCase__   )}, but is'
                f' {type(lowerCamelCase__   )}'   )

         UpperCamelCase       =  (
             ([mask_token_sent] + additional_special_tokens)
             if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
             else additional_special_tokens
         )
         # fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
         additional_special_tokens_extended += [
             f'<unk_{i}>' for i in range(len(lowerCamelCase__   ) ,   self.offset - 1   )
         ]

         if len(set(lowerCamelCase__   )   ) != len(lowerCamelCase__   ):
            raise ValueError(
                '''Please make sure that the provided additional_special_tokens do not contain an incorrectly'''
                f' shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}.'   )
         UpperCamelCase       =  additional_special_tokens_extended
      else:
         UpperCamelCase       =  [mask_token_sent] if mask_token_sent is not None else []
         additional_special_tokens += [f'<unk_{i}>' for i in range(2 ,   self.offset   )]

      super().__init__(
          lowerCamelCase__ ,   tokenizer_file=lowerCamelCase__ ,   pad_token=lowerCamelCase__ ,   eos_token=lowerCamelCase__ ,   unk_token=lowerCamelCase__ ,   mask_token=lowerCamelCase__ ,   mask_token_sent=lowerCamelCase__ ,   offset=lowerCamelCase__ ,   additional_special_tokens=lowerCamelCase__ ,   **lowerCamelCase__ ,   )
      UpperCamelCase       =  vocab_file
      UpperCamelCase       =  False if not self.vocab_file else True







   def       UpperCAmelCase				(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      UpperCamelCase       =  set(self.all_special_ids   )  # call it once instead of inside list comp
      all_special_ids.remove(self.unk_token_id   )  # <unk> is only sometimes special

      if all_special_ids != set(range(len(self.additional_special_tokens   ) + 3   )   ):
         raise ValueError(
             '''There should be 3 special tokens: mask_token, pad_token, and eos_token +'''
             f' {len(self.additional_special_tokens   )} additional_special_tokens, but got {all_special_ids}'   )

      return [1 if x in all_special_ids else 0 for x in seq]







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__ = None ,   lowerCamelCase__ = False   ):

      '''simple docstring'''







      if already_has_special_tokens:
         return self._special_token_mask(lowerCamelCase__   )
      elif token_ids_a is None:
         return self._special_token_mask(lowerCamelCase__   ) + [1]
      else:
         return self._special_token_mask(token_ids_a + token_ids_a   ) + [1]







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__=None   ):

      '''simple docstring'''







      if token_ids_a is None:
         return token_ids_a + [self.eos_token_id]
      # We don't expect to process pairs, but leave the pair logic for API consistency
      return token_ids_a + token_ids_a + [self.eos_token_id]







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__ = None   ):

      '''simple docstring'''







      if not self.can_save_slow_tokenizer:
         raise ValueError(
             '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
             '''tokenizer.'''   )

      if not os.path.isdir(lowerCamelCase__   ):
         logger.error(f'Vocabulary path ({save_directory}) should be a directory'   )
         return
      UpperCamelCase       =  os.path.join(
          lowerCamelCase__ ,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']   )

      if os.path.abspath(self.vocab_file   ) != os.path.abspath(lowerCamelCase__   ):
         copyfile(self.vocab_file ,   lowerCamelCase__   )

      return (out_vocab_file,)





