






"""simple docstring"""







import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_UpperCamelCase	     =    logging.get_logger(__name__)

_UpperCamelCase	     =    {
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

_UpperCamelCase	     =    {
    """vocab_file""": {"""ctrl""": """https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json"""},
    """merges_file""": {"""ctrl""": """https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt"""},
}

_UpperCamelCase	     =    {
    """ctrl""": 256,
}

_UpperCamelCase	     =    {
    """Pregnancy""": 168629,
    """Christianity""": 7675,
    """Explain""": 106423,
    """Fitness""": 63440,
    """Saving""": 63163,
    """Ask""": 27171,
    """Ass""": 95985,
    """Joke""": 163509,
    """Questions""": 45622,
    """Thoughts""": 49605,
    """Retail""": 52342,
    """Feminism""": 164338,
    """Writing""": 11992,
    """Atheism""": 192263,
    """Netflix""": 48616,
    """Computing""": 39639,
    """Opinion""": 43213,
    """Alone""": 44967,
    """Funny""": 58917,
    """Gaming""": 40358,
    """Human""": 4088,
    """India""": 1331,
    """Joker""": 77138,
    """Diet""": 36206,
    """Legal""": 11859,
    """Norman""": 4939,
    """Tip""": 72689,
    """Weight""": 52343,
    """Movies""": 46273,
    """Running""": 23425,
    """Science""": 2090,
    """Horror""": 37793,
    """Confession""": 60572,
    """Finance""": 12250,
    """Politics""": 16360,
    """Scary""": 191985,
    """Support""": 12654,
    """Technologies""": 32516,
    """Teenage""": 66160,
    """Event""": 32769,
    """Learned""": 67460,
    """Notion""": 182770,
    """Wikipedia""": 37583,
    """Books""": 6665,
    """Extract""": 76050,
    """Confessions""": 102701,
    """Conspiracy""": 75932,
    """Links""": 63674,
    """Narcissus""": 150425,
    """Relationship""": 54766,
    """Relationships""": 134796,
    """Reviews""": 41671,
    """News""": 4256,
    """Translation""": 26820,
    """multilingual""": 128406,
}

def _a					(						_snake_case				):







      """simple docstring"""






      UpperCAmelCase							     =			set()
      UpperCAmelCase							     =			word[0]
      for char in word[1:]:
            pairs.add((prev_char, char)				)
            UpperCAmelCase							     =			char

      UpperCAmelCase							     =			set(_A				)
      return pairs







class    lowerCamelCase__   (			lowerCAmelCase__					):
    SCREAMING_SNAKE_CASE						=     VOCAB_FILES_NAMES
    SCREAMING_SNAKE_CASE						=     PRETRAINED_VOCAB_FILES_MAP
    SCREAMING_SNAKE_CASE						=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    SCREAMING_SNAKE_CASE						=     CONTROL_CODES






    def __init__(					self      ,A      ,A      ,A="<unk>"      ,**A   ):
          super().__init__(unk_token=__lowerCAmelCase      ,**__lowerCAmelCase   )

          with open(__lowerCAmelCase      ,encoding="""utf-8"""   ) as vocab_handle:
                UpperCAmelCase							     =			json.load(__lowerCAmelCase   )
          UpperCAmelCase							     =			{v: k for k, v in self.encoder.items()}
          with open(__lowerCAmelCase      ,encoding="""utf-8"""   ) as merges_handle:
                UpperCAmelCase							     =			merges_handle.read().split("""\n"""   )[1:-1]
          UpperCAmelCase							     =			[tuple(merge.split()   ) for merge in merges]
          UpperCAmelCase							     =			dict(zip(__lowerCAmelCase      ,range(len(__lowerCAmelCase   )   )   )   )
          UpperCAmelCase							     =			{}






    @property
    def 							_UpperCamelCase				(					self   ):
          return len(self.encoder   )






    def 							_UpperCamelCase				(					self   ):
          return dict(self.encoder      ,**self.added_tokens_encoder   )






    def 							_UpperCamelCase				(					self      ,A   ):


          if token in self.cache:
                return self.cache[token]
          UpperCAmelCase							     =			tuple(__lowerCAmelCase   )
          UpperCAmelCase							     =			tuple(list(word[:-1]   ) + [word[-1] + """</w>"""]   )
          UpperCAmelCase							     =			get_pairs(__lowerCAmelCase   )

          if not pairs:
                return token

          while True:
                UpperCAmelCase							     =			min(__lowerCAmelCase      ,key=lambda A   : self.bpe_ranks.get(__lowerCAmelCase      ,float("""inf"""   )   )   )
                if bigram not in self.bpe_ranks:
                      break
                UpperCAmelCase     ,   UpperCAmelCase							     =			bigram
                UpperCAmelCase							     =			[]
                UpperCAmelCase							     =			0
                while i < len(__lowerCAmelCase   ):
                      try:
                            UpperCAmelCase							     =			word.index(__lowerCAmelCase      ,__lowerCAmelCase   )
                      except ValueError:
                            new_word.extend(word[i:]   )
                            break
                      else:
                            new_word.extend(word[i:j]   )
                            UpperCAmelCase							     =			j

                      if word[i] == first and i < len(__lowerCAmelCase   ) - 1 and word[i + 1] == second:
                            new_word.append(first + second   )
                            i += 2
                      else:
                            new_word.append(word[i]   )
                            i += 1
                UpperCAmelCase							     =			tuple(__lowerCAmelCase   )
                UpperCAmelCase							     =			new_word
                if len(__lowerCAmelCase   ) == 1:
                      break
                else:
                      UpperCAmelCase							     =			get_pairs(__lowerCAmelCase   )
          UpperCAmelCase							     =			"""@@ """.join(__lowerCAmelCase   )
          UpperCAmelCase							     =			word[:-4]
          UpperCAmelCase							     =			word
          return word






    def 							_UpperCamelCase				(					self      ,A   ):
          UpperCAmelCase							     =			[]

          UpperCAmelCase							     =			re.findall(r"""\S+\n?"""      ,__lowerCAmelCase   )

          for token in words:
                split_tokens.extend(list(self.bpe(__lowerCAmelCase   ).split(""" """   )   )   )
          return split_tokens






    def 							_UpperCamelCase				(					self      ,A   ):
          return self.encoder.get(__lowerCAmelCase      ,self.encoder.get(self.unk_token   )   )






    def 							_UpperCamelCase				(					self      ,A   ):
          return self.decoder.get(__lowerCAmelCase      ,self.unk_token   )






    def 							_UpperCamelCase				(					self      ,A   ):
          UpperCAmelCase							     =			""" """.join(__lowerCAmelCase   ).replace("""@@ """      ,""""""   ).strip()
          return out_string






    def 							_UpperCamelCase				(					self      ,A      ,A = None   ):


          if not os.path.isdir(__lowerCAmelCase   ):
                logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''   )
                return
          UpperCAmelCase							     =			os.path.join(
              __lowerCAmelCase      ,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]   )
          UpperCAmelCase							     =			os.path.join(
              __lowerCAmelCase      ,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]   )

          with open(__lowerCAmelCase      ,"""w"""      ,encoding="""utf-8"""   ) as f:
                f.write(json.dumps(self.encoder      ,indent=2      ,sort_keys=__lowerCAmelCase      ,ensure_ascii=__lowerCAmelCase   ) + """\n"""   )

          UpperCAmelCase							     =			0
          with open(__lowerCAmelCase      ,"""w"""      ,encoding="""utf-8"""   ) as writer:
                writer.write("""#version: 0.2\n"""   )
                for bpe_tokens, token_index in sorted(self.bpe_ranks.items()      ,key=lambda A   : kv[1]   ):
                      if index != token_index:
                            logger.warning(
                                F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                                """ Please check that the tokenizer is not corrupted!"""   )
                            UpperCAmelCase							     =			token_index
                      writer.write(""" """.join(__lowerCAmelCase   ) + """\n"""   )
                      index += 1

          return vocab_file, merge_file

    # def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
    #     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
    #     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
    #     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
    #     return ''.join(tokens_generated_so_far)




'''simple docstring'''




import warnings
from typing import List, Optional, Union

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
from ...utils import TensorType






class 							a__(   lowerCAmelCase__      ):


			'''simple docstring'''






			UpperCAmelCase_     :	str						=			['''image_processor''', '''tokenizer''']
			UpperCAmelCase_     :	str						=			'''ViltImageProcessor'''
			UpperCAmelCase_     :	Any						=			('''BertTokenizer''', '''BertTokenizerFast''')



			def __init__(  self			,       __lowerCAmelCase=None			,       __lowerCAmelCase=None			,       **__lowerCAmelCase):




					"""simple docstring"""


					lowerCAmelCase     =    None
					if "feature_extractor" in kwargs:
							warnings.warn(
							    """The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`"""
							    """ instead."""			,       __lowerCAmelCase			,       )
							lowerCAmelCase     =    kwargs.pop("""feature_extractor""")

					lowerCAmelCase     =    image_processor if image_processor is not None else feature_extractor
					if image_processor is None:
							raise ValueError("""You need to specify an `image_processor`.""")
					if tokenizer is None:
							raise ValueError("""You need to specify a `tokenizer`.""")

					super().__init__(__lowerCAmelCase			,       __lowerCAmelCase)
					lowerCAmelCase     =    self.image_processor



			def __call__(  self			,       __lowerCAmelCase			,       __lowerCAmelCase = None			,       __lowerCAmelCase = True			,       __lowerCAmelCase = False			,       __lowerCAmelCase = None			,       __lowerCAmelCase = None			,       __lowerCAmelCase = 0			,       __lowerCAmelCase = None			,       __lowerCAmelCase = None			,       __lowerCAmelCase = None			,       __lowerCAmelCase = False			,       __lowerCAmelCase = False			,       __lowerCAmelCase = False			,       __lowerCAmelCase = False			,       __lowerCAmelCase = True			,       __lowerCAmelCase = None			,       **__lowerCAmelCase			,       ):




					"""simple docstring"""


					lowerCAmelCase     =    self.tokenizer(
					    text=__lowerCAmelCase			,       add_special_tokens=__lowerCAmelCase			,       padding=__lowerCAmelCase			,       truncation=__lowerCAmelCase			,       max_length=__lowerCAmelCase			,       stride=__lowerCAmelCase			,       pad_to_multiple_of=__lowerCAmelCase			,       return_token_type_ids=__lowerCAmelCase			,       return_attention_mask=__lowerCAmelCase			,       return_overflowing_tokens=__lowerCAmelCase			,       return_special_tokens_mask=__lowerCAmelCase			,       return_offsets_mapping=__lowerCAmelCase			,       return_length=__lowerCAmelCase			,       verbose=__lowerCAmelCase			,       return_tensors=__lowerCAmelCase			,       **__lowerCAmelCase			,       )
					# add pixel_values + pixel_mask
					lowerCAmelCase     =    self.image_processor(__lowerCAmelCase			,       return_tensors=__lowerCAmelCase)
					encoding.update(__lowerCAmelCase)

					return encoding



			def 				a_			(  self			,       *__lowerCAmelCase			,       **__lowerCAmelCase):




					"""simple docstring"""


					return self.tokenizer.batch_decode(*__lowerCAmelCase			,       **__lowerCAmelCase)



			def 				a_			(  self			,       *__lowerCAmelCase			,       **__lowerCAmelCase):




					"""simple docstring"""


					return self.tokenizer.decode(*__lowerCAmelCase			,       **__lowerCAmelCase)



			@property
			def 				a_			(  self):




					"""simple docstring"""


					lowerCAmelCase     =    self.tokenizer.model_input_names
					lowerCAmelCase     =    self.image_processor.model_input_names
					return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names))



			@property
			def 				a_			(  self):




					"""simple docstring"""


					warnings.warn(
					    """`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead."""			,       __lowerCAmelCase			,       )
					return self.image_processor_class







			@property
			def 				a_			(  self):




					"""simple docstring"""


					warnings.warn(
					    """`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead."""			,       __lowerCAmelCase			,       )
					return self.image_processor




