


from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
    is_vision_available,
)


a : str		=				{
    '''configuration_owlvit''': [
        '''OWLVIT_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''OwlViTConfig''',
        '''OwlViTOnnxConfig''',
        '''OwlViTTextConfig''',
        '''OwlViTVisionConfig''',
    ],
    '''processing_owlvit''': ['''OwlViTProcessor'''],
}


try:
	if not is_vision_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	a : Optional[int]		=				['''OwlViTFeatureExtractor''']
	a : Optional[int]		=				['''OwlViTImageProcessor''']

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	a : Dict		=				[
	    '''OWLVIT_PRETRAINED_MODEL_ARCHIVE_LIST''',
	    '''OwlViTModel''',
	    '''OwlViTPreTrainedModel''',
	    '''OwlViTTextModel''',
	    '''OwlViTVisionModel''',
	    '''OwlViTForObjectDetection''',
	]

if TYPE_CHECKING:
	from .configuration_owlvit import (
	    OWLVIT_PRETRAINED_CONFIG_ARCHIVE_MAP,
	    OwlViTConfig,
	    OwlViTOnnxConfig,
	    OwlViTTextConfig,
	    OwlViTVisionConfig,
	)
	from .processing_owlvit import OwlViTProcessor

	try:
		if not is_vision_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .feature_extraction_owlvit import OwlViTFeatureExtractor
		from .image_processing_owlvit import OwlViTImageProcessor

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_owlvit import (
		    OWLVIT_PRETRAINED_MODEL_ARCHIVE_LIST,
		    OwlViTForObjectDetection,
		    OwlViTModel,
		    OwlViTPreTrainedModel,
		    OwlViTTextModel,
		    OwlViTVisionModel,
		)

else:
	import sys

	a : Optional[Any]		=				_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)






import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


a : Union[str, Any]		=				logging.get_logger(__name__)


a : Union[str, Any]		=				{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt'''}

a : Union[str, Any]		=				{
    '''vocab_file''': {
        '''allenai/longformer-base-4096''': '''https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json''',
        '''allenai/longformer-large-4096''': (
            '''https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json'''
        ),
        '''allenai/longformer-large-4096-finetuned-triviaqa''': (
            '''https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json'''
        ),
        '''allenai/longformer-base-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json'''
        ),
        '''allenai/longformer-large-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json'''
        ),
    },
    '''merges_file''': {
        '''allenai/longformer-base-4096''': '''https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt''',
        '''allenai/longformer-large-4096''': (
            '''https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt'''
        ),
        '''allenai/longformer-large-4096-finetuned-triviaqa''': (
            '''https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt'''
        ),
        '''allenai/longformer-base-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt'''
        ),
        '''allenai/longformer-large-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt'''
        ),
    },
}

a : Dict		=				{
    '''allenai/longformer-base-4096''': 4096,
    '''allenai/longformer-large-4096''': 4096,
    '''allenai/longformer-large-4096-finetuned-triviaqa''': 4096,
    '''allenai/longformer-base-4096-extra.pos.embd.only''': 4096,
    '''allenai/longformer-large-4096-extra.pos.embd.only''': 4096,
}


@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def 		lowercase_  (     ):


			'''simple docstring'''







			__lowercase 					=   (
			    list(range(ord('''!'''  )       ,			ord('''~'''  ) + 1  )  ) + list(range(ord('''¡'''  )       ,			ord('''¬'''  ) + 1  )  ) + list(range(ord('''®'''  )       ,			ord('''ÿ'''  ) + 1  )  )
			)
			__lowercase 					=   bs[:]
			__lowercase 					=   0
			for b in range(2**8  ):
						if b not in bs:
									bs.append(_UpperCamelCase  )
									cs.append(2**8 + n  )
									n += 1
			__lowercase 					=   [chr(_UpperCamelCase  ) for n in cs]
			return dict(zip(_UpperCamelCase       ,			_UpperCamelCase  )  )


def 		lowercase_  (     _UpperCamelCase  ):


			'''simple docstring'''







			__lowercase 					=   set()
			__lowercase 					=   word[0]
			for char in word[1:]:
						pairs.add((prev_char, char)  )
						__lowercase 					=   char
			return pairs





class 	lowerCamelCase_       (				lowerCAmelCase__				):



					'''simple docstring'''







					__UpperCAmelCase            =				VOCAB_FILES_NAMES
					__UpperCAmelCase            =				PRETRAINED_VOCAB_FILES_MAP
					__UpperCAmelCase            =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					__UpperCAmelCase            =				["input_ids", "attention_mask"]






					def __init__(				self ,       snake_case_ ,       snake_case_ ,       snake_case_="replace" ,       snake_case_="<s>" ,       snake_case_="</s>" ,       snake_case_="</s>" ,       snake_case_="<s>" ,       snake_case_="<unk>" ,       snake_case_="<pad>" ,       snake_case_="<mask>" ,       snake_case_=False ,       **snake_case_ ,       )   ->			Optional[int]:







								'''simple docstring'''

								__lowercase 					=   AddedToken(snake_case_ ,       lstrip=snake_case_ ,       rstrip=snake_case_						) if isinstance(snake_case_ ,       snake_case_						) else bos_token
								__lowercase 					=   AddedToken(snake_case_ ,       lstrip=snake_case_ ,       rstrip=snake_case_						) if isinstance(snake_case_ ,       snake_case_						) else eos_token
								__lowercase 					=   AddedToken(snake_case_ ,       lstrip=snake_case_ ,       rstrip=snake_case_						) if isinstance(snake_case_ ,       snake_case_						) else sep_token
								__lowercase 					=   AddedToken(snake_case_ ,       lstrip=snake_case_ ,       rstrip=snake_case_						) if isinstance(snake_case_ ,       snake_case_						) else cls_token
								__lowercase 					=   AddedToken(snake_case_ ,       lstrip=snake_case_ ,       rstrip=snake_case_						) if isinstance(snake_case_ ,       snake_case_						) else unk_token
								__lowercase 					=   AddedToken(snake_case_ ,       lstrip=snake_case_ ,       rstrip=snake_case_						) if isinstance(snake_case_ ,       snake_case_						) else pad_token

								# Mask token behave like a normal word, i.e. include the space before it
								__lowercase 					=   AddedToken(snake_case_ ,       lstrip=snake_case_ ,       rstrip=snake_case_						) if isinstance(snake_case_ ,       snake_case_						) else mask_token

								super().__init__(
								    errors=snake_case_ ,       bos_token=snake_case_ ,       eos_token=snake_case_ ,       unk_token=snake_case_ ,       sep_token=snake_case_ ,       cls_token=snake_case_ ,       pad_token=snake_case_ ,       mask_token=snake_case_ ,       add_prefix_space=snake_case_ ,       **snake_case_ ,       )

								with open(snake_case_ ,       encoding='''utf-8'''						) as vocab_handle:
											__lowercase 					=   json.load(snake_case_						)
								__lowercase 					=   {v: k for k, v in self.encoder.items()}
								__lowercase 					=   errors  # how to handle errors in decoding
								__lowercase 					=   bytes_to_unicode()
								__lowercase 					=   {v: k for k, v in self.byte_encoder.items()}
								with open(snake_case_ ,       encoding='''utf-8'''						) as merges_handle:
											__lowercase 					=   merges_handle.read().split('''\n'''						)[1:-1]
								__lowercase 					=   [tuple(merge.split()						) for merge in bpe_merges]
								__lowercase 					=   dict(zip(snake_case_ ,       range(len(snake_case_						)						)						)						)
								__lowercase 					=   {}
								__lowercase 					=   add_prefix_space

								# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
								__lowercase 					=   re.compile(r'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''						)






					@property
					def A      (				self						)   ->			List[str]:







								'''simple docstring'''

								return len(self.encoder						)






					def A      (				self						)   ->			Union[str, Any]:







								'''simple docstring'''

								return dict(self.encoder ,       **self.added_tokens_encoder						)






					def A      (				self ,       snake_case_						)   ->			List[str]:







								'''simple docstring'''

								if token in self.cache:
											return self.cache[token]
								__lowercase 					=   tuple(snake_case_						)
								__lowercase 					=   get_pairs(snake_case_						)

								if not pairs:
											return token

								while True:
											__lowercase 					=   min(snake_case_ ,       key=lambda snake_case_						: self.bpe_ranks.get(snake_case_ ,       float('''inf'''						)						)						)
											if bigram not in self.bpe_ranks:
														break
											__lowercase					, __lowercase 					=   bigram
											__lowercase 					=   []
											__lowercase 					=   0
											while i < len(snake_case_						):
														try:
																	__lowercase 					=   word.index(snake_case_ ,       snake_case_						)
														except ValueError:
																	new_word.extend(word[i:]						)
																	break
														else:
																	new_word.extend(word[i:j]						)
																	__lowercase 					=   j

														if word[i] == first and i < len(snake_case_						) - 1 and word[i + 1] == second:
																	new_word.append(first + second						)
																	i += 2
														else:
																	new_word.append(word[i]						)
																	i += 1
											__lowercase 					=   tuple(snake_case_						)
											__lowercase 					=   new_word
											if len(snake_case_						) == 1:
														break
											else:
														__lowercase 					=   get_pairs(snake_case_						)
								__lowercase 					=   ''' '''.join(snake_case_						)
								__lowercase 					=   word
								return word






					def A      (				self ,       snake_case_						)   ->			Union[str, Any]:







								'''simple docstring'''

								__lowercase 					=   []
								for token in re.findall(self.pat ,       snake_case_						):
											__lowercase 					=   ''''''.join(
											    self.byte_encoder[b] for b in token.encode('''utf-8'''						)						)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
											bpe_tokens.extend(bpe_token for bpe_token in self.bpe(snake_case_						).split(''' '''						)						)
								return bpe_tokens






					def A      (				self ,       snake_case_						)   ->			Any:







								'''simple docstring'''

								return self.encoder.get(snake_case_ ,       self.encoder.get(self.unk_token						)						)






					def A      (				self ,       snake_case_						)   ->			Dict:







								'''simple docstring'''

								return self.decoder.get(snake_case_						)






					def A      (				self ,       snake_case_						)   ->			List[str]:







								'''simple docstring'''

								__lowercase 					=   ''''''.join(snake_case_						)
								__lowercase 					=   bytearray([self.byte_decoder[c] for c in text]						).decode('''utf-8''' ,       errors=self.errors						)
								return text






					def A      (				self ,       snake_case_ ,       snake_case_ = None						)   ->			Tuple[str]:







								'''simple docstring'''

								if not os.path.isdir(snake_case_						):
											logger.error(F'Vocabulary path ({save_directory}) should be a directory'						)
											return
								__lowercase 					=   os.path.join(
								    snake_case_ ,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']						)
								__lowercase 					=   os.path.join(
								    snake_case_ ,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']						)

								with open(snake_case_ ,       '''w''' ,       encoding='''utf-8'''						) as f:
											f.write(json.dumps(self.encoder ,       indent=2 ,       sort_keys=snake_case_ ,       ensure_ascii=snake_case_						) + '''\n'''						)

								__lowercase 					=   0
								with open(snake_case_ ,       '''w''' ,       encoding='''utf-8'''						) as writer:
											writer.write('''#version: 0.2\n'''						)
											for bpe_tokens, token_index in sorted(self.bpe_ranks.items() ,       key=lambda snake_case_						: kv[1]						):
														if index != token_index:
																	logger.warning(
																	    F'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
																	    ''' Please check that the tokenizer is not corrupted!'''						)
																	__lowercase 					=   token_index
														writer.write(''' '''.join(snake_case_						) + '''\n'''						)
														index += 1

								return vocab_file, merge_file






					def A      (				self ,       snake_case_ ,       snake_case_ = None						)   ->			List[int]:







								'''simple docstring'''

								if token_ids_a is None:
											return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
								__lowercase 					=   [self.cls_token_id]
								__lowercase 					=   [self.sep_token_id]
								return cls + token_ids_a + sep + sep + token_ids_a + sep






					def A      (				self ,       snake_case_ ,       snake_case_ = None ,       snake_case_ = False						)   ->			List[int]:







								'''simple docstring'''

								if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=snake_case_ ,       token_ids_a=snake_case_ ,       already_has_special_tokens=snake_case_						)

								if token_ids_a is None:
											return [1] + ([0] * len(snake_case_						)) + [1]
								return [1] + ([0] * len(snake_case_						)) + [1, 1] + ([0] * len(snake_case_						)) + [1]






					def A      (				self ,       snake_case_ ,       snake_case_ = None						)   ->			List[int]:







								'''simple docstring'''

								__lowercase 					=   [self.sep_token_id]
								__lowercase 					=   [self.cls_token_id]

								if token_ids_a is None:
											return len(cls + token_ids_a + sep						) * [0]
								return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]






					def A      (				self ,       snake_case_ ,       snake_case_=False ,       **snake_case_						)   ->			Union[str, Any]:







								'''simple docstring'''

								__lowercase 					=   kwargs.pop('''add_prefix_space''' ,       self.add_prefix_space						)
								if (is_split_into_words or add_prefix_space) and (len(snake_case_						) > 0 and not text[0].isspace()):
											__lowercase 					=   ''' ''' + text
								return (text, kwargs)


