

'''simple docstring'''





import os
import unittest

from transformers import BatchEncoding
from transformers.models.bert.tokenization_bert import (
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.prophetnet.tokenization_prophetnet import VOCAB_FILES_NAMES, ProphetNetTokenizer
from transformers.testing_utils import require_torch, slow

from ...test_tokenization_common import TokenizerTesterMixin


class  UpperCAmelCase (							UpperCAmelCase__  , unittest.TestCase    ):

		'''simple docstring'''






		SCREAMING_SNAKE_CASE_           =					ProphetNetTokenizer
		SCREAMING_SNAKE_CASE_           =					False
		def 			UpperCamelCase(					self		)  ->     int:






							'''simple docstring'''



							super().setUp()

							lowerCamelCase_  				=      [
							    '[UNK]',
							    '[CLS]',
							    '[SEP]',
							    '[PAD]',
							    '[MASK]',
							    'want',
							    '##want',
							    '##ed',
							    'wa',
							    'un',
							    'runn',
							    '##ing',
							    ',',
							    'low',
							    'lowest',
							]
							lowerCamelCase_  				=      os.path.join(self.tmpdirname   ,			VOCAB_FILES_NAMES['vocab_file']		)
							with open(self.vocab_file   ,			'w'   ,			encoding='utf-8'		) as vocab_writer:
												vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]		)		)
		def 			UpperCamelCase(					self   ,			SCREAMING_SNAKE_CASE_		)  ->     Tuple:






							'''simple docstring'''



							lowerCamelCase_  				=      'UNwant\u00E9d,running'
							lowerCamelCase_  				=      'unwanted, running'
							return input_text, output_text
		def 			UpperCamelCase(					self		)  ->     Dict:






							'''simple docstring'''



							lowerCamelCase_  				=      self.tokenizer_class(self.vocab_file		)

							lowerCamelCase_  				=      tokenizer.tokenize('UNwant\u00E9d,running'		)
							self.assertListEqual(SCREAMING_SNAKE_CASE_   ,			['un', '##want', '##ed', ',', 'runn', '##ing']		)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_		)   ,			[9, 6, 7, 12, 10, 11]		)
		def 			UpperCamelCase(					self		)  ->     str:






							'''simple docstring'''



							lowerCamelCase_  				=      BasicTokenizer()

							self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'		)   ,			['ah', '\u535A', '\u63A8', 'zz']		)
		def 			UpperCamelCase(					self		)  ->     List[str]:






							'''simple docstring'''



							lowerCamelCase_  				=      BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_		)

							self.assertListEqual(
							    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '		)   ,			['hello', '!', 'how', 'are', 'you', '?']		)
							self.assertListEqual(tokenizer.tokenize('H\u00E9llo'		)   ,			['hello']		)
		def 			UpperCamelCase(					self		)  ->     Optional[int]:






							'''simple docstring'''



							lowerCamelCase_  				=      BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_   ,			strip_accents=SCREAMING_SNAKE_CASE_		)

							self.assertListEqual(
							    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '		)   ,			['hällo', '!', 'how', 'are', 'you', '?']		)
							self.assertListEqual(tokenizer.tokenize('H\u00E9llo'		)   ,			['h\u00E9llo']		)
		def 			UpperCamelCase(					self		)  ->     Optional[int]:






							'''simple docstring'''



							lowerCamelCase_  				=      BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_   ,			strip_accents=SCREAMING_SNAKE_CASE_		)

							self.assertListEqual(
							    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '		)   ,			['hallo', '!', 'how', 'are', 'you', '?']		)
							self.assertListEqual(tokenizer.tokenize('H\u00E9llo'		)   ,			['hello']		)
		def 			UpperCamelCase(					self		)  ->     Dict:






							'''simple docstring'''



							lowerCamelCase_  				=      BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_		)

							self.assertListEqual(
							    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '		)   ,			['hallo', '!', 'how', 'are', 'you', '?']		)
							self.assertListEqual(tokenizer.tokenize('H\u00E9llo'		)   ,			['hello']		)
		def 			UpperCamelCase(					self		)  ->     Optional[Any]:






							'''simple docstring'''



							lowerCamelCase_  				=      BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_		)

							self.assertListEqual(
							    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '		)   ,			['HeLLo', '!', 'how', 'Are', 'yoU', '?']		)
		def 			UpperCamelCase(					self		)  ->     List[str]:






							'''simple docstring'''



							lowerCamelCase_  				=      BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_   ,			strip_accents=SCREAMING_SNAKE_CASE_		)

							self.assertListEqual(
							    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '		)   ,			['HäLLo', '!', 'how', 'Are', 'yoU', '?']		)
		def 			UpperCamelCase(					self		)  ->     Tuple:






							'''simple docstring'''



							lowerCamelCase_  				=      BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_   ,			strip_accents=SCREAMING_SNAKE_CASE_		)

							self.assertListEqual(
							    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '		)   ,			['HaLLo', '!', 'how', 'Are', 'yoU', '?']		)
		def 			UpperCamelCase(					self		)  ->     Tuple:






							'''simple docstring'''



							lowerCamelCase_  				=      BasicTokenizer(do_lower_case=SCREAMING_SNAKE_CASE_   ,			never_split=['[UNK]']		)

							self.assertListEqual(
							    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'		)   ,			['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']		)
		def 			UpperCamelCase(					self		)  ->     List[Any]:






							'''simple docstring'''



							lowerCamelCase_  				=      ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

							lowerCamelCase_  				=      {}
							for i, token in enumerate(SCREAMING_SNAKE_CASE_		):
												lowerCamelCase_  				=      i
							lowerCamelCase_  				=      WordpieceTokenizer(vocab=SCREAMING_SNAKE_CASE_   ,			unk_token='[UNK]'		)

							self.assertListEqual(tokenizer.tokenize(''		)   ,			[]		)

							self.assertListEqual(tokenizer.tokenize('unwanted running'		)   ,			['un', '##want', '##ed', 'runn', '##ing']		)

							self.assertListEqual(tokenizer.tokenize('unwantedX running'		)   ,			['[UNK]', 'runn', '##ing']		)
		@require_torch
		def 			UpperCamelCase(					self		)  ->     Optional[int]:






							'''simple docstring'''



							lowerCamelCase_  				=      self.tokenizer_class.from_pretrained('microsoft/prophetnet-large-uncased'		)

							lowerCamelCase_  				=      ['A long paragraph for summarization.', 'Another paragraph for summarization.']
							lowerCamelCase_  				=      [1037, 2146, 20423, 2005, 7680, 7849, 3989, 1012, 102]
							lowerCamelCase_  				=      tokenizer(SCREAMING_SNAKE_CASE_   ,			padding=SCREAMING_SNAKE_CASE_   ,			return_tensors='pt'		)
							self.assertIsInstance(SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_		)
							lowerCamelCase_  				=      list(batch.input_ids.numpy()[0]		)
							self.assertListEqual(SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_		)

							self.assertEqual((2, 9)   ,			batch.input_ids.shape		)
							self.assertEqual((2, 9)   ,			batch.attention_mask.shape		)
		def 			UpperCamelCase(					self		)  ->     Dict:






							'''simple docstring'''



							self.assertTrue(_is_whitespace(' '		)		)
							self.assertTrue(_is_whitespace('\t'		)		)
							self.assertTrue(_is_whitespace('\r'		)		)
							self.assertTrue(_is_whitespace('\n'		)		)
							self.assertTrue(_is_whitespace('\u00A0'		)		)

							self.assertFalse(_is_whitespace('A'		)		)
							self.assertFalse(_is_whitespace('-'		)		)
		def 			UpperCamelCase(					self		)  ->     Optional[int]:






							'''simple docstring'''



							self.assertTrue(_is_control('\u0005'		)		)

							self.assertFalse(_is_control('A'		)		)
							self.assertFalse(_is_control(' '		)		)
							self.assertFalse(_is_control('\t'		)		)
							self.assertFalse(_is_control('\r'		)		)
		def 			UpperCamelCase(					self		)  ->     List[Any]:






							'''simple docstring'''



							self.assertTrue(_is_punctuation('-'		)		)
							self.assertTrue(_is_punctuation('$'		)		)
							self.assertTrue(_is_punctuation('`'		)		)
							self.assertTrue(_is_punctuation('.'		)		)

							self.assertFalse(_is_punctuation('A'		)		)
							self.assertFalse(_is_punctuation(' '		)		)






		@slow
		def 			UpperCamelCase(					self		)  ->     List[Any]:






							'''simple docstring'''



							lowerCamelCase_  				=      self.tokenizer_class.from_pretrained('microsoft/prophetnet-large-uncased'		)

							lowerCamelCase_  				=      tokenizer.encode('sequence builders'   ,			add_special_tokens=SCREAMING_SNAKE_CASE_		)
							lowerCamelCase_  				=      tokenizer.encode('multi-sequence build'   ,			add_special_tokens=SCREAMING_SNAKE_CASE_		)

							lowerCamelCase_  				=      tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE_		)
							lowerCamelCase_  				=      tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE_   ,			SCREAMING_SNAKE_CASE_		)

							assert encoded_sentence == text + [102]
							assert encoded_pair == text + [102] + text_a + [102]






'''simple docstring'''







import warnings
from typing import List

import numpy as np

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding
from ...utils import is_flax_available, is_tf_available, is_torch_available


class        A__   (   _snake_case      ):
    lowercase						       =		["image_processor", "tokenizer"]
    lowercase						       =		"OwlViTImageProcessor"
    lowercase						       =		("CLIPTokenizer", "CLIPTokenizerFast")



    def __init__(       self		,					UpperCamelCase__=None		,					UpperCamelCase__=None		,					**UpperCamelCase__							)    ->   str:





           '''simple docstring'''
           A_						       =   None
           if "feature_extractor" in kwargs:
                  warnings.warn(
                      """The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`"""
                      """ instead."""		,					UpperCamelCase__		,					)
                  A_						       =   kwargs.pop("""feature_extractor"""							)

           A_						       =   image_processor if image_processor is not None else feature_extractor
           if image_processor is None:
                  raise ValueError("""You need to specify an `image_processor`."""							)
           if tokenizer is None:
                  raise ValueError("""You need to specify a `tokenizer`."""							)

           super().__init__(UpperCamelCase__		,					UpperCamelCase__							)



    def __call__(       self		,					UpperCamelCase__=None		,					UpperCamelCase__=None		,					UpperCamelCase__=None		,					UpperCamelCase__="max_length"		,					UpperCamelCase__="np"		,					**UpperCamelCase__							)    ->   int:





           '''simple docstring'''

           if text is None and query_images is None and images is None:
                  raise ValueError(
                      """You have to specify at least one text or query image or image. All three cannot be none."""							)

           if text is not None:
                  if isinstance(UpperCamelCase__		,					UpperCamelCase__							) or (isinstance(UpperCamelCase__		,					UpperCamelCase__							) and not isinstance(text[0]		,					UpperCamelCase__							)):
                         A_						       =   [self.tokenizer(UpperCamelCase__		,					padding=UpperCamelCase__		,					return_tensors=UpperCamelCase__		,					**UpperCamelCase__							)]

                  elif isinstance(UpperCamelCase__		,					UpperCamelCase__							) and isinstance(text[0]		,					UpperCamelCase__							):
                         A_						       =   []

                         # Maximum number of queries across batch
                         A_						       =   max([len(UpperCamelCase__							) for t in text]							)

                         # Pad all batch samples to max number of text queries
                         for t in text:
                                if len(UpperCamelCase__							) != max_num_queries:
                                       A_						       =   t + [""" """] * (max_num_queries - len(UpperCamelCase__							))

                                A_						       =   self.tokenizer(UpperCamelCase__		,					padding=UpperCamelCase__		,					return_tensors=UpperCamelCase__		,					**UpperCamelCase__							)
                                encodings.append(UpperCamelCase__							)
                  else:
                         raise TypeError("""Input text should be a string, a list of strings or a nested list of strings"""							)

                  if return_tensors == "np":
                         A_						       =   np.concatenate([encoding["""input_ids"""] for encoding in encodings]		,					axis=0							)
                         A_						       =   np.concatenate([encoding["""attention_mask"""] for encoding in encodings]		,					axis=0							)

                  elif return_tensors == "jax" and is_flax_available():
                         import jax.numpy as jnp

                         A_						       =   jnp.concatenate([encoding["""input_ids"""] for encoding in encodings]		,					axis=0							)
                         A_						       =   jnp.concatenate([encoding["""attention_mask"""] for encoding in encodings]		,					axis=0							)

                  elif return_tensors == "pt" and is_torch_available():
                         import torch

                         A_						       =   torch.cat([encoding["""input_ids"""] for encoding in encodings]		,					dim=0							)
                         A_						       =   torch.cat([encoding["""attention_mask"""] for encoding in encodings]		,					dim=0							)

                  elif return_tensors == "tf" and is_tf_available():
                         import tensorflow as tf

                         A_						       =   tf.stack([encoding["""input_ids"""] for encoding in encodings]		,					axis=0							)
                         A_						       =   tf.stack([encoding["""attention_mask"""] for encoding in encodings]		,					axis=0							)

                  else:
                         raise ValueError("""Target return tensor type could not be returned"""							)

                  A_						       =   BatchEncoding()
                  A_						       =   input_ids
                  A_						       =   attention_mask

           if query_images is not None:
                  A_						       =   BatchEncoding()
                  A_						       =   self.image_processor(
                      UpperCamelCase__		,					return_tensors=UpperCamelCase__		,					**UpperCamelCase__							).pixel_values
                  A_						       =   query_pixel_values

           if images is not None:
                  A_						       =   self.image_processor(UpperCamelCase__		,					return_tensors=UpperCamelCase__		,					**UpperCamelCase__							)

           if text is not None and images is not None:
                  A_						       =   image_features.pixel_values
                  return encoding
           elif query_images is not None and images is not None:
                  A_						       =   image_features.pixel_values
                  return encoding
           elif text is not None or query_images is not None:
                  return encoding
           else:
                  return BatchEncoding(data=dict(**UpperCamelCase__							)		,					tensor_type=UpperCamelCase__							)



    def        snake_case_      (       self		,					*UpperCamelCase__		,					**UpperCamelCase__							)    ->   int:





           '''simple docstring'''
           return self.image_processor.post_process(*UpperCamelCase__		,					**UpperCamelCase__							)



    def        snake_case_      (       self		,					*UpperCamelCase__		,					**UpperCamelCase__							)    ->   List[Any]:





           '''simple docstring'''
           return self.image_processor.post_process_object_detection(*UpperCamelCase__		,					**UpperCamelCase__							)



    def        snake_case_      (       self		,					*UpperCamelCase__		,					**UpperCamelCase__							)    ->   Tuple:





           '''simple docstring'''
           return self.image_processor.post_process_image_guided_detection(*UpperCamelCase__		,					**UpperCamelCase__							)



    def        snake_case_      (       self		,					*UpperCamelCase__		,					**UpperCamelCase__							)    ->   Any:





           '''simple docstring'''
           return self.tokenizer.batch_decode(*UpperCamelCase__		,					**UpperCamelCase__							)



    def        snake_case_      (       self		,					*UpperCamelCase__		,					**UpperCamelCase__							)    ->   List[str]:





           '''simple docstring'''
           return self.tokenizer.decode(*UpperCamelCase__		,					**UpperCamelCase__							)



    @property
    def        snake_case_      (       self							)    ->   Optional[int]:





           '''simple docstring'''
           warnings.warn(
               """`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead."""		,					UpperCamelCase__		,					)
           return self.image_processor_class







    @property
    def        snake_case_      (       self							)    ->   List[str]:





           '''simple docstring'''
           warnings.warn(
               """`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead."""		,					UpperCamelCase__		,					)
           return self.image_processor
