



import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase							 =					logging.get_logger(__name__)

UpperCAmelCase							 =					"""▁"""

UpperCAmelCase							 =					{"""vocab_file""": """sentencepiece.bpe.model"""}

UpperCAmelCase							 =					{
    """vocab_file""": {
        """facebook/mbart-large-en-ro""": (
            """https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model"""
        ),
        """facebook/mbart-large-cc25""": (
            """https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model"""
        ),
    }
}

UpperCAmelCase							 =					{
    """facebook/mbart-large-en-ro""": 1_024,
    """facebook/mbart-large-cc25""": 1_024,
}

# fmt: off
UpperCAmelCase							 =					["""ar_AR""", """cs_CZ""", """de_DE""", """en_XX""", """es_XX""", """et_EE""", """fi_FI""", """fr_XX""", """gu_IN""", """hi_IN""", """it_IT""", """ja_XX""", """kk_KZ""", """ko_KR""", """lt_LT""", """lv_LV""", """my_MM""", """ne_NP""", """nl_XX""", """ro_RO""", """ru_RU""", """si_LK""", """tr_TR""", """vi_VN""", """zh_CN"""]

class  __snake_case      (   _UpperCamelCase):







							'''simple docstring'''






							UpperCamelCase__				:      Tuple         =					VOCAB_FILES_NAMES
							UpperCamelCase__				:      int         =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							UpperCamelCase__				:      Union[str, Any]         =					PRETRAINED_VOCAB_FILES_MAP
							UpperCamelCase__				:      Union[str, Any]         =					["""input_ids""", """attention_mask"""]

							UpperCamelCase__				:      Any         =					[]
							UpperCamelCase__				:      str         =					[]


							def __init__(							self  ,  a_  ,  a_="<s>"  ,  a_="</s>"  ,  a_="</s>"  ,  a_="<s>"  ,  a_="<unk>"  ,  a_="<pad>"  ,  a_="<mask>"  ,  a_=None  ,  a_=None  ,  a_=None  ,  a_ = None  ,  a_=None  ,  **a_  ,  ):
											# Mask token behave like a normal word, i.e. include the space before it
											a__				      =		AddedToken(_UpperCAmelCase  ,  lstrip=_UpperCAmelCase  ,  rstrip=_UpperCAmelCase		) if isinstance(_UpperCAmelCase  ,  _UpperCAmelCase		) else mask_token

											a__				      =		{} if sp_model_kwargs is None else sp_model_kwargs

											super().__init__(
											    bos_token=_UpperCAmelCase  ,  eos_token=_UpperCAmelCase  ,  unk_token=_UpperCAmelCase  ,  sep_token=_UpperCAmelCase  ,  cls_token=_UpperCAmelCase  ,  pad_token=_UpperCAmelCase  ,  mask_token=_UpperCAmelCase  ,  tokenizer_file=_UpperCAmelCase  ,  src_lang=_UpperCAmelCase  ,  tgt_lang=_UpperCAmelCase  ,  additional_special_tokens=_UpperCAmelCase  ,  sp_model_kwargs=self.sp_model_kwargs  ,  **_UpperCAmelCase  ,  )

											a__				      =		spm.SentencePieceProcessor(**self.sp_model_kwargs		)
											self.sp_model.Load(str(_UpperCAmelCase		)		)
											a__				      =		vocab_file

											# Original fairseq vocab and spm vocab must be "aligned":
											# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
											# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
											# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
											# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

											# Mimic fairseq token-to-id alignment for the first 4 token
											a__				      =		{'''<s>''': 0, '''<pad>''': 1, '''</s>''': 2, '''<unk>''': 3}

											# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
											a__				      =		1

											a__				      =		len(self.sp_model		)
											a__				      =		{
											    code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(_UpperCAmelCase		)
											}
											a__				      =		{v: k for k, v in self.lang_code_to_id.items()}
											a__				      =		len(self.sp_model		) + len(self.lang_code_to_id		) + self.fairseq_offset

											self.fairseq_tokens_to_ids.update(self.lang_code_to_id		)
											a__				      =		{v: k for k, v in self.fairseq_tokens_to_ids.items()}
											a__				      =		list(self.lang_code_to_id.keys()		)

											if additional_special_tokens is not None:
															# Only add those special tokens if they are not already there.
															self._additional_special_tokens.extend(
															    [t for t in additional_special_tokens if t not in self._additional_special_tokens]		)

											a__				      =		src_lang if src_lang is not None else '''en_XX'''
											a__				      =		self.lang_code_to_id[self._src_lang]
											a__				      =		tgt_lang
											self.set_src_lang_special_tokens(self._src_lang		)


							def __getstate__(							self		):
											a__				      =		self.__dict__.copy()
											a__				      =		None
											a__				      =		self.sp_model.serialized_model_proto()
											return state


							def __setstate__(							self  ,  a_		):
											a__				      =		d

											# for backward compatibility
											if not hasattr(self  ,  """sp_model_kwargs"""		):
															a__				      =		{}

											a__				      =		spm.SentencePieceProcessor(**self.sp_model_kwargs		)
											self.sp_model.LoadFromSerializedProto(self.sp_model_proto		)


							@property
							def _a      (							self		):
											return len(self.sp_model		) + len(self.lang_code_to_id		) + self.fairseq_offset + 1  # Plus 1 for the mask token


							@property
							def _a      (							self		):
											return self._src_lang


							@src_lang.setter
							def _a      (							self  ,  a_		):
											a__				      =		new_src_lang
											self.set_src_lang_special_tokens(self._src_lang		)


							def _a      (							self  ,  a_  ,  a_ = None  ,  a_ = False		):

											if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=_UpperCAmelCase  ,  token_ids_a=_UpperCAmelCase  ,  already_has_special_tokens=_UpperCAmelCase		)

											a__				      =		[1] * len(self.prefix_tokens		)
											a__				      =		[1] * len(self.suffix_tokens		)
											if token_ids_a is None:
															return prefix_ones + ([0] * len(_UpperCAmelCase		)) + suffix_ones
											return prefix_ones + ([0] * len(_UpperCAmelCase		)) + ([0] * len(_UpperCAmelCase		)) + suffix_ones


							def _a      (							self  ,  a_  ,  a_ = None		):
											if token_ids_a is None:
															return self.prefix_tokens + token_ids_a + self.suffix_tokens
											# We don't expect to process pairs, but leave the pair logic for API consistency
											return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens


							def _a      (							self  ,  a_  ,  a_ = None		):
											a__				      =		[self.sep_token_id]
											a__				      =		[self.cls_token_id]

											if token_ids_a is None:
															return len(cls + token_ids_a + sep		) * [0]
											return len(cls + token_ids_a + sep + sep + token_ids_a + sep		) * [0]


							def _a      (							self  ,  a_  ,  a_  ,  a_  ,  a_  ,  **a_		):
											if src_lang is None or tgt_lang is None:
															raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""		)
											a__				      =		src_lang
											a__				      =		self(_UpperCAmelCase  ,  add_special_tokens=_UpperCAmelCase  ,  return_tensors=_UpperCAmelCase  ,  **_UpperCAmelCase		)
											a__				      =		self.convert_tokens_to_ids(_UpperCAmelCase		)
											a__				      =		tgt_lang_id
											return inputs


							def _a      (							self		):
											a__				      =		{self.convert_ids_to_tokens(_UpperCAmelCase		): i for i in range(self.vocab_size		)}
											vocab.update(self.added_tokens_encoder		)
											return vocab


							def _a      (							self  ,  a_		):
											return self.sp_model.encode(_UpperCAmelCase  ,  out_type=_UpperCAmelCase		)


							def _a      (							self  ,  a_		):
											if token in self.fairseq_tokens_to_ids:
															return self.fairseq_tokens_to_ids[token]
											a__				      =		self.sp_model.PieceToId(_UpperCAmelCase		)

											# Need to return unknown token if the SP model returned 0
											return spm_id + self.fairseq_offset if spm_id else self.unk_token_id


							def _a      (							self  ,  a_		):
											if index in self.fairseq_ids_to_tokens:
															return self.fairseq_ids_to_tokens[index]
											return self.sp_model.IdToPiece(index - self.fairseq_offset		)


							def _a      (							self  ,  a_		):
											a__				      =		''''''.join(_UpperCAmelCase		).replace(_UpperCAmelCase  ,  """ """		).strip()
											return out_string


							def _a      (							self  ,  a_  ,  a_ = None		):
											if not os.path.isdir(_UpperCAmelCase		):
															logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''		)
															return
											a__				      =		os.path.join(
											    _UpperCAmelCase  ,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]		)

											if os.path.abspath(self.vocab_file		) != os.path.abspath(_UpperCAmelCase		) and os.path.isfile(self.vocab_file		):
															copyfile(self.vocab_file  ,  _UpperCAmelCase		)
											elif not os.path.isfile(self.vocab_file		):
															with open(_UpperCAmelCase  ,  """wb"""		) as fi:
																			a__				      =		self.sp_model.serialized_model_proto()
																			fi.write(_UpperCAmelCase		)

											return (out_vocab_file,)


							def _a      (							self  ,  a_  ,  a_ = "en_XX"  ,  a_ = None  ,  a_ = "ro_RO"  ,  **a_  ,  ):
											a__				      =		src_lang
											a__				      =		tgt_lang
											return super().prepare_seqaseq_batch(_UpperCAmelCase  ,  _UpperCAmelCase  ,  **_UpperCAmelCase		)


							def _a      (							self		):
											return self.set_src_lang_special_tokens(self.src_lang		)


							def _a      (							self		):
											return self.set_tgt_lang_special_tokens(self.tgt_lang		)


							def _a      (							self  ,  a_		):
											a__				      =		self.lang_code_to_id[src_lang]
											a__				      =		[]
											a__				      =		[self.eos_token_id, self.cur_lang_code]







							def _a      (							self  ,  a_		):
											a__				      =		self.lang_code_to_id[lang]
											a__				      =		[]
											a__				      =		[self.eos_token_id, self.cur_lang_code]






import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
	from PIL import Image

	from transformers import OwlViTImageProcessor, OwlViTProcessor

@require_vision
class  __snake_case      (   unittest.TestCase):







							'''simple docstring'''






							def _a      (							self		):
											a__				      =		tempfile.mkdtemp()

											# fmt: off
											a__				      =		["""""", """l""", """o""", """w""", """e""", """r""", """s""", """t""", """i""", """d""", """n""", """lo""", """l</w>""", """w</w>""", """r</w>""", """t</w>""", """low</w>""", """er</w>""", """lowest</w>""", """newer</w>""", """wider""", """<unk>""", """<|startoftext|>""", """<|endoftext|>"""]
											# fmt: on
											a__				      =		dict(zip(a_  ,  range(len(a_		)		)		)		)
											a__				      =		["""#version: 0.2""", """l o""", """lo w</w>""", """e r</w>""", """"""]
											a__				      =		{"""unk_token""": """<unk>"""}

											a__				      =		os.path.join(self.tmpdirname  ,  VOCAB_FILES_NAMES["""vocab_file"""]		)
											a__				      =		os.path.join(self.tmpdirname  ,  VOCAB_FILES_NAMES["""merges_file"""]		)
											with open(self.vocab_file  ,  """w"""  ,  encoding="""utf-8"""		) as fp:
															fp.write(json.dumps(a_		) + """\n"""		)
											with open(self.merges_file  ,  """w"""  ,  encoding="""utf-8"""		) as fp:
															fp.write("""\n""".join(a_		)		)

											a__				      =		{
											    """do_resize""": True,
											    """size""": 20,
											    """do_center_crop""": True,
											    """crop_size""": 18,
											    """do_normalize""": True,
											    """image_mean""": [0.48_145_466, 0.4_578_275, 0.40_821_073],
											    """image_std""": [0.26_862_954, 0.26_130_258, 0.27_577_711],
											}
											a__				      =		os.path.join(self.tmpdirname  ,  a_		)
											with open(self.image_processor_file  ,  """w"""  ,  encoding="""utf-8"""		) as fp:
															json.dump(a_  ,  a_		)


							def _a      (							self  ,  **a_		):
											return CLIPTokenizer.from_pretrained(self.tmpdirname  ,  pad_token="""!"""  ,  **a_		)


							def _a      (							self  ,  **a_		):
											return CLIPTokenizerFast.from_pretrained(self.tmpdirname  ,  pad_token="""!"""  ,  **a_		)


							def _a      (							self  ,  **a_		):
											return OwlViTImageProcessor.from_pretrained(self.tmpdirname  ,  **a_		)


							def _a      (							self		):
											shutil.rmtree(self.tmpdirname		)


							def _a      (							self		):
											a__				      =		[np.random.randint(255  ,  size=(3, 30, 400)  ,  dtype=np.uinta		)]

											a__				      =		[Image.fromarray(np.moveaxis(a_  ,  0  ,  -1		)		) for x in image_inputs]

											return image_inputs


							def _a      (							self		):
											a__				      =		self.get_tokenizer()
											a__				      =		self.get_rust_tokenizer()
											a__				      =		self.get_image_processor()

											a__				      =		OwlViTProcessor(tokenizer=a_  ,  image_processor=a_		)
											processor_slow.save_pretrained(self.tmpdirname		)
											a__				      =		OwlViTProcessor.from_pretrained(self.tmpdirname  ,  use_fast=a_		)

											a__				      =		OwlViTProcessor(tokenizer=a_  ,  image_processor=a_		)
											processor_fast.save_pretrained(self.tmpdirname		)
											a__				      =		OwlViTProcessor.from_pretrained(self.tmpdirname		)

											self.assertEqual(processor_slow.tokenizer.get_vocab()  ,  tokenizer_slow.get_vocab()		)
											self.assertEqual(processor_fast.tokenizer.get_vocab()  ,  tokenizer_fast.get_vocab()		)
											self.assertEqual(tokenizer_slow.get_vocab()  ,  tokenizer_fast.get_vocab()		)
											self.assertIsInstance(processor_slow.tokenizer  ,  a_		)
											self.assertIsInstance(processor_fast.tokenizer  ,  a_		)

											self.assertEqual(processor_slow.image_processor.to_json_string()  ,  image_processor.to_json_string()		)
											self.assertEqual(processor_fast.image_processor.to_json_string()  ,  image_processor.to_json_string()		)
											self.assertIsInstance(processor_slow.image_processor  ,  a_		)
											self.assertIsInstance(processor_fast.image_processor  ,  a_		)


							def _a      (							self		):
											a__				      =		OwlViTProcessor(tokenizer=self.get_tokenizer()  ,  image_processor=self.get_image_processor()		)
											processor.save_pretrained(self.tmpdirname		)

											a__				      =		self.get_tokenizer(bos_token="""(BOS)"""  ,  eos_token="""(EOS)"""		)
											a__				      =		self.get_image_processor(do_normalize=a_		)

											a__				      =		OwlViTProcessor.from_pretrained(
											    self.tmpdirname  ,  bos_token="""(BOS)"""  ,  eos_token="""(EOS)"""  ,  do_normalize=a_		)

											self.assertEqual(processor.tokenizer.get_vocab()  ,  tokenizer_add_kwargs.get_vocab()		)
											self.assertIsInstance(processor.tokenizer  ,  a_		)

											self.assertEqual(processor.image_processor.to_json_string()  ,  image_processor_add_kwargs.to_json_string()		)
											self.assertIsInstance(processor.image_processor  ,  a_		)


							def _a      (							self		):
											a__				      =		self.get_image_processor()
											a__				      =		self.get_tokenizer()

											a__				      =		OwlViTProcessor(tokenizer=a_  ,  image_processor=a_		)

											a__				      =		self.prepare_image_inputs()

											a__				      =		image_processor(a_  ,  return_tensors="""np"""		)
											a__				      =		processor(images=a_  ,  return_tensors="""np"""		)

											for key in input_image_proc.keys():
															self.assertAlmostEqual(input_image_proc[key].sum()  ,  input_processor[key].sum()  ,  delta=1E-2		)


							def _a      (							self		):
											a__				      =		self.get_image_processor()
											a__				      =		self.get_tokenizer()

											a__				      =		OwlViTProcessor(tokenizer=a_  ,  image_processor=a_		)

											a__				      =		"""lower newer"""

											a__				      =		processor(text=a_  ,  return_tensors="""np"""		)

											a__				      =		tokenizer(a_  ,  return_tensors="""np"""		)

											for key in encoded_tok.keys():
															self.assertListEqual(encoded_tok[key][0].tolist()  ,  encoded_processor[key][0].tolist()		)


							def _a      (							self		):
											a__				      =		self.get_image_processor()
											a__				      =		self.get_tokenizer()

											a__				      =		OwlViTProcessor(tokenizer=a_  ,  image_processor=a_		)

											a__				      =		"""lower newer"""
											a__				      =		self.prepare_image_inputs()

											a__				      =		processor(text=a_  ,  images=a_		)

											self.assertListEqual(list(inputs.keys()		)  ,  ["""input_ids""", """attention_mask""", """pixel_values"""]		)

											# test if it raises when no input is passed
											with pytest.raises(a_		):
															processor()


							def _a      (							self		):
											a__				      =		"""google/owlvit-base-patch32"""
											a__				      =		OwlViTProcessor.from_pretrained(a_		)

											a__				      =		["""cat""", """nasa badge"""]
											a__				      =		processor(text=a_		)

											a__				      =		16
											self.assertListEqual(list(inputs.keys()		)  ,  ["""input_ids""", """attention_mask"""]		)
											self.assertEqual(inputs["""input_ids"""].shape  ,  (2, seq_length)		)

											# test if it raises when no input is passed
											with pytest.raises(a_		):
															processor()


							def _a      (							self		):
											a__				      =		"""google/owlvit-base-patch32"""
											a__				      =		OwlViTProcessor.from_pretrained(a_		)

											a__				      =		[["""cat""", """nasa badge"""], ["""person"""]]
											a__				      =		processor(text=a_		)

											a__				      =		16
											a__				      =		len(a_		)
											a__				      =		max([len(a_		) for texts in input_texts]		)

											self.assertListEqual(list(inputs.keys()		)  ,  ["""input_ids""", """attention_mask"""]		)
											self.assertEqual(inputs["""input_ids"""].shape  ,  (batch_size * num_max_text_queries, seq_length)		)

											# test if it raises when no input is passed
											with pytest.raises(a_		):
															processor()


							def _a      (							self		):
											a__				      =		"""google/owlvit-base-patch32"""
											a__				      =		OwlViTProcessor.from_pretrained(a_		)

											a__				      =		["""cat""", """nasa badge"""]
											a__				      =		processor(text=a_		)

											a__				      =		16
											a__				      =		inputs["""input_ids"""]
											a__				      =		[
											    [49_406, 2_368, 49_407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
											    [49_406, 6_841, 11_301, 49_407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
											]

											self.assertListEqual(list(inputs.keys()		)  ,  ["""input_ids""", """attention_mask"""]		)
											self.assertEqual(inputs["""input_ids"""].shape  ,  (2, seq_length)		)
											self.assertListEqual(list(input_ids[0]		)  ,  predicted_ids[0]		)
											self.assertListEqual(list(input_ids[1]		)  ,  predicted_ids[1]		)


							def _a      (							self		):
											a__				      =		self.get_image_processor()
											a__				      =		self.get_tokenizer()

											a__				      =		OwlViTProcessor(tokenizer=a_  ,  image_processor=a_		)

											a__				      =		self.prepare_image_inputs()
											a__				      =		self.prepare_image_inputs()

											a__				      =		processor(images=a_  ,  query_images=a_		)

											self.assertListEqual(list(inputs.keys()		)  ,  ["""query_pixel_values""", """pixel_values"""]		)

											# test if it raises when no input is passed
											with pytest.raises(a_		):
															processor()







							def _a      (							self		):
											a__				      =		self.get_image_processor()
											a__				      =		self.get_tokenizer()

											a__				      =		OwlViTProcessor(tokenizer=a_  ,  image_processor=a_		)

											a__				      =		[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

											a__				      =		processor.batch_decode(a_		)
											a__				      =		tokenizer.batch_decode(a_		)

											self.assertListEqual(a_  ,  a_		)

