




import copy
import inspect
import unittest

import numpy as np
from huggingface_hub import hf_hub_download

from transformers import VideoMAEConfig
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
	import torch
	from torch import nn

	from transformers import (
	    MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING,
	    VideoMAEForPreTraining,
	    VideoMAEForVideoClassification,
	    VideoMAEModel,
	)
	from transformers.models.videomae.modeling_videomae import VIDEOMAE_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
	from transformers import VideoMAEImageProcessor


class    SCREAMING_SNAKE_CASE_				:



	"""simple docstring"""


	def __init__(					self		:						str					,     lowerCAmelCase		:						int					,     lowerCAmelCase		:						Tuple=13					,     lowerCAmelCase		:						List[str]=10					,     lowerCAmelCase		:						Optional[Any]=3					,     lowerCAmelCase		:						List[str]=2					,     lowerCAmelCase		:						Optional[int]=2					,     lowerCAmelCase		:						List[str]=2					,     lowerCAmelCase		:						Dict=True					,     lowerCAmelCase		:						Optional[int]=True					,     lowerCAmelCase		:						List[Any]=32					,     lowerCAmelCase		:						Any=5					,     lowerCAmelCase		:						List[Any]=4					,     lowerCAmelCase		:						List[Any]=37					,     lowerCAmelCase		:						Optional[int]="gelu"					,     lowerCAmelCase		:						Dict=0.1					,     lowerCAmelCase		:						Optional[int]=0.1					,     lowerCAmelCase		:						Tuple=10					,     lowerCAmelCase		:						Dict=0.02					,     lowerCAmelCase		:						Tuple=0.9					,     lowerCAmelCase		:						Dict=None					,     )	->       Union[str, Any]:







						"""simple docstring"""







						__UpperCamelCase					:	Optional[int]             =      parent
						__UpperCamelCase					:	int             =      batch_size
						__UpperCamelCase					:	Optional[int]             =      image_size
						__UpperCamelCase					:	Optional[int]             =      num_channels
						__UpperCamelCase					:	Tuple             =      patch_size
						__UpperCamelCase					:	Tuple             =      tubelet_size
						__UpperCamelCase					:	Any             =      num_frames
						__UpperCamelCase					:	List[Any]             =      is_training
						__UpperCamelCase					:	int             =      use_labels
						__UpperCamelCase					:	str             =      hidden_size
						__UpperCamelCase					:	Tuple             =      num_hidden_layers
						__UpperCamelCase					:	Dict             =      num_attention_heads
						__UpperCamelCase					:	Optional[int]             =      intermediate_size
						__UpperCamelCase					:	Any             =      hidden_act
						__UpperCamelCase					:	int             =      hidden_dropout_prob
						__UpperCamelCase					:	Optional[Any]             =      attention_probs_dropout_prob
						__UpperCamelCase					:	Optional[Any]             =      type_sequence_label_size
						__UpperCamelCase					:	Union[str, Any]             =      initializer_range
						__UpperCamelCase					:	Dict             =      mask_ratio
						__UpperCamelCase					:	List[str]             =      scope

						# in VideoMAE, the number of tokens equals num_frames/tubelet_size * num_patches per frame
						__UpperCamelCase					:	int             =      (image_size // patch_size) ** 2
						__UpperCamelCase					:	Tuple             =      (num_frames // tubelet_size) * self.num_patches_per_frame

						# use this variable to define bool_masked_pos
						__UpperCamelCase					:	Optional[Any]             =      int(mask_ratio * self.seq_length	)






	def 					lowerCamelCase__     (					self		:						Union[str, Any]	)	->       Optional[int]:







						"""simple docstring"""







						__UpperCamelCase					:	Optional[Any]             =      floats_tensor(
						    [self.batch_size, self.num_frames, self.num_channels, self.image_size, self.image_size]	)

						__UpperCamelCase					:	List[Any]             =      None
						if self.use_labels:
											__UpperCamelCase					:	Any             =      ids_tensor([self.batch_size]					,     self.type_sequence_label_size	)

						__UpperCamelCase					:	Any             =      self.get_config()

						return config, pixel_values, labels






	def 					lowerCamelCase__     (					self		:						Union[str, Any]	)	->       Union[str, Any]:







						"""simple docstring"""







						return VideoMAEConfig(
						    image_size=self.image_size					,     patch_size=self.patch_size					,     num_channels=self.num_channels					,     num_frames=self.num_frames					,     tubelet_size=self.tubelet_size					,     hidden_size=self.hidden_size					,     num_hidden_layers=self.num_hidden_layers					,     num_attention_heads=self.num_attention_heads					,     intermediate_size=self.intermediate_size					,     hidden_act=self.hidden_act					,     hidden_dropout_prob=self.hidden_dropout_prob					,     attention_probs_dropout_prob=self.attention_probs_dropout_prob					,     is_decoder=lowerCAmelCase					,     initializer_range=self.initializer_range					,     )






	def 					lowerCamelCase__     (					self		:						List[str]					,     lowerCAmelCase		:						List[Any]					,     lowerCAmelCase		:						List[Any]					,     lowerCAmelCase		:						Any	)	->       Any:







						"""simple docstring"""







						__UpperCamelCase					:	Dict             =      VideoMAEModel(config=lowerCAmelCase	)
						model.to(lowerCAmelCase	)
						model.eval()
						__UpperCamelCase					:	Optional[int]             =      model(lowerCAmelCase	)
						self.parent.assertEqual(result.last_hidden_state.shape					,     (self.batch_size, self.seq_length, self.hidden_size)	)






	def 					lowerCamelCase__     (					self		:						str					,     lowerCAmelCase		:						Any					,     lowerCAmelCase		:						Union[str, Any]					,     lowerCAmelCase		:						str	)	->       Any:







						"""simple docstring"""







						__UpperCamelCase					:	Optional[Any]             =      VideoMAEForPreTraining(lowerCAmelCase	)
						model.to(lowerCAmelCase	)
						model.eval()
						# important: each video needs to have the same number of masked patches
						# hence we define a single mask, which we then repeat for each example in the batch
						__UpperCamelCase					:	Tuple             =      torch.ones((self.num_masks,)	)
						__UpperCamelCase					:	int             =      torch.cat([mask, torch.zeros(self.seq_length - mask.size(0	)	)]	)
						__UpperCamelCase					:	Optional[Any]             =      mask.expand(self.batch_size					,     -1	).bool()

						__UpperCamelCase					:	Any             =      model(lowerCAmelCase					,     lowerCAmelCase	)
						# model only returns predictions for masked patches
						__UpperCamelCase					:	int             =      mask.sum().item()
						__UpperCamelCase					:	Tuple             =      3 * self.tubelet_size * self.patch_size**2
						self.parent.assertEqual(result.logits.shape					,     (self.batch_size, num_masked_patches, decoder_num_labels)	)






	def 					lowerCamelCase__     (					self		:						Dict	)	->       List[str]:







						"""simple docstring"""







						__UpperCamelCase					:	Optional[Any]             =      self.prepare_config_and_inputs()
						__UpperCamelCase      ,			__UpperCamelCase      ,			__UpperCamelCase					:	Optional[int]             =      config_and_inputs
						__UpperCamelCase					:	int             =      {"""pixel_values""": pixel_values}
						return config, inputs_dict






@require_torch
class    SCREAMING_SNAKE_CASE_				(       _UpperCamelCase	,  _UpperCamelCase	,  unittest.TestCase   ):



	"""simple docstring"""


	__magic_name__   :       Union[str, Any]      	=		(
	    (VideoMAEModel, VideoMAEForPreTraining, VideoMAEForVideoClassification) if is_torch_available() else ()
	)
	__magic_name__   :       Union[str, Any]      	=		(
	    {'feature-extraction': VideoMAEModel, 'video-classification': VideoMAEForVideoClassification}
	    if is_torch_available()
	    else {}
	)

	__magic_name__   :       str      	=		False
	__magic_name__   :       List[str]      	=		False
	__magic_name__   :       int      	=		False
	__magic_name__   :       Tuple      	=		False






	def 					lowerCamelCase__     (					self		:						Optional[int]	)	->       Tuple:







						"""simple docstring"""







						__UpperCamelCase					:	Any             =      VideoMAEModelTester(self	)
						__UpperCamelCase					:	List[str]             =      ConfigTester(self					,     config_class=lowerCAmelCase					,     has_text_modality=lowerCAmelCase					,     hidden_size=37	)






	def 					lowerCamelCase__     (					self		:						str					,     lowerCAmelCase		:						int					,     lowerCAmelCase		:						Optional[Any]					,     lowerCAmelCase		:						List[Any]=False	)	->       Union[str, Any]:







						"""simple docstring"""







						__UpperCamelCase					:	Any             =      copy.deepcopy(lowerCAmelCase	)

						if model_class == VideoMAEForPreTraining:
											# important: each video needs to have the same number of masked patches
											# hence we define a single mask, which we then repeat for each example in the batch
											__UpperCamelCase					:	Any             =      torch.ones((self.model_tester.num_masks,)	)
											__UpperCamelCase					:	str             =      torch.cat([mask, torch.zeros(self.model_tester.seq_length - mask.size(0	)	)]	)
											__UpperCamelCase					:	Dict             =      mask.expand(self.model_tester.batch_size					,     -1	).bool()
											__UpperCamelCase					:	int             =      bool_masked_pos.to(lowerCAmelCase	)

						if return_labels:
											if model_class in [
											    *get_values(lowerCAmelCase	),
											]:
																__UpperCamelCase					:	List[Any]             =      torch.zeros(
																    self.model_tester.batch_size					,     dtype=torch.long					,     device=lowerCAmelCase	)

						return inputs_dict






	def 					lowerCamelCase__     (					self		:						Optional[Any]	)	->       str:







						"""simple docstring"""







						self.config_tester.run_common_tests()






	@unittest.skip(reason="""VideoMAE does not use inputs_embeds"""	)
	def 					lowerCamelCase__     (					self		:						List[Any]	)	->       Any:







						"""simple docstring"""







						pass






	def 					lowerCamelCase__     (					self		:						Dict	)	->       Union[str, Any]:







						"""simple docstring"""







						__UpperCamelCase      ,			__UpperCamelCase					:	str             =      self.model_tester.prepare_config_and_inputs_for_common()

						for model_class in self.all_model_classes:
											__UpperCamelCase					:	Dict             =      model_class(lowerCAmelCase	)
											self.assertIsInstance(model.get_input_embeddings()					,     (nn.Module)	)
											__UpperCamelCase					:	Any             =      model.get_output_embeddings()
											self.assertTrue(x is None or isinstance(lowerCAmelCase					,     nn.Linear	)	)






	def 					lowerCamelCase__     (					self		:						List[str]	)	->       Dict:







						"""simple docstring"""







						__UpperCamelCase      ,			__UpperCamelCase					:	Optional[int]             =      self.model_tester.prepare_config_and_inputs_for_common()

						for model_class in self.all_model_classes:
											__UpperCamelCase					:	Any             =      model_class(lowerCAmelCase	)
											__UpperCamelCase					:	int             =      inspect.signature(model.forward	)
											# signature.parameters is an OrderedDict => so arg_names order is deterministic
											__UpperCamelCase					:	Optional[Any]             =      [*signature.parameters.keys()]

											__UpperCamelCase					:	Union[str, Any]             =      ["""pixel_values"""]
											self.assertListEqual(arg_names[:1]					,     lowerCAmelCase	)






	def 					lowerCamelCase__     (					self		:						int	)	->       Optional[Any]:







						"""simple docstring"""







						__UpperCamelCase					:	Tuple             =      self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_model(*lowerCAmelCase	)






	def 					lowerCamelCase__     (					self		:						Optional[int]	)	->       str:







						"""simple docstring"""







						__UpperCamelCase					:	Tuple             =      self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_pretraining(*lowerCAmelCase	)






	@slow
	def 					lowerCamelCase__     (					self		:						str	)	->       Dict:







						"""simple docstring"""







						for model_name in VIDEOMAE_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											__UpperCamelCase					:	Dict             =      VideoMAEModel.from_pretrained(lowerCAmelCase	)
											self.assertIsNotNone(lowerCAmelCase	)






	def 					lowerCamelCase__     (					self		:						Any	)	->       int:







						"""simple docstring"""







						if not self.has_attentions:
											pass

						else:
											__UpperCamelCase      ,			__UpperCamelCase					:	int             =      self.model_tester.prepare_config_and_inputs_for_common()
											__UpperCamelCase					:	Optional[Any]             =      True

											for model_class in self.all_model_classes:
																__UpperCamelCase					:	List[str]             =      self.model_tester.seq_length - self.model_tester.num_masks
																__UpperCamelCase					:	Tuple             =      (
																    num_visible_patches if model_class == VideoMAEForPreTraining else self.model_tester.seq_length
																)

																__UpperCamelCase					:	Optional[Any]             =      True
																__UpperCamelCase					:	str             =      False
																__UpperCamelCase					:	Optional[int]             =      True
																__UpperCamelCase					:	int             =      model_class(lowerCAmelCase	)
																model.to(lowerCAmelCase	)
																model.eval()
																with torch.no_grad():
																					__UpperCamelCase					:	Tuple             =      model(**self._prepare_for_class(lowerCAmelCase					,     lowerCAmelCase	)	)
																__UpperCamelCase					:	str             =      outputs.attentions
																self.assertEqual(len(lowerCAmelCase	)					,     self.model_tester.num_hidden_layers	)

																# check that output_attentions also work using config
																del inputs_dict["output_attentions"]
																__UpperCamelCase					:	List[str]             =      True
																__UpperCamelCase					:	str             =      model_class(lowerCAmelCase	)
																model.to(lowerCAmelCase	)
																model.eval()
																with torch.no_grad():
																					__UpperCamelCase					:	Optional[Any]             =      model(**self._prepare_for_class(lowerCAmelCase					,     lowerCAmelCase	)	)
																__UpperCamelCase					:	Tuple             =      outputs.attentions
																self.assertEqual(len(lowerCAmelCase	)					,     self.model_tester.num_hidden_layers	)

																self.assertListEqual(
																    list(attentions[0].shape[-3:]	)					,     [self.model_tester.num_attention_heads, seq_len, seq_len]					,     )
																__UpperCamelCase					:	Tuple             =      len(lowerCAmelCase	)

																# Check attention is always last and order is fine
																__UpperCamelCase					:	Dict             =      True
																__UpperCamelCase					:	Tuple             =      True
																__UpperCamelCase					:	str             =      model_class(lowerCAmelCase	)
																model.to(lowerCAmelCase	)
																model.eval()
																with torch.no_grad():
																					__UpperCamelCase					:	Optional[int]             =      model(**self._prepare_for_class(lowerCAmelCase					,     lowerCAmelCase	)	)

																self.assertEqual(out_len + 1					,     len(lowerCAmelCase	)	)

																__UpperCamelCase					:	Optional[Any]             =      outputs.attentions

																self.assertEqual(len(lowerCAmelCase	)					,     self.model_tester.num_hidden_layers	)
																self.assertListEqual(
																    list(self_attentions[0].shape[-3:]	)					,     [self.model_tester.num_attention_heads, seq_len, seq_len]					,     )






	def 					lowerCamelCase__     (					self		:						int	)	->       List[str]:







						"""simple docstring"""







						def check_hidden_states_output(lowerCAmelCase		:						Optional[Any]					,     lowerCAmelCase		:						Union[str, Any]					,     lowerCAmelCase		:						Tuple	):
											__UpperCamelCase					:	int             =      model_class(lowerCAmelCase	)
											model.to(lowerCAmelCase	)
											model.eval()

											with torch.no_grad():
																__UpperCamelCase					:	str             =      model(**self._prepare_for_class(lowerCAmelCase					,     lowerCAmelCase	)	)

											__UpperCamelCase					:	Optional[int]             =      outputs.hidden_states
											__UpperCamelCase					:	Union[str, Any]             =      self.model_tester.num_hidden_layers + 1
											self.assertEqual(len(lowerCAmelCase	)					,     lowerCAmelCase	)

											__UpperCamelCase					:	Any             =      self.model_tester.seq_length - self.model_tester.num_masks
											__UpperCamelCase					:	Dict             =      num_visible_patches if model_class == VideoMAEForPreTraining else self.model_tester.seq_length

											self.assertListEqual(
											    list(hidden_states[0].shape[-2:]	)					,     [seq_length, self.model_tester.hidden_size]					,     )

						__UpperCamelCase      ,			__UpperCamelCase					:	Optional[int]             =      self.model_tester.prepare_config_and_inputs_for_common()

						for model_class in self.all_model_classes:
											__UpperCamelCase					:	Union[str, Any]             =      True
											check_hidden_states_output(lowerCAmelCase					,     lowerCAmelCase					,     lowerCAmelCase	)

											# check that output_hidden_states also work using config
											del inputs_dict["output_hidden_states"]
											__UpperCamelCase					:	Union[str, Any]             =      True

											check_hidden_states_output(lowerCAmelCase					,     lowerCAmelCase					,     lowerCAmelCase	)






	@unittest.skip("""Will be fixed soon by reducing the size of the model used for common tests."""	)
	def 					lowerCamelCase__     (					self		:						Tuple	)	->       Optional[Any]:







						"""simple docstring"""







						pass






def 				A__ ()     ->						Any:
					__UpperCamelCase					:	Union[str, Any]             =      hf_hub_download(
					    repo_id="""hf-internal-testing/spaghetti-video"""		,	filename="""eating_spaghetti.npy"""		,	repo_type="""dataset"""				)
					__UpperCamelCase					:	Any             =      np.load(snake_case				)
					return list(snake_case				)







@require_torch
@require_vision
class    SCREAMING_SNAKE_CASE_				(       unittest.TestCase   ):



	"""simple docstring"""


	@cached_property
	def 					lowerCamelCase__     (					self		:						int	)	->       List[str]:







						"""simple docstring"""







						return (
						    VideoMAEImageProcessor(image_mean=[0.5, 0.5, 0.5]					,     image_std=[0.5, 0.5, 0.5]	)
						    if is_vision_available()
						    else None
						)






	@slow
	def 					lowerCamelCase__     (					self		:						List[Any]	)	->       Optional[Any]:







						"""simple docstring"""







						__UpperCamelCase					:	Union[str, Any]             =      VideoMAEForVideoClassification.from_pretrained("""MCG-NJU/videomae-base-finetuned-kinetics"""	).to(
						    lowerCAmelCase	)

						__UpperCamelCase					:	Any             =      self.default_image_processor
						__UpperCamelCase					:	str             =      prepare_video()
						__UpperCamelCase					:	Dict             =      image_processor(lowerCAmelCase					,     return_tensors="""pt"""	).to(lowerCAmelCase	)

						# forward pass
						with torch.no_grad():
											__UpperCamelCase					:	Optional[int]             =      model(**lowerCAmelCase	)

						# verify the logits
						__UpperCamelCase					:	Optional[int]             =      torch.Size((1, 400)	)
						self.assertEqual(outputs.logits.shape					,     lowerCAmelCase	)

						__UpperCamelCase					:	Any             =      torch.tensor([0.36_69, -0.06_88, -0.24_21]	).to(lowerCAmelCase	)

						self.assertTrue(torch.allclose(outputs.logits[0, :3]					,     lowerCAmelCase					,     atol=1E-4	)	)






	@slow
	def 					lowerCamelCase__     (					self		:						Tuple	)	->       Any:







						"""simple docstring"""







						__UpperCamelCase					:	Tuple             =      VideoMAEForPreTraining.from_pretrained("""MCG-NJU/videomae-base-short"""	).to(lowerCAmelCase	)

						__UpperCamelCase					:	int             =      self.default_image_processor
						__UpperCamelCase					:	Optional[int]             =      prepare_video()
						__UpperCamelCase					:	int             =      image_processor(lowerCAmelCase					,     return_tensors="""pt"""	).to(lowerCAmelCase	)

						# add boolean mask, indicating which patches to mask
						__UpperCamelCase					:	Tuple             =      hf_hub_download(repo_id="""hf-internal-testing/bool-masked-pos"""					,     filename="""bool_masked_pos.pt"""	)
						__UpperCamelCase					:	int             =      torch.load(lowerCAmelCase	)

						# forward pass
						with torch.no_grad():
											__UpperCamelCase					:	int             =      model(**lowerCAmelCase	)

						# verify the logits
						__UpperCamelCase					:	str             =      torch.Size([1, 1408, 1536]	)
						__UpperCamelCase					:	Union[str, Any]             =      torch.tensor(
						    [[0.79_94, 0.96_12, 0.85_08], [0.74_01, 0.89_58, 0.83_02], [0.58_62, 0.74_68, 0.73_25]]					,     device=lowerCAmelCase	)
						self.assertEqual(outputs.logits.shape					,     lowerCAmelCase	)
						self.assertTrue(torch.allclose(outputs.logits[0, :3, :3]					,     lowerCAmelCase					,     atol=1E-4	)	)

						# verify the loss (`config.norm_pix_loss` = `True`)
						__UpperCamelCase					:	Tuple             =      torch.tensor([0.51_42]					,     device=lowerCAmelCase	)
						self.assertTrue(torch.allclose(outputs.loss					,     lowerCAmelCase					,     atol=1E-4	)	)

						# verify the loss (`config.norm_pix_loss` = `False`)
						__UpperCamelCase					:	Optional[Any]             =      VideoMAEForPreTraining.from_pretrained("""MCG-NJU/videomae-base-short"""					,     norm_pix_loss=lowerCAmelCase	).to(
						    lowerCAmelCase	)

						with torch.no_grad():
											__UpperCamelCase					:	Union[str, Any]             =      model(**lowerCAmelCase	)

						__UpperCamelCase					:	Tuple             =      torch.tensor(torch.tensor([0.64_69]	)					,     device=lowerCAmelCase	)
						self.assertTrue(torch.allclose(outputs.loss					,     lowerCAmelCase					,     atol=1E-4	)	)












import warnings
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


a__		     =     logging.get_logger(__name__)


class    SCREAMING_SNAKE_CASE_				(       _UpperCamelCase   ):



	"""simple docstring"""


	__magic_name__   :       int      	=		['input_ids', 'attention_mask']






	def __init__(					self		:						List[Any]					,     lowerCAmelCase		:						Union[str, Any]="</s>"					,     lowerCAmelCase		:						Tuple="<unk>"					,     lowerCAmelCase		:						Union[str, Any]="<pad>"					,     lowerCAmelCase		:						Tuple=125					,     lowerCAmelCase		:						str=None					,     **lowerCAmelCase		:						Optional[Any]					,     )	->       None:







						"""simple docstring"""







						if extra_ids > 0 and additional_special_tokens is None:
											__UpperCamelCase					:	str             =      [F'''<extra_id_{i}>''' for i in range(lowerCAmelCase	)]
						elif extra_ids > 0 and additional_special_tokens is not None:
											# Check that we have the right number of extra_id special tokens
											__UpperCamelCase					:	Tuple             =      len(set(filter(lambda lowerCAmelCase	: bool("""extra_id""" in str(lowerCAmelCase	)	)					,     lowerCAmelCase	)	)	)
											if extra_tokens != extra_ids:
																raise ValueError(
																    F'''Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are'''
																    """ provided to ByT5Tokenizer. In this case the additional_special_tokens must include the"""
																    """ extra_ids tokens"""	)

						__UpperCamelCase					:	Tuple             =      AddedToken(lowerCAmelCase					,     lstrip=lowerCAmelCase					,     rstrip=lowerCAmelCase	) if isinstance(lowerCAmelCase					,     lowerCAmelCase	) else pad_token
						__UpperCamelCase					:	Tuple             =      AddedToken(lowerCAmelCase					,     lstrip=lowerCAmelCase					,     rstrip=lowerCAmelCase	) if isinstance(lowerCAmelCase					,     lowerCAmelCase	) else eos_token
						__UpperCamelCase					:	int             =      AddedToken(lowerCAmelCase					,     lstrip=lowerCAmelCase					,     rstrip=lowerCAmelCase	) if isinstance(lowerCAmelCase					,     lowerCAmelCase	) else unk_token

						super().__init__(
						    eos_token=lowerCAmelCase					,     unk_token=lowerCAmelCase					,     pad_token=lowerCAmelCase					,     extra_ids=lowerCAmelCase					,     additional_special_tokens=lowerCAmelCase					,     **lowerCAmelCase					,     )

						__UpperCamelCase					:	Any             =      extra_ids

						__UpperCamelCase					:	List[Any]             =      2**8  # utf is 8 bits

						# define special tokens dict
						__UpperCamelCase					:	Dict[int, str]             =      {
						    self.pad_token: 0,
						    self.eos_token: 1,
						    self.unk_token: 2,
						}
						__UpperCamelCase					:	int             =      len(self.special_tokens_encoder	)
						__UpperCamelCase					:	Tuple             =      len(lowerCAmelCase	)
						for i, token in enumerate(lowerCAmelCase	):
											__UpperCamelCase					:	Optional[int]             =      self.vocab_size + i - n
						__UpperCamelCase					:	Dict[str, int]             =      {v: k for k, v in self.special_tokens_encoder.items()}






	@property
	def 					lowerCamelCase__     (					self		:						Any	)	->       Optional[int]:







						"""simple docstring"""







						return self._utf_vocab_size + self._num_special_tokens + self._extra_ids






	def 					lowerCamelCase__     (					self		:						Optional[int]					,     lowerCAmelCase		:						List[int]					,     lowerCAmelCase		:						Optional[List[int]] = None					,     lowerCAmelCase		:						bool = False	)	->       List[int]:







						"""simple docstring"""







						if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=lowerCAmelCase					,     token_ids_a=lowerCAmelCase					,     already_has_special_tokens=lowerCAmelCase	)

						# normal case: some special tokens
						if token_ids_a is None:
											return ([0] * len(lowerCAmelCase	)) + [1]
						return ([0] * len(lowerCAmelCase	)) + [1] + ([0] * len(lowerCAmelCase	)) + [1]






	def 					lowerCamelCase__     (					self		:						Tuple					,     lowerCAmelCase		:						List[int]	)	->       List[int]:







						"""simple docstring"""







						if len(lowerCAmelCase	) > 0 and token_ids[-1] == self.eos_token_id:
											warnings.warn(
											    F'''This sequence already has {self.eos_token}. In future versions this behavior may lead to duplicated'''
											    """ eos tokens being added."""	)
											return token_ids
						else:
											return token_ids + [self.eos_token_id]






	def 					lowerCamelCase__     (					self		:						List[Any]					,     lowerCAmelCase		:						List[int]					,     lowerCAmelCase		:						Optional[List[int]] = None	)	->       List[int]:







						"""simple docstring"""







						__UpperCamelCase					:	Any             =      [self.eos_token_id]

						if token_ids_a is None:
											return len(token_ids_a + eos	) * [0]
						return len(token_ids_a + eos + token_ids_a + eos	) * [0]






	def 					lowerCamelCase__     (					self		:						List[str]					,     lowerCAmelCase		:						List[int]					,     lowerCAmelCase		:						Optional[List[int]] = None	)	->       List[int]:







						"""simple docstring"""







						__UpperCamelCase					:	List[str]             =      self._add_eos_if_not_present(lowerCAmelCase	)
						if token_ids_a is None:
											return token_ids_a
						else:
											__UpperCamelCase					:	List[Any]             =      self._add_eos_if_not_present(lowerCAmelCase	)
											return token_ids_a + token_ids_a






	def 					lowerCamelCase__     (					self		:						str					,     lowerCAmelCase		:						str	)	->       List[str]:







						"""simple docstring"""







						__UpperCamelCase					:	Optional[int]             =      [chr(lowerCAmelCase	) for i in text.encode("""utf-8"""	)]
						return tokens






	def 					lowerCamelCase__     (					self		:						Union[str, Any]					,     lowerCAmelCase		:						Optional[int]	)	->       int:







						"""simple docstring"""







						if token in self.special_tokens_encoder:
											__UpperCamelCase					:	Dict             =      self.special_tokens_encoder[token]
						elif token in self.added_tokens_encoder:
											__UpperCamelCase					:	Tuple             =      self.added_tokens_encoder[token]
						elif len(lowerCAmelCase	) != 1:
											__UpperCamelCase					:	Dict             =      self.unk_token_id
						else:
											__UpperCamelCase					:	Union[str, Any]             =      ord(lowerCAmelCase	) + self._num_special_tokens
						return token_id






	def 					lowerCamelCase__     (					self		:						Union[str, Any]					,     lowerCAmelCase		:						Dict	)	->       Optional[Any]:







						"""simple docstring"""







						if index in self.special_tokens_decoder:
											__UpperCamelCase					:	str             =      self.special_tokens_decoder[index]
						else:
											__UpperCamelCase					:	Any             =      chr(index - self._num_special_tokens	)
						return token






	def 					lowerCamelCase__     (					self		:						str					,     lowerCAmelCase		:						List[Any]	)	->       List[Any]:







						"""simple docstring"""







						__UpperCamelCase					:	Union[str, Any]             =      b""""""
						for token in tokens:
											if token in self.special_tokens_decoder:
																__UpperCamelCase					:	Optional[Any]             =      self.special_tokens_decoder[token].encode("""utf-8"""	)
											elif token in self.added_tokens_decoder:
																__UpperCamelCase					:	Optional[int]             =      self.special_tokens_decoder[token].encode("""utf-8"""	)
											elif token in self.special_tokens_encoder:
																__UpperCamelCase					:	Tuple             =      token.encode("""utf-8"""	)
											elif token in self.added_tokens_encoder:
																__UpperCamelCase					:	Tuple             =      token.encode("""utf-8"""	)
											else:
																__UpperCamelCase					:	Union[str, Any]             =      bytes([ord(lowerCAmelCase	)]	)
											bstring += tok_string
						__UpperCamelCase					:	Optional[int]             =      bstring.decode("""utf-8"""					,     errors="""ignore"""	)
						return string






	def 					lowerCamelCase__     (					self		:						Optional[int]					,     lowerCAmelCase		:						str					,     lowerCAmelCase		:						Optional[str] = None	)	->       Tuple[str]:







						"""simple docstring"""







						return ()






