





'''simple docstring'''






import inspect
import unittest

from transformers import MobileViTVaConfig
from transformers.testing_utils import require_torch, require_torch_multi_gpu, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
					import torch

					from transformers import MobileViTVaForImageClassification, MobileViTVaForSemanticSegmentation, MobileViTVaModel
					from transformers.models.mobilevitva.modeling_mobilevitva import (
					    MOBILEVITV2_PRETRAINED_MODEL_ARCHIVE_LIST,
					    make_divisible,
					)


if is_vision_available():
					from PIL import Image

					from transformers import MobileViTImageProcessor
class 		lowerCamelCase  (      _A       ):



	def   _lowerCamelCase    (   self	):
			lowerCAmelCase     :    Union[str, Any]						=   self.config_class(**self.inputs_dict	)
			self.parent.assertTrue(hasattr(a_		,      "width_multiplier"	)	)






class 		lowerCamelCase  :



	def __init__(   self		,      a_		,      a_=13		,      a_=64		,      a_=2		,      a_=3		,      a_="swish"		,      a_=3		,      a_=32		,      a_=0.1		,      a_=0.02		,      a_=True		,      a_=True		,      a_=10		,      a_=None		,      a_=0.25		,      a_=0.0		,      a_=0.0		,      ):
			lowerCAmelCase     :    Optional[Any]						=   parent
			lowerCAmelCase     :    Any						=   batch_size
			lowerCAmelCase     :    List[str]						=   image_size
			lowerCAmelCase     :    Tuple						=   patch_size
			lowerCAmelCase     :    Union[str, Any]						=   num_channels
			lowerCAmelCase     :    Tuple						=   make_divisible(512 * width_multiplier		,      divisor=8	)
			lowerCAmelCase     :    str						=   hidden_act
			lowerCAmelCase     :    Dict						=   conv_kernel_size
			lowerCAmelCase     :    Optional[int]						=   output_stride
			lowerCAmelCase     :    Dict						=   classifier_dropout_prob
			lowerCAmelCase     :    List[str]						=   use_labels
			lowerCAmelCase     :    List[str]						=   is_training
			lowerCAmelCase     :    Optional[int]						=   num_labels
			lowerCAmelCase     :    Any						=   initializer_range
			lowerCAmelCase     :    Optional[Any]						=   scope
			lowerCAmelCase     :    Union[str, Any]						=   width_multiplier
			lowerCAmelCase     :    Any						=   ffn_dropout
			lowerCAmelCase     :    Dict						=   attn_dropout



	def   _lowerCamelCase    (   self	):
			lowerCAmelCase     :    Any						=   floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]	)

			lowerCAmelCase     :    Any						=   None
			lowerCAmelCase     :    str						=   None
			if self.use_labels:
					lowerCAmelCase     :    Dict						=   ids_tensor([self.batch_size]		,      self.num_labels	)
					lowerCAmelCase     :    List[str]						=   ids_tensor([self.batch_size, self.image_size, self.image_size]		,      self.num_labels	)

			lowerCAmelCase     :    Optional[Any]						=   self.get_config()

			return config, pixel_values, labels, pixel_labels



	def   _lowerCamelCase    (   self	):
			return MobileViTVaConfig(
			    image_size=self.image_size		,      patch_size=self.patch_size		,      num_channels=self.num_channels		,      hidden_act=self.hidden_act		,      conv_kernel_size=self.conv_kernel_size		,      output_stride=self.output_stride		,      classifier_dropout_prob=self.classifier_dropout_prob		,      initializer_range=self.initializer_range		,      width_multiplier=self.width_multiplier		,      ffn_dropout=self.ffn_dropout_prob		,      attn_dropout=self.attn_dropout_prob		,      )



	def   _lowerCamelCase    (   self		,      a_		,      a_		,      a_		,      a_	):
			lowerCAmelCase     :    List[Any]						=   MobileViTVaModel(config=a_	)
			model.to(a_	)
			model.eval()
			lowerCAmelCase     :    Optional[int]						=   model(a_	)
			self.parent.assertEqual(
			    result.last_hidden_state.shape		,      (
			        self.batch_size,
			        self.last_hidden_size,
			        self.image_size // self.output_stride,
			        self.image_size // self.output_stride,
			    )		,      )



	def   _lowerCamelCase    (   self		,      a_		,      a_		,      a_		,      a_	):
			lowerCAmelCase     :    str						=   self.num_labels
			lowerCAmelCase     :    Dict						=   MobileViTVaForImageClassification(a_	)
			model.to(a_	)
			model.eval()
			lowerCAmelCase     :    str						=   model(a_		,      labels=a_	)
			self.parent.assertEqual(result.logits.shape		,      (self.batch_size, self.num_labels)	)



	def   _lowerCamelCase    (   self		,      a_		,      a_		,      a_		,      a_	):
			lowerCAmelCase     :    Tuple						=   self.num_labels
			lowerCAmelCase     :    int						=   MobileViTVaForSemanticSegmentation(a_	)
			model.to(a_	)
			model.eval()
			lowerCAmelCase     :    Optional[Any]						=   model(a_	)
			self.parent.assertEqual(
			    result.logits.shape		,      (
			        self.batch_size,
			        self.num_labels,
			        self.image_size // self.output_stride,
			        self.image_size // self.output_stride,
			    )		,      )
			lowerCAmelCase     :    Optional[int]						=   model(a_		,      labels=a_	)
			self.parent.assertEqual(
			    result.logits.shape		,      (
			        self.batch_size,
			        self.num_labels,
			        self.image_size // self.output_stride,
			        self.image_size // self.output_stride,
			    )		,      )





	def   _lowerCamelCase    (   self	):
			lowerCAmelCase     :    List[Any]						=   self.prepare_config_and_inputs()
			lowerCAmelCase	,		lowerCAmelCase	,		lowerCAmelCase	,		lowerCAmelCase     :    List[Any]						=   config_and_inputs
			lowerCAmelCase     :    Optional[int]						=   {"pixel_values": pixel_values}
			return config, inputs_dict






@require_torch
class 		lowerCamelCase  (      _A      ,   _A      ,   unittest.TestCase       ):
	snake_case_      =   (
	    (MobileViTVaModel, MobileViTVaForImageClassification, MobileViTVaForSemanticSegmentation)
	    if is_torch_available()
	    else ()
	)

	snake_case_      =   (
	    {
	        "feature-extraction": MobileViTVaModel,
	        "image-classification": MobileViTVaForImageClassification,
	        "image-segmentation": MobileViTVaForSemanticSegmentation,
	    }
	    if is_torch_available()
	    else {}
	)

	snake_case_      =   False
	snake_case_      =   False
	snake_case_      =   False
	snake_case_      =   False



	def   _lowerCamelCase    (   self	):
			lowerCAmelCase     :    Union[str, Any]						=   MobileViTVaModelTester(self	)
			lowerCAmelCase     :    Union[str, Any]						=   MobileViTVaConfigTester(self		,      config_class=a_		,      has_text_modality=a_	)



	def   _lowerCamelCase    (   self	):
			self.config_tester.run_common_tests()



	@unittest.skip(reason="MobileViTV2 does not use inputs_embeds"	)
	def   _lowerCamelCase    (   self	):
			pass



	@unittest.skip(reason="MobileViTV2 does not support input and output embeddings"	)
	def   _lowerCamelCase    (   self	):
			pass



	@unittest.skip(reason="MobileViTV2 does not output attentions"	)
	def   _lowerCamelCase    (   self	):
			pass



	@require_torch_multi_gpu
	@unittest.skip(reason="Got `CUDA error: misaligned address` for tests after this one being run."	)
	def   _lowerCamelCase    (   self	):
			pass



	@unittest.skip("Will be fixed soon by reducing the size of the model used for common tests."	)
	def   _lowerCamelCase    (   self	):
			pass



	def   _lowerCamelCase    (   self	):
			lowerCAmelCase	,		lowerCAmelCase     :    int						=   self.model_tester.prepare_config_and_inputs_for_common()

			for model_class in self.all_model_classes:
					lowerCAmelCase     :    List[Any]						=   model_class(a_	)
					lowerCAmelCase     :    str						=   inspect.signature(model.forward	)
					# signature.parameters is an OrderedDict => so arg_names order is deterministic
					lowerCAmelCase     :    List[Any]						=   [*signature.parameters.keys()]

					lowerCAmelCase     :    Union[str, Any]						=   ["pixel_values"]
					self.assertListEqual(arg_names[:1]		,      a_	)



	def   _lowerCamelCase    (   self	):
			lowerCAmelCase     :    List[str]						=   self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_model(*a_	)



	def   _lowerCamelCase    (   self	):
			def check_hidden_states_output(a_		,      a_		,      a_	):
					lowerCAmelCase     :    List[str]						=   model_class(a_	)
					model.to(a_	)
					model.eval()

					with torch.no_grad():
							lowerCAmelCase     :    List[str]						=   model(**self._prepare_for_class(a_		,      a_	)	)

					lowerCAmelCase     :    int						=   outputs.hidden_states

					lowerCAmelCase     :    Tuple						=   5
					self.assertEqual(len(a_	)		,      a_	)

					# MobileViTV2's feature maps are of shape (batch_size, num_channels, height, width)
					# with the width and height being successively divided by 2.
					lowerCAmelCase     :    Optional[int]						=   2
					for i in range(len(a_	)	):
							self.assertListEqual(
							    list(hidden_states[i].shape[-2:]	)		,      [self.model_tester.image_size // divisor, self.model_tester.image_size // divisor]		,      )
							divisor *= 2

					self.assertEqual(self.model_tester.output_stride		,      divisor // 2	)

			lowerCAmelCase	,		lowerCAmelCase     :    Optional[int]						=   self.model_tester.prepare_config_and_inputs_for_common()

			for model_class in self.all_model_classes:
					lowerCAmelCase     :    int						=   True
					check_hidden_states_output(a_		,      a_		,      a_	)

					# check that output_hidden_states also work using config
					del inputs_dict["output_hidden_states"]
					lowerCAmelCase     :    Dict						=   True

					check_hidden_states_output(a_		,      a_		,      a_	)



	def   _lowerCamelCase    (   self	):
			lowerCAmelCase     :    Optional[Any]						=   self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_image_classification(*a_	)



	def   _lowerCamelCase    (   self	):
			lowerCAmelCase     :    Tuple						=   self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_for_semantic_segmentation(*a_	)





	@slow
	def   _lowerCamelCase    (   self	):
			for model_name in MOBILEVITV2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
					lowerCAmelCase     :    Optional[int]						=   MobileViTVaModel.from_pretrained(a_	)
					self.assertIsNotNone(a_	)






def      __A						(							):
		lowerCAmelCase     :    Any						=   Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"	)
		return image

@require_torch
@require_vision
class 		lowerCamelCase  (      unittest.TestCase       ):



	@cached_property
	def   _lowerCamelCase    (   self	):
			return (
			    MobileViTImageProcessor.from_pretrained("apple/mobilevitv2-1.0-imagenet1k-256"	)
			    if is_vision_available()
			    else None
			)



	@slow
	def   _lowerCamelCase    (   self	):
			lowerCAmelCase     :    Union[str, Any]						=   MobileViTVaForImageClassification.from_pretrained("apple/mobilevitv2-1.0-imagenet1k-256"	).to(
			    a_	)

			lowerCAmelCase     :    List[str]						=   self.default_image_processor
			lowerCAmelCase     :    Any						=   prepare_img()
			lowerCAmelCase     :    Any						=   image_processor(images=a_		,      return_tensors="pt"	).to(a_	)

			# forward pass
			with torch.no_grad():
					lowerCAmelCase     :    Dict						=   model(**a_	)

			# verify the logits
			lowerCAmelCase     :    Optional[int]						=   torch.Size((1, 1_000)	)
			self.assertEqual(outputs.logits.shape		,      a_	)

			lowerCAmelCase     :    Optional[Any]						=   torch.tensor([-1.6_3_3_6e0_0, -7.3_2_0_4e-0_2, -5.1_8_8_3e-0_1]	).to(a_	)

			self.assertTrue(torch.allclose(outputs.logits[0, :3]		,      a_		,      atol=1e-4	)	)



	@slow
	def   _lowerCamelCase    (   self	):
			lowerCAmelCase     :    List[str]						=   MobileViTVaForSemanticSegmentation.from_pretrained("shehan97/mobilevitv2-1.0-voc-deeplabv3"	)
			lowerCAmelCase     :    Any						=   model.to(a_	)

			lowerCAmelCase     :    Union[str, Any]						=   MobileViTImageProcessor.from_pretrained("shehan97/mobilevitv2-1.0-voc-deeplabv3"	)

			lowerCAmelCase     :    Optional[int]						=   prepare_img()
			lowerCAmelCase     :    Any						=   image_processor(images=a_		,      return_tensors="pt"	).to(a_	)

			# forward pass
			with torch.no_grad():
					lowerCAmelCase     :    int						=   model(**a_	)
			lowerCAmelCase     :    str						=   outputs.logits

			# verify the logits
			lowerCAmelCase     :    Union[str, Any]						=   torch.Size((1, 21, 32, 32)	)
			self.assertEqual(logits.shape		,      a_	)

			lowerCAmelCase     :    str						=   torch.tensor(
			    [
			        [[7.0863, 7.1525, 6.8201], [6.6931, 6.8770, 6.8933], [6.2978, 7.0366, 6.9636]],
			        [[-3.7134, -3.6712, -3.6675], [-3.5825, -3.3549, -3.4777], [-3.3435, -3.3979, -3.2857]],
			        [[-2.9329, -2.8003, -2.7369], [-3.0564, -2.4780, -2.0207], [-2.6889, -1.9298, -1.7640]],
			    ]		,      device=a_		,      )

			self.assertTrue(torch.allclose(logits[0, :3, :3, :3]		,      a_		,      atol=1e-4	)	)





	@slow
	def   _lowerCamelCase    (   self	):
			lowerCAmelCase     :    Any						=   MobileViTVaForSemanticSegmentation.from_pretrained("shehan97/mobilevitv2-1.0-voc-deeplabv3"	)
			lowerCAmelCase     :    List[str]						=   model.to(a_	)

			lowerCAmelCase     :    Optional[Any]						=   MobileViTImageProcessor.from_pretrained("shehan97/mobilevitv2-1.0-voc-deeplabv3"	)

			lowerCAmelCase     :    Union[str, Any]						=   prepare_img()
			lowerCAmelCase     :    str						=   image_processor(images=a_		,      return_tensors="pt"	).to(a_	)

			# forward pass
			with torch.no_grad():
					lowerCAmelCase     :    List[Any]						=   model(**a_	)

			lowerCAmelCase     :    List[str]						=   outputs.logits.detach().cpu()

			lowerCAmelCase     :    Any						=   image_processor.post_process_semantic_segmentation(outputs=a_		,      target_sizes=[(50, 60)]	)
			lowerCAmelCase     :    Tuple						=   torch.Size((50, 60)	)
			self.assertEqual(segmentation[0].shape		,      a_	)

			lowerCAmelCase     :    Tuple						=   image_processor.post_process_semantic_segmentation(outputs=a_	)
			lowerCAmelCase     :    Dict						=   torch.Size((32, 32)	)
			self.assertEqual(segmentation[0].shape		,      a_	)










'''simple docstring'''






import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase       		=      logging.get_logger(__name__)

lowerCAmelCase       		=      """▁"""

lowerCAmelCase       		=      {"""vocab_file""": """sentencepiece.bpe.model"""}

lowerCAmelCase       		=      {
    """vocab_file""": {
        """xlm-roberta-base""": """https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model""",
        """xlm-roberta-large""": """https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model""",
        """xlm-roberta-large-finetuned-conll02-dutch""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model"""
        ),
        """xlm-roberta-large-finetuned-conll02-spanish""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model"""
        ),
        """xlm-roberta-large-finetuned-conll03-english""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model"""
        ),
        """xlm-roberta-large-finetuned-conll03-german""": (
            """https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model"""
        ),
    }
}

lowerCAmelCase       		=      {
    """xlm-roberta-base""": 5_12,
    """xlm-roberta-large""": 5_12,
    """xlm-roberta-large-finetuned-conll02-dutch""": 5_12,
    """xlm-roberta-large-finetuned-conll02-spanish""": 5_12,
    """xlm-roberta-large-finetuned-conll03-english""": 5_12,
    """xlm-roberta-large-finetuned-conll03-german""": 5_12,
}
class 		lowerCamelCase  (      _A       ):
	snake_case_      =   VOCAB_FILES_NAMES
	snake_case_      =   PRETRAINED_VOCAB_FILES_MAP
	snake_case_      =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	snake_case_      =   ["input_ids", "attention_mask"]



	def __init__(   self		,      a_		,      a_="<s>"		,      a_="</s>"		,      a_="</s>"		,      a_="<s>"		,      a_="<unk>"		,      a_="<pad>"		,      a_="<mask>"		,      a_ = None		,      **a_		,      ):
			# Mask token behave like a normal word, i.e. include the space before it
			lowerCAmelCase     :    str						=   AddedToken(a_		,      lstrip=a_		,      rstrip=a_	) if isinstance(a_		,      a_	) else mask_token

			lowerCAmelCase     :    Tuple						=   {} if sp_model_kwargs is None else sp_model_kwargs

			super().__init__(
			    bos_token=a_		,      eos_token=a_		,      unk_token=a_		,      sep_token=a_		,      cls_token=a_		,      pad_token=a_		,      mask_token=a_		,      sp_model_kwargs=self.sp_model_kwargs		,      **a_		,      )

			lowerCAmelCase     :    Optional[int]						=   spm.SentencePieceProcessor(**self.sp_model_kwargs	)
			self.sp_model.Load(str(a_	)	)
			lowerCAmelCase     :    int						=   vocab_file

			# Original fairseq vocab and spm vocab must be "aligned":
			# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
			# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
			# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
			# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

			# Mimic fairseq token-to-id alignment for the first 4 token
			lowerCAmelCase     :    Any						=   {"<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3}

			# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
			lowerCAmelCase     :    Dict						=   1

			lowerCAmelCase     :    Optional[Any]						=   len(self.sp_model	) + self.fairseq_offset
			lowerCAmelCase     :    List[Any]						=   {v: k for k, v in self.fairseq_tokens_to_ids.items()}



	def __getstate__(   self	):
			lowerCAmelCase     :    Any						=   self.__dict__.copy()
			lowerCAmelCase     :    Optional[int]						=   None
			lowerCAmelCase     :    List[str]						=   self.sp_model.serialized_model_proto()
			return state



	def __setstate__(   self		,      a_	):
			lowerCAmelCase     :    Any						=   d

			# for backward compatibility
			if not hasattr(self		,      "sp_model_kwargs"	):
					lowerCAmelCase     :    Any						=   {}

			lowerCAmelCase     :    Optional[Any]						=   spm.SentencePieceProcessor(**self.sp_model_kwargs	)
			self.sp_model.LoadFromSerializedProto(self.sp_model_proto	)



	def   _lowerCamelCase    (   self		,      a_		,      a_ = None	):

			if token_ids_a is None:
					return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
			lowerCAmelCase     :    Optional[Any]						=   [self.cls_token_id]
			lowerCAmelCase     :    List[Any]						=   [self.sep_token_id]
			return cls + token_ids_a + sep + sep + token_ids_a + sep



	def   _lowerCamelCase    (   self		,      a_		,      a_ = None		,      a_ = False	):

			if already_has_special_tokens:
					return super().get_special_tokens_mask(
					    token_ids_a=a_		,      token_ids_a=a_		,      already_has_special_tokens=a_	)

			if token_ids_a is None:
					return [1] + ([0] * len(a_	)) + [1]
			return [1] + ([0] * len(a_	)) + [1, 1] + ([0] * len(a_	)) + [1]



	def   _lowerCamelCase    (   self		,      a_		,      a_ = None	):
			lowerCAmelCase     :    Dict						=   [self.sep_token_id]
			lowerCAmelCase     :    List[Any]						=   [self.cls_token_id]

			if token_ids_a is None:
					return len(cls + token_ids_a + sep	) * [0]
			return len(cls + token_ids_a + sep + sep + token_ids_a + sep	) * [0]



	@property
	def   _lowerCamelCase    (   self	):
			return len(self.sp_model	) + self.fairseq_offset + 1  # Add the <mask> token



	def   _lowerCamelCase    (   self	):
			lowerCAmelCase     :    Dict						=   {self.convert_ids_to_tokens(a_	): i for i in range(self.vocab_size	)}
			vocab.update(self.added_tokens_encoder	)
			return vocab



	def   _lowerCamelCase    (   self		,      a_	):
			return self.sp_model.encode(a_		,      out_type=a_	)



	def   _lowerCamelCase    (   self		,      a_	):
			if token in self.fairseq_tokens_to_ids:
					return self.fairseq_tokens_to_ids[token]
			lowerCAmelCase     :    int						=   self.sp_model.PieceToId(a_	)

			# Need to return unknown token if the SP model returned 0
			return spm_id + self.fairseq_offset if spm_id else self.unk_token_id



	def   _lowerCamelCase    (   self		,      a_	):
			if index in self.fairseq_ids_to_tokens:
					return self.fairseq_ids_to_tokens[index]
			return self.sp_model.IdToPiece(index - self.fairseq_offset	)



	def   _lowerCamelCase    (   self		,      a_	):
			lowerCAmelCase     :    Dict						=   "".join(a_	).replace(a_		,      " "	).strip()
			return out_string





	def   _lowerCamelCase    (   self		,      a_		,      a_ = None	):
			if not os.path.isdir(a_	):
					logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''	)
					return
			lowerCAmelCase     :    int						=   os.path.join(
			    a_		,      (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]	)

			if os.path.abspath(self.vocab_file	) != os.path.abspath(a_	) and os.path.isfile(self.vocab_file	):
					copyfile(self.vocab_file		,      a_	)
			elif not os.path.isfile(self.vocab_file	):
					with open(a_		,      "wb"	) as fi:
							lowerCAmelCase     :    Dict						=   self.sp_model.serialized_model_proto()
							fi.write(a_	)

			return (out_vocab_file,)



