






'''simple docstring'''



import torch
from torch import nn
class UpperCAmelCase_       (     nn.Module					):




      """simple docstring"""




      def __init__(			self      :		List[Any]  ,   UpperCAmelCase      :		Union[str, Any]  ,   UpperCAmelCase      :		Union[str, Any]  ,   UpperCAmelCase      :		str  ,   UpperCAmelCase      :		Union[str, Any]  ,   UpperCAmelCase      :		List[str]=1  ,   UpperCAmelCase      :		Union[str, Any]=False		)	->	Dict:


       '''simple docstring'''

       super().__init__()

       lowercase       : str						=n_token
       lowercase       : int						=d_embed
       lowercase       : Tuple						=d_proj

       lowercase       : Union[str, Any]						=cutoffs + [n_token]
       lowercase       : Dict						=[0] + self.cutoffs
       lowercase       : Union[str, Any]						=div_val

       lowercase       : Union[str, Any]						=self.cutoffs[0]
       lowercase       : Any						=len(self.cutoffs		) - 1
       lowercase       : Dict						=self.shortlist_size + self.n_clusters

       if self.n_clusters > 0:
        lowercase       : Tuple						=nn.Parameter(torch.zeros(self.n_clusters  ,   self.d_embed		)		)
        lowercase       : Union[str, Any]						=nn.Parameter(torch.zeros(self.n_clusters		)		)

       lowercase       : Tuple						=nn.ModuleList()
       lowercase       : Optional[Any]						=nn.ParameterList()

       if div_val == 1:
        for i in range(len(self.cutoffs		)		):
         if d_proj != d_embed:
          self.out_projs.append(nn.Parameter(torch.FloatTensor(UpperCAmelCase  ,   UpperCAmelCase		)		)		)
         else:
          self.out_projs.append(UpperCAmelCase		)

        self.out_layers.append(nn.Linear(UpperCAmelCase  ,   UpperCAmelCase		)		)
       else:
        for i in range(len(self.cutoffs		)		):
         lowercase	,		lowercase       : Union[str, Any]						=self.cutoff_ends[i], self.cutoff_ends[i + 1]
         lowercase       : Optional[int]						=d_embed // (div_val**i)

         self.out_projs.append(nn.Parameter(torch.FloatTensor(UpperCAmelCase  ,   UpperCAmelCase		)		)		)

         self.out_layers.append(nn.Linear(UpperCAmelCase  ,   r_idx - l_idx		)		)

       lowercase       : int						=keep_order






      def 					A__   (			self      :		List[str]  ,   UpperCAmelCase      :		str  ,   UpperCAmelCase      :		List[Any]  ,   UpperCAmelCase      :		Any  ,   UpperCAmelCase      :		List[str]		)	->	Optional[int]:


       '''simple docstring'''

       if proj is None:
        lowercase       : Optional[int]						=nn.functional.linear(UpperCAmelCase  ,   UpperCAmelCase  ,   bias=UpperCAmelCase		)
       else:
        # if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
        lowercase       : List[Any]						=nn.functional.linear(UpperCAmelCase  ,   proj.t().contiguous()		)
        lowercase       : Optional[Any]						=nn.functional.linear(UpperCAmelCase  ,   UpperCAmelCase  ,   bias=UpperCAmelCase		)
        # else:
        #     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
        #     if bias is not None:
        #         logit = logit + bias

       return logit






      def 					A__   (			self      :		Dict  ,   UpperCAmelCase      :		Optional[int]  ,   UpperCAmelCase      :		Union[str, Any]=None  ,   UpperCAmelCase      :		Union[str, Any]=False		)	->	List[str]:


       '''simple docstring'''

       if labels is not None:
        # Shift so that tokens < n predict n
        lowercase       : int						=hidden[..., :-1, :].contiguous()
        lowercase       : Dict						=labels[..., 1:].contiguous()
        lowercase       : str						=hidden.view(-1  ,   hidden.size(-1		)		)
        lowercase       : List[Any]						=labels.view(-1		)
        if hidden.size(0		) != labels.size(0		):
         raise RuntimeError('''Input and labels should have the same size in the batch dimension.'''		)
       else:
        lowercase       : Union[str, Any]						=hidden.view(-1  ,   hidden.size(-1		)		)

       if self.n_clusters == 0:
        lowercase       : Any						=self._compute_logit(UpperCAmelCase  ,   self.out_layers[0].weight  ,   self.out_layers[0].bias  ,   self.out_projs[0]		)
        if labels is not None:
         lowercase       : Optional[Any]						=labels != -100
         lowercase       : Optional[Any]						=torch.zeros_like(UpperCAmelCase  ,   dtype=hidden.dtype  ,   device=hidden.device		)
         lowercase       : str						=(
             -nn.functional.log_softmax(UpperCAmelCase  ,   dim=-1		)[mask].gather(1  ,   labels[mask].unsqueeze(1		)		).squeeze(1		)
         )
        else:
         lowercase       : int						=nn.functional.log_softmax(UpperCAmelCase  ,   dim=-1		)
       else:
        # construct weights and biases
        lowercase	,		lowercase       : Optional[int]						=[], []
        for i in range(len(self.cutoffs		)		):
         if self.div_val == 1:
          lowercase	,		lowercase       : str						=self.cutoff_ends[i], self.cutoff_ends[i + 1]
          lowercase       : List[Any]						=self.out_layers[0].weight[l_idx:r_idx]
          lowercase       : Optional[int]						=self.out_layers[0].bias[l_idx:r_idx]
         else:
          lowercase       : int						=self.out_layers[i].weight
          lowercase       : str						=self.out_layers[i].bias

         if i == 0:
          lowercase       : int						=torch.cat([weight_i, self.cluster_weight]  ,   dim=0		)
          lowercase       : Tuple						=torch.cat([bias_i, self.cluster_bias]  ,   dim=0		)

         weights.append(UpperCAmelCase		)
         biases.append(UpperCAmelCase		)

        lowercase	,		lowercase	,		lowercase       : List[Any]						=weights[0], biases[0], self.out_projs[0]

        lowercase       : Any						=self._compute_logit(UpperCAmelCase  ,   UpperCAmelCase  ,   UpperCAmelCase  ,   UpperCAmelCase		)
        lowercase       : Tuple						=nn.functional.log_softmax(UpperCAmelCase  ,   dim=1		)

        if labels is None:
         lowercase       : Optional[Any]						=hidden.new_empty((head_logit.size(0		), self.n_token)		)
        else:
         lowercase       : List[str]						=torch.zeros_like(UpperCAmelCase  ,   dtype=hidden.dtype  ,   device=hidden.device		)

        lowercase       : Union[str, Any]						=0
        lowercase       : Optional[Any]						=[0] + self.cutoffs
        for i in range(len(UpperCAmelCase		) - 1		):
         lowercase	,		lowercase       : Union[str, Any]						=cutoff_values[i], cutoff_values[i + 1]

         if labels is not None:
          lowercase       : str						=(labels >= l_idx) & (labels < r_idx)
          lowercase       : Dict						=mask_i.nonzero().squeeze()

          if indices_i.numel() == 0:
           continue

          lowercase       : List[str]						=labels.index_select(0  ,   UpperCAmelCase		) - l_idx
          lowercase       : Optional[int]						=head_logprob.index_select(0  ,   UpperCAmelCase		)
          lowercase       : Union[str, Any]						=hidden.index_select(0  ,   UpperCAmelCase		)
         else:
          lowercase       : str						=hidden

         if i == 0:
          if labels is not None:
           lowercase       : Dict						=head_logprob_i.gather(1  ,   target_i[:, None]		).squeeze(1		)
          else:
           lowercase       : List[str]						=head_logprob[:, : self.cutoffs[0]]
         else:
          lowercase	,		lowercase	,		lowercase       : int						=weights[i], biases[i], self.out_projs[i]

          lowercase       : List[Any]						=self._compute_logit(UpperCAmelCase  ,   UpperCAmelCase  ,   UpperCAmelCase  ,   UpperCAmelCase		)
          lowercase       : List[Any]						=nn.functional.log_softmax(UpperCAmelCase  ,   dim=1		)
          lowercase       : Tuple						=self.cutoffs[0] + i - 1  # No probability for the head cluster
          if labels is not None:
           lowercase       : Any						=head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
               1  ,   target_i[:, None]		).squeeze(1		)
          else:
           lowercase       : List[str]						=head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
           lowercase       : Union[str, Any]						=logprob_i

         if labels is not None:
          if (hasattr(self  ,   '''keep_order'''		) and self.keep_order) or keep_order:
           out.index_copy_(0  ,   UpperCAmelCase  ,   -logprob_i		)
          else:
           out[offset : offset + logprob_i.size(0		)].copy_(-logprob_i		)
          offset += logprob_i.size(0		)

       return out






      def 					A__   (			self      :		Tuple  ,   UpperCAmelCase      :		List[Any]		)	->	Optional[Any]:


       '''simple docstring'''

       if self.n_clusters == 0:
        lowercase       : Optional[Any]						=self._compute_logit(UpperCAmelCase  ,   self.out_layers[0].weight  ,   self.out_layers[0].bias  ,   self.out_projs[0]		)
        return nn.functional.log_softmax(UpperCAmelCase  ,   dim=-1		)
       else:
        # construct weights and biases
        lowercase	,		lowercase       : Tuple						=[], []
        for i in range(len(self.cutoffs		)		):
         if self.div_val == 1:
          lowercase	,		lowercase       : Union[str, Any]						=self.cutoff_ends[i], self.cutoff_ends[i + 1]
          lowercase       : Tuple						=self.out_layers[0].weight[l_idx:r_idx]
          lowercase       : str						=self.out_layers[0].bias[l_idx:r_idx]
         else:
          lowercase       : Dict						=self.out_layers[i].weight
          lowercase       : Optional[Any]						=self.out_layers[i].bias

         if i == 0:
          lowercase       : Tuple						=torch.cat([weight_i, self.cluster_weight]  ,   dim=0		)
          lowercase       : Union[str, Any]						=torch.cat([bias_i, self.cluster_bias]  ,   dim=0		)

         weights.append(UpperCAmelCase		)
         biases.append(UpperCAmelCase		)

        lowercase	,		lowercase	,		lowercase       : List[Any]						=weights[0], biases[0], self.out_projs[0]
        lowercase       : List[Any]						=self._compute_logit(UpperCAmelCase  ,   UpperCAmelCase  ,   UpperCAmelCase  ,   UpperCAmelCase		)

        lowercase       : List[Any]						=hidden.new_empty((head_logit.size(0		), self.n_token)		)
        lowercase       : int						=nn.functional.log_softmax(UpperCAmelCase  ,   dim=1		)

        lowercase       : Tuple						=[0] + self.cutoffs
        for i in range(len(UpperCAmelCase		) - 1		):
         lowercase	,		lowercase       : Any						=cutoff_values[i], cutoff_values[i + 1]

         if i == 0:
          lowercase       : Any						=head_logprob[:, : self.cutoffs[0]]
         else:
          lowercase	,		lowercase	,		lowercase       : Any						=weights[i], biases[i], self.out_projs[i]

          lowercase       : List[str]						=self._compute_logit(UpperCAmelCase  ,   UpperCAmelCase  ,   UpperCAmelCase  ,   UpperCAmelCase		)
          lowercase       : List[str]						=nn.functional.log_softmax(UpperCAmelCase  ,   dim=1		)

          lowercase       : Tuple						=head_logprob[:, -i] + tail_logprob_i
          lowercase       : Optional[int]						=logprob_i

        return out








"""simple docstring"""







import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import BertTokenizer, BertTokenizerFast
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
			from PIL import Image

			from transformers import AlignProcessor, EfficientNetImageProcessor






@require_vision
class       UpperCamelCase__			(   unittest.TestCase      ):

			"""simple docstring"""


			def   snake_case__	(      self		)      ->	Any:
				A__											=					tempfile.mkdtemp()

				A__											=					[
				    "[UNK]",
				    "[CLS]",
				    "[SEP]",
				    "[PAD]",
				    "[MASK]",
				    "want",
				    "##want",
				    "##ed",
				    "wa",
				    "un",
				    "runn",
				    "##ing",
				    ",",
				    "low",
				    "lowest",
				]
				A__											=					os.path.join(self.tmpdirname						,	VOCAB_FILES_NAMES["vocab_file"]		)
				with open(self.vocab_file						,	"w"						,	encoding="utf-8"		) as vocab_writer:
					vocab_writer.write("".join([x + "\n" for x in vocab_tokens]		)		)

				A__											=					{
				    "do_resize": True,
				    "size": 20,
				    "do_center_crop": True,
				    "crop_size": 18,
				    "do_normalize": True,
				    "image_mean": [0.4_8_1_4_5_4_6_6, 0.4_5_7_8_2_7_5, 0.4_0_8_2_1_0_7_3],
				    "image_std": [0.2_6_8_6_2_9_5_4, 0.2_6_1_3_0_2_5_8, 0.2_7_5_7_7_7_1_1],
				}
				A__											=					os.path.join(self.tmpdirname						,	SCREAMING_SNAKE_CASE__		)
				with open(self.image_processor_file						,	"w"						,	encoding="utf-8"		) as fp:
					json.dump(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__		)


			def   snake_case__	(      self						,	**SCREAMING_SNAKE_CASE__		)      ->	List[Any]:
				return BertTokenizer.from_pretrained(self.tmpdirname						,	**SCREAMING_SNAKE_CASE__		)


			def   snake_case__	(      self						,	**SCREAMING_SNAKE_CASE__		)      ->	Tuple:
				return BertTokenizerFast.from_pretrained(self.tmpdirname						,	**SCREAMING_SNAKE_CASE__		)


			def   snake_case__	(      self						,	**SCREAMING_SNAKE_CASE__		)      ->	str:
				return EfficientNetImageProcessor.from_pretrained(self.tmpdirname						,	**SCREAMING_SNAKE_CASE__		)


			def   snake_case__	(      self		)      ->	List[str]:
				shutil.rmtree(self.tmpdirname		)


			def   snake_case__	(      self		)      ->	Any:
				A__											=					[np.random.randint(255						,	size=(3, 30, 400)						,	dtype=np.uinta		)]
				A__											=					[Image.fromarray(np.moveaxis(SCREAMING_SNAKE_CASE__						,	0						,	-1		)		) for x in image_inputs]
				return image_inputs


			def   snake_case__	(      self		)      ->	Dict:
				A__											=					self.get_tokenizer()
				A__											=					self.get_rust_tokenizer()
				A__											=					self.get_image_processor()

				A__											=					AlignProcessor(tokenizer=SCREAMING_SNAKE_CASE__						,	image_processor=SCREAMING_SNAKE_CASE__		)
				processor_slow.save_pretrained(self.tmpdirname		)
				A__											=					AlignProcessor.from_pretrained(self.tmpdirname						,	use_fast=SCREAMING_SNAKE_CASE__		)

				A__											=					AlignProcessor(tokenizer=SCREAMING_SNAKE_CASE__						,	image_processor=SCREAMING_SNAKE_CASE__		)
				processor_fast.save_pretrained(self.tmpdirname		)
				A__											=					AlignProcessor.from_pretrained(self.tmpdirname		)

				self.assertEqual(processor_slow.tokenizer.get_vocab()						,	tokenizer_slow.get_vocab()		)
				self.assertEqual(processor_fast.tokenizer.get_vocab()						,	tokenizer_fast.get_vocab()		)
				self.assertEqual(tokenizer_slow.get_vocab()						,	tokenizer_fast.get_vocab()		)
				self.assertIsInstance(processor_slow.tokenizer						,	SCREAMING_SNAKE_CASE__		)
				self.assertIsInstance(processor_fast.tokenizer						,	SCREAMING_SNAKE_CASE__		)

				self.assertEqual(processor_slow.image_processor.to_json_string()						,	image_processor.to_json_string()		)
				self.assertEqual(processor_fast.image_processor.to_json_string()						,	image_processor.to_json_string()		)
				self.assertIsInstance(processor_slow.image_processor						,	SCREAMING_SNAKE_CASE__		)
				self.assertIsInstance(processor_fast.image_processor						,	SCREAMING_SNAKE_CASE__		)


			def   snake_case__	(      self		)      ->	Union[str, Any]:
				A__											=					AlignProcessor(tokenizer=self.get_tokenizer()						,	image_processor=self.get_image_processor()		)
				processor.save_pretrained(self.tmpdirname		)

				A__											=					self.get_tokenizer(bos_token="(BOS)"						,	eos_token="(EOS)"		)
				A__											=					self.get_image_processor(do_normalize=SCREAMING_SNAKE_CASE__						,	padding_value=1.0		)

				A__											=					AlignProcessor.from_pretrained(
				    self.tmpdirname						,	bos_token="(BOS)"						,	eos_token="(EOS)"						,	do_normalize=SCREAMING_SNAKE_CASE__						,	padding_value=1.0		)

				self.assertEqual(processor.tokenizer.get_vocab()						,	tokenizer_add_kwargs.get_vocab()		)
				self.assertIsInstance(processor.tokenizer						,	SCREAMING_SNAKE_CASE__		)

				self.assertEqual(processor.image_processor.to_json_string()						,	image_processor_add_kwargs.to_json_string()		)
				self.assertIsInstance(processor.image_processor						,	SCREAMING_SNAKE_CASE__		)


			def   snake_case__	(      self		)      ->	List[Any]:
				A__											=					self.get_image_processor()
				A__											=					self.get_tokenizer()

				A__											=					AlignProcessor(tokenizer=SCREAMING_SNAKE_CASE__						,	image_processor=SCREAMING_SNAKE_CASE__		)

				A__											=					self.prepare_image_inputs()

				A__											=					image_processor(SCREAMING_SNAKE_CASE__						,	return_tensors="np"		)
				A__											=					processor(images=SCREAMING_SNAKE_CASE__						,	return_tensors="np"		)

				for key in input_image_proc.keys():
					self.assertAlmostEqual(input_image_proc[key].sum()						,	input_processor[key].sum()						,	delta=1e-2		)


			def   snake_case__	(      self		)      ->	List[Any]:
				A__											=					self.get_image_processor()
				A__											=					self.get_tokenizer()

				A__											=					AlignProcessor(tokenizer=SCREAMING_SNAKE_CASE__						,	image_processor=SCREAMING_SNAKE_CASE__		)

				A__											=					"lower newer"

				A__											=					processor(text=SCREAMING_SNAKE_CASE__		)

				A__											=					tokenizer(SCREAMING_SNAKE_CASE__						,	padding="max_length"						,	max_length=64		)

				for key in encoded_tok.keys():
					self.assertListEqual(encoded_tok[key]						,	encoded_processor[key]		)


			def   snake_case__	(      self		)      ->	int:
				A__											=					self.get_image_processor()
				A__											=					self.get_tokenizer()

				A__											=					AlignProcessor(tokenizer=SCREAMING_SNAKE_CASE__						,	image_processor=SCREAMING_SNAKE_CASE__		)

				A__											=					"lower newer"
				A__											=					self.prepare_image_inputs()

				A__											=					processor(text=SCREAMING_SNAKE_CASE__						,	images=SCREAMING_SNAKE_CASE__		)

				self.assertListEqual(list(inputs.keys()		)						,	["input_ids", "token_type_ids", "attention_mask", "pixel_values"]		)

				# test if it raises when no input is passed
				with pytest.raises(SCREAMING_SNAKE_CASE__		):
					processor()


			def   snake_case__	(      self		)      ->	Union[str, Any]:
				A__											=					self.get_image_processor()
				A__											=					self.get_tokenizer()

				A__											=					AlignProcessor(tokenizer=SCREAMING_SNAKE_CASE__						,	image_processor=SCREAMING_SNAKE_CASE__		)

				A__											=					[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

				A__											=					processor.batch_decode(SCREAMING_SNAKE_CASE__		)
				A__											=					tokenizer.batch_decode(SCREAMING_SNAKE_CASE__		)

				self.assertListEqual(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__		)







			def   snake_case__	(      self		)      ->	Union[str, Any]:
				A__											=					self.get_image_processor()
				A__											=					self.get_tokenizer()

				A__											=					AlignProcessor(tokenizer=SCREAMING_SNAKE_CASE__						,	image_processor=SCREAMING_SNAKE_CASE__		)

				A__											=					"lower newer"
				A__											=					self.prepare_image_inputs()

				A__											=					processor(text=SCREAMING_SNAKE_CASE__						,	images=SCREAMING_SNAKE_CASE__		)

				self.assertListEqual(list(inputs.keys()		)						,	processor.model_input_names		)




