
'''simple docstring'''


import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class 						lowercase__	(  lowercase__,					unittest.TestCase   ):






   '''simple docstring'''







   _snake_case		   =							None
   _snake_case		   =							BloomTokenizerFast
   _snake_case		   =							BloomTokenizerFast
   _snake_case		   =							True
   _snake_case		   =							False
   _snake_case		   =							'''tokenizer_file'''
   _snake_case		   =							{'''bos_token''': '''<s>''', '''eos_token''': '''</s>''', '''unk_token''': '''<unk>''', '''pad_token''': '''<pad>'''}







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      super().setUp()
      UpperCamelCase       =  BloomTokenizerFast.from_pretrained('''bigscience/tokenizer'''   )
      tokenizer.save_pretrained(self.tmpdirname   )







   def       UpperCAmelCase				(			self ,   **lowerCamelCase__   ):

      '''simple docstring'''







      kwargs.update(self.special_tokens_map   )
      return BloomTokenizerFast.from_pretrained(self.tmpdirname ,   **UpperCAmelCase__   )







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  self.get_rust_tokenizer()

      UpperCamelCase       =  ['''The quick brown fox</s>''', '''jumps over the lazy dog</s>''']
      UpperCamelCase       =  [[2_1_7_5, 2_3_7_1_4, 7_3_1_7_3, 1_4_4_2_5_2, 2], [7_7, 1_3_2_6_1_9, 3_4_7_8, 3_6_8, 1_0_9_5_8_6, 3_5_4_3_3, 2]]

      UpperCamelCase       =  tokenizer.batch_encode_plus(UpperCAmelCase__   )['''input_ids''']
      self.assertListEqual(UpperCAmelCase__ ,   UpperCAmelCase__   )

      UpperCamelCase       =  tokenizer.batch_decode(UpperCAmelCase__   )
      self.assertListEqual(UpperCAmelCase__ ,   UpperCAmelCase__   )







   def       UpperCAmelCase				(			self ,   lowerCamelCase__=6   ):

      '''simple docstring'''







      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'   ):
            UpperCamelCase       =  self.rust_tokenizer_class.from_pretrained(UpperCAmelCase__ ,   **UpperCAmelCase__   )
            # tokenizer_r.pad_token = None # Hotfixing padding = None
            # Simple input
            UpperCamelCase       =  '''This is a simple input'''
            UpperCamelCase       =  ['''This is a simple input 1''', '''This is a simple input 2''']
            UpperCamelCase       =  ('''This is a simple input''', '''This is a pair''')
            UpperCamelCase       =  [
                ('''This is a simple input 1''', '''This is a simple input 2'''),
                ('''This is a simple pair 1''', '''This is a simple pair 2'''),
            ]

            # Simple input tests
            try:
               tokenizer_r.encode(UpperCAmelCase__ ,   max_length=UpperCAmelCase__   )
               tokenizer_r.encode_plus(UpperCAmelCase__ ,   max_length=UpperCAmelCase__   )

               tokenizer_r.batch_encode_plus(UpperCAmelCase__ ,   max_length=UpperCAmelCase__   )
               tokenizer_r.encode(UpperCAmelCase__ ,   max_length=UpperCAmelCase__   )
               tokenizer_r.batch_encode_plus(UpperCAmelCase__ ,   max_length=UpperCAmelCase__   )
            except ValueError:
               self.fail('''Bloom Tokenizer should be able to deal with padding'''   )

            UpperCamelCase       =  None  # Hotfixing padding = None
            self.assertRaises(UpperCAmelCase__ ,   tokenizer_r.encode ,   UpperCAmelCase__ ,   max_length=UpperCAmelCase__ ,   padding='''max_length'''   )

            # Simple input
            self.assertRaises(UpperCAmelCase__ ,   tokenizer_r.encode_plus ,   UpperCAmelCase__ ,   max_length=UpperCAmelCase__ ,   padding='''max_length'''   )

            # Simple input
            self.assertRaises(
                UpperCAmelCase__ ,   tokenizer_r.batch_encode_plus ,   UpperCAmelCase__ ,   max_length=UpperCAmelCase__ ,   padding='''max_length''' ,   )

            # Pair input
            self.assertRaises(UpperCAmelCase__ ,   tokenizer_r.encode ,   UpperCAmelCase__ ,   max_length=UpperCAmelCase__ ,   padding='''max_length'''   )

            # Pair input
            self.assertRaises(UpperCAmelCase__ ,   tokenizer_r.encode_plus ,   UpperCAmelCase__ ,   max_length=UpperCAmelCase__ ,   padding='''max_length'''   )

            # Pair input
            self.assertRaises(
                UpperCAmelCase__ ,   tokenizer_r.batch_encode_plus ,   UpperCAmelCase__ ,   max_length=UpperCAmelCase__ ,   padding='''max_length''' ,   )







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  self.get_rust_tokenizer()
      UpperCamelCase       =  load_dataset('''xnli''' ,   '''all_languages''' ,   split='''test''' ,   streaming=UpperCAmelCase__   )

      UpperCamelCase       =  next(iter(UpperCAmelCase__   )   )['''premise''']  # pick up one data
      UpperCamelCase       =  list(sample_data.values()   )

      UpperCamelCase       =  list(map(tokenizer.encode ,   UpperCAmelCase__   )   )
      UpperCamelCase       =  [tokenizer.decode(UpperCAmelCase__ ,   clean_up_tokenization_spaces=UpperCAmelCase__   ) for x in output_tokens]
      self.assertListEqual(UpperCAmelCase__ ,   UpperCAmelCase__   )







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map   ) ,   1   )
      self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values()   )[0]   ) ,   1   )


'''simple docstring'''


import torch
from torch import nn


class 						lowercase__	(  nn.Module   ):






   '''simple docstring'''







   def __init__(			self ,   lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__=1 ,   lowerCamelCase__=False   ):

      '''simple docstring'''







      super().__init__()

      UpperCamelCase       =  n_token
      UpperCamelCase       =  d_embed
      UpperCamelCase       =  d_proj

      UpperCamelCase       =  cutoffs + [n_token]
      UpperCamelCase       =  [0] + self.cutoffs
      UpperCamelCase       =  div_val

      UpperCamelCase       =  self.cutoffs[0]
      UpperCamelCase       =  len(self.cutoffs   ) - 1
      UpperCamelCase       =  self.shortlist_size + self.n_clusters

      if self.n_clusters > 0:
         UpperCamelCase       =  nn.Parameter(torch.zeros(self.n_clusters ,   self.d_embed   )   )
         UpperCamelCase       =  nn.Parameter(torch.zeros(self.n_clusters   )   )

      UpperCamelCase       =  nn.ModuleList()
      UpperCamelCase       =  nn.ParameterList()

      if div_val == 1:
         for i in range(len(self.cutoffs   )   ):
            if d_proj != d_embed:
               self.out_projs.append(nn.Parameter(torch.FloatTensor(lowerCamelCase__ ,   lowerCamelCase__   )   )   )
            else:
               self.out_projs.append(lowerCamelCase__   )

         self.out_layers.append(nn.Linear(lowerCamelCase__ ,   lowerCamelCase__   )   )
      else:
         for i in range(len(self.cutoffs   )   ):
            UpperCamelCase     , UpperCamelCase       =  self.cutoff_ends[i], self.cutoff_ends[i + 1]
            UpperCamelCase       =  d_embed // (div_val**i)

            self.out_projs.append(nn.Parameter(torch.FloatTensor(lowerCamelCase__ ,   lowerCamelCase__   )   )   )

            self.out_layers.append(nn.Linear(lowerCamelCase__ ,   r_idx - l_idx   )   )

      UpperCamelCase       =  keep_order







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__   ):

      '''simple docstring'''







      if proj is None:
         UpperCamelCase       =  nn.functional.linear(lowerCamelCase__ ,   lowerCamelCase__ ,   bias=lowerCamelCase__   )
      else:
         # if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
         UpperCamelCase       =  nn.functional.linear(lowerCamelCase__ ,   proj.t().contiguous()   )
         UpperCamelCase       =  nn.functional.linear(lowerCamelCase__ ,   lowerCamelCase__ ,   bias=lowerCamelCase__   )
         # else:
         #     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
         #     if bias is not None:
         #         logit = logit + bias

      return logit







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__=None ,   lowerCamelCase__=False   ):

      '''simple docstring'''







      if labels is not None:
         # Shift so that tokens < n predict n
         UpperCamelCase       =  hidden[..., :-1, :].contiguous()
         UpperCamelCase       =  labels[..., 1:].contiguous()
         UpperCamelCase       =  hidden.view(-1 ,   hidden.size(-1   )   )
         UpperCamelCase       =  labels.view(-1   )
         if hidden.size(0   ) != labels.size(0   ):
            raise RuntimeError('''Input and labels should have the same size in the batch dimension.'''   )
      else:
         UpperCamelCase       =  hidden.view(-1 ,   hidden.size(-1   )   )

      if self.n_clusters == 0:
         UpperCamelCase       =  self._compute_logit(lowerCamelCase__ ,   self.out_layers[0].weight ,   self.out_layers[0].bias ,   self.out_projs[0]   )
         if labels is not None:
            UpperCamelCase       =  labels != -1_0_0
            UpperCamelCase       =  torch.zeros_like(lowerCamelCase__ ,   dtype=hidden.dtype ,   device=hidden.device   )
            UpperCamelCase       =  (
                -nn.functional.log_softmax(lowerCamelCase__ ,   dim=-1   )[mask].gather(1 ,   labels[mask].unsqueeze(1   )   ).squeeze(1   )
            )
         else:
            UpperCamelCase       =  nn.functional.log_softmax(lowerCamelCase__ ,   dim=-1   )
      else:
         # construct weights and biases
         UpperCamelCase     , UpperCamelCase       =  [], []
         for i in range(len(self.cutoffs   )   ):
            if self.div_val == 1:
               UpperCamelCase     , UpperCamelCase       =  self.cutoff_ends[i], self.cutoff_ends[i + 1]
               UpperCamelCase       =  self.out_layers[0].weight[l_idx:r_idx]
               UpperCamelCase       =  self.out_layers[0].bias[l_idx:r_idx]
            else:
               UpperCamelCase       =  self.out_layers[i].weight
               UpperCamelCase       =  self.out_layers[i].bias

            if i == 0:
               UpperCamelCase       =  torch.cat([weight_i, self.cluster_weight] ,   dim=0   )
               UpperCamelCase       =  torch.cat([bias_i, self.cluster_bias] ,   dim=0   )

            weights.append(lowerCamelCase__   )
            biases.append(lowerCamelCase__   )

         UpperCamelCase     , UpperCamelCase     , UpperCamelCase       =  weights[0], biases[0], self.out_projs[0]

         UpperCamelCase       =  self._compute_logit(lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__   )
         UpperCamelCase       =  nn.functional.log_softmax(lowerCamelCase__ ,   dim=1   )

         if labels is None:
            UpperCamelCase       =  hidden.new_empty((head_logit.size(0   ), self.n_token)   )
         else:
            UpperCamelCase       =  torch.zeros_like(lowerCamelCase__ ,   dtype=hidden.dtype ,   device=hidden.device   )

         UpperCamelCase       =  0
         UpperCamelCase       =  [0] + self.cutoffs
         for i in range(len(lowerCamelCase__   ) - 1   ):
            UpperCamelCase     , UpperCamelCase       =  cutoff_values[i], cutoff_values[i + 1]

            if labels is not None:
               UpperCamelCase       =  (labels >= l_idx) & (labels < r_idx)
               UpperCamelCase       =  mask_i.nonzero().squeeze()

               if indices_i.numel() == 0:
                  continue

               UpperCamelCase       =  labels.index_select(0 ,   lowerCamelCase__   ) - l_idx
               UpperCamelCase       =  head_logprob.index_select(0 ,   lowerCamelCase__   )
               UpperCamelCase       =  hidden.index_select(0 ,   lowerCamelCase__   )
            else:
               UpperCamelCase       =  hidden

            if i == 0:
               if labels is not None:
                  UpperCamelCase       =  head_logprob_i.gather(1 ,   target_i[:, None]   ).squeeze(1   )
               else:
                  UpperCamelCase       =  head_logprob[:, : self.cutoffs[0]]
            else:
               UpperCamelCase     , UpperCamelCase     , UpperCamelCase       =  weights[i], biases[i], self.out_projs[i]

               UpperCamelCase       =  self._compute_logit(lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__   )
               UpperCamelCase       =  nn.functional.log_softmax(lowerCamelCase__ ,   dim=1   )
               UpperCamelCase       =  self.cutoffs[0] + i - 1  # No probability for the head cluster
               if labels is not None:
                  UpperCamelCase       =  head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
                      1 ,   target_i[:, None]   ).squeeze(1   )
               else:
                  UpperCamelCase       =  head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
                  UpperCamelCase       =  logprob_i

            if labels is not None:
               if (hasattr(self ,   '''keep_order'''   ) and self.keep_order) or keep_order:
                  out.index_copy_(0 ,   lowerCamelCase__ ,   -logprob_i   )
               else:
                  out[offset : offset + logprob_i.size(0   )].copy_(-logprob_i   )
               offset += logprob_i.size(0   )

      return out







   def       UpperCAmelCase				(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      if self.n_clusters == 0:
         UpperCamelCase       =  self._compute_logit(lowerCamelCase__ ,   self.out_layers[0].weight ,   self.out_layers[0].bias ,   self.out_projs[0]   )
         return nn.functional.log_softmax(lowerCamelCase__ ,   dim=-1   )
      else:
         # construct weights and biases
         UpperCamelCase     , UpperCamelCase       =  [], []
         for i in range(len(self.cutoffs   )   ):
            if self.div_val == 1:
               UpperCamelCase     , UpperCamelCase       =  self.cutoff_ends[i], self.cutoff_ends[i + 1]
               UpperCamelCase       =  self.out_layers[0].weight[l_idx:r_idx]
               UpperCamelCase       =  self.out_layers[0].bias[l_idx:r_idx]
            else:
               UpperCamelCase       =  self.out_layers[i].weight
               UpperCamelCase       =  self.out_layers[i].bias

            if i == 0:
               UpperCamelCase       =  torch.cat([weight_i, self.cluster_weight] ,   dim=0   )
               UpperCamelCase       =  torch.cat([bias_i, self.cluster_bias] ,   dim=0   )

            weights.append(lowerCamelCase__   )
            biases.append(lowerCamelCase__   )

         UpperCamelCase     , UpperCamelCase     , UpperCamelCase       =  weights[0], biases[0], self.out_projs[0]
         UpperCamelCase       =  self._compute_logit(lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__   )

         UpperCamelCase       =  hidden.new_empty((head_logit.size(0   ), self.n_token)   )
         UpperCamelCase       =  nn.functional.log_softmax(lowerCamelCase__ ,   dim=1   )

         UpperCamelCase       =  [0] + self.cutoffs
         for i in range(len(lowerCamelCase__   ) - 1   ):
            UpperCamelCase     , UpperCamelCase       =  cutoff_values[i], cutoff_values[i + 1]

            if i == 0:
               UpperCamelCase       =  head_logprob[:, : self.cutoffs[0]]
            else:
               UpperCamelCase     , UpperCamelCase     , UpperCamelCase       =  weights[i], biases[i], self.out_projs[i]

               UpperCamelCase       =  self._compute_logit(lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__   )
               UpperCamelCase       =  nn.functional.log_softmax(lowerCamelCase__ ,   dim=1   )

               UpperCamelCase       =  head_logprob[:, -i] + tail_logprob_i
               UpperCamelCase       =  logprob_i

         return out





