



import gc
import unittest

from transformers import CTRLConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
       import torch

       from transformers import (
           CTRL_PRETRAINED_MODEL_ARCHIVE_LIST,
           CTRLForSequenceClassification,
           CTRLLMHeadModel,
           CTRLModel,
       )


class    SCREAMING_SNAKE_CASE__ :

    def __init__(				self			:				int ,  lowerCAmelCase_			:				Optional[Any] ,  lowerCAmelCase_			:				Union[str, Any]=1_4 ,  lowerCAmelCase_			:				Tuple=7 ,  lowerCAmelCase_			:				Optional[Any]=True ,  lowerCAmelCase_			:				List[Any]=True ,  lowerCAmelCase_			:				Any=True ,  lowerCAmelCase_			:				Any=True ,  lowerCAmelCase_			:				int=True ,  lowerCAmelCase_			:				Optional[int]=9_9 ,  lowerCAmelCase_			:				str=3_2 ,  lowerCAmelCase_			:				List[Any]=5 ,  lowerCAmelCase_			:				List[str]=4 ,  lowerCAmelCase_			:				Any=3_7 ,  lowerCAmelCase_			:				List[str]="gelu" ,  lowerCAmelCase_			:				Tuple=0.1 ,  lowerCAmelCase_			:				str=0.1 ,  lowerCAmelCase_			:				Optional[Any]=5_1_2 ,  lowerCAmelCase_			:				int=1_6 ,  lowerCAmelCase_			:				Dict=2 ,  lowerCAmelCase_			:				Optional[int]=0.02 ,  lowerCAmelCase_			:				Union[str, Any]=3 ,  lowerCAmelCase_			:				Any=4 ,  lowerCAmelCase_			:				int=None ,  ):







      """simple docstring"""







      lowercase_								= parent
      lowercase_								= batch_size
      lowercase_								= seq_length
      lowercase_								= is_training
      lowercase_								= use_token_type_ids
      lowercase_								= use_input_mask
      lowercase_								= use_labels
      lowercase_								= use_mc_token_ids
      lowercase_								= vocab_size
      lowercase_								= hidden_size
      lowercase_								= num_hidden_layers
      lowercase_								= num_attention_heads
      lowercase_								= intermediate_size
      lowercase_								= hidden_act
      lowercase_								= hidden_dropout_prob
      lowercase_								= attention_probs_dropout_prob
      lowercase_								= max_position_embeddings
      lowercase_								= type_vocab_size
      lowercase_								= type_sequence_label_size
      lowercase_								= initializer_range
      lowercase_								= num_labels
      lowercase_								= num_choices
      lowercase_								= scope
      lowercase_								= self.vocab_size - 1

    def 					_UpperCAmelCase	(				self			:				Optional[int]):







      """simple docstring"""







      lowercase_								= ids_tensor([self.batch_size, self.seq_length] ,  self.vocab_size)

      lowercase_								= None
      if self.use_input_mask:
        lowercase_								= random_attention_mask([self.batch_size, self.seq_length])

      lowercase_								= None
      if self.use_token_type_ids:
        lowercase_								= ids_tensor([self.batch_size, self.seq_length] ,  self.type_vocab_size)

      lowercase_								= None
      if self.use_mc_token_ids:
        lowercase_								= ids_tensor([self.batch_size, self.num_choices] ,  self.seq_length)

      lowercase_								= None
      lowercase_								= None
      lowercase_								= None
      if self.use_labels:
        lowercase_								= ids_tensor([self.batch_size] ,  self.type_sequence_label_size)
        lowercase_								= ids_tensor([self.batch_size, self.seq_length] ,  self.num_labels)
        lowercase_								= ids_tensor([self.batch_size] ,  self.num_choices)

      lowercase_								= self.get_config()

      lowercase_								= ids_tensor([self.num_hidden_layers, self.num_attention_heads] ,  2)

      return (
          config,
          input_ids,
          input_mask,
          head_mask,
          token_type_ids,
          mc_token_ids,
          sequence_labels,
          token_labels,
          choice_labels,
      )

    def 					_UpperCAmelCase	(				self			:				Union[str, Any]):







      """simple docstring"""







      return CTRLConfig(
          vocab_size=self.vocab_size ,  n_embd=self.hidden_size ,  n_layer=self.num_hidden_layers ,  n_head=self.num_attention_heads ,  n_positions=self.max_position_embeddings ,  pad_token_id=self.pad_token_id ,  )

    def 					_UpperCAmelCase	(				self			:				Any ,  lowerCAmelCase_			:				Dict ,  lowerCAmelCase_			:				Dict ,  lowerCAmelCase_			:				int ,  lowerCAmelCase_			:				Union[str, Any] ,  lowerCAmelCase_			:				Any ,  *lowerCAmelCase_			:				Any):







      """simple docstring"""







      lowercase_								= CTRLModel(config=lowerCAmelCase_)
      model.to(lowerCAmelCase_)
      model.eval()

      model(lowerCAmelCase_ ,  token_type_ids=lowerCAmelCase_ ,  head_mask=lowerCAmelCase_)
      model(lowerCAmelCase_ ,  token_type_ids=lowerCAmelCase_)
      lowercase_								= model(lowerCAmelCase_)
      self.parent.assertEqual(result.last_hidden_state.shape ,  (self.batch_size, self.seq_length, self.hidden_size))
      self.parent.assertEqual(len(result.past_key_values) ,  config.n_layer)

    def 					_UpperCAmelCase	(				self			:				Union[str, Any] ,  lowerCAmelCase_			:				Dict ,  lowerCAmelCase_			:				Any ,  lowerCAmelCase_			:				Union[str, Any] ,  lowerCAmelCase_			:				Union[str, Any] ,  lowerCAmelCase_			:				Optional[Any] ,  *lowerCAmelCase_			:				Any):







      """simple docstring"""







      lowercase_								= CTRLLMHeadModel(lowerCAmelCase_)
      model.to(lowerCAmelCase_)
      model.eval()

      lowercase_								= model(lowerCAmelCase_ ,  token_type_ids=lowerCAmelCase_ ,  labels=lowerCAmelCase_)
      self.parent.assertEqual(result.loss.shape ,  ())
      self.parent.assertEqual(result.logits.shape ,  (self.batch_size, self.seq_length, self.vocab_size))

    def 					_UpperCAmelCase	(				self			:				int):







      """simple docstring"""







      lowercase_								= self.prepare_config_and_inputs()

      (
          (
          lowercase_    
      )					,    (
          lowercase_    
      )					,    (
          lowercase_    
      )					,    (
          lowercase_    
      )					,    (
          lowercase_    
      )					,    (
          lowercase_    
      )					,    (
          lowercase_    
      )					,    (
          lowercase_    
      )					,    (
          lowercase_    
      )					,        
      )								= config_and_inputs

      lowercase_								= {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """head_mask""": head_mask}

      return config, inputs_dict




    def 					_UpperCAmelCase	(				self			:				str ,  lowerCAmelCase_			:				Dict ,  lowerCAmelCase_			:				List[str] ,  lowerCAmelCase_			:				Tuple ,  lowerCAmelCase_			:				Tuple ,  *lowerCAmelCase_			:				List[Any]):







      """simple docstring"""







      lowercase_								= self.num_labels
      lowercase_								= CTRLForSequenceClassification(lowerCAmelCase_)
      model.to(lowerCAmelCase_)
      model.eval()
      lowercase_								= ids_tensor([self.batch_size] ,  self.type_sequence_label_size)
      lowercase_								= model(lowerCAmelCase_ ,  token_type_ids=lowerCAmelCase_ ,  labels=lowerCAmelCase_)
      self.parent.assertEqual(result.logits.shape ,  (self.batch_size, self.num_labels))







@require_torch
class    SCREAMING_SNAKE_CASE__ (							__UpperCAmelCase      ,  __UpperCAmelCase      ,  __UpperCAmelCase      ,  unittest.TestCase   ):
    lowercase__      =   (CTRLModel, CTRLLMHeadModel, CTRLForSequenceClassification) if is_torch_available() else ()
    lowercase__      =   (CTRLLMHeadModel,) if is_torch_available() else ()
    lowercase__      =   (
        {
            "feature-extraction": CTRLModel,
            "text-classification": CTRLForSequenceClassification,
            "text-generation": CTRLLMHeadModel,
            "zero-shot": CTRLForSequenceClassification,
        }
        if is_torch_available()
        else {}
    )
    lowercase__      =   True
    lowercase__      =   False
    lowercase__      =   False

    def 					_UpperCAmelCase	(				self			:				List[str] ,  lowerCAmelCase_			:				Union[str, Any] ,  lowerCAmelCase_			:				int ,  lowerCAmelCase_			:				Union[str, Any] ,  lowerCAmelCase_			:				Optional[Any] ,  lowerCAmelCase_			:				Union[str, Any]):







      """simple docstring"""







      if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
        # Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
        # `CTRLConfig` was never used in pipeline tests, either because of a missing checkpoint or because a tiny
        # config could not be created.
        return True

      return False

    def 					_UpperCAmelCase	(				self			:				str):







      """simple docstring"""







      lowercase_								= CTRLModelTester(self)
      lowercase_								= ConfigTester(self ,  config_class=lowerCAmelCase_ ,  n_embd=3_7)

    def 					_UpperCAmelCase	(				self			:				List[str]):







      """simple docstring"""







      super().tearDown()
      # clean-up as much as possible GPU memory occupied by PyTorch
      gc.collect()
      torch.cuda.empty_cache()

    def 					_UpperCAmelCase	(				self			:				Any):







      """simple docstring"""







      self.config_tester.run_common_tests()

    def 					_UpperCAmelCase	(				self			:				Optional[Any]):







      """simple docstring"""







      lowercase_								= self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_ctrl_model(*lowerCAmelCase_)

    def 					_UpperCAmelCase	(				self			:				Dict):







      """simple docstring"""







      lowercase_								= self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_lm_head_model(*lowerCAmelCase_)

    @unittest.skip("""Will be fixed soon by reducing the size of the model used for common tests.""")
    def 					_UpperCAmelCase	(				self			:				Optional[Any]):







      """simple docstring"""







      pass

    @slow
    def 					_UpperCAmelCase	(				self			:				Any):







      """simple docstring"""







      for model_name in CTRL_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
        lowercase_								= CTRLModel.from_pretrained(lowerCAmelCase_)
        self.assertIsNotNone(lowerCAmelCase_)




    @unittest.skip("""The model doesn't support left padding""")  # and it's not used enough to be worth fixing :)
    def 					_UpperCAmelCase	(				self			:				int):







      """simple docstring"""







      pass







@require_torch
class    SCREAMING_SNAKE_CASE__ (							unittest.TestCase   ):

    def 					_UpperCAmelCase	(				self			:				Dict):







      """simple docstring"""







      super().tearDown()
      # clean-up as much as possible GPU memory occupied by PyTorch
      gc.collect()
      torch.cuda.empty_cache()




    @slow
    def 					_UpperCAmelCase	(				self			:				Dict):







      """simple docstring"""







      lowercase_								= CTRLLMHeadModel.from_pretrained("""ctrl""")
      model.to(lowerCAmelCase_)
      lowercase_								= torch.tensor(
          [[1_1_8_5_9, 0, 1_6_1_1, 8]] ,  dtype=torch.long ,  device=lowerCAmelCase_)  # Legal the president is
      lowercase_								= [
          1_1_8_5_9,
          0,
          1_6_1_1,
          8,
          5,
          1_5_0,
          2_6_4_4_9,
          2,
          1_9,
          3_4_8,
          4_6_9,
          3,
          2_5_9_5,
          4_8,
          2_0_7_4_0,
          2_4_6_5_3_3,
          2_4_6_5_3_3,
          1_9,
          3_0,
          5,
      ]  # Legal the president is a good guy and I don't want to lose my job. \n \n I have a

      lowercase_								= model.generate(lowerCAmelCase_ ,  do_sample=lowerCAmelCase_)
      self.assertListEqual(output_ids[0].tolist() ,  lowerCAmelCase_)






"""simple docstring"""
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase						:		List[Any]			= logging.get_logger(__name__)


UpperCAmelCase						:		List[Any]			= "▁"

UpperCAmelCase						:		Union[str, Any]			= {"vocab_file": "spiece.model"}

UpperCAmelCase						:		List[Any]			= {
    "vocab_file": {
        "google/reformer-crime-and-punishment": (
            "https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/spiece.model"
        )
    }
}

UpperCAmelCase						:		Optional[Any]			= {
    "google/reformer-crime-and-punishment": 52_4288,
}


class    SCREAMING_SNAKE_CASE__ (							__UpperCAmelCase   ):
    lowercase__      =   VOCAB_FILES_NAMES
    lowercase__      =   PRETRAINED_VOCAB_FILES_MAP
    lowercase__      =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowercase__      =   ["input_ids", "attention_mask"]

    def __init__(				self			:				int ,  lowerCAmelCase_			:				Any ,  lowerCAmelCase_			:				Dict="</s>" ,  lowerCAmelCase_			:				Dict="<unk>" ,  lowerCAmelCase_			:				Dict=[] ,  lowerCAmelCase_			:				Optional[Dict[str, Any]] = None ,  **lowerCAmelCase_			:				List[str] ,  ):







      """simple docstring"""







      lowercase_								= {} if sp_model_kwargs is None else sp_model_kwargs

      super().__init__(
          eos_token=lowerCAmelCase_ ,  unk_token=lowerCAmelCase_ ,  additional_special_tokens=lowerCAmelCase_ ,  sp_model_kwargs=self.sp_model_kwargs ,  **lowerCAmelCase_ ,  )

      lowercase_								= vocab_file
      lowercase_								= spm.SentencePieceProcessor(**self.sp_model_kwargs)
      self.sp_model.Load(lowerCAmelCase_)

    @property
    def 					_UpperCAmelCase	(				self			:				List[Any]):







      """simple docstring"""







      return self.sp_model.get_piece_size()

    def 					_UpperCAmelCase	(				self			:				Dict):







      """simple docstring"""







      lowercase_								= {self.convert_ids_to_tokens(lowerCAmelCase_): i for i in range(self.vocab_size)}
      vocab.update(self.added_tokens_encoder)
      return vocab

    def __getstate__(				self			:				Union[str, Any]):







      """simple docstring"""







      lowercase_								= self.__dict__.copy()
      lowercase_								= None
      return state

    def __setstate__(				self			:				Optional[Any] ,  lowerCAmelCase_			:				int):







      """simple docstring"""







      lowercase_								= d

      # for backward compatibility
      if not hasattr(self ,  """sp_model_kwargs"""):
        lowercase_								= {}

      lowercase_								= spm.SentencePieceProcessor(**self.sp_model_kwargs)
      self.sp_model.Load(self.vocab_file)

    def 					_UpperCAmelCase	(				self			:				Union[str, Any] ,  lowerCAmelCase_			:				str):







      """simple docstring"""







      return self.sp_model.encode(lowerCAmelCase_ ,  out_type=lowerCAmelCase_)

    def 					_UpperCAmelCase	(				self			:				Tuple ,  lowerCAmelCase_			:				List[str]):







      """simple docstring"""







      return self.sp_model.piece_to_id(lowerCAmelCase_)

    def 					_UpperCAmelCase	(				self			:				str ,  lowerCAmelCase_			:				Any):







      """simple docstring"""







      if index < self.sp_model.get_piece_size():
        lowercase_								= self.sp_model.IdToPiece(lowerCAmelCase_)
      return token

    def 					_UpperCAmelCase	(				self			:				int ,  lowerCAmelCase_			:				Any):







      """simple docstring"""







      lowercase_								= []
      lowercase_								= """"""
      for token in tokens:
        # make sure that special tokens are not decoded using sentencepiece model
        if token in self.all_special_tokens:
          out_string += self.sp_model.decode(lowerCAmelCase_) + token
          lowercase_								= []
        else:
          current_sub_tokens.append(lowerCAmelCase_)
      out_string += self.sp_model.decode(lowerCAmelCase_)
      return out_string.strip()




    def 					_UpperCAmelCase	(				self			:				Optional[int] ,  lowerCAmelCase_			:				str ,  lowerCAmelCase_			:				Optional[str] = None):







      """simple docstring"""







      if not os.path.isdir(lowerCAmelCase_):
        logger.error(F'''Vocabulary path ({save_directory}) should be a directory''')
        return
      lowercase_								= os.path.join(
          lowerCAmelCase_ ,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""])

      if os.path.abspath(self.vocab_file) != os.path.abspath(lowerCAmelCase_) and os.path.isfile(self.vocab_file):
        copyfile(self.vocab_file ,  lowerCAmelCase_)
      elif not os.path.isfile(self.vocab_file):
        with open(lowerCAmelCase_ ,  """wb""") as fi:
          lowercase_								= self.sp_model.serialized_model_proto()
          fi.write(lowerCAmelCase_)

      return (out_vocab_file,)

