




'''simple docstring'''







import os
import re
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


A									=logging.get_logger(__name__)

A									={'vocab_file': 'spiece.model'}

A									={
    'vocab_file': {
        'google/bigbird-roberta-base': 'https://huggingface.co/google/bigbird-roberta-base/resolve/main/spiece.model',
        'google/bigbird-roberta-large': (
            'https://huggingface.co/google/bigbird-roberta-large/resolve/main/spiece.model'
        ),
        'google/bigbird-base-trivia-itc': (
            'https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/spiece.model'
        ),
    }
}

A									={
    'google/bigbird-roberta-base': 40_96,
    'google/bigbird-roberta-large': 40_96,
    'google/bigbird-base-trivia-itc': 40_96,
}


class     _a						(			__a				):
							__a       :  Any            =      VOCAB_FILES_NAMES
							__a       :  str            =      PRETRAINED_VOCAB_FILES_MAP
							__a       :  List[Any]            =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							__a       :  int            =      ["""input_ids""", """attention_mask"""]
							__a       :  List[int]            =      []
							def __init__(			self		:			int     ,       lowercase		:			Tuple     ,       lowercase		:			Any="<unk>"     ,       lowercase		:			Tuple="<s>"     ,       lowercase		:			Union[str, Any]="</s>"     ,       lowercase		:			str="<pad>"     ,       lowercase		:			Any="[SEP]"     ,       lowercase		:			Optional[int]="[MASK]"     ,       lowercase		:			Union[str, Any]="[CLS]"     ,       lowercase		:			Optional[Dict[str, Any]] = None     ,       **lowercase		:			str     ,       ):



											'''simple docstring'''

											UpperCAmelCase     =	AddedToken(lowercase     ,       lstrip=lowercase     ,       rstrip=lowercase       ) if isinstance(lowercase     ,       lowercase       ) else bos_token
											UpperCAmelCase     =	AddedToken(lowercase     ,       lstrip=lowercase     ,       rstrip=lowercase       ) if isinstance(lowercase     ,       lowercase       ) else eos_token
											UpperCAmelCase     =	AddedToken(lowercase     ,       lstrip=lowercase     ,       rstrip=lowercase       ) if isinstance(lowercase     ,       lowercase       ) else unk_token
											UpperCAmelCase     =	AddedToken(lowercase     ,       lstrip=lowercase     ,       rstrip=lowercase       ) if isinstance(lowercase     ,       lowercase       ) else pad_token
											UpperCAmelCase     =	AddedToken(lowercase     ,       lstrip=lowercase     ,       rstrip=lowercase       ) if isinstance(lowercase     ,       lowercase       ) else cls_token
											UpperCAmelCase     =	AddedToken(lowercase     ,       lstrip=lowercase     ,       rstrip=lowercase       ) if isinstance(lowercase     ,       lowercase       ) else sep_token

											# Mask token behave like a normal word, i.e. include the space before it
											UpperCAmelCase     =	AddedToken(lowercase     ,       lstrip=lowercase     ,       rstrip=lowercase       ) if isinstance(lowercase     ,       lowercase       ) else mask_token

											UpperCAmelCase     =	{} if sp_model_kwargs is None else sp_model_kwargs

											super().__init__(
											    bos_token=lowercase     ,       eos_token=lowercase     ,       unk_token=lowercase     ,       pad_token=lowercase     ,       sep_token=lowercase     ,       mask_token=lowercase     ,       cls_token=lowercase     ,       sp_model_kwargs=self.sp_model_kwargs     ,       **lowercase     ,       )

											UpperCAmelCase     =	vocab_file

											UpperCAmelCase     =	spm.SentencePieceProcessor(**self.sp_model_kwargs       )
											self.sp_model.Load(lowercase       )
							@property
							def 					A   (			self		:			Optional[int]       ):



											'''simple docstring'''

											return self.sp_model.get_piece_size()
							def 					A   (			self		:			Union[str, Any]       ):



											'''simple docstring'''

											UpperCAmelCase     =	{self.convert_ids_to_tokens(lowercase       ): i for i in range(self.vocab_size       )}
											vocab.update(self.added_tokens_encoder       )
											return vocab
							def __getstate__(			self		:			Optional[Any]       ):



											'''simple docstring'''

											UpperCAmelCase     =	self.__dict__.copy()
											UpperCAmelCase     =	None
											return state
							def __setstate__(			self		:			str     ,       lowercase		:			Optional[Any]       ):



											'''simple docstring'''

											UpperCAmelCase     =	d

											# for backward compatibility
											if not hasattr(self     ,       '''sp_model_kwargs'''       ):
															UpperCAmelCase     =	{}

											UpperCAmelCase     =	spm.SentencePieceProcessor(**self.sp_model_kwargs       )
											self.sp_model.Load(self.vocab_file       )
							def 					A   (			self		:			Dict     ,       lowercase		:			str       ):



											'''simple docstring'''

											return self.sp_model.encode(lowercase     ,       out_type=lowercase       )
							def 					A   (			self		:			Tuple     ,       lowercase		:			Any       ):



											'''simple docstring'''

											return self.sp_model.piece_to_id(lowercase       )
							def 					A   (			self		:			str     ,       lowercase		:			List[Any]       ):



											'''simple docstring'''

											UpperCAmelCase     =	self.sp_model.IdToPiece(lowercase       )
											return token
							def 					A   (			self		:			List[str]     ,       lowercase		:			List[str]       ):



											'''simple docstring'''

											UpperCAmelCase     =	[]
											UpperCAmelCase     =	''''''
											UpperCAmelCase     =	False
											for token in tokens:
															# make sure that special tokens are not decoded using sentencepiece model
															if token in self.all_special_tokens:
																			if not prev_is_special:
																							out_string += " "
																			out_string += self.sp_model.decode(lowercase       ) + token
																			UpperCAmelCase     =	True
																			UpperCAmelCase     =	[]
															else:
																			current_sub_tokens.append(lowercase       )
																			UpperCAmelCase     =	False
											out_string += self.sp_model.decode(lowercase       )
											return out_string.strip()
							def 					A   (			self		:			str     ,       lowercase		:			List[int]     ,       lowercase		:			bool = False     ,       lowercase		:			bool = None     ,       lowercase		:			bool = True     ,       **lowercase		:			List[str]     ,       ):



											'''simple docstring'''

											UpperCAmelCase     =	kwargs.pop('''use_source_tokenizer'''     ,       lowercase       )

											UpperCAmelCase     =	self.convert_ids_to_tokens(lowercase     ,       skip_special_tokens=lowercase       )

											# To avoid mixing byte-level and unicode for byte-level BPT
											# we need to build string separately for added tokens and byte-level tokens
											# cf. https://github.com/huggingface/transformers/issues/1133
											UpperCAmelCase     =	[]
											UpperCAmelCase     =	[]
											for token in filtered_tokens:
															if skip_special_tokens and token in self.all_special_ids:
																			continue
															if token in self.added_tokens_encoder:
																			if current_sub_text:
																							sub_texts.append(self.convert_tokens_to_string(lowercase       )       )
																							UpperCAmelCase     =	[]
																			sub_texts.append(lowercase       )
															else:
																			current_sub_text.append(lowercase       )
											if current_sub_text:
															sub_texts.append(self.convert_tokens_to_string(lowercase       )       )

											# Mimic the behavior of the Rust tokenizer:
											# No space before [MASK] and [SEP]
											if spaces_between_special_tokens:
															UpperCAmelCase     =	re.sub(R''' (\[(MASK|SEP)\])'''     ,       R'''\1'''     ,       ''' '''.join(lowercase       )       )
											else:
															UpperCAmelCase     =	''''''.join(lowercase       )

											UpperCAmelCase     =	(
											    clean_up_tokenization_spaces
											    if clean_up_tokenization_spaces is not None
											    else self.clean_up_tokenization_spaces
											)
											if clean_up_tokenization_spaces:
															UpperCAmelCase     =	self.clean_up_tokenization(lowercase       )
															return clean_text
											else:
															return text
							def 					A   (			self		:			int     ,       lowercase		:			str     ,       lowercase		:			Optional[str] = None       ):



											'''simple docstring'''

											if not os.path.isdir(lowercase       ):
															logger.error(f"Vocabulary path ({save_directory}) should be a directory"       )
															return
											UpperCAmelCase     =	os.path.join(
											    lowercase     ,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']       )

											if os.path.abspath(self.vocab_file       ) != os.path.abspath(lowercase       ) and os.path.isfile(self.vocab_file       ):
															copyfile(self.vocab_file     ,       lowercase       )
											elif not os.path.isfile(self.vocab_file       ):
															with open(lowercase     ,       '''wb'''       ) as fi:
																			UpperCAmelCase     =	self.sp_model.serialized_model_proto()
																			fi.write(lowercase       )

											return (out_vocab_file,)
							def 					A   (			self		:			List[str]     ,       lowercase		:			List[int]     ,       lowercase		:			Optional[List[int]] = None       ):



											'''simple docstring'''

											if token_ids_a is None:
															return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
											UpperCAmelCase     =	[self.cls_token_id]
											UpperCAmelCase     =	[self.sep_token_id]
											return cls + token_ids_a + sep + token_ids_a + sep
							def 					A   (			self		:			int     ,       lowercase		:			List[int]     ,       lowercase		:			Optional[List[int]] = None     ,       lowercase		:			bool = False       ):



											'''simple docstring'''

											if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=lowercase     ,       token_ids_a=lowercase     ,       already_has_special_tokens=lowercase       )

											if token_ids_a is None:
															return [1] + ([0] * len(lowercase       )) + [1]
											return [1] + ([0] * len(lowercase       )) + [1] + ([0] * len(lowercase       )) + [1]







							def 					A   (			self		:			Union[str, Any]     ,       lowercase		:			List[int]     ,       lowercase		:			Optional[List[int]] = None       ):



											'''simple docstring'''

											UpperCAmelCase     =	[self.sep_token_id]
											UpperCAmelCase     =	[self.cls_token_id]
											if token_ids_a is None:
															return len(cls + token_ids_a + sep       ) * [0]
											return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]










'''simple docstring'''







import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
   import torch

   from transformers import (
       OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST,
       OpenAIGPTConfig,
       OpenAIGPTDoubleHeadsModel,
       OpenAIGPTForSequenceClassification,
       OpenAIGPTLMHeadModel,
       OpenAIGPTModel,
   )


class     _a						:
       def __init__(			self		:			Optional[Any]     ,       lowercase		:			int     ,       lowercase		:			str=13     ,       lowercase		:			int=7     ,       lowercase		:			Optional[int]=True     ,       lowercase		:			Dict=True     ,       lowercase		:			Union[str, Any]=True     ,       lowercase		:			Union[str, Any]=99     ,       lowercase		:			Optional[Any]=32     ,       lowercase		:			List[Any]=5     ,       lowercase		:			List[Any]=4     ,       lowercase		:			int=37     ,       lowercase		:			int="gelu"     ,       lowercase		:			Tuple=0.1     ,       lowercase		:			Dict=0.1     ,       lowercase		:			Tuple=512     ,       lowercase		:			List[str]=16     ,       lowercase		:			Optional[int]=2     ,       lowercase		:			int=0.02     ,       lowercase		:			str=3     ,       lowercase		:			List[Any]=4     ,       lowercase		:			List[str]=None     ,       ):



           '''simple docstring'''

           UpperCAmelCase     =	parent
           UpperCAmelCase     =	batch_size
           UpperCAmelCase     =	seq_length
           UpperCAmelCase     =	is_training
           UpperCAmelCase     =	use_token_type_ids
           UpperCAmelCase     =	use_labels
           UpperCAmelCase     =	vocab_size
           UpperCAmelCase     =	hidden_size
           UpperCAmelCase     =	num_hidden_layers
           UpperCAmelCase     =	num_attention_heads
           UpperCAmelCase     =	intermediate_size
           UpperCAmelCase     =	hidden_act
           UpperCAmelCase     =	hidden_dropout_prob
           UpperCAmelCase     =	attention_probs_dropout_prob
           UpperCAmelCase     =	max_position_embeddings
           UpperCAmelCase     =	type_vocab_size
           UpperCAmelCase     =	type_sequence_label_size
           UpperCAmelCase     =	initializer_range
           UpperCAmelCase     =	num_labels
           UpperCAmelCase     =	num_choices
           UpperCAmelCase     =	scope
           UpperCAmelCase     =	self.vocab_size - 1
       def 					A   (			self		:			List[Any]       ):



           '''simple docstring'''

           UpperCAmelCase     =	ids_tensor([self.batch_size, self.seq_length]     ,       self.vocab_size       )

           UpperCAmelCase     =	None
           if self.use_token_type_ids:
               UpperCAmelCase     =	ids_tensor([self.batch_size, self.seq_length]     ,       self.type_vocab_size       )

           UpperCAmelCase     =	None
           UpperCAmelCase     =	None
           UpperCAmelCase     =	None
           if self.use_labels:
               UpperCAmelCase     =	ids_tensor([self.batch_size]     ,       self.type_sequence_label_size       )
               UpperCAmelCase     =	ids_tensor([self.batch_size, self.seq_length]     ,       self.num_labels       )
               UpperCAmelCase     =	ids_tensor([self.batch_size]     ,       self.num_choices       )

           UpperCAmelCase     =	OpenAIGPTConfig(
               vocab_size=self.vocab_size     ,       n_embd=self.hidden_size     ,       n_layer=self.num_hidden_layers     ,       n_head=self.num_attention_heads     ,       n_positions=self.max_position_embeddings     ,       pad_token_id=self.pad_token_id     ,       )

           UpperCAmelCase     =	ids_tensor([self.num_hidden_layers, self.num_attention_heads]     ,       2       )

           return (
               config,
               input_ids,
               head_mask,
               token_type_ids,
               sequence_labels,
               token_labels,
               choice_labels,
           )
       def 					A   (			self		:			Any     ,       lowercase		:			Tuple     ,       lowercase		:			List[Any]     ,       lowercase		:			Optional[Any]     ,       lowercase		:			List[str]     ,       *lowercase		:			Optional[int]       ):



           '''simple docstring'''

           UpperCAmelCase     =	OpenAIGPTModel(config=lowercase       )
           model.to(lowercase       )
           model.eval()

           UpperCAmelCase     =	model(lowercase     ,       token_type_ids=lowercase     ,       head_mask=lowercase       )
           UpperCAmelCase     =	model(lowercase     ,       token_type_ids=lowercase       )
           UpperCAmelCase     =	model(lowercase       )

           self.parent.assertEqual(result.last_hidden_state.shape     ,       (self.batch_size, self.seq_length, self.hidden_size)       )
       def 					A   (			self		:			List[str]     ,       lowercase		:			Optional[int]     ,       lowercase		:			List[Any]     ,       lowercase		:			str     ,       lowercase		:			Optional[int]     ,       *lowercase		:			List[str]       ):



           '''simple docstring'''

           UpperCAmelCase     =	OpenAIGPTLMHeadModel(lowercase       )
           model.to(lowercase       )
           model.eval()

           UpperCAmelCase     =	model(lowercase     ,       token_type_ids=lowercase     ,       labels=lowercase       )
           self.parent.assertEqual(result.loss.shape     ,       ()       )
           self.parent.assertEqual(result.logits.shape     ,       (self.batch_size, self.seq_length, self.vocab_size)       )
       def 					A   (			self		:			Optional[int]     ,       lowercase		:			Tuple     ,       lowercase		:			Tuple     ,       lowercase		:			Optional[int]     ,       lowercase		:			Optional[int]     ,       *lowercase		:			Optional[int]       ):



           '''simple docstring'''

           UpperCAmelCase     =	OpenAIGPTDoubleHeadsModel(lowercase       )
           model.to(lowercase       )
           model.eval()

           UpperCAmelCase     =	model(lowercase     ,       token_type_ids=lowercase     ,       labels=lowercase       )
           self.parent.assertEqual(result.loss.shape     ,       ()       )
           self.parent.assertEqual(result.logits.shape     ,       (self.batch_size, self.seq_length, self.vocab_size)       )
       def 					A   (			self		:			Tuple     ,       lowercase		:			Dict     ,       lowercase		:			Any     ,       lowercase		:			Any     ,       lowercase		:			Optional[int]     ,       *lowercase		:			int       ):



           '''simple docstring'''

           UpperCAmelCase     =	self.num_labels
           UpperCAmelCase     =	OpenAIGPTForSequenceClassification(lowercase       )
           model.to(lowercase       )
           model.eval()

           UpperCAmelCase     =	ids_tensor([self.batch_size]     ,       self.type_sequence_label_size       )
           UpperCAmelCase     =	model(lowercase     ,       token_type_ids=lowercase     ,       labels=lowercase       )
           self.parent.assertEqual(result.logits.shape     ,       (self.batch_size, self.num_labels)       )







       def 					A   (			self		:			List[str]       ):



           '''simple docstring'''

           UpperCAmelCase     =	self.prepare_config_and_inputs()
           (
               (
               UpperCAmelCase
           ) ,  (
               UpperCAmelCase
           ) ,  (
               UpperCAmelCase
           ) ,  (
               UpperCAmelCase
           ) ,  (
               UpperCAmelCase
           ) ,  (
               UpperCAmelCase
           ) ,  (
               UpperCAmelCase
           ) ,  
           )     =	config_and_inputs
           UpperCAmelCase     =	{
               '''input_ids''': input_ids,
               '''token_type_ids''': token_type_ids,
               '''head_mask''': head_mask,
           }

           return config, inputs_dict






@require_torch
class     _a						(			__a							,							__a							,							__a							,							unittest.TestCase				):
       __a       :  List[str]            =      (
           (OpenAIGPTModel, OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel, OpenAIGPTForSequenceClassification)
           if is_torch_available()
           else ()
       )
       __a       :  List[Any]            =      (
           (OpenAIGPTLMHeadModel,) if is_torch_available() else ()
       )  # TODO (PVP): Add Double HeadsModel when generate() function is changed accordingly
       __a       :  Dict            =      (
           {
               """feature-extraction""": OpenAIGPTModel,
               """text-classification""": OpenAIGPTForSequenceClassification,
               """text-generation""": OpenAIGPTLMHeadModel,
               """zero-shot""": OpenAIGPTForSequenceClassification,
           }
           if is_torch_available()
           else {}
       )
       def 					A   (			self		:			Optional[Any]     ,       lowercase		:			Union[str, Any]     ,       lowercase		:			Tuple     ,       lowercase		:			int     ,       lowercase		:			List[Any]     ,       lowercase		:			Optional[Any]       ):



           '''simple docstring'''

           if pipeline_test_casse_name == "ZeroShotClassificationPipelineTests":
               # Get `tokenizer does not have a padding token` error for both fast/slow tokenizers.
               # `OpenAIGPTConfig` was never used in pipeline tests, either because of a missing checkpoint or because a
               # tiny config could not be created.
               return True

           return False
       def 					A   (			self		:			List[Any]     ,       lowercase		:			int     ,       lowercase		:			List[Any]     ,       lowercase		:			Optional[Any]=False       ):



           '''simple docstring'''

           UpperCAmelCase     =	super()._prepare_for_class(lowercase     ,       lowercase     ,       return_labels=lowercase       )

           if return_labels:
               if model_class.__name__ == "OpenAIGPTDoubleHeadsModel":
                   UpperCAmelCase     =	torch.zeros(
                       (self.model_tester.batch_size, self.model_tester.num_choices, self.model_tester.seq_length)     ,       dtype=torch.long     ,       device=lowercase     ,       )
                   UpperCAmelCase     =	inputs_dict['''labels''']
                   UpperCAmelCase     =	inputs_dict['''labels''']
                   UpperCAmelCase     =	torch.zeros(
                       (self.model_tester.batch_size, self.model_tester.num_choices)     ,       dtype=torch.long     ,       device=lowercase     ,       )
                   UpperCAmelCase     =	torch.zeros(
                       self.model_tester.batch_size     ,       dtype=torch.long     ,       device=lowercase       )
           return inputs_dict
       def 					A   (			self		:			List[Any]       ):



           '''simple docstring'''

           UpperCAmelCase     =	OpenAIGPTModelTester(self       )
           UpperCAmelCase     =	ConfigTester(self     ,       config_class=lowercase     ,       n_embd=37       )
       def 					A   (			self		:			str       ):



           '''simple docstring'''

           self.config_tester.run_common_tests()
       def 					A   (			self		:			Tuple       ):



           '''simple docstring'''

           UpperCAmelCase     =	self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_openai_gpt_model(*lowercase       )
       def 					A   (			self		:			Optional[int]       ):



           '''simple docstring'''

           UpperCAmelCase     =	self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_lm_head_model(*lowercase       )
       def 					A   (			self		:			List[Any]       ):



           '''simple docstring'''

           UpperCAmelCase     =	self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_double_lm_head_model(*lowercase       )
       def 					A   (			self		:			Optional[int]       ):



           '''simple docstring'''

           UpperCAmelCase     =	self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_openai_gpt_for_sequence_classification(*lowercase       )







       @slow
       def 					A   (			self		:			List[Any]       ):



           '''simple docstring'''

           for model_name in OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
               UpperCAmelCase     =	OpenAIGPTModel.from_pretrained(lowercase       )
               self.assertIsNotNone(lowercase       )






@require_torch
class     _a						(			unittest.TestCase				):
       @slow
       def 					A   (			self		:			Any       ):



           '''simple docstring'''

           UpperCAmelCase     =	OpenAIGPTLMHeadModel.from_pretrained('''openai-gpt'''       )
           model.to(lowercase       )
           UpperCAmelCase     =	torch.tensor([[481, 4_735, 544]]     ,       dtype=torch.long     ,       device=lowercase       )  # the president is
           UpperCAmelCase     =	[
               481,
               4_735,
               544,
               246,
               963,
               870,
               762,
               239,
               244,
               40_477,
               244,
               249,
               719,
               881,
               487,
               544,
               240,
               244,
               603,
               481,
           ]  # the president is a very good man. " \n " i\'m sure he is, " said the

           UpperCAmelCase     =	model.generate(lowercase     ,       do_sample=lowercase       )
           self.assertListEqual(output_ids[0].tolist()     ,       lowercase       )




