import unittest

from parameterized import parameterized

from transformers import OpenLlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
			import torch

			from transformers import OpenLlamaForCausalLM, OpenLlamaForSequenceClassification, OpenLlamaModel







class __snake_case       :




			def __init__(					self    :       Dict ,			_lowercase    :       Dict ,			_lowercase    :       Optional[int]=13 ,			_lowercase    :       Optional[Any]=7 ,			_lowercase    :       str=True ,			_lowercase    :       List[str]=True ,			_lowercase    :       List[Any]=False ,			_lowercase    :       int=True ,			_lowercase    :       Union[str, Any]=99 ,			_lowercase    :       List[str]=32 ,			_lowercase    :       Optional[Any]=5 ,			_lowercase    :       List[Any]=4 ,			_lowercase    :       List[str]=37 ,			_lowercase    :       str="gelu" ,			_lowercase    :       str=0.1 ,			_lowercase    :       Tuple=0.1 ,			_lowercase    :       str=5_12 ,			_lowercase    :       str=16 ,			_lowercase    :       Dict=2 ,			_lowercase    :       Union[str, Any]=0.02 ,			_lowercase    :       Union[str, Any]=3 ,			_lowercase    :       Dict=4 ,			_lowercase    :       Optional[int]=None ,			):



					"""simple docstring"""






					SCREAMING_SNAKE_CASE__    							=   parent
					SCREAMING_SNAKE_CASE__    							=   batch_size
					SCREAMING_SNAKE_CASE__    							=   seq_length
					SCREAMING_SNAKE_CASE__    							=   is_training
					SCREAMING_SNAKE_CASE__    							=   use_input_mask
					SCREAMING_SNAKE_CASE__    							=   use_token_type_ids
					SCREAMING_SNAKE_CASE__    							=   use_labels
					SCREAMING_SNAKE_CASE__    							=   vocab_size
					SCREAMING_SNAKE_CASE__    							=   hidden_size
					SCREAMING_SNAKE_CASE__    							=   num_hidden_layers
					SCREAMING_SNAKE_CASE__    							=   num_attention_heads
					SCREAMING_SNAKE_CASE__    							=   intermediate_size
					SCREAMING_SNAKE_CASE__    							=   hidden_act
					SCREAMING_SNAKE_CASE__    							=   hidden_dropout_prob
					SCREAMING_SNAKE_CASE__    							=   attention_probs_dropout_prob
					SCREAMING_SNAKE_CASE__    							=   max_position_embeddings
					SCREAMING_SNAKE_CASE__    							=   type_vocab_size
					SCREAMING_SNAKE_CASE__    							=   type_sequence_label_size
					SCREAMING_SNAKE_CASE__    							=   initializer_range
					SCREAMING_SNAKE_CASE__    							=   num_labels
					SCREAMING_SNAKE_CASE__    							=   num_choices
					SCREAMING_SNAKE_CASE__    							=   scope




			def 			__a		(					self    :       str   ):



					"""simple docstring"""






					SCREAMING_SNAKE_CASE__    							=   ids_tensor([self.batch_size, self.seq_length] ,			self.vocab_size   )

					SCREAMING_SNAKE_CASE__    							=   None
					if self.use_input_mask:
							SCREAMING_SNAKE_CASE__    							=   random_attention_mask([self.batch_size, self.seq_length]   )

					SCREAMING_SNAKE_CASE__    							=   None
					if self.use_token_type_ids:
							SCREAMING_SNAKE_CASE__    							=   ids_tensor([self.batch_size, self.seq_length] ,			self.type_vocab_size   )

					SCREAMING_SNAKE_CASE__    							=   None
					SCREAMING_SNAKE_CASE__    							=   None
					SCREAMING_SNAKE_CASE__    							=   None
					if self.use_labels:
							SCREAMING_SNAKE_CASE__    							=   ids_tensor([self.batch_size] ,			self.type_sequence_label_size   )
							SCREAMING_SNAKE_CASE__    							=   ids_tensor([self.batch_size, self.seq_length] ,			self.num_labels   )
							SCREAMING_SNAKE_CASE__    							=   ids_tensor([self.batch_size] ,			self.num_choices   )

					SCREAMING_SNAKE_CASE__    							=   self.get_config()

					return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




			def 			__a		(					self    :       int   ):



					"""simple docstring"""






					return OpenLlamaConfig(
					    vocab_size=self.vocab_size ,			hidden_size=self.hidden_size ,			num_hidden_layers=self.num_hidden_layers ,			num_attention_heads=self.num_attention_heads ,			intermediate_size=self.intermediate_size ,			hidden_act=self.hidden_act ,			hidden_dropout_prob=self.hidden_dropout_prob ,			attention_probs_dropout_prob=self.attention_probs_dropout_prob ,			max_position_embeddings=self.max_position_embeddings ,			type_vocab_size=self.type_vocab_size ,			is_decoder=snake_case__ ,			initializer_range=self.initializer_range ,			use_stable_embedding=snake_case__ ,			)




			def 			__a		(					self    :       Optional[Any] ,			_lowercase    :       Any ,			_lowercase    :       Any ,			_lowercase    :       Tuple ,			_lowercase    :       Union[str, Any] ,			_lowercase    :       List[Any] ,			_lowercase    :       Optional[int] ,			_lowercase    :       Union[str, Any]   ):



					"""simple docstring"""






					SCREAMING_SNAKE_CASE__    							=   OpenLlamaModel(config=snake_case__   )
					model.to(snake_case__   )
					model.eval()
					SCREAMING_SNAKE_CASE__    							=   model(snake_case__ ,			attention_mask=snake_case__   )
					SCREAMING_SNAKE_CASE__    							=   model(snake_case__   )
					self.parent.assertEqual(result.last_hidden_state.shape ,			(self.batch_size, self.seq_length, self.hidden_size)   )




			def 			__a		(					self    :       str ,			_lowercase    :       Optional[int] ,			_lowercase    :       Any ,			_lowercase    :       Dict ,			_lowercase    :       List[str] ,			_lowercase    :       Any ,			_lowercase    :       Tuple ,			_lowercase    :       List[Any] ,			_lowercase    :       Dict ,			_lowercase    :       List[str] ,			):



					"""simple docstring"""






					SCREAMING_SNAKE_CASE__    							=   True
					SCREAMING_SNAKE_CASE__    							=   OpenLlamaModel(snake_case__   )
					model.to(snake_case__   )
					model.eval()
					SCREAMING_SNAKE_CASE__    							=   model(
					    snake_case__ ,			attention_mask=snake_case__ ,			encoder_hidden_states=snake_case__ ,			encoder_attention_mask=snake_case__ ,			)
					SCREAMING_SNAKE_CASE__    							=   model(
					    snake_case__ ,			attention_mask=snake_case__ ,			encoder_hidden_states=snake_case__ ,			)
					SCREAMING_SNAKE_CASE__    							=   model(snake_case__ ,			attention_mask=snake_case__   )
					self.parent.assertEqual(result.last_hidden_state.shape ,			(self.batch_size, self.seq_length, self.hidden_size)   )




			def 			__a		(					self    :       Any ,			_lowercase    :       Union[str, Any] ,			_lowercase    :       int ,			_lowercase    :       Union[str, Any] ,			_lowercase    :       List[str] ,			_lowercase    :       Union[str, Any] ,			_lowercase    :       int ,			_lowercase    :       Union[str, Any] ,			_lowercase    :       Union[str, Any] ,			_lowercase    :       Dict ,			):



					"""simple docstring"""






					SCREAMING_SNAKE_CASE__    							=   OpenLlamaForCausalLM(config=snake_case__   )
					model.to(snake_case__   )
					model.eval()
					SCREAMING_SNAKE_CASE__    							=   model(snake_case__ ,			attention_mask=snake_case__ ,			labels=snake_case__   )
					self.parent.assertEqual(result.logits.shape ,			(self.batch_size, self.seq_length, self.vocab_size)   )




			def 			__a		(					self    :       Any ,			_lowercase    :       Optional[int] ,			_lowercase    :       Optional[int] ,			_lowercase    :       Union[str, Any] ,			_lowercase    :       Optional[int] ,			_lowercase    :       Union[str, Any] ,			_lowercase    :       Optional[int] ,			_lowercase    :       Optional[Any] ,			_lowercase    :       Tuple ,			_lowercase    :       str ,			):



					"""simple docstring"""






					SCREAMING_SNAKE_CASE__    							=   True
					SCREAMING_SNAKE_CASE__    							=   True
					SCREAMING_SNAKE_CASE__    							=   OpenLlamaForCausalLM(config=snake_case__   )
					model.to(snake_case__   )
					model.eval()

					# first forward pass
					SCREAMING_SNAKE_CASE__    							=   model(
					    snake_case__ ,			attention_mask=snake_case__ ,			encoder_hidden_states=snake_case__ ,			encoder_attention_mask=snake_case__ ,			use_cache=snake_case__ ,			)
					SCREAMING_SNAKE_CASE__    							=   outputs.past_key_values

					# create hypothetical multiple next token and extent to next_input_ids
					SCREAMING_SNAKE_CASE__    							=   ids_tensor((self.batch_size, 3) ,			config.vocab_size   )
					SCREAMING_SNAKE_CASE__    							=   ids_tensor((self.batch_size, 3) ,			vocab_size=2   )

					# append to next input_ids and
					SCREAMING_SNAKE_CASE__    							=   torch.cat([input_ids, next_tokens] ,			dim=-1   )
					SCREAMING_SNAKE_CASE__    							=   torch.cat([input_mask, next_mask] ,			dim=-1   )

					SCREAMING_SNAKE_CASE__    							=   model(
					    snake_case__ ,			attention_mask=snake_case__ ,			encoder_hidden_states=snake_case__ ,			encoder_attention_mask=snake_case__ ,			output_hidden_states=snake_case__ ,			)["""hidden_states"""][0]
					SCREAMING_SNAKE_CASE__    							=   model(
					    snake_case__ ,			attention_mask=snake_case__ ,			encoder_hidden_states=snake_case__ ,			encoder_attention_mask=snake_case__ ,			past_key_values=snake_case__ ,			output_hidden_states=snake_case__ ,			)["""hidden_states"""][0]

					# select random slice
					SCREAMING_SNAKE_CASE__    							=   ids_tensor((1,) ,			output_from_past.shape[-1]   ).item()
					SCREAMING_SNAKE_CASE__    							=   output_from_no_past[:, -3:, random_slice_idx].detach()
					SCREAMING_SNAKE_CASE__    							=   output_from_past[:, :, random_slice_idx].detach()

					self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]   )

					# test that outputs are equal for slice
					self.parent.assertTrue(torch.allclose(snake_case__ ,			snake_case__ ,			atol=1E-3   )   )




			def 			__a		(					self    :       Tuple   ):



					"""simple docstring"""






					SCREAMING_SNAKE_CASE__    							=   self.prepare_config_and_inputs()
					(
					    (
					    SCREAMING_SNAKE_CASE__						
					)							,  (
					    SCREAMING_SNAKE_CASE__						
					)							,  (
					    SCREAMING_SNAKE_CASE__						
					)							,  (
					    SCREAMING_SNAKE_CASE__						
					)							,  (
					    SCREAMING_SNAKE_CASE__						
					)							,  (
					    SCREAMING_SNAKE_CASE__						
					)							,  (
					    SCREAMING_SNAKE_CASE__						
					)							,  						
					)    							=   config_and_inputs
					SCREAMING_SNAKE_CASE__    							=   {"""input_ids""": input_ids, """attention_mask""": input_mask}
					return config, inputs_dict







@require_torch
class __snake_case       (		__a     ,	__a     ,	__a     ,	unittest.TestCase	):
			lowerCAmelCase_														=							(
			    (OpenLlamaModel, OpenLlamaForCausalLM, OpenLlamaForSequenceClassification) if is_torch_available() else ()
			)
			lowerCAmelCase_														=							(OpenLlamaForCausalLM,) if is_torch_available() else ()
			lowerCAmelCase_														=							(
			    {
			        '''feature-extraction''': OpenLlamaModel,
			        '''text-classification''': OpenLlamaForSequenceClassification,
			        '''text-generation''': OpenLlamaForCausalLM,
			        '''zero-shot''': OpenLlamaForSequenceClassification,
			    }
			    if is_torch_available()
			    else {}
			)
			lowerCAmelCase_														=							False
			lowerCAmelCase_														=							False




			def 			__a		(					self    :       int   ):



					"""simple docstring"""






					SCREAMING_SNAKE_CASE__    							=   OpenLlamaModelTester(self   )
					SCREAMING_SNAKE_CASE__    							=   ConfigTester(self ,			config_class=snake_case__ ,			hidden_size=37   )




			def 			__a		(					self    :       Tuple   ):



					"""simple docstring"""






					self.config_tester.run_common_tests()




			def 			__a		(					self    :       str   ):



					"""simple docstring"""






					SCREAMING_SNAKE_CASE__    							=   self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_model(*snake_case__   )




			def 			__a		(					self    :       Optional[int]   ):



					"""simple docstring"""






					SCREAMING_SNAKE_CASE__    							=   self.model_tester.prepare_config_and_inputs()
					for type in ["absolute", "relative_key", "relative_key_query"]:
							SCREAMING_SNAKE_CASE__    							=   type
							self.model_tester.create_and_check_model(*snake_case__   )




			def 			__a		(					self    :       Tuple   ):



					"""simple docstring"""






					SCREAMING_SNAKE_CASE__							,  SCREAMING_SNAKE_CASE__    							=   self.model_tester.prepare_config_and_inputs_for_common()
					SCREAMING_SNAKE_CASE__    							=   3
					SCREAMING_SNAKE_CASE__    							=   input_dict["""input_ids"""]
					SCREAMING_SNAKE_CASE__    							=   input_ids.ne(1   ).to(snake_case__   )
					SCREAMING_SNAKE_CASE__    							=   ids_tensor([self.model_tester.batch_size] ,			self.model_tester.type_sequence_label_size   )
					SCREAMING_SNAKE_CASE__    							=   OpenLlamaForSequenceClassification(snake_case__   )
					model.to(snake_case__   )
					model.eval()
					SCREAMING_SNAKE_CASE__    							=   model(snake_case__ ,			attention_mask=snake_case__ ,			labels=snake_case__   )
					self.assertEqual(result.logits.shape ,			(self.model_tester.batch_size, self.model_tester.num_labels)   )




			def 			__a		(					self    :       List[str]   ):



					"""simple docstring"""






					SCREAMING_SNAKE_CASE__							,  SCREAMING_SNAKE_CASE__    							=   self.model_tester.prepare_config_and_inputs_for_common()
					SCREAMING_SNAKE_CASE__    							=   3
					SCREAMING_SNAKE_CASE__    							=   """single_label_classification"""
					SCREAMING_SNAKE_CASE__    							=   input_dict["""input_ids"""]
					SCREAMING_SNAKE_CASE__    							=   input_ids.ne(1   ).to(snake_case__   )
					SCREAMING_SNAKE_CASE__    							=   ids_tensor([self.model_tester.batch_size] ,			self.model_tester.type_sequence_label_size   )
					SCREAMING_SNAKE_CASE__    							=   OpenLlamaForSequenceClassification(snake_case__   )
					model.to(snake_case__   )
					model.eval()
					SCREAMING_SNAKE_CASE__    							=   model(snake_case__ ,			attention_mask=snake_case__ ,			labels=snake_case__   )
					self.assertEqual(result.logits.shape ,			(self.model_tester.batch_size, self.model_tester.num_labels)   )




			def 			__a		(					self    :       Union[str, Any]   ):



					"""simple docstring"""






					SCREAMING_SNAKE_CASE__							,  SCREAMING_SNAKE_CASE__    							=   self.model_tester.prepare_config_and_inputs_for_common()
					SCREAMING_SNAKE_CASE__    							=   3
					SCREAMING_SNAKE_CASE__    							=   """multi_label_classification"""
					SCREAMING_SNAKE_CASE__    							=   input_dict["""input_ids"""]
					SCREAMING_SNAKE_CASE__    							=   input_ids.ne(1   ).to(snake_case__   )
					SCREAMING_SNAKE_CASE__    							=   ids_tensor(
					    [self.model_tester.batch_size, config.num_labels] ,			self.model_tester.type_sequence_label_size   ).to(torch.float   )
					SCREAMING_SNAKE_CASE__    							=   OpenLlamaForSequenceClassification(snake_case__   )
					model.to(snake_case__   )
					model.eval()
					SCREAMING_SNAKE_CASE__    							=   model(snake_case__ ,			attention_mask=snake_case__ ,			labels=snake_case__   )
					self.assertEqual(result.logits.shape ,			(self.model_tester.batch_size, self.model_tester.num_labels)   )




			@unittest.skip("""Open-Llama buffers include complex numbers, which breaks this test"""   )
			def 			__a		(					self    :       int   ):



					"""simple docstring"""






					pass




			@parameterized.expand([("""linear""",), ("""dynamic""",)]   )
			def 			__a		(					self    :       Union[str, Any] ,			_lowercase    :       int   ):



					"""simple docstring"""






					SCREAMING_SNAKE_CASE__							,  SCREAMING_SNAKE_CASE__    							=   self.model_tester.prepare_config_and_inputs_for_common()
					SCREAMING_SNAKE_CASE__    							=   ids_tensor([1, 10] ,			config.vocab_size   )
					SCREAMING_SNAKE_CASE__    							=   ids_tensor([1, int(config.max_position_embeddings * 1.5   )] ,			config.vocab_size   )

					set_seed(42   )  # Fixed seed at init time so the two models get the same random weights
					SCREAMING_SNAKE_CASE__    							=   OpenLlamaModel(snake_case__   )
					original_model.to(snake_case__   )
					original_model.eval()
					SCREAMING_SNAKE_CASE__    							=   original_model(snake_case__   ).last_hidden_state
					SCREAMING_SNAKE_CASE__    							=   original_model(snake_case__   ).last_hidden_state

					set_seed(42   )  # Fixed seed at init time so the two models get the same random weights
					SCREAMING_SNAKE_CASE__    							=   {"""type""": scaling_type, """factor""": 10.0}
					SCREAMING_SNAKE_CASE__    							=   OpenLlamaModel(snake_case__   )
					scaled_model.to(snake_case__   )
					scaled_model.eval()
					SCREAMING_SNAKE_CASE__    							=   scaled_model(snake_case__   ).last_hidden_state
					SCREAMING_SNAKE_CASE__    							=   scaled_model(snake_case__   ).last_hidden_state

					# Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
					# maximum sequence length, so the outputs for the short input should match.
					if scaling_type == "dynamic":
							self.assertTrue(torch.allclose(snake_case__ ,			snake_case__ ,			atol=1E-5   )   )
					else:
							self.assertFalse(torch.allclose(snake_case__ ,			snake_case__ ,			atol=1E-5   )   )

					# The output should be different for long inputs
					self.assertFalse(torch.allclose(snake_case__ ,			snake_case__ ,			atol=1E-5   )   )

import collections
import os
from typing import List, Optional, Tuple

from transformers.utils import is_jieba_available, requires_backends


if is_jieba_available():
   import jieba

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__lowerCamelCase    :				Any							     =	logging.get_logger(__name__)

__lowerCamelCase    :				Optional[Any]							     =	{'''vocab_file''': '''vocab.txt'''}

__lowerCamelCase    :				Union[str, Any]							     =	{
    '''vocab_file''': {
        '''openbmb/cpm-ant-10b''': '''https://huggingface.co/openbmb/cpm-ant-10b/blob/main/vocab.txt''',
    },
}

__lowerCamelCase    :				Optional[Any]							     =	{
    '''openbmb/cpm-ant-10b''': 1024,
}



def 			__SCREAMING_SNAKE_CASE						(      __UpperCamelCase					: str			)							->      Optional[Any]:







  """simple docstring"""





  SCREAMING_SNAKE_CASE__    							=   collections.OrderedDict()
  with open(__UpperCamelCase				,  """r"""				,  encoding="""utf-8"""			) as reader:
    SCREAMING_SNAKE_CASE__    							=   reader.readlines()
  for index, token in enumerate(__UpperCamelCase			):
    SCREAMING_SNAKE_CASE__    							=   token.rstrip("""\n"""			)
    SCREAMING_SNAKE_CASE__    							=   index
  return vocab







class __snake_case       (		lowerCamelCase_	):




   def __init__(					self    :       Optional[int] ,			_lowercase    :       Optional[Any] ,			_lowercase    :       int="<unk>" ,			_lowercase    :       int=2_00   ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__    							=   vocab
     SCREAMING_SNAKE_CASE__    							=   unk_token
     SCREAMING_SNAKE_CASE__    							=   max_input_chars_per_word




   def 			__a		(					self    :       Optional[int] ,			_lowercase    :       List[Any]   ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__    							=   list(_lowercase   )
     if len(_lowercase   ) > self.max_input_chars_per_word:
       return [self.unk_token]

     SCREAMING_SNAKE_CASE__    							=   0
     SCREAMING_SNAKE_CASE__    							=   []
     while start < len(_lowercase   ):
       SCREAMING_SNAKE_CASE__    							=   len(_lowercase   )
       SCREAMING_SNAKE_CASE__    							=   None
       while start < end:
         SCREAMING_SNAKE_CASE__    							=   """""".join(chars[start:end]   )
         if substr in self.vocab:
           SCREAMING_SNAKE_CASE__    							=   substr
           break
         end -= 1
       if cur_substr is None:
         sub_tokens.append(self.unk_token   )
         start += 1
       else:
         sub_tokens.append(_lowercase   )
         SCREAMING_SNAKE_CASE__    							=   end

     return sub_tokens







class __snake_case       (		lowerCamelCase_	):
   lowerCAmelCase_														=							VOCAB_FILES_NAMES
   lowerCAmelCase_														=							PRETRAINED_VOCAB_FILES_MAP
   lowerCAmelCase_														=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   lowerCAmelCase_														=							["input_ids", "attention_mask"]
   lowerCAmelCase_														=							False




   def __init__(					self    :       int ,			_lowercase    :       str ,			_lowercase    :       List[Any]="<d>" ,			_lowercase    :       List[Any]="</d>" ,			_lowercase    :       Union[str, Any]="<s>" ,			_lowercase    :       List[str]="</s>" ,			_lowercase    :       str="<pad>" ,			_lowercase    :       int="<unk>" ,			_lowercase    :       List[str]="</n>" ,			_lowercase    :       Tuple="</_>" ,			_lowercase    :       Any="left" ,			**_lowercase    :       Any ,			):



     """simple docstring"""






     requires_backends(self ,			["""jieba"""]   )
     super().__init__(
         bod_token=_lowercase ,			eod_token=_lowercase ,			bos_token=_lowercase ,			eos_token=_lowercase ,			pad_token=_lowercase ,			unk_token=_lowercase ,			line_token=_lowercase ,			space_token=_lowercase ,			padding_side=_lowercase ,			**_lowercase ,			)
     SCREAMING_SNAKE_CASE__    							=   bod_token
     SCREAMING_SNAKE_CASE__    							=   eod_token
     SCREAMING_SNAKE_CASE__    							=   load_vocab(_lowercase   )
     SCREAMING_SNAKE_CASE__    							=   self.encoder[space_token]
     SCREAMING_SNAKE_CASE__    							=   self.encoder[line_token]

     del self.encoder[space_token]
     del self.encoder[line_token]

     SCREAMING_SNAKE_CASE__    							=   collections.OrderedDict(sorted(self.encoder.items() ,			key=lambda _lowercase   : x[1]   )   )
     SCREAMING_SNAKE_CASE__    							=   {v: k for k, v in self.encoder.items()}

     SCREAMING_SNAKE_CASE__    							=   WordpieceTokenizer(vocab=self.encoder ,			unk_token=self.unk_token   )




   @property
   def 			__a		(					self    :       Optional[Any]   ):



     """simple docstring"""






     return self.encoder[self.bod_token]




   @property
   def 			__a		(					self    :       List[Any]   ):



     """simple docstring"""






     return self.encoder[self.eod_token]




   @property
   def 			__a		(					self    :       Any   ):



     """simple docstring"""






     return self.encoder["\n"]




   @property
   def 			__a		(					self    :       Union[str, Any]   ):



     """simple docstring"""






     return len(self.encoder   )




   def 			__a		(					self    :       int   ):



     """simple docstring"""






     return dict(self.encoder ,			**self.added_tokens_encoder   )




   def 			__a		(					self    :       Union[str, Any] ,			_lowercase    :       str   ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__    							=   []
     for x in jieba.cut(_lowercase ,			cut_all=_lowercase   ):
       output_tokens.extend(self.wordpiece_tokenizer.tokenize(_lowercase   )   )
     return output_tokens




   def 			__a		(					self    :       int ,			_lowercase    :       Any ,			**_lowercase    :       Any   ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__    							=   [i for i in token_ids if i >= 0]
     SCREAMING_SNAKE_CASE__    							=   [
         x for x in token_ids if x != self.pad_token_id and x != self.eos_token_id and x != self.bos_token_id
     ]
     return super()._decode(_lowercase ,			**_lowercase   )




   def 			__a		(					self    :       Optional[int] ,			_lowercase    :       List[Any]   ):



     """simple docstring"""






     return token in self.encoder




   def 			__a		(					self    :       List[str] ,			_lowercase    :       List[str]   ):



     """simple docstring"""






     return "".join(_lowercase   )




   def 			__a		(					self    :       Optional[int] ,			_lowercase    :       Any   ):



     """simple docstring"""






     return self.encoder.get(_lowercase ,			self.encoder.get(self.unk_token   )   )




   def 			__a		(					self    :       Tuple ,			_lowercase    :       List[Any]   ):



     """simple docstring"""






     return self.decoder.get(_lowercase ,			self.unk_token   )




   def 			__a		(					self    :       Optional[Any] ,			_lowercase    :       str ,			_lowercase    :       Optional[str] = None   ):



     """simple docstring"""






     if os.path.isdir(_lowercase   ):
       SCREAMING_SNAKE_CASE__    							=   os.path.join(
           _lowercase ,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]   )
     else:
       SCREAMING_SNAKE_CASE__    							=   (filename_prefix + """-""" if filename_prefix else """""") + save_directory
     SCREAMING_SNAKE_CASE__    							=   0
     if " " in self.encoder:
       SCREAMING_SNAKE_CASE__    							=   self.encoder[""" """]
       del self.encoder[" "]
     if "\n" in self.encoder:
       SCREAMING_SNAKE_CASE__    							=   self.encoder["""\n"""]
       del self.encoder["\n"]
     SCREAMING_SNAKE_CASE__    							=   collections.OrderedDict(sorted(self.encoder.items() ,			key=lambda _lowercase   : x[1]   )   )
     with open(_lowercase ,			"""w""" ,			encoding="""utf-8"""   ) as writer:
       for token, token_index in self.encoder.items():
         if index != token_index:
           logger.warning(
               f"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
               """ Please check that the vocabulary is not corrupted!"""   )
           SCREAMING_SNAKE_CASE__    							=   token_index
         writer.write(token + """\n"""   )
         index += 1
     return (vocab_file,)




   def 			__a		(					self    :       int ,			_lowercase    :       List[int] ,			_lowercase    :       List[int] = None   ):



     """simple docstring"""






     if token_ids_a is None:
       return [self.bos_token_id] + token_ids_a
     return [self.bos_token_id] + token_ids_a + [self.bos_token_id] + token_ids_a




   def 			__a		(					self    :       Union[str, Any] ,			_lowercase    :       List[int] ,			_lowercase    :       Optional[List[int]] = None ,			_lowercase    :       bool = False   ):



     """simple docstring"""






     if already_has_special_tokens:
       return super().get_special_tokens_mask(
           token_ids_a=_lowercase ,			token_ids_a=_lowercase ,			already_has_special_tokens=_lowercase   )

     if token_ids_a is not None:
       return [1] + ([0] * len(_lowercase   )) + [1] + ([0] * len(_lowercase   ))
     return [1] + ([0] * len(_lowercase   ))
