'''simple docstring'''





from __future__ import annotations

import os
import tempfile
import unittest

from transformers import ConvBertConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
			import tensorflow as tf

			from transformers import (
			    TFConvBertForMaskedLM,
			    TFConvBertForMultipleChoice,
			    TFConvBertForQuestionAnswering,
			    TFConvBertForSequenceClassification,
			    TFConvBertForTokenClassification,
			    TFConvBertModel,
			)




class 		SCREAMING_SNAKE_CASE     :







				'''simple docstring'''





				def __init__(						self					:						Tuple       ,		lowercase__					:						List[Any]       ,		lowercase__					:						List[Any]=13       ,		lowercase__					:						str=7       ,		lowercase__					:						Optional[int]=True       ,		lowercase__					:						Optional[int]=True       ,		lowercase__					:						str=True       ,		lowercase__					:						List[Any]=True       ,		lowercase__					:						List[str]=99       ,		lowercase__					:						Optional[Any]=32       ,		lowercase__					:						Any=2       ,		lowercase__					:						List[Any]=4       ,		lowercase__					:						Any=37       ,		lowercase__					:						Dict="gelu"       ,		lowercase__					:						Optional[int]=0.1       ,		lowercase__					:						Optional[int]=0.1       ,		lowercase__					:						List[Any]=512       ,		lowercase__					:						Union[str, Any]=16       ,		lowercase__					:						Any=2       ,		lowercase__					:						List[str]=0.0_2       ,		lowercase__					:						Optional[int]=3       ,		lowercase__					:						List[Any]=4       ,		lowercase__					:						Tuple=None       ,		)							->Optional[Any]:


										'''simple docstring'''




										_UpperCamelCase			:			int   			=						parent
										_UpperCamelCase			:			Optional[Any]   			=						13
										_UpperCamelCase			:			Optional[int]   			=						7
										_UpperCamelCase			:			int   			=						True
										_UpperCamelCase			:			List[str]   			=						True
										_UpperCamelCase			:			str   			=						True
										_UpperCamelCase			:			str   			=						True
										_UpperCamelCase			:			str   			=						99
										_UpperCamelCase			:			int   			=						384
										_UpperCamelCase			:			Dict   			=						2
										_UpperCamelCase			:			Union[str, Any]   			=						4
										_UpperCamelCase			:			List[str]   			=						37
										_UpperCamelCase			:			List[Any]   			=						"gelu"
										_UpperCamelCase			:			str   			=						0.1
										_UpperCamelCase			:			Union[str, Any]   			=						0.1
										_UpperCamelCase			:			Optional[Any]   			=						512
										_UpperCamelCase			:			Any   			=						16
										_UpperCamelCase			:			Tuple   			=						2
										_UpperCamelCase			:			str   			=						0.0_2
										_UpperCamelCase			:			str   			=						3
										_UpperCamelCase			:			str   			=						4
										_UpperCamelCase			:			Optional[Any]   			=						128
										_UpperCamelCase			:			Tuple   			=						2
										_UpperCamelCase			:			Any   			=						9
										_UpperCamelCase			:			Union[str, Any]   			=						1
										_UpperCamelCase			:			Dict   			=						None





				def  snake_case__				(						self					:						List[Any]					)							->Optional[Any]:


										'''simple docstring'''




										_UpperCamelCase			:			List[Any]   			=						ids_tensor([self.batch_size, self.seq_length]       ,		self.vocab_size					)

										_UpperCamelCase			:			Tuple   			=						None
										if self.use_input_mask:
																_UpperCamelCase			:			List[Any]   			=						random_attention_mask([self.batch_size, self.seq_length]					)

										_UpperCamelCase			:			Tuple   			=						None
										if self.use_token_type_ids:
																_UpperCamelCase			:			Optional[Any]   			=						ids_tensor([self.batch_size, self.seq_length]       ,		self.type_vocab_size					)

										_UpperCamelCase			:			str   			=						None
										_UpperCamelCase			:			List[Any]   			=						None
										_UpperCamelCase			:			List[Any]   			=						None
										if self.use_labels:
																_UpperCamelCase			:			Dict   			=						ids_tensor([self.batch_size]       ,		self.type_sequence_label_size					)
																_UpperCamelCase			:			Optional[int]   			=						ids_tensor([self.batch_size, self.seq_length]       ,		self.num_labels					)
																_UpperCamelCase			:			Optional[int]   			=						ids_tensor([self.batch_size]       ,		self.num_choices					)

										_UpperCamelCase			:			Optional[Any]   			=						ConvBertConfig(
										    vocab_size=self.vocab_size       ,		hidden_size=self.hidden_size       ,		num_hidden_layers=self.num_hidden_layers       ,		num_attention_heads=self.num_attention_heads       ,		intermediate_size=self.intermediate_size       ,		hidden_act=self.hidden_act       ,		hidden_dropout_prob=self.hidden_dropout_prob       ,		attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,		max_position_embeddings=self.max_position_embeddings       ,		type_vocab_size=self.type_vocab_size       ,		initializer_range=self.initializer_range       ,		return_dict=lowercase__       ,		)

										return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels





				def  snake_case__				(						self					:						int       ,		lowercase__					:						List[str]       ,		lowercase__					:						List[str]       ,		lowercase__					:						Dict       ,		lowercase__					:						Dict       ,		lowercase__					:						Union[str, Any]       ,		lowercase__					:						str       ,		lowercase__					:						List[Any]					)							->Any:


										'''simple docstring'''




										_UpperCamelCase			:			Tuple   			=						TFConvBertModel(config=lowercase__					)
										_UpperCamelCase			:			Union[str, Any]   			=						{"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}

										_UpperCamelCase			:			Dict   			=						[input_ids, input_mask]
										_UpperCamelCase			:			Tuple   			=						model(lowercase__					)

										_UpperCamelCase			:			Optional[Any]   			=						model(lowercase__					)

										self.parent.assertEqual(result.last_hidden_state.shape       ,		(self.batch_size, self.seq_length, self.hidden_size)					)





				def  snake_case__				(						self					:						Dict       ,		lowercase__					:						List[Any]       ,		lowercase__					:						Union[str, Any]       ,		lowercase__					:						Union[str, Any]       ,		lowercase__					:						Optional[int]       ,		lowercase__					:						str       ,		lowercase__					:						int       ,		lowercase__					:						Tuple					)							->Dict:


										'''simple docstring'''




										_UpperCamelCase			:			int   			=						TFConvBertForMaskedLM(config=lowercase__					)
										_UpperCamelCase			:			List[Any]   			=						{
										    "input_ids": input_ids,
										    "attention_mask": input_mask,
										    "token_type_ids": token_type_ids,
										}
										_UpperCamelCase			:			Dict   			=						model(lowercase__					)
										self.parent.assertEqual(result.logits.shape       ,		(self.batch_size, self.seq_length, self.vocab_size)					)





				def  snake_case__				(						self					:						Tuple       ,		lowercase__					:						List[Any]       ,		lowercase__					:						Optional[Any]       ,		lowercase__					:						Union[str, Any]       ,		lowercase__					:						Optional[int]       ,		lowercase__					:						List[str]       ,		lowercase__					:						str       ,		lowercase__					:						List[Any]					)							->Dict:


										'''simple docstring'''




										_UpperCamelCase			:			Any   			=						self.num_labels
										_UpperCamelCase			:			Optional[Any]   			=						TFConvBertForSequenceClassification(config=lowercase__					)
										_UpperCamelCase			:			Optional[Any]   			=						{
										    "input_ids": input_ids,
										    "attention_mask": input_mask,
										    "token_type_ids": token_type_ids,
										}

										_UpperCamelCase			:			Optional[int]   			=						model(lowercase__					)
										self.parent.assertEqual(result.logits.shape       ,		(self.batch_size, self.num_labels)					)





				def  snake_case__				(						self					:						int       ,		lowercase__					:						Optional[int]       ,		lowercase__					:						Tuple       ,		lowercase__					:						Tuple       ,		lowercase__					:						Optional[Any]       ,		lowercase__					:						Dict       ,		lowercase__					:						Dict       ,		lowercase__					:						int					)							->Optional[int]:


										'''simple docstring'''




										_UpperCamelCase			:			Tuple   			=						self.num_choices
										_UpperCamelCase			:			str   			=						TFConvBertForMultipleChoice(config=lowercase__					)
										_UpperCamelCase			:			List[str]   			=						tf.tile(tf.expand_dims(lowercase__       ,		1					)       ,		(1, self.num_choices, 1)					)
										_UpperCamelCase			:			Union[str, Any]   			=						tf.tile(tf.expand_dims(lowercase__       ,		1					)       ,		(1, self.num_choices, 1)					)
										_UpperCamelCase			:			Tuple   			=						tf.tile(tf.expand_dims(lowercase__       ,		1					)       ,		(1, self.num_choices, 1)					)
										_UpperCamelCase			:			Any   			=						{
										    "input_ids": multiple_choice_inputs_ids,
										    "attention_mask": multiple_choice_input_mask,
										    "token_type_ids": multiple_choice_token_type_ids,
										}
										_UpperCamelCase			:			List[str]   			=						model(lowercase__					)
										self.parent.assertEqual(result.logits.shape       ,		(self.batch_size, self.num_choices)					)





				def  snake_case__				(						self					:						Tuple       ,		lowercase__					:						Optional[Any]       ,		lowercase__					:						Tuple       ,		lowercase__					:						int       ,		lowercase__					:						Tuple       ,		lowercase__					:						Union[str, Any]       ,		lowercase__					:						Union[str, Any]       ,		lowercase__					:						Dict					)							->Any:


										'''simple docstring'''




										_UpperCamelCase			:			Dict   			=						self.num_labels
										_UpperCamelCase			:			Tuple   			=						TFConvBertForTokenClassification(config=lowercase__					)
										_UpperCamelCase			:			Any   			=						{
										    "input_ids": input_ids,
										    "attention_mask": input_mask,
										    "token_type_ids": token_type_ids,
										}
										_UpperCamelCase			:			Tuple   			=						model(lowercase__					)
										self.parent.assertEqual(result.logits.shape       ,		(self.batch_size, self.seq_length, self.num_labels)					)





				def  snake_case__				(						self					:						List[str]       ,		lowercase__					:						Dict       ,		lowercase__					:						Dict       ,		lowercase__					:						List[Any]       ,		lowercase__					:						Tuple       ,		lowercase__					:						Optional[Any]       ,		lowercase__					:						Any       ,		lowercase__					:						str					)							->Optional[int]:


										'''simple docstring'''




										_UpperCamelCase			:			int   			=						TFConvBertForQuestionAnswering(config=lowercase__					)
										_UpperCamelCase			:			Any   			=						{
										    "input_ids": input_ids,
										    "attention_mask": input_mask,
										    "token_type_ids": token_type_ids,
										}

										_UpperCamelCase			:			Optional[int]   			=						model(lowercase__					)
										self.parent.assertEqual(result.start_logits.shape       ,		(self.batch_size, self.seq_length)					)
										self.parent.assertEqual(result.end_logits.shape       ,		(self.batch_size, self.seq_length)					)







				def  snake_case__				(						self					:						Union[str, Any]					)							->List[str]:


										'''simple docstring'''




										_UpperCamelCase			:			str   			=						self.prepare_config_and_inputs()
										(
										    (
										    _UpperCamelCase
										)			,       (
										    _UpperCamelCase
										)			,       (
										    _UpperCamelCase
										)			,       (
										    _UpperCamelCase
										)			,       (
										    _UpperCamelCase
										)			,       (
										    _UpperCamelCase
										)			,       (
										    _UpperCamelCase
										)			,       
										)			:			Optional[Any]   			=						config_and_inputs
										_UpperCamelCase			:			str   			=						{"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
										return config, inputs_dict





@require_tf
class 		SCREAMING_SNAKE_CASE     (					SCREAMING_SNAKE_CASE   , SCREAMING_SNAKE_CASE   , unittest.TestCase	):







				'''simple docstring'''
				UpperCAmelCase__							=							(
				    (
				        TFConvBertModel,
				        TFConvBertForMaskedLM,
				        TFConvBertForQuestionAnswering,
				        TFConvBertForSequenceClassification,
				        TFConvBertForTokenClassification,
				        TFConvBertForMultipleChoice,
				    )
				    if is_tf_available()
				    else ()
				)
				UpperCAmelCase__							=							(
				    {
				        '''feature-extraction''': TFConvBertModel,
				        '''fill-mask''': TFConvBertForMaskedLM,
				        '''question-answering''': TFConvBertForQuestionAnswering,
				        '''text-classification''': TFConvBertForSequenceClassification,
				        '''token-classification''': TFConvBertForTokenClassification,
				        '''zero-shot''': TFConvBertForSequenceClassification,
				    }
				    if is_tf_available()
				    else {}
				)
				UpperCAmelCase__							=							False
				UpperCAmelCase__							=							False
				UpperCAmelCase__							=							False





				def  snake_case__				(						self					:						Union[str, Any]					)							->int:


										'''simple docstring'''




										_UpperCamelCase			:			Optional[Any]   			=						TFConvBertModelTester(self					)
										_UpperCamelCase			:			Any   			=						ConfigTester(self       ,		config_class=lowercase__       ,		hidden_size=37					)





				def  snake_case__				(						self					:						Optional[Any]					)							->int:


										'''simple docstring'''




										self.config_tester.run_common_tests()





				def  snake_case__				(						self					:						Optional[Any]					)							->List[str]:


										'''simple docstring'''




										_UpperCamelCase			:			List[Any]   			=						self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_model(*lowercase__					)





				def  snake_case__				(						self					:						int					)							->Tuple:


										'''simple docstring'''




										_UpperCamelCase			:			str   			=						self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_masked_lm(*lowercase__					)





				def  snake_case__				(						self					:						Optional[int]					)							->Optional[Any]:


										'''simple docstring'''




										_UpperCamelCase			:			int   			=						self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_multiple_choice(*lowercase__					)





				def  snake_case__				(						self					:						List[Any]					)							->str:


										'''simple docstring'''




										_UpperCamelCase			:			int   			=						self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_question_answering(*lowercase__					)





				def  snake_case__				(						self					:						str					)							->int:


										'''simple docstring'''




										_UpperCamelCase			:			List[Any]   			=						self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_sequence_classification(*lowercase__					)





				def  snake_case__				(						self					:						Optional[int]					)							->Union[str, Any]:


										'''simple docstring'''




										_UpperCamelCase			:			Dict   			=						self.model_tester.prepare_config_and_inputs()
										self.model_tester.create_and_check_for_token_classification(*lowercase__					)





				@slow
				def  snake_case__				(						self					:						List[str]					)							->Dict:


										'''simple docstring'''




										_UpperCamelCase			,       _UpperCamelCase			:			Any   			=						self.model_tester.prepare_config_and_inputs_for_common()
										_UpperCamelCase			:			Any   			=						True
										_UpperCamelCase			:			List[str]   			=						True

										if hasattr(lowercase__       ,		"use_cache"					):
																_UpperCamelCase			:			int   			=						True

										_UpperCamelCase			:			List[str]   			=						getattr(self.model_tester       ,		"encoder_seq_length"       ,		self.model_tester.seq_length					)
										_UpperCamelCase			:			Union[str, Any]   			=						getattr(self.model_tester       ,		"key_length"       ,		lowercase__					)

										for model_class in self.all_model_classes:
																_UpperCamelCase			:			List[str]   			=						self._prepare_for_class(lowercase__       ,		lowercase__					)
																_UpperCamelCase			:			int   			=						model_class(lowercase__					)
																_UpperCamelCase			:			Union[str, Any]   			=						len(model(lowercase__					)					)

																with tempfile.TemporaryDirectory() as tmpdirname:
																						model.save_pretrained(lowercase__       ,		saved_model=lowercase__					)
																						_UpperCamelCase			:			List[str]   			=						os.path.join(lowercase__       ,		"saved_model"       ,		"1"					)
																						_UpperCamelCase			:			List[str]   			=						tf.keras.models.load_model(lowercase__					)
																						_UpperCamelCase			:			Tuple   			=						model(lowercase__					)

																						if self.is_encoder_decoder:
																												_UpperCamelCase			:			Optional[int]   			=						outputs["encoder_hidden_states"]
																												_UpperCamelCase			:			int   			=						outputs["encoder_attentions"]
																						else:
																												_UpperCamelCase			:			Any   			=						outputs["hidden_states"]
																												_UpperCamelCase			:			List[Any]   			=						outputs["attentions"]

																						self.assertEqual(len(lowercase__					)       ,		lowercase__					)

																						_UpperCamelCase			:			Any   			=						getattr(
																						    self.model_tester       ,		"expected_num_hidden_layers"       ,		self.model_tester.num_hidden_layers + 1					)

																						self.assertEqual(len(lowercase__					)       ,		lowercase__					)
																						self.assertListEqual(
																						    list(output_hidden_states[0].shape[-2:]					)       ,		[self.model_tester.seq_length, self.model_tester.hidden_size]       ,		)

																						self.assertEqual(len(lowercase__					)       ,		self.model_tester.num_hidden_layers					)
																						self.assertListEqual(
																						    list(output_attentions[0].shape[-3:]					)       ,		[self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length]       ,		)





				@slow
				def  snake_case__				(						self					:						List[Any]					)							->str:


										'''simple docstring'''




										_UpperCamelCase			:			List[Any]   			=						TFConvBertModel.from_pretrained("YituTech/conv-bert-base"					)
										self.assertIsNotNone(lowercase__					)







				def  snake_case__				(						self					:						List[str]					)							->List[str]:


										'''simple docstring'''




										_UpperCamelCase			,       _UpperCamelCase			:			Optional[int]   			=						self.model_tester.prepare_config_and_inputs_for_common()
										_UpperCamelCase			:			Any   			=						True
										_UpperCamelCase			:			Optional[Any]   			=						getattr(self.model_tester       ,		"decoder_seq_length"       ,		self.model_tester.seq_length					)
										_UpperCamelCase			:			Dict   			=						getattr(self.model_tester       ,		"encoder_seq_length"       ,		self.model_tester.seq_length					)
										_UpperCamelCase			:			Union[str, Any]   			=						getattr(self.model_tester       ,		"key_length"       ,		lowercase__					)
										_UpperCamelCase			:			List[str]   			=						getattr(self.model_tester       ,		"key_length"       ,		lowercase__					)

										def check_decoder_attentions_output(lowercase__					:						List[Any]					):
																_UpperCamelCase			:			str   			=						len(lowercase__					)
																self.assertEqual(out_len % 2       ,		0					)
																_UpperCamelCase			:			Union[str, Any]   			=						outputs.decoder_attentions
																self.assertEqual(len(lowercase__					)       ,		self.model_tester.num_hidden_layers					)
																self.assertListEqual(
																    list(decoder_attentions[0].shape[-3:]					)       ,		[self.model_tester.num_attention_heads / 2, decoder_seq_length, decoder_key_length]       ,		)

										def check_encoder_attentions_output(lowercase__					:						Union[str, Any]					):
																_UpperCamelCase			:			Optional[int]   			=						[
																    t.numpy() for t in (outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions)
																]
																self.assertEqual(len(lowercase__					)       ,		self.model_tester.num_hidden_layers					)
																self.assertListEqual(
																    list(attentions[0].shape[-3:]					)       ,		[self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length]       ,		)

										for model_class in self.all_model_classes:
																_UpperCamelCase			:			Optional[int]   			=						True
																_UpperCamelCase			:			Any   			=						False
																_UpperCamelCase			:			Union[str, Any]   			=						model_class(lowercase__					)
																_UpperCamelCase			:			Dict   			=						model(self._prepare_for_class(lowercase__       ,		lowercase__					)					)
																_UpperCamelCase			:			Dict   			=						len(lowercase__					)
																self.assertEqual(config.output_hidden_states       ,		lowercase__					)
																check_encoder_attentions_output(lowercase__					)

																if self.is_encoder_decoder:
																						_UpperCamelCase			:			Optional[Any]   			=						model_class(lowercase__					)
																						_UpperCamelCase			:			Union[str, Any]   			=						model(self._prepare_for_class(lowercase__       ,		lowercase__					)					)
																						self.assertEqual(config.output_hidden_states       ,		lowercase__					)
																						check_decoder_attentions_output(lowercase__					)

																# Check that output attentions can also be changed via the config
																del inputs_dict["output_attentions"]
																_UpperCamelCase			:			Any   			=						True
																_UpperCamelCase			:			Any   			=						model_class(lowercase__					)
																_UpperCamelCase			:			Optional[int]   			=						model(self._prepare_for_class(lowercase__       ,		lowercase__					)					)
																self.assertEqual(config.output_hidden_states       ,		lowercase__					)
																check_encoder_attentions_output(lowercase__					)

																# Check attention is always last and order is fine
																_UpperCamelCase			:			Dict   			=						True
																_UpperCamelCase			:			List[Any]   			=						True
																_UpperCamelCase			:			List[Any]   			=						model_class(lowercase__					)
																_UpperCamelCase			:			Tuple   			=						model(self._prepare_for_class(lowercase__       ,		lowercase__					)					)

																self.assertEqual(out_len + (2 if self.is_encoder_decoder else 1)       ,		len(lowercase__					)					)
																self.assertEqual(model.config.output_hidden_states       ,		lowercase__					)
																check_encoder_attentions_output(lowercase__					)





@require_tf
class 		SCREAMING_SNAKE_CASE     (					unittest.TestCase	):







				'''simple docstring'''





				@slow
				def  snake_case__				(						self					:						Union[str, Any]					)							->str:


										'''simple docstring'''




										_UpperCamelCase			:			Optional[int]   			=						TFConvBertModel.from_pretrained("YituTech/conv-bert-base"					)
										_UpperCamelCase			:			Dict   			=						tf.constant([[0, 1, 2, 3, 4, 5]]					)
										_UpperCamelCase			:			Union[str, Any]   			=						model(lowercase__					)[0]

										_UpperCamelCase			:			Union[str, Any]   			=						[1, 6, 768]
										self.assertEqual(output.shape       ,		lowercase__					)

										_UpperCamelCase			:			List[Any]   			=						tf.constant(
										    [
										        [
										            [-0.0_3_4_7_5_4_9_3, -0.4_6_8_6_0_3_4, -0.3_0_6_3_8_8_3_2],
										            [0.2_2_6_3_7_2_4_8, -0.2_6_9_8_8_6_4_6, -0.7_4_2_3_4_2_4],
										            [0.1_0_3_2_4_8_6_8, -0.4_5_0_1_3_5_0_8, -0.5_8_2_8_0_7_8_4],
										        ]
										    ]					)
										tf.debugging.assert_near(output[:, :3, :3]       ,		lowercase__       ,		atol=1e-4					)



'''simple docstring'''





import collections
import os
from typing import List, Optional, Tuple

from transformers.utils import is_jieba_available, requires_backends


if is_jieba_available():
			import jieba

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase_	:		int			=     logging.get_logger(__name__)

lowerCAmelCase_	:		List[str]			=     {"""vocab_file""": """vocab.txt"""}

lowerCAmelCase_	:		Optional[Any]			=     {
    """vocab_file""": {
        """openbmb/cpm-ant-10b""": """https://huggingface.co/openbmb/cpm-ant-10b/blob/main/vocab.txt""",
    },
}

lowerCAmelCase_	:		int			=     {
    """openbmb/cpm-ant-10b""": 1024,
}
def 						__A							(     UpperCAmelCase					)     ->			List[str]:




						'''simple docstring'''
						_UpperCamelCase			:			Optional[int]   			=						collections.OrderedDict()
						with open(UpperCAmelCase     ,"r"     ,encoding="utf-8"					) as reader:
												_UpperCamelCase			:			int   			=						reader.readlines()
						for index, token in enumerate(UpperCAmelCase					):
												_UpperCamelCase			:			Optional[int]   			=						token.rstrip("\n"					)
												_UpperCamelCase			:			List[str]   			=						index
						return vocab




class 		SCREAMING_SNAKE_CASE     (					SCREAMING_SNAKE_CASE	):







				'''simple docstring'''





				def __init__(						self					:						Dict       ,		lowercase__					:						List[str]       ,		lowercase__					:						Any="<unk>"       ,		lowercase__					:						str=200					)							->Optional[int]:


										'''simple docstring'''




										_UpperCamelCase			:			Any   			=						vocab
										_UpperCamelCase			:			str   			=						unk_token
										_UpperCamelCase			:			List[Any]   			=						max_input_chars_per_word







				def  snake_case__				(						self					:						Union[str, Any]       ,		lowercase__					:						Any					)							->Dict:


										'''simple docstring'''




										_UpperCamelCase			:			int   			=						list(lowercase__					)
										if len(lowercase__					) > self.max_input_chars_per_word:
																return [self.unk_token]

										_UpperCamelCase			:			int   			=						0
										_UpperCamelCase			:			List[str]   			=						[]
										while start < len(lowercase__					):
																_UpperCamelCase			:			Union[str, Any]   			=						len(lowercase__					)
																_UpperCamelCase			:			Union[str, Any]   			=						None
																while start < end:
																						_UpperCamelCase			:			Dict   			=						"".join(chars[start:end]					)
																						if substr in self.vocab:
																												_UpperCamelCase			:			int   			=						substr
																												break
																						end -= 1
																if cur_substr is None:
																						sub_tokens.append(self.unk_token					)
																						start += 1
																else:
																						sub_tokens.append(lowercase__					)
																						_UpperCamelCase			:			List[str]   			=						end

										return sub_tokens





class 		SCREAMING_SNAKE_CASE     (					SCREAMING_SNAKE_CASE	):







				'''simple docstring'''
				UpperCAmelCase__							=							VOCAB_FILES_NAMES
				UpperCAmelCase__							=							PRETRAINED_VOCAB_FILES_MAP
				UpperCAmelCase__							=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				UpperCAmelCase__							=							['''input_ids''', '''attention_mask''']
				UpperCAmelCase__							=							False





				def __init__(						self					:						int       ,		lowercase__					:						str       ,		lowercase__					:						int="<d>"       ,		lowercase__					:						Tuple="</d>"       ,		lowercase__					:						Optional[int]="<s>"       ,		lowercase__					:						Optional[int]="</s>"       ,		lowercase__					:						List[str]="<pad>"       ,		lowercase__					:						Optional[int]="<unk>"       ,		lowercase__					:						int="</n>"       ,		lowercase__					:						Optional[Any]="</_>"       ,		lowercase__					:						int="left"       ,		**lowercase__					:						List[str]       ,		)							->Union[str, Any]:


										'''simple docstring'''




										requires_backends(self       ,		["jieba"]					)
										super().__init__(
										    bod_token=lowercase__       ,		eod_token=lowercase__       ,		bos_token=lowercase__       ,		eos_token=lowercase__       ,		pad_token=lowercase__       ,		unk_token=lowercase__       ,		line_token=lowercase__       ,		space_token=lowercase__       ,		padding_side=lowercase__       ,		**lowercase__       ,		)
										_UpperCamelCase			:			Union[str, Any]   			=						bod_token
										_UpperCamelCase			:			Union[str, Any]   			=						eod_token
										_UpperCamelCase			:			int   			=						load_vocab(lowercase__					)
										_UpperCamelCase			:			Any   			=						self.encoder[space_token]
										_UpperCamelCase			:			List[str]   			=						self.encoder[line_token]

										del self.encoder[space_token]
										del self.encoder[line_token]

										_UpperCamelCase			:			Dict   			=						collections.OrderedDict(sorted(self.encoder.items()       ,		key=lambda lowercase__					: x[1]					)					)
										_UpperCamelCase			:			Any   			=						{v: k for k, v in self.encoder.items()}

										_UpperCamelCase			:			Any   			=						WordpieceTokenizer(vocab=self.encoder       ,		unk_token=self.unk_token					)





				@property
				def  snake_case__				(						self					:						Tuple					)							->List[str]:


										'''simple docstring'''




										return self.encoder[self.bod_token]





				@property
				def  snake_case__				(						self					:						Tuple					)							->List[Any]:


										'''simple docstring'''




										return self.encoder[self.eod_token]





				@property
				def  snake_case__				(						self					:						List[str]					)							->int:


										'''simple docstring'''




										return self.encoder["\n"]





				@property
				def  snake_case__				(						self					:						List[str]					)							->int:


										'''simple docstring'''




										return len(self.encoder					)





				def  snake_case__				(						self					:						Dict					)							->Tuple:


										'''simple docstring'''




										return dict(self.encoder       ,		**self.added_tokens_encoder					)





				def  snake_case__				(						self					:						Optional[int]       ,		lowercase__					:						Tuple					)							->Tuple:


										'''simple docstring'''




										_UpperCamelCase			:			Tuple   			=						[]
										for x in jieba.cut(lowercase__       ,		cut_all=lowercase__					):
																output_tokens.extend(self.wordpiece_tokenizer.tokenize(lowercase__					)					)
										return output_tokens





				def  snake_case__				(						self					:						Optional[Any]       ,		lowercase__					:						Any       ,		**lowercase__					:						Tuple					)							->Union[str, Any]:


										'''simple docstring'''




										_UpperCamelCase			:			Optional[int]   			=						[i for i in token_ids if i >= 0]
										_UpperCamelCase			:			Union[str, Any]   			=						[
										    x for x in token_ids if x != self.pad_token_id and x != self.eos_token_id and x != self.bos_token_id
										]
										return super()._decode(lowercase__       ,		**lowercase__					)





				def  snake_case__				(						self					:						Optional[Any]       ,		lowercase__					:						Tuple					)							->Union[str, Any]:


										'''simple docstring'''




										return token in self.encoder





				def  snake_case__				(						self					:						str       ,		lowercase__					:						List[str]					)							->str:


										'''simple docstring'''




										return "".join(lowercase__					)





				def  snake_case__				(						self					:						List[Any]       ,		lowercase__					:						Optional[Any]					)							->int:


										'''simple docstring'''




										return self.encoder.get(lowercase__       ,		self.encoder.get(self.unk_token					)					)





				def  snake_case__				(						self					:						Any       ,		lowercase__					:						Dict					)							->Dict:


										'''simple docstring'''




										return self.decoder.get(lowercase__       ,		self.unk_token					)





				def  snake_case__				(						self					:						Tuple       ,		lowercase__					:						str       ,		lowercase__					:						Optional[str] = None					)							->Tuple[str]:


										'''simple docstring'''




										if os.path.isdir(lowercase__					):
																_UpperCamelCase			:			Tuple   			=						os.path.join(
																    lowercase__       ,		(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]					)
										else:
																_UpperCamelCase			:			Optional[int]   			=						(filename_prefix + "-" if filename_prefix else "") + save_directory
										_UpperCamelCase			:			List[str]   			=						0
										if " " in self.encoder:
																_UpperCamelCase			:			Dict   			=						self.encoder[" "]
																del self.encoder[" "]
										if "\n" in self.encoder:
																_UpperCamelCase			:			int   			=						self.encoder["\n"]
																del self.encoder["\n"]
										_UpperCamelCase			:			List[str]   			=						collections.OrderedDict(sorted(self.encoder.items()       ,		key=lambda lowercase__					: x[1]					)					)
										with open(lowercase__       ,		"w"       ,		encoding="utf-8"					) as writer:
																for token, token_index in self.encoder.items():
																						if index != token_index:
																												logger.warning(
																												    f'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
																												    " Please check that the vocabulary is not corrupted!"					)
																												_UpperCamelCase			:			str   			=						token_index
																						writer.write(token + "\n"					)
																						index += 1
										return (vocab_file,)





				def  snake_case__				(						self					:						List[Any]       ,		lowercase__					:						List[int]       ,		lowercase__					:						List[int] = None					)							->List[int]:


										'''simple docstring'''




										if token_ids_a is None:
																return [self.bos_token_id] + token_ids_a
										return [self.bos_token_id] + token_ids_a + [self.bos_token_id] + token_ids_a







				def  snake_case__				(						self					:						Optional[int]       ,		lowercase__					:						List[int]       ,		lowercase__					:						Optional[List[int]] = None       ,		lowercase__					:						bool = False					)							->List[int]:


										'''simple docstring'''




										if already_has_special_tokens:
																return super().get_special_tokens_mask(
																    token_ids_a=lowercase__       ,		token_ids_a=lowercase__       ,		already_has_special_tokens=lowercase__					)

										if token_ids_a is not None:
																return [1] + ([0] * len(lowercase__					)) + [1] + ([0] * len(lowercase__					))
										return [1] + ([0] * len(lowercase__					))


