
'''simple docstring'''
from __future__ import annotations

import os
import tempfile
import unittest

from transformers import ConvBertConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
		import tensorflow as tf

		from transformers import (
		    TFConvBertForMaskedLM,
		    TFConvBertForMultipleChoice,
		    TFConvBertForQuestionAnswering,
		    TFConvBertForSequenceClassification,
		    TFConvBertForTokenClassification,
		    TFConvBertModel,
		)







class   a_ :




			def __init__(       self					,   UpperCAmelCase					,   UpperCAmelCase=13					,   UpperCAmelCase=7					,   UpperCAmelCase=True					,   UpperCAmelCase=True					,   UpperCAmelCase=True					,   UpperCAmelCase=True					,   UpperCAmelCase=99					,   UpperCAmelCase=32					,   UpperCAmelCase=2					,   UpperCAmelCase=4					,   UpperCAmelCase=37					,   UpperCAmelCase="gelu"					,   UpperCAmelCase=0.1					,   UpperCAmelCase=0.1					,   UpperCAmelCase=5_12					,   UpperCAmelCase=16					,   UpperCAmelCase=2					,   UpperCAmelCase=0.02					,   UpperCAmelCase=3					,   UpperCAmelCase=4					,   UpperCAmelCase=None					,   ):
				a_      				=						parent
				a_      				=						13
				a_      				=						7
				a_      				=						True
				a_      				=						True
				a_      				=						True
				a_      				=						True
				a_      				=						99
				a_      				=						3_84
				a_      				=						2
				a_      				=						4
				a_      				=						37
				a_      				=						"""gelu"""
				a_      				=						0.1
				a_      				=						0.1
				a_      				=						5_12
				a_      				=						16
				a_      				=						2
				a_      				=						0.02
				a_      				=						3
				a_      				=						4
				a_      				=						1_28
				a_      				=						2
				a_      				=						9
				a_      				=						1
				a_      				=						None




			def 					lowerCAmelCase__					(       self	):
				a_      				=						ids_tensor([self.batch_size, self.seq_length]					,   self.vocab_size	)

				a_      				=						None
				if self.use_input_mask:
					a_      				=						random_attention_mask([self.batch_size, self.seq_length]	)

				a_      				=						None
				if self.use_token_type_ids:
					a_      				=						ids_tensor([self.batch_size, self.seq_length]					,   self.type_vocab_size	)

				a_      				=						None
				a_      				=						None
				a_      				=						None
				if self.use_labels:
					a_      				=						ids_tensor([self.batch_size]					,   self.type_sequence_label_size	)
					a_      				=						ids_tensor([self.batch_size, self.seq_length]					,   self.num_labels	)
					a_      				=						ids_tensor([self.batch_size]					,   self.num_choices	)

				a_      				=						ConvBertConfig(
				    vocab_size=self.vocab_size					,   hidden_size=self.hidden_size					,   num_hidden_layers=self.num_hidden_layers					,   num_attention_heads=self.num_attention_heads					,   intermediate_size=self.intermediate_size					,   hidden_act=self.hidden_act					,   hidden_dropout_prob=self.hidden_dropout_prob					,   attention_probs_dropout_prob=self.attention_probs_dropout_prob					,   max_position_embeddings=self.max_position_embeddings					,   type_vocab_size=self.type_vocab_size					,   initializer_range=self.initializer_range					,   return_dict=UpperCAmelCase					,   )

				return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




			def 					lowerCAmelCase__					(       self					,   UpperCAmelCase					,   UpperCAmelCase					,   UpperCAmelCase					,   UpperCAmelCase					,   UpperCAmelCase					,   UpperCAmelCase					,   UpperCAmelCase	):
				a_      				=						TFConvBertModel(config=UpperCAmelCase	)
				a_      				=						{"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}

				a_      				=						[input_ids, input_mask]
				a_      				=						model(UpperCAmelCase	)

				a_      				=						model(UpperCAmelCase	)

				self.parent.assertEqual(result.last_hidden_state.shape					,   (self.batch_size, self.seq_length, self.hidden_size)	)




			def 					lowerCAmelCase__					(       self					,   UpperCAmelCase					,   UpperCAmelCase					,   UpperCAmelCase					,   UpperCAmelCase					,   UpperCAmelCase					,   UpperCAmelCase					,   UpperCAmelCase	):
				a_      				=						TFConvBertForMaskedLM(config=UpperCAmelCase	)
				a_      				=						{
				    """input_ids""": input_ids,
				    """attention_mask""": input_mask,
				    """token_type_ids""": token_type_ids,
				}
				a_      				=						model(UpperCAmelCase	)
				self.parent.assertEqual(result.logits.shape					,   (self.batch_size, self.seq_length, self.vocab_size)	)




			def 					lowerCAmelCase__					(       self					,   UpperCAmelCase					,   UpperCAmelCase					,   UpperCAmelCase					,   UpperCAmelCase					,   UpperCAmelCase					,   UpperCAmelCase					,   UpperCAmelCase	):
				a_      				=						self.num_labels
				a_      				=						TFConvBertForSequenceClassification(config=UpperCAmelCase	)
				a_      				=						{
				    """input_ids""": input_ids,
				    """attention_mask""": input_mask,
				    """token_type_ids""": token_type_ids,
				}

				a_      				=						model(UpperCAmelCase	)
				self.parent.assertEqual(result.logits.shape					,   (self.batch_size, self.num_labels)	)




			def 					lowerCAmelCase__					(       self					,   UpperCAmelCase					,   UpperCAmelCase					,   UpperCAmelCase					,   UpperCAmelCase					,   UpperCAmelCase					,   UpperCAmelCase					,   UpperCAmelCase	):
				a_      				=						self.num_choices
				a_      				=						TFConvBertForMultipleChoice(config=UpperCAmelCase	)
				a_      				=						tf.tile(tf.expand_dims(UpperCAmelCase					,   1	)					,   (1, self.num_choices, 1)	)
				a_      				=						tf.tile(tf.expand_dims(UpperCAmelCase					,   1	)					,   (1, self.num_choices, 1)	)
				a_      				=						tf.tile(tf.expand_dims(UpperCAmelCase					,   1	)					,   (1, self.num_choices, 1)	)
				a_      				=						{
				    """input_ids""": multiple_choice_inputs_ids,
				    """attention_mask""": multiple_choice_input_mask,
				    """token_type_ids""": multiple_choice_token_type_ids,
				}
				a_      				=						model(UpperCAmelCase	)
				self.parent.assertEqual(result.logits.shape					,   (self.batch_size, self.num_choices)	)




			def 					lowerCAmelCase__					(       self					,   UpperCAmelCase					,   UpperCAmelCase					,   UpperCAmelCase					,   UpperCAmelCase					,   UpperCAmelCase					,   UpperCAmelCase					,   UpperCAmelCase	):
				a_      				=						self.num_labels
				a_      				=						TFConvBertForTokenClassification(config=UpperCAmelCase	)
				a_      				=						{
				    """input_ids""": input_ids,
				    """attention_mask""": input_mask,
				    """token_type_ids""": token_type_ids,
				}
				a_      				=						model(UpperCAmelCase	)
				self.parent.assertEqual(result.logits.shape					,   (self.batch_size, self.seq_length, self.num_labels)	)




			def 					lowerCAmelCase__					(       self					,   UpperCAmelCase					,   UpperCAmelCase					,   UpperCAmelCase					,   UpperCAmelCase					,   UpperCAmelCase					,   UpperCAmelCase					,   UpperCAmelCase	):
				a_      				=						TFConvBertForQuestionAnswering(config=UpperCAmelCase	)
				a_      				=						{
				    """input_ids""": input_ids,
				    """attention_mask""": input_mask,
				    """token_type_ids""": token_type_ids,
				}

				a_      				=						model(UpperCAmelCase	)
				self.parent.assertEqual(result.start_logits.shape					,   (self.batch_size, self.seq_length)	)
				self.parent.assertEqual(result.end_logits.shape					,   (self.batch_size, self.seq_length)	)






			def 					lowerCAmelCase__					(       self	):
				a_      				=						self.prepare_config_and_inputs()
				(
				    (
				    a_
				)				,  (
				    a_
				)				,  (
				    a_
				)				,  (
				    a_
				)				,  (
				    a_
				)				,  (
				    a_
				)				,  (
				    a_
				)				,  
				)      				=						config_and_inputs
				a_      				=						{"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
				return config, inputs_dict







@require_tf
class   a_ (      UpperCamelCase__		,  UpperCamelCase__		,  unittest.TestCase					):
			lowerCamelCase__	:			Tuple       =    (
			    (
			        TFConvBertModel,
			        TFConvBertForMaskedLM,
			        TFConvBertForQuestionAnswering,
			        TFConvBertForSequenceClassification,
			        TFConvBertForTokenClassification,
			        TFConvBertForMultipleChoice,
			    )
			    if is_tf_available()
			    else ()
			)
			lowerCamelCase__	:			Tuple       =    (
			    {
			        'feature-extraction': TFConvBertModel,
			        'fill-mask': TFConvBertForMaskedLM,
			        'question-answering': TFConvBertForQuestionAnswering,
			        'text-classification': TFConvBertForSequenceClassification,
			        'token-classification': TFConvBertForTokenClassification,
			        'zero-shot': TFConvBertForSequenceClassification,
			    }
			    if is_tf_available()
			    else {}
			)
			lowerCamelCase__	:			Tuple       =    False
			lowerCamelCase__	:			Tuple       =    False
			lowerCamelCase__	:			Union[str, Any]       =    False




			def 					lowerCAmelCase__					(       self	):
				a_      				=						TFConvBertModelTester(self	)
				a_      				=						ConfigTester(self					,   config_class=UpperCAmelCase					,   hidden_size=37	)




			def 					lowerCAmelCase__					(       self	):
				self.config_tester.run_common_tests()




			def 					lowerCAmelCase__					(       self	):
				a_      				=						self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_model(*UpperCAmelCase	)




			def 					lowerCAmelCase__					(       self	):
				a_      				=						self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_masked_lm(*UpperCAmelCase	)




			def 					lowerCAmelCase__					(       self	):
				a_      				=						self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_multiple_choice(*UpperCAmelCase	)




			def 					lowerCAmelCase__					(       self	):
				a_      				=						self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_question_answering(*UpperCAmelCase	)




			def 					lowerCAmelCase__					(       self	):
				a_      				=						self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_sequence_classification(*UpperCAmelCase	)




			def 					lowerCAmelCase__					(       self	):
				a_      				=						self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_for_token_classification(*UpperCAmelCase	)




			@slow
			def 					lowerCAmelCase__					(       self	):
				a_				,  a_      				=						self.model_tester.prepare_config_and_inputs_for_common()
				a_      				=						True
				a_      				=						True

				if hasattr(UpperCAmelCase					,   """use_cache"""	):
					a_      				=						True

				a_      				=						getattr(self.model_tester					,   """encoder_seq_length"""					,   self.model_tester.seq_length	)
				a_      				=						getattr(self.model_tester					,   """key_length"""					,   UpperCAmelCase	)

				for model_class in self.all_model_classes:
					a_      				=						self._prepare_for_class(UpperCAmelCase					,   UpperCAmelCase	)
					a_      				=						model_class(UpperCAmelCase	)
					a_      				=						len(model(UpperCAmelCase	)	)

					with tempfile.TemporaryDirectory() as tmpdirname:
						model.save_pretrained(UpperCAmelCase					,   saved_model=UpperCAmelCase	)
						a_      				=						os.path.join(UpperCAmelCase					,   """saved_model"""					,   """1"""	)
						a_      				=						tf.keras.models.load_model(UpperCAmelCase	)
						a_      				=						model(UpperCAmelCase	)

						if self.is_encoder_decoder:
							a_      				=						outputs["""encoder_hidden_states"""]
							a_      				=						outputs["""encoder_attentions"""]
						else:
							a_      				=						outputs["""hidden_states"""]
							a_      				=						outputs["""attentions"""]

						self.assertEqual(len(UpperCAmelCase	)					,   UpperCAmelCase	)

						a_      				=						getattr(
						    self.model_tester					,   """expected_num_hidden_layers"""					,   self.model_tester.num_hidden_layers + 1	)

						self.assertEqual(len(UpperCAmelCase	)					,   UpperCAmelCase	)
						self.assertListEqual(
						    list(output_hidden_states[0].shape[-2:]	)					,   [self.model_tester.seq_length, self.model_tester.hidden_size]					,   )

						self.assertEqual(len(UpperCAmelCase	)					,   self.model_tester.num_hidden_layers	)
						self.assertListEqual(
						    list(output_attentions[0].shape[-3:]	)					,   [self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length]					,   )




			@slow
			def 					lowerCAmelCase__					(       self	):
				a_      				=						TFConvBertModel.from_pretrained("""YituTech/conv-bert-base"""	)
				self.assertIsNotNone(UpperCAmelCase	)






			def 					lowerCAmelCase__					(       self	):
				a_				,  a_      				=						self.model_tester.prepare_config_and_inputs_for_common()
				a_      				=						True
				a_      				=						getattr(self.model_tester					,   """decoder_seq_length"""					,   self.model_tester.seq_length	)
				a_      				=						getattr(self.model_tester					,   """encoder_seq_length"""					,   self.model_tester.seq_length	)
				a_      				=						getattr(self.model_tester					,   """key_length"""					,   UpperCAmelCase	)
				a_      				=						getattr(self.model_tester					,   """key_length"""					,   UpperCAmelCase	)

				def check_decoder_attentions_output(UpperCAmelCase	):
					a_      				=						len(UpperCAmelCase	)
					self.assertEqual(out_len % 2					,   0	)
					a_      				=						outputs.decoder_attentions
					self.assertEqual(len(UpperCAmelCase	)					,   self.model_tester.num_hidden_layers	)
					self.assertListEqual(
					    list(decoder_attentions[0].shape[-3:]	)					,   [self.model_tester.num_attention_heads / 2, decoder_seq_length, decoder_key_length]					,   )

				def check_encoder_attentions_output(UpperCAmelCase	):
					a_      				=						[
					    t.numpy() for t in (outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions)
					]
					self.assertEqual(len(UpperCAmelCase	)					,   self.model_tester.num_hidden_layers	)
					self.assertListEqual(
					    list(attentions[0].shape[-3:]	)					,   [self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length]					,   )

				for model_class in self.all_model_classes:
					a_      				=						True
					a_      				=						False
					a_      				=						model_class(UpperCAmelCase	)
					a_      				=						model(self._prepare_for_class(UpperCAmelCase					,   UpperCAmelCase	)	)
					a_      				=						len(UpperCAmelCase	)
					self.assertEqual(config.output_hidden_states					,   UpperCAmelCase	)
					check_encoder_attentions_output(UpperCAmelCase	)

					if self.is_encoder_decoder:
						a_      				=						model_class(UpperCAmelCase	)
						a_      				=						model(self._prepare_for_class(UpperCAmelCase					,   UpperCAmelCase	)	)
						self.assertEqual(config.output_hidden_states					,   UpperCAmelCase	)
						check_decoder_attentions_output(UpperCAmelCase	)

					# Check that output attentions can also be changed via the config
					del inputs_dict["output_attentions"]
					a_      				=						True
					a_      				=						model_class(UpperCAmelCase	)
					a_      				=						model(self._prepare_for_class(UpperCAmelCase					,   UpperCAmelCase	)	)
					self.assertEqual(config.output_hidden_states					,   UpperCAmelCase	)
					check_encoder_attentions_output(UpperCAmelCase	)

					# Check attention is always last and order is fine
					a_      				=						True
					a_      				=						True
					a_      				=						model_class(UpperCAmelCase	)
					a_      				=						model(self._prepare_for_class(UpperCAmelCase					,   UpperCAmelCase	)	)

					self.assertEqual(out_len + (2 if self.is_encoder_decoder else 1)					,   len(UpperCAmelCase	)	)
					self.assertEqual(model.config.output_hidden_states					,   UpperCAmelCase	)
					check_encoder_attentions_output(UpperCAmelCase	)







@require_tf
class   a_ (      unittest.TestCase					):




			@slow
			def 					lowerCAmelCase__					(       self	):
				a_      				=						TFConvBertModel.from_pretrained("""YituTech/conv-bert-base"""	)
				a_      				=						tf.constant([[0, 1, 2, 3, 4, 5]]	)
				a_      				=						model(UpperCAmelCase	)[0]

				a_      				=						[1, 6, 7_68]
				self.assertEqual(output.shape					,   UpperCAmelCase	)

				a_      				=						tf.constant(
				    [
				        [
				            [-0.03_47_54_93, -0.4_68_60_34, -0.30_63_88_32],
				            [0.22_63_72_48, -0.26_98_86_46, -0.7_42_34_24],
				            [0.10_32_48_68, -0.45_01_35_08, -0.58_28_07_84],
				        ]
				    ]	)
				tf.debugging.assert_near(output[:, :3, :3]					,   UpperCAmelCase					,   atol=1e-4	)


'''simple docstring'''
import json
import os
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowercase__											=logging.get_logger(__name__)


lowercase__											={
    'vocab_file': 'vocab.json',
    'tokenizer_config_file': 'tokenizer_config.json',
    'merges_file': 'merges.txt',
}

lowercase__											={
    'vocab_file': {
        'facebook/s2t-wav2vec2-large-en-de': (
            'https://huggingface.co/facebook/s2t-wav2vec2-large-en-de/resolve/main/vocab.json'
        ),
    },
    'tokenizer_config_file': {
        'facebook/s2t-wav2vec2-large-en-de': (
            'https://huggingface.co/facebook/s2t-wav2vec2-large-en-de/resolve/main/tokenizer_config.json'
        ),
    },
    'merges_file': {
        'facebook/s2t-wav2vec2-large-en-de': (
            'https://huggingface.co/facebook/s2t-wav2vec2-large-en-de/resolve/main/merges.txt'
        ),
    },
}

lowercase__											='</w>'
lowercase__											='@@ '






def      UpperCamelCase_				(      A__						):
	a_      				=						set()
	a_      				=						word[0]
	for char in word[1:]:
		pairs.add((prev_char, char)						)
		a_      				=						char
	return pairs


# Speech2Text2 has no max input length
lowercase__											={'facebook/s2t-wav2vec2-large-en-de': 10_24}







class   a_ (      UpperCamelCase__					):
			lowerCamelCase__	:			List[str]       =    VOCAB_FILES_NAMES
			lowerCamelCase__	:			Optional[Any]       =    PRETRAINED_VOCAB_FILES_MAP
			lowerCamelCase__	:			List[Any]       =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCamelCase__	:			int       =    ['input_ids', 'attention_mask']




			def __init__(       self					,   UpperCAmelCase					,   UpperCAmelCase="<s>"					,   UpperCAmelCase="<pad>"					,   UpperCAmelCase="</s>"					,   UpperCAmelCase="<unk>"					,   UpperCAmelCase=False					,   UpperCAmelCase=None					,   **UpperCAmelCase					,   ):
				super().__init__(
				    unk_token=UpperCAmelCase					,   bos_token=UpperCAmelCase					,   eos_token=UpperCAmelCase					,   pad_token=UpperCAmelCase					,   do_lower_case=UpperCAmelCase					,   **UpperCAmelCase					,   )

				a_      				=						do_lower_case

				with open(UpperCAmelCase					,   encoding="""utf-8"""	) as vocab_handle:
					a_      				=						json.load(UpperCAmelCase	)
				a_      				=						{v: k for k, v in self.encoder.items()}

				if merges_file is None:
					logger.info(f'''No merges files provided. {self.__class__.__name__} can only be used for decoding.'''	)

					a_      				=						None
					a_      				=						None
				else:
					with open(UpperCAmelCase					,   encoding="""utf-8"""	) as merges_handle:
						a_      				=						merges_handle.read().split("""\n"""	)[:-1]

					a_      				=						[tuple(merge.split()[:2]	) for merge in merges]
					a_      				=						dict(zip(UpperCAmelCase					,   range(len(UpperCAmelCase	)	)	)	)
					a_      				=						{}




			@property
			def 					lowerCAmelCase__					(       self	):
				return len(self.decoder	)




			def 					lowerCAmelCase__					(       self	):
				return dict(self.encoder					,   **self.added_tokens_encoder	)




			def 					lowerCAmelCase__					(       self					,   UpperCAmelCase	):
				a_      				=						tuple(token[:-1]	) + (token[-1] + BPE_TOKEN_MERGES,)
				if token in self.cache:
					return self.cache[token]
				a_      				=						get_pairs(UpperCAmelCase	)

				if not pairs:
					return token

				while True:
					a_      				=						min(UpperCAmelCase					,   key=lambda UpperCAmelCase	: self.bpe_ranks.get(UpperCAmelCase					,   float("""inf"""	)	)	)
					if bigram not in self.bpe_ranks:
						break
					a_				,  a_      				=						bigram
					a_      				=						[]
					a_      				=						0
					while i < len(UpperCAmelCase	):
						try:
							a_      				=						word.index(UpperCAmelCase					,   UpperCAmelCase	)
						except ValueError:
							new_word.extend(word[i:]	)
							break
						else:
							new_word.extend(word[i:j]	)
							a_      				=						j

						if word[i] == first and i < len(UpperCAmelCase	) - 1 and word[i + 1] == second:
							new_word.append(first + second	)
							i += 2
						else:
							new_word.append(word[i]	)
							i += 1
					a_      				=						tuple(UpperCAmelCase	)
					a_      				=						new_word
					if len(UpperCAmelCase	) == 1:
						break
					else:
						a_      				=						get_pairs(UpperCAmelCase	)
				a_      				=						""" """.join(UpperCAmelCase	)
				if word == "\n  " + BPE_TOKEN_MERGES:
					a_      				=						"""\n""" + BPE_TOKEN_MERGES

				if word.endswith(UpperCAmelCase	):
					a_      				=						word.replace(UpperCAmelCase					,   """"""	)

				a_      				=						word.replace(""" """					,   UpperCAmelCase	)
				a_      				=						word
				return word




			def 					lowerCAmelCase__					(       self					,   UpperCAmelCase	):

				if self.bpe_ranks is None:
					raise ValueError(
					    """This tokenizer was instantiated without a `merges.txt` file, so"""
					    """ that it can only be used for decoding, not for encoding."""
					    """Make sure to provide `merges.txt` file at instantiation to enable """
					    """encoding."""	)

				if self.do_lower_case:
					a_      				=						text.lower()

				a_      				=						text.split()

				a_      				=						[]
				for token in text:
					if token:
						split_tokens.extend(list(self.bpe(UpperCAmelCase	).split(""" """	)	)	)

				return split_tokens




			def 					lowerCAmelCase__					(       self					,   UpperCAmelCase	):
				return self.encoder.get(UpperCAmelCase					,   self.encoder.get(self.unk_token	)	)




			def 					lowerCAmelCase__					(       self					,   UpperCAmelCase	):
				a_      				=						self.decoder.get(UpperCAmelCase					,   self.unk_token	)
				return result




			def 					lowerCAmelCase__					(       self					,   UpperCAmelCase	):
				a_      				=						""" """.join(UpperCAmelCase	)

				# make sure @@ tokens are concatenated
				a_      				=						"""""".join(string.split(UpperCAmelCase	)	)

				return string






			def 					lowerCAmelCase__					(       self					,   UpperCAmelCase					,   UpperCAmelCase = None	):
				if not os.path.isdir(UpperCAmelCase	):
					logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''	)
					return
				a_      				=						os.path.join(
				    UpperCAmelCase					,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]	)
				a_      				=						os.path.join(
				    UpperCAmelCase					,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]	)

				with open(UpperCAmelCase					,   """w"""					,   encoding="""utf-8"""	) as f:
					f.write(json.dumps(self.encoder					,   indent=2					,   sort_keys=UpperCAmelCase					,   ensure_ascii=UpperCAmelCase	) + """\n"""	)

				a_      				=						0
				if self.bpe_ranks is None:
					return (vocab_file,)

				with open(UpperCAmelCase					,   """w"""					,   encoding="""utf-8"""	) as writer:
					for bpe_tokens, token_index in sorted(self.bpe_ranks.items()					,   key=lambda UpperCAmelCase	: kv[1]	):
						if index != token_index:
							logger.warning(
							    f'''Saving vocabulary to {merges_file}: BPE merge indices are not consecutive.'''
							    """ Please check that the tokenizer is not corrupted!"""	)
							a_      				=						token_index
						writer.write(""" """.join(UpperCAmelCase	) + """\n"""	)
						index += 1

				return (vocab_file, merges_file)
