"""simple docstring"""
from __future__ import annotations

import unittest

import numpy as np

from transformers import OPTConfig, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
					import tensorflow as tf

					from transformers import GPTaTokenizer, TFOPTForCausalLM, TFOPTModel





def    lowerCAmelCase__    (			_UpperCamelCase       :				Dict ,			_UpperCamelCase       :				Optional[int] ,			_UpperCamelCase       :				Optional[int]=None ,			_UpperCamelCase       :				List[str]=None					)					->				Tuple:





						"""simple docstring"""






						if attention_mask is None:
												snake_case								=			tf.cast(tf.math.not_equal(_UpperCamelCase ,			config.pad_token_id					) ,			tf.inta					)
						return {"input_ids": input_ids, "attention_mask": attention_mask}



@require_tf
class   lowerCAmelCase_    :



				"""simple docstring"""
				_lowerCAmelCase     :    Dict			  =						OPTConfig
				_lowerCAmelCase     :    Any			  =						{}
				_lowerCAmelCase     :    Any			  =						"""gelu"""

				def __init__(				self		,    lowerCAmelCase		,    lowerCAmelCase=13		,    lowerCAmelCase=7		,    lowerCAmelCase=True		,    lowerCAmelCase=False		,    lowerCAmelCase=99		,    lowerCAmelCase=16		,    lowerCAmelCase=2		,    lowerCAmelCase=4		,    lowerCAmelCase=4		,    lowerCAmelCase="gelu"		,    lowerCAmelCase=0.1		,    lowerCAmelCase=0.1		,    lowerCAmelCase=20		,    lowerCAmelCase=2		,    lowerCAmelCase=1		,    lowerCAmelCase=0		,    lowerCAmelCase=16		,    lowerCAmelCase=16		,    ):






										"""simple docstring"""

										snake_case								=			parent
										snake_case								=			batch_size
										snake_case								=			seq_length
										snake_case								=			is_training
										snake_case								=			use_labels
										snake_case								=			vocab_size
										snake_case								=			hidden_size
										snake_case								=			num_hidden_layers
										snake_case								=			num_attention_heads
										snake_case								=			intermediate_size
										snake_case								=			hidden_act
										snake_case								=			hidden_dropout_prob
										snake_case								=			attention_probs_dropout_prob
										snake_case								=			max_position_embeddings
										snake_case								=			eos_token_id
										snake_case								=			pad_token_id
										snake_case								=			bos_token_id
										snake_case								=			embed_dim
										snake_case								=			word_embed_proj_dim
										snake_case								=			False

				def 		snake_case (				self    ):






										"""simple docstring"""

										snake_case								=			ids_tensor([self.batch_size, self.seq_length - 1]		,    self.vocab_size    )
										snake_case								=			tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size    )		,    1    )
										snake_case								=			tf.concat([input_ids, eos_tensor]		,    axis=1    )

										snake_case								=			self.config_cls(
										    vocab_size=self.vocab_size		,    hidden_size=self.hidden_size		,    num_hidden_layers=self.num_hidden_layers		,    num_attention_heads=self.num_attention_heads		,    ffn_dim=self.intermediate_size		,    dropout=self.hidden_dropout_prob		,    attention_dropout=self.attention_probs_dropout_prob		,    max_position_embeddings=self.max_position_embeddings		,    eos_token_id=self.eos_token_id		,    bos_token_id=self.bos_token_id		,    pad_token_id=self.pad_token_id		,    embed_dim=self.embed_dim		,    word_embed_proj_dim=self.word_embed_proj_dim		,    is_encoder_decoder=snake_case__		,    **self.config_updates		,    )
										snake_case								=			prepare_opt_inputs_dict(snake_case__		,    snake_case__    )
										return config, inputs_dict

				def 		snake_case (				self		,    lowerCAmelCase		,    lowerCAmelCase    ):






										"""simple docstring"""

										snake_case								=			TFOPTModel(config=snake_case__    )
										snake_case								=			inputs_dict["input_ids"]

										snake_case								=			input_ids[:1, :]
										snake_case								=			inputs_dict["attention_mask"][:1, :]
										snake_case								=			1

										# first forward pass
										snake_case								=			model(snake_case__		,    attention_mask=snake_case__		,    use_cache=snake_case__    )

										snake_case								=			outputs.to_tuple()

										# create hypothetical next token and extent to next_input_ids
										snake_case								=			ids_tensor((self.batch_size, 3)		,    config.vocab_size    )
										snake_case								=			tf.cast(ids_tensor((self.batch_size, 3)		,    2    )		,    tf.inta    )

										# append to next input_ids and
										snake_case								=			tf.concat([input_ids, next_tokens]		,    axis=-1    )
										snake_case								=			tf.concat([attention_mask, next_attn_mask]		,    axis=-1    )

										snake_case								=			model(snake_case__		,    attention_mask=snake_case__    )[0]
										snake_case								=			model(snake_case__		,    attention_mask=snake_case__		,    past_key_values=snake_case__    )[0]

										self.parent.assertEqual(next_tokens.shape[1]		,    output_from_past.shape[1]    )

										# select random slice
										snake_case								=			int(ids_tensor((1,)		,    output_from_past.shape[-1]    )    )
										snake_case								=			output_from_no_past[:, -3:, random_slice_idx]
										snake_case								=			output_from_past[:, :, random_slice_idx]

										# test that outputs are equal for slice
										tf.debugging.assert_near(snake_case__		,    snake_case__		,    rtol=1E-3    )





@require_tf
class   lowerCAmelCase_    (			SCREAMING_SNAKE_CASE__     ,    SCREAMING_SNAKE_CASE__     ,    unittest.TestCase					):



				"""simple docstring"""
				_lowerCAmelCase     :    str			  =						(TFOPTModel, TFOPTForCausalLM) if is_tf_available() else ()
				_lowerCAmelCase     :    List[Any]			  =						(TFOPTForCausalLM,) if is_tf_available() else ()
				_lowerCAmelCase     :    Tuple			  =						(
				    {"""feature-extraction""": TFOPTModel, """text-generation""": TFOPTForCausalLM} if is_tf_available() else {}
				)
				_lowerCAmelCase     :    List[Any]			  =						False
				_lowerCAmelCase     :    Dict			  =						False
				_lowerCAmelCase     :    int			  =						False
				_lowerCAmelCase     :    Tuple			  =						10

				def 		snake_case (				self    ):






										"""simple docstring"""

										snake_case								=			TFOPTModelTester(self    )
										snake_case								=			ConfigTester(self		,    config_class=snake_case__    )

				def 		snake_case (				self    ):






										"""simple docstring"""

										self.config_tester.run_common_tests()

				def 		snake_case (				self    ):






										"""simple docstring"""

										snake_case								=			self.model_tester.prepare_config_and_inputs_for_common()
										self.model_tester.check_decoder_model_past_large_inputs(*snake_case__    )

				def 		snake_case (				self    ):






										"""simple docstring"""

										snake_case								=			self.model_tester.prepare_config_and_inputs_for_common()

										def _get_word_embedding_weight(lowerCAmelCase		,    lowerCAmelCase    ):
																if hasattr(snake_case__		,    'weight'    ):
																						return embedding_layer.weight
																else:
																						# Here we build the word embeddings weights if not exists.
																						# And then we retry to get the attribute once built.
																						model.build()
																						if hasattr(snake_case__		,    'weight'    ):
																												return embedding_layer.weight
																						else:
																												return None

										for model_class in self.all_model_classes:
																for size in [config.vocab_size - 10, config.vocab_size + 10]:
																						# build the embeddings
																						snake_case								=			model_class(config=snake_case__    )
																						snake_case								=			_get_word_embedding_weight(snake_case__		,    model.get_input_embeddings()    )
																						snake_case								=			_get_word_embedding_weight(snake_case__		,    model.get_output_embeddings()    )

																						# reshape the embeddings
																						model.resize_token_embeddings(snake_case__    )
																						snake_case								=			_get_word_embedding_weight(snake_case__		,    model.get_input_embeddings()    )
																						snake_case								=			_get_word_embedding_weight(snake_case__		,    model.get_output_embeddings()    )

																						# check that the resized embeddings size matches the desired size.
																						snake_case								=			size if size is not None else config.vocab_size

																						self.assertEqual(new_input_embeddings.shape[0]		,    snake_case__    )

																						# check that weights remain the same after resizing
																						snake_case								=			True
																						for pa, pa in zip(old_input_embeddings.value()		,    new_input_embeddings.value()    ):
																												if tf.math.reduce_sum(tf.math.abs(pa - pa    )    ) > 0:
																																		snake_case								=			False
																						self.assertTrue(snake_case__    )

																						if old_output_embeddings is not None and new_output_embeddings is not None:
																												self.assertEqual(new_output_embeddings.shape[0]		,    snake_case__    )

																												snake_case								=			True
																												for pa, pa in zip(old_output_embeddings.value()		,    new_output_embeddings.value()    ):
																																		if tf.math.reduce_sum(tf.math.abs(pa - pa    )    ) > 0:
																																								snake_case								=			False
																												self.assertTrue(snake_case__    )





def    lowerCAmelCase__    (			_UpperCamelCase       :				Tuple					)					->				int:





						"""simple docstring"""






						return tf.constant(_UpperCamelCase ,			dtype=tf.intaa					)



@require_tf
class   lowerCAmelCase_    (			unittest.TestCase					):



				"""simple docstring"""
				_lowerCAmelCase     :    Dict			  =						99

				def 		snake_case (				self    ):






										"""simple docstring"""

										snake_case								=			tf.ones((4, 1)		,    dtype=tf.intaa    ) * 2
										snake_case								=			tf.concat([ids_tensor((4, 6)		,    self.vocab_size - 3    ) + 3, eos_column_vector]		,    axis=1    )
										snake_case								=			input_ids.shape[0]
										snake_case								=			OPTConfig(
										    vocab_size=self.vocab_size		,    hidden_size=24		,    num_hidden_layers=2		,    num_attention_heads=2		,    ffn_dim=32		,    max_position_embeddings=48		,    eos_token_id=2		,    pad_token_id=1		,    bos_token_id=0		,    )
										return config, input_ids, batch_size





@require_sentencepiece
@require_tf
class   lowerCAmelCase_    (			unittest.TestCase					):



				"""simple docstring"""

				@slow
				def 		snake_case (				self    ):






										"""simple docstring"""

										snake_case								=			TFOPTModel.from_pretrained('facebook/opt-350m'    )
										snake_case								=			_long_tensor([[0, 3_14_14, 2_32, 3_28, 7_40, 11_40, 1_26_95, 69, 4_60_78, 15_88, 2]]    )
										snake_case								=			tf.not_equal(snake_case__		,    model.config.pad_token_id    )
										with tf.GradientTape():
																snake_case								=			model(input_ids=snake_case__		,    attention_mask=snake_case__    ).last_hidden_state
										snake_case								=			(1, 11, 5_12)
										self.assertEqual(output.shape		,    snake_case__    )
										snake_case								=			tf.constant(
										    [[-0.28_73, -1.92_18, -0.30_33], [-1.27_10, -0.13_38, -0.19_02], [0.40_95, 0.12_14, -1.31_21]]    )
										self.assertTrue(np.allclose(output[:, :3, :3]		,    snake_case__		,    atol=4E-3    )    )

										snake_case								=			tf.function(snake_case__		,    jit_compile=snake_case__    )
										snake_case								=			xla_generate(snake_case__		,    snake_case__    )[0]
										self.assertTrue(np.allclose(output[:, :3, :3]		,    snake_case__		,    atol=4E-2    )    )





@require_tf
@slow
class   lowerCAmelCase_    (			unittest.TestCase					):



				"""simple docstring"""

				def 		snake_case (				self    ):






										"""simple docstring"""

										super().setUp()
										snake_case								=			"facebook/opt-350m"

				def 		snake_case (				self    ):






										"""simple docstring"""

										snake_case								=			TFOPTForCausalLM.from_pretrained(self.path_model    )
										snake_case								=			GPTaTokenizer.from_pretrained(self.path_model    )

										snake_case								=			[
										    "Today is a beautiful day and I want to",
										    "In the city of",
										    "Paris is the capital of France and",
										    "Computers and mobile phones have taken",
										]
										# verify that prompt without BOS token is identical to Metaseq -> add_special_tokens=False
										snake_case								=			tokenizer(snake_case__		,    return_tensors='tf'		,    padding=snake_case__		,    add_special_tokens=snake_case__    )
										snake_case								=			tf.math.reduce_mean(model(inputs.input_ids		,    attention_mask=inputs.attention_mask    )[0]		,    axis=-1    )
										snake_case								=			tf.constant(
										    [
										        [1.38_51, -13.89_23, -10.52_29, -10.75_33, -0.23_09, -10.23_84, -0.53_65, -9.09_47, -5.16_70],
										        [-4.70_73, -10.62_76, -3.94_15, -21.52_42, -0.28_22, -0.28_22, -0.28_22, -0.28_22, -0.28_22],
										        [0.62_47, -3.42_29, -8.91_79, -1.42_97, -14.16_50, 1.41_46, -9.02_18, -0.27_03, -0.27_03],
										        [6.47_83, -1.99_13, -10.79_26, -2.33_36, 1.50_92, -0.99_74, -6.82_13, 1.34_77, 1.34_77],
										    ]    )
										self.assertTrue(np.allclose(snake_case__		,    snake_case__		,    atol=1E-4    )    )

										snake_case								=			tf.function(snake_case__		,    jit_compile=snake_case__    )
										snake_case								=			tf.math.reduce_mean(xla_generate(inputs.input_ids		,    attention_mask=inputs.attention_mask    )[0]		,    axis=-1    )
										self.assertTrue(np.allclose(snake_case__		,    snake_case__		,    atol=1E-4    )    )





@require_tf
@slow
class   lowerCAmelCase_    (			unittest.TestCase					):



				"""simple docstring"""

				@property
				def 		snake_case (				self    ):






										"""simple docstring"""

										return [
										    "Today is a beautiful day and I want",
										    "In the city of",
										    "Paris is the capital of France and",
										    "Computers and mobile phones have taken",
										]

				def 		snake_case (				self    ):






										"""simple docstring"""

										snake_case								=			"facebook/opt-125m"

										snake_case								=			[
										    "Today is a beautiful day and I want to",
										    "In the city of New York, the city",
										    "Paris is the capital of France and the capital",
										    "Computers and mobile phones have taken over the",
										]

										snake_case								=			[]
										snake_case								=			GPTaTokenizer.from_pretrained(snake_case__    )
										snake_case								=			TFOPTForCausalLM.from_pretrained(snake_case__    )

										for prompt in self.prompts:
																snake_case								=			tokenizer(snake_case__		,    return_tensors='tf'    ).input_ids

																snake_case								=			model.generate(snake_case__		,    max_length=10    )

																snake_case								=			tokenizer.batch_decode(snake_case__		,    skip_special_tokens=snake_case__    )
																predicted_outputs += generated_string

										self.assertListEqual(snake_case__		,    snake_case__    )

				def 		snake_case (				self    ):






										"""simple docstring"""

										snake_case								=			"facebook/opt-350m"

										snake_case								=			GPTaTokenizer.from_pretrained(snake_case__    )
										snake_case								=			TFOPTForCausalLM.from_pretrained(snake_case__    )

										snake_case								=			"left"

										# use different length sentences to test batching
										snake_case								=			[
										    "Hello, my dog is a little",
										    "Today, I",
										]

										snake_case								=			tokenizer(snake_case__		,    return_tensors='tf'		,    padding=snake_case__    )
										snake_case								=			inputs["input_ids"]

										snake_case								=			model.generate(input_ids=snake_case__		,    attention_mask=inputs['attention_mask']    )

										snake_case								=			tokenizer(sentences[0]		,    return_tensors='tf'    ).input_ids
										snake_case								=			model.generate(input_ids=snake_case__    )

										snake_case								=			inputs_non_padded.shape[-1] - tf.math.reduce_sum(
										    tf.cast(inputs['attention_mask'][-1]		,    tf.intaa    )    )
										snake_case								=			tokenizer(sentences[1]		,    return_tensors='tf'    ).input_ids
										snake_case								=			model.generate(input_ids=snake_case__		,    max_length=model.config.max_length - num_paddings    )

										snake_case								=			tokenizer.batch_decode(snake_case__		,    skip_special_tokens=snake_case__    )
										snake_case								=			tokenizer.decode(output_non_padded[0]		,    skip_special_tokens=snake_case__    )
										snake_case								=			tokenizer.decode(output_padded[0]		,    skip_special_tokens=snake_case__    )

										snake_case								=			[
										    "Hello, my dog is a little bit of a dork.\nI'm a little bit",
										    "Today, I was in the middle of a conversation with a friend about the",
										]
										self.assertListEqual(snake_case__		,    snake_case__    )
										self.assertListEqual(snake_case__		,    [non_padded_sentence, padded_sentence]    )

				def 		snake_case (				self    ):






										"""simple docstring"""

										snake_case								=			"facebook/opt-350m"

										snake_case								=			[
										    "Today is a beautiful day and I want to",
										    "In the city of San Francisco, the city",
										    "Paris is the capital of France and the capital",
										    "Computers and mobile phones have taken over the",
										]

										snake_case								=			[]
										snake_case								=			GPTaTokenizer.from_pretrained(snake_case__    )
										snake_case								=			TFOPTForCausalLM.from_pretrained(snake_case__    )

										for prompt in self.prompts:
																snake_case								=			tokenizer(snake_case__		,    return_tensors='tf'    ).input_ids

																snake_case								=			model.generate(snake_case__		,    max_length=10    )

																snake_case								=			tokenizer.batch_decode(snake_case__		,    skip_special_tokens=snake_case__    )
																predicted_outputs += generated_string

										self.assertListEqual(snake_case__		,    snake_case__    )



"""simple docstring"""
import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE__      						=  logging.get_logger(__name__)


SCREAMING_SNAKE_CASE__      						=  {"vocab_file": "vocab.json", "merges_file": "merges.txt"}

# See all BART models at https://huggingface.co/models?filter=bart
SCREAMING_SNAKE_CASE__      						=  {
    "vocab_file": {
        "facebook/bart-base": "https://huggingface.co/facebook/bart-base/resolve/main/vocab.json",
        "facebook/bart-large": "https://huggingface.co/facebook/bart-large/resolve/main/vocab.json",
        "facebook/bart-large-mnli": "https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json",
        "facebook/bart-large-cnn": "https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json",
        "facebook/bart-large-xsum": "https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json",
        "yjernite/bart_eli5": "https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json",
    },
    "merges_file": {
        "facebook/bart-base": "https://huggingface.co/facebook/bart-base/resolve/main/merges.txt",
        "facebook/bart-large": "https://huggingface.co/facebook/bart-large/resolve/main/merges.txt",
        "facebook/bart-large-mnli": "https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt",
        "facebook/bart-large-cnn": "https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt",
        "facebook/bart-large-xsum": "https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt",
        "yjernite/bart_eli5": "https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt",
    },
}

SCREAMING_SNAKE_CASE__      						=  {
    "facebook/bart-base": 1_024,
    "facebook/bart-large": 1_024,
    "facebook/bart-large-mnli": 1_024,
    "facebook/bart-large-cnn": 1_024,
    "facebook/bart-large-xsum": 1_024,
    "yjernite/bart_eli5": 1_024,
}





@lru_cache()
def    lowerCAmelCase__    (			)					->				Union[str, Any]:





						"""simple docstring"""






						snake_case								=			(
						    list(range(ord('!'					) ,			ord('~'					) + 1					)					) + list(range(ord('¡'					) ,			ord('¬'					) + 1					)					) + list(range(ord('®'					) ,			ord('ÿ'					) + 1					)					)
						)
						snake_case								=			bs[:]
						snake_case								=			0
						for b in range(2**8					):
												if b not in bs:
																		bs.append(_UpperCamelCase					)
																		cs.append(2**8 + n					)
																		n += 1
						snake_case								=			[chr(_UpperCamelCase					) for n in cs]
						return dict(zip(_UpperCamelCase ,			_UpperCamelCase					)					)





def    lowerCAmelCase__    (			_UpperCamelCase       :				List[Any]					)					->				List[str]:





						"""simple docstring"""






						snake_case								=			set()
						snake_case								=			word[0]
						for char in word[1:]:
												pairs.add((prev_char, char)					)
												snake_case								=			char
						return pairs



class   lowerCAmelCase_    (			lowerCAmelCase					):



				"""simple docstring"""
				_lowerCAmelCase     :    Union[str, Any]			  =						VOCAB_FILES_NAMES
				_lowerCAmelCase     :    str			  =						PRETRAINED_VOCAB_FILES_MAP
				_lowerCAmelCase     :    int			  =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				_lowerCAmelCase     :    str			  =						["""input_ids""", """attention_mask"""]

				def __init__(				self		,    lowerCAmelCase		,    lowerCAmelCase		,    lowerCAmelCase="replace"		,    lowerCAmelCase="<s>"		,    lowerCAmelCase="</s>"		,    lowerCAmelCase="</s>"		,    lowerCAmelCase="<s>"		,    lowerCAmelCase="<unk>"		,    lowerCAmelCase="<pad>"		,    lowerCAmelCase="<mask>"		,    lowerCAmelCase=False		,    **lowerCAmelCase		,    ):






										"""simple docstring"""

										snake_case								=			AddedToken(lowerCAmelCase		,    lstrip=lowerCAmelCase		,    rstrip=lowerCAmelCase    ) if isinstance(lowerCAmelCase		,    lowerCAmelCase    ) else bos_token
										snake_case								=			AddedToken(lowerCAmelCase		,    lstrip=lowerCAmelCase		,    rstrip=lowerCAmelCase    ) if isinstance(lowerCAmelCase		,    lowerCAmelCase    ) else eos_token
										snake_case								=			AddedToken(lowerCAmelCase		,    lstrip=lowerCAmelCase		,    rstrip=lowerCAmelCase    ) if isinstance(lowerCAmelCase		,    lowerCAmelCase    ) else sep_token
										snake_case								=			AddedToken(lowerCAmelCase		,    lstrip=lowerCAmelCase		,    rstrip=lowerCAmelCase    ) if isinstance(lowerCAmelCase		,    lowerCAmelCase    ) else cls_token
										snake_case								=			AddedToken(lowerCAmelCase		,    lstrip=lowerCAmelCase		,    rstrip=lowerCAmelCase    ) if isinstance(lowerCAmelCase		,    lowerCAmelCase    ) else unk_token
										snake_case								=			AddedToken(lowerCAmelCase		,    lstrip=lowerCAmelCase		,    rstrip=lowerCAmelCase    ) if isinstance(lowerCAmelCase		,    lowerCAmelCase    ) else pad_token

										# Mask token behave like a normal word, i.e. include the space before it
										snake_case								=			AddedToken(lowerCAmelCase		,    lstrip=lowerCAmelCase		,    rstrip=lowerCAmelCase    ) if isinstance(lowerCAmelCase		,    lowerCAmelCase    ) else mask_token

										super().__init__(
										    errors=lowerCAmelCase		,    bos_token=lowerCAmelCase		,    eos_token=lowerCAmelCase		,    unk_token=lowerCAmelCase		,    sep_token=lowerCAmelCase		,    cls_token=lowerCAmelCase		,    pad_token=lowerCAmelCase		,    mask_token=lowerCAmelCase		,    add_prefix_space=lowerCAmelCase		,    **lowerCAmelCase		,    )

										with open(lowerCAmelCase		,    encoding='utf-8'    ) as vocab_handle:
																snake_case								=			json.load(lowerCAmelCase    )
										snake_case								=			{v: k for k, v in self.encoder.items()}
										snake_case								=			errors  # how to handle errors in decoding
										snake_case								=			bytes_to_unicode()
										snake_case								=			{v: k for k, v in self.byte_encoder.items()}
										with open(lowerCAmelCase		,    encoding='utf-8'    ) as merges_handle:
																snake_case								=			merges_handle.read().split('\n'    )[1:-1]
										snake_case								=			[tuple(merge.split()    ) for merge in bpe_merges]
										snake_case								=			dict(zip(lowerCAmelCase		,    range(len(lowerCAmelCase    )    )    )    )
										snake_case								=			{}
										snake_case								=			add_prefix_space

										# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
										snake_case								=			re.compile(R'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'    )

				@property
				def 		snake_case (				self    ):






										"""simple docstring"""

										return len(self.encoder    )

				def 		snake_case (				self    ):






										"""simple docstring"""

										return dict(self.encoder		,    **self.added_tokens_encoder    )

				def 		snake_case (				self		,    lowerCAmelCase    ):






										"""simple docstring"""

										if token in self.cache:
																return self.cache[token]
										snake_case								=			tuple(lowerCAmelCase    )
										snake_case								=			get_pairs(lowerCAmelCase    )

										if not pairs:
																return token

										while True:
																snake_case								=			min(lowerCAmelCase		,    key=lambda lowerCAmelCase    : self.bpe_ranks.get(lowerCAmelCase		,    float('inf'    )    )    )
																if bigram not in self.bpe_ranks:
																						break
																snake_case					,snake_case								=			bigram
																snake_case								=			[]
																snake_case								=			0
																while i < len(lowerCAmelCase    ):
																						try:
																												snake_case								=			word.index(lowerCAmelCase		,    lowerCAmelCase    )
																						except ValueError:
																												new_word.extend(word[i:]    )
																												break
																						else:
																												new_word.extend(word[i:j]    )
																												snake_case								=			j

																						if word[i] == first and i < len(lowerCAmelCase    ) - 1 and word[i + 1] == second:
																												new_word.append(first + second    )
																												i += 2
																						else:
																												new_word.append(word[i]    )
																												i += 1
																snake_case								=			tuple(lowerCAmelCase    )
																snake_case								=			new_word
																if len(lowerCAmelCase    ) == 1:
																						break
																else:
																						snake_case								=			get_pairs(lowerCAmelCase    )
										snake_case								=			' '.join(lowerCAmelCase    )
										snake_case								=			word
										return word

				def 		snake_case (				self		,    lowerCAmelCase    ):






										"""simple docstring"""

										snake_case								=			[]
										for token in re.findall(self.pat		,    lowerCAmelCase    ):
																snake_case								=			''.join(
																    self.byte_encoder[b] for b in token.encode('utf-8'    )    )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
																bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowerCAmelCase    ).split(' '    )    )
										return bpe_tokens

				def 		snake_case (				self		,    lowerCAmelCase    ):






										"""simple docstring"""

										return self.encoder.get(lowerCAmelCase		,    self.encoder.get(self.unk_token    )    )

				def 		snake_case (				self		,    lowerCAmelCase    ):






										"""simple docstring"""

										return self.decoder.get(lowerCAmelCase    )

				def 		snake_case (				self		,    lowerCAmelCase    ):






										"""simple docstring"""

										snake_case								=			''.join(lowerCAmelCase    )
										snake_case								=			bytearray([self.byte_decoder[c] for c in text]    ).decode('utf-8'		,    errors=self.errors    )
										return text

				def 		snake_case (				self		,    lowerCAmelCase		,    lowerCAmelCase = None    ):






										"""simple docstring"""

										if not os.path.isdir(lowerCAmelCase    ):
																logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""    )
																return
										snake_case								=			os.path.join(
										    lowerCAmelCase		,    (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']    )
										snake_case								=			os.path.join(
										    lowerCAmelCase		,    (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']    )

										with open(lowerCAmelCase		,    'w'		,    encoding='utf-8'    ) as f:
																f.write(json.dumps(self.encoder		,    indent=2		,    sort_keys=lowerCAmelCase		,    ensure_ascii=lowerCAmelCase    ) + '\n'    )

										snake_case								=			0
										with open(lowerCAmelCase		,    'w'		,    encoding='utf-8'    ) as writer:
																writer.write('#version: 0.2\n'    )
																for bpe_tokens, token_index in sorted(self.bpe_ranks.items()		,    key=lambda lowerCAmelCase    : kv[1]    ):
																						if index != token_index:
																												logger.warning(
																												    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																												    ' Please check that the tokenizer is not corrupted!'    )
																												snake_case								=			token_index
																						writer.write(' '.join(lowerCAmelCase    ) + '\n'    )
																						index += 1

										return vocab_file, merge_file

				def 		snake_case (				self		,    lowerCAmelCase		,    lowerCAmelCase = None    ):






										"""simple docstring"""

										if token_ids_a is None:
																return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
										snake_case								=			[self.cls_token_id]
										snake_case								=			[self.sep_token_id]
										return cls + token_ids_a + sep + sep + token_ids_a + sep

				def 		snake_case (				self		,    lowerCAmelCase		,    lowerCAmelCase = None		,    lowerCAmelCase = False    ):






										"""simple docstring"""

										if already_has_special_tokens:
																return super().get_special_tokens_mask(
																    token_ids_a=lowerCAmelCase		,    token_ids_a=lowerCAmelCase		,    already_has_special_tokens=lowerCAmelCase    )

										if token_ids_a is None:
																return [1] + ([0] * len(lowerCAmelCase    )) + [1]
										return [1] + ([0] * len(lowerCAmelCase    )) + [1, 1] + ([0] * len(lowerCAmelCase    )) + [1]

				def 		snake_case (				self		,    lowerCAmelCase		,    lowerCAmelCase = None    ):






										"""simple docstring"""

										snake_case								=			[self.sep_token_id]
										snake_case								=			[self.cls_token_id]

										if token_ids_a is None:
																return len(cls + token_ids_a + sep    ) * [0]
										return len(cls + token_ids_a + sep + sep + token_ids_a + sep    ) * [0]

				def 		snake_case (				self		,    lowerCAmelCase		,    lowerCAmelCase=False		,    **lowerCAmelCase    ):






										"""simple docstring"""

										snake_case								=			kwargs.pop('add_prefix_space'		,    self.add_prefix_space    )
										if (is_split_into_words or add_prefix_space) and (len(lowerCAmelCase    ) > 0 and not text[0].isspace()):
																snake_case								=			' ' + text
										return (text, kwargs)


