






'''simple docstring'''





from dataclasses import asdict, dataclass
from typing import Optional

from ...configuration_utils import PretrainedConfig
from ...utils import logging


a_							=    logging.get_logger(__name__)

# TODO Update this
a_							=    {
    '''facebook/esm-1b''': '''https://huggingface.co/facebook/esm-1b/resolve/main/config.json''',
    # See all ESM models at https://huggingface.co/models?filter=esm
}


class      lowercase__     (			__lowercase	):
	a_         ="""esm"""

	def __init__(							self		,    __UpperCAmelCase=None		,    __UpperCAmelCase=None		,    __UpperCAmelCase=None		,    __UpperCAmelCase=768		,    __UpperCAmelCase=12		,    __UpperCAmelCase=12		,    __UpperCAmelCase=3072		,    __UpperCAmelCase=0.1		,    __UpperCAmelCase=0.1		,    __UpperCAmelCase=1026		,    __UpperCAmelCase=0.02		,    __UpperCAmelCase=1E-1_2		,    __UpperCAmelCase="absolute"		,    __UpperCAmelCase=True		,    __UpperCAmelCase=None		,    __UpperCAmelCase=False		,    __UpperCAmelCase=False		,    __UpperCAmelCase=None		,    __UpperCAmelCase=None		,    **__UpperCAmelCase		,    )->					List[str]:







							'''simple docstring'''





							super().__init__(pad_token_id=_A		,    mask_token_id=_A		,    **_A       )

							lowerCAmelCase__       =						vocab_size
							lowerCAmelCase__       =						hidden_size
							lowerCAmelCase__       =						num_hidden_layers
							lowerCAmelCase__       =						num_attention_heads
							lowerCAmelCase__       =						intermediate_size
							lowerCAmelCase__       =						hidden_dropout_prob
							lowerCAmelCase__       =						attention_probs_dropout_prob
							lowerCAmelCase__       =						max_position_embeddings
							lowerCAmelCase__       =						initializer_range
							lowerCAmelCase__       =						layer_norm_eps
							lowerCAmelCase__       =						position_embedding_type
							lowerCAmelCase__       =						use_cache
							lowerCAmelCase__       =						emb_layer_norm_before
							lowerCAmelCase__       =						token_dropout
							lowerCAmelCase__       =						is_folding_model
							if is_folding_model:
													if esmfold_config is None:
																			logger.info("No esmfold_config supplied for folding model, using default values."       )
																			lowerCAmelCase__       =						EsmFoldConfig()
													elif isinstance(_A		,    _A       ):
																			lowerCAmelCase__       =						EsmFoldConfig(**_A       )
													lowerCAmelCase__       =						esmfold_config
													if vocab_list is None:
																			logger.warning("No vocab_list supplied for folding model, assuming the ESM-2 vocabulary!"       )
																			lowerCAmelCase__       =						get_default_vocab_list()
													else:
																			lowerCAmelCase__       =						vocab_list
							else:
													lowerCAmelCase__       =						None
													lowerCAmelCase__       =						None
							if self.esmfold_config is not None and getattr(self.esmfold_config		,    "use_esm_attn_map"		,    _A       ):
													raise ValueError("The HuggingFace port of ESMFold does not support use_esm_attn_map at this time!"       )


	def 		UpperCAmelCase  (							self       )->					str:







							'''simple docstring'''





							lowerCAmelCase__       =						super().to_dict()
							if isinstance(self.esmfold_config		,    _A       ):
													lowerCAmelCase__       =						self.esmfold_config.to_dict()
							return output





@dataclass
class      lowercase__     :
	a_         =None
	a_         =True
	a_         =False
	a_         =False
	a_         =False
	a_         =0

	a_         =True
	a_         =False

	a_         =128
	a_         =None

	def 		UpperCAmelCase  (							self       )->					Union[str, Any]:







							'''simple docstring'''





							if self.trunk is None:
													lowerCAmelCase__       =						TrunkConfig()
							elif isinstance(self.trunk		,    _A       ):
													lowerCAmelCase__       =						TrunkConfig(**self.trunk       )


	def 		UpperCAmelCase  (							self       )->					Optional[int]:







							'''simple docstring'''





							lowerCAmelCase__       =						asdict(self       )
							lowerCAmelCase__       =						self.trunk.to_dict()
							return output





@dataclass
class      lowercase__     :
	a_         =48
	a_         =1_024
	a_         =128
	a_         =32
	a_         =32
	a_         =32
	a_         =0
	a_         =0
	a_         =False
	a_         =4
	a_         =128
	a_         =None

	def 		UpperCAmelCase  (							self       )->					Tuple:







							'''simple docstring'''





							if self.structure_module is None:
													lowerCAmelCase__       =						StructureModuleConfig()
							elif isinstance(self.structure_module		,    _A       ):
													lowerCAmelCase__       =						StructureModuleConfig(**self.structure_module       )

							if self.max_recycles <= 0:
													raise ValueError(F"`max_recycles` should be positive, got {self.max_recycles}."       )
							if self.sequence_state_dim % self.sequence_state_dim != 0:
													raise ValueError(
													    "`sequence_state_dim` should be a round multiple of `sequence_state_dim`, got"
													    F" {self.sequence_state_dim} and {self.sequence_state_dim}."       )
							if self.pairwise_state_dim % self.pairwise_state_dim != 0:
													raise ValueError(
													    "`pairwise_state_dim` should be a round multiple of `pairwise_state_dim`, got"
													    F" {self.pairwise_state_dim} and {self.pairwise_state_dim}."       )

							lowerCAmelCase__       =						self.sequence_state_dim // self.sequence_head_width
							lowerCAmelCase__       =						self.pairwise_state_dim // self.pairwise_head_width

							if self.sequence_state_dim != sequence_num_heads * self.sequence_head_width:
													raise ValueError(
													    "`sequence_state_dim` should be equal to `sequence_num_heads * sequence_head_width, got"
													    F" {self.sequence_state_dim} != {sequence_num_heads} * {self.sequence_head_width}."       )
							if self.pairwise_state_dim != pairwise_num_heads * self.pairwise_head_width:
													raise ValueError(
													    "`pairwise_state_dim` should be equal to `pairwise_num_heads * pairwise_head_width, got"
													    F" {self.pairwise_state_dim} != {pairwise_num_heads} * {self.pairwise_head_width}."       )
							if self.pairwise_state_dim % 2 != 0:
													raise ValueError(F"`pairwise_state_dim` should be even, got {self.pairwise_state_dim}."       )

							if self.dropout >= 0.4:
													raise ValueError(F"`dropout` should not be greater than 0.4, got {self.dropout}."       )


	def 		UpperCAmelCase  (							self       )->					List[str]:







							'''simple docstring'''





							lowerCAmelCase__       =						asdict(self       )
							lowerCAmelCase__       =						self.structure_module.to_dict()
							return output





@dataclass
class      lowercase__     :
	a_         =384
	a_         =128
	a_         =16
	a_         =128
	a_         =12
	a_         =4
	a_         =8
	a_         =0.1
	a_         =8
	a_         =1
	a_         =2
	a_         =7
	a_         =10
	a_         =1e-8
	a_         =1e5

	def 		UpperCAmelCase  (							self       )->					Any:







							'''simple docstring'''





							return asdict(self       )





def        _a  (     ) ->   Union[str, Any]:
						"""simple docstring"""







						return (
						    "<cls>",
						    "<pad>",
						    "<eos>",
						    "<unk>",
						    "L",
						    "A",
						    "G",
						    "V",
						    "S",
						    "E",
						    "R",
						    "T",
						    "I",
						    "D",
						    "P",
						    "K",
						    "Q",
						    "N",
						    "F",
						    "Y",
						    "M",
						    "H",
						    "W",
						    "C",
						    "X",
						    "B",
						    "U",
						    "Z",
						    "O",
						    ".",
						    "-",
						    "<null_1>",
						    "<mask>",
						)














import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class      lowercase__     (			_UpperCAmelCase,					unittest.TestCase	):
	a_         =None
	a_         =BloomTokenizerFast
	a_         =BloomTokenizerFast
	a_         =True
	a_         =False
	a_         ="""tokenizer_file"""
	a_         ={"""bos_token""": """<s>""", """eos_token""": """</s>""", """unk_token""": """<unk>""", """pad_token""": """<pad>"""}

	def 		UpperCAmelCase  (							self       )->					int:







							'''simple docstring'''





							super().setUp()
							lowerCAmelCase__       =						BloomTokenizerFast.from_pretrained("bigscience/tokenizer"       )
							tokenizer.save_pretrained(self.tmpdirname       )

	def 		UpperCAmelCase  (							self		,    **__UpperCAmelCase       )->					int:







							'''simple docstring'''





							kwargs.update(self.special_tokens_map       )
							return BloomTokenizerFast.from_pretrained(self.tmpdirname		,    **__UpperCAmelCase       )

	def 		UpperCAmelCase  (							self       )->					Any:







							'''simple docstring'''





							lowerCAmelCase__       =						self.get_rust_tokenizer()

							lowerCAmelCase__       =						["The quick brown fox</s>", "jumps over the lazy dog</s>"]
							lowerCAmelCase__       =						[[2175, 23714, 73173, 144252, 2], [77, 132619, 3478, 368, 109586, 35433, 2]]

							lowerCAmelCase__       =						tokenizer.batch_encode_plus(__UpperCAmelCase       )["input_ids"]
							self.assertListEqual(__UpperCAmelCase		,    __UpperCAmelCase       )

							lowerCAmelCase__       =						tokenizer.batch_decode(__UpperCAmelCase       )
							self.assertListEqual(__UpperCAmelCase		,    __UpperCAmelCase       )

	def 		UpperCAmelCase  (							self		,    __UpperCAmelCase=6       )->					str:







							'''simple docstring'''





							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"       ):
																			lowerCAmelCase__       =						self.rust_tokenizer_class.from_pretrained(__UpperCAmelCase		,    **__UpperCAmelCase       )
																			# tokenizer_r.pad_token = None # Hotfixing padding = None
																			# Simple input
																			lowerCAmelCase__       =						"This is a simple input"
																			lowerCAmelCase__       =						["This is a simple input 1", "This is a simple input 2"]
																			lowerCAmelCase__       =						("This is a simple input", "This is a pair")
																			lowerCAmelCase__       =						[
																			    ("This is a simple input 1", "This is a simple input 2"),
																			    ("This is a simple pair 1", "This is a simple pair 2"),
																			]

																			# Simple input tests
																			try:
																									tokenizer_r.encode(__UpperCAmelCase		,    max_length=__UpperCAmelCase       )
																									tokenizer_r.encode_plus(__UpperCAmelCase		,    max_length=__UpperCAmelCase       )

																									tokenizer_r.batch_encode_plus(__UpperCAmelCase		,    max_length=__UpperCAmelCase       )
																									tokenizer_r.encode(__UpperCAmelCase		,    max_length=__UpperCAmelCase       )
																									tokenizer_r.batch_encode_plus(__UpperCAmelCase		,    max_length=__UpperCAmelCase       )
																			except ValueError:
																									self.fail("Bloom Tokenizer should be able to deal with padding"       )

																			lowerCAmelCase__       =						None  # Hotfixing padding = None
																			self.assertRaises(__UpperCAmelCase		,    tokenizer_r.encode		,    __UpperCAmelCase		,    max_length=__UpperCAmelCase		,    padding="max_length"       )

																			# Simple input
																			self.assertRaises(__UpperCAmelCase		,    tokenizer_r.encode_plus		,    __UpperCAmelCase		,    max_length=__UpperCAmelCase		,    padding="max_length"       )

																			# Simple input
																			self.assertRaises(
																			    __UpperCAmelCase		,    tokenizer_r.batch_encode_plus		,    __UpperCAmelCase		,    max_length=__UpperCAmelCase		,    padding="max_length"		,    )

																			# Pair input
																			self.assertRaises(__UpperCAmelCase		,    tokenizer_r.encode		,    __UpperCAmelCase		,    max_length=__UpperCAmelCase		,    padding="max_length"       )

																			# Pair input
																			self.assertRaises(__UpperCAmelCase		,    tokenizer_r.encode_plus		,    __UpperCAmelCase		,    max_length=__UpperCAmelCase		,    padding="max_length"       )

																			# Pair input
																			self.assertRaises(
																			    __UpperCAmelCase		,    tokenizer_r.batch_encode_plus		,    __UpperCAmelCase		,    max_length=__UpperCAmelCase		,    padding="max_length"		,    )

	def 		UpperCAmelCase  (							self       )->					int:







							'''simple docstring'''





							lowerCAmelCase__       =						self.get_rust_tokenizer()
							lowerCAmelCase__       =						load_dataset("xnli"		,    "all_languages"		,    split="test"		,    streaming=__UpperCAmelCase       )

							lowerCAmelCase__       =						next(iter(__UpperCAmelCase       )       )["premise"]  # pick up one data
							lowerCAmelCase__       =						list(sample_data.values()       )

							lowerCAmelCase__       =						list(map(tokenizer.encode		,    __UpperCAmelCase       )       )
							lowerCAmelCase__       =						[tokenizer.decode(__UpperCAmelCase		,    clean_up_tokenization_spaces=__UpperCAmelCase       ) for x in output_tokens]
							self.assertListEqual(__UpperCAmelCase		,    __UpperCAmelCase       )


	def 		UpperCAmelCase  (							self       )->					List[str]:







							'''simple docstring'''





							self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map       )		,    1       )
							self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values()       )[0]       )		,    1       )






