
import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__lowerCAmelCase     :	int       			=  get_tests_dir('fixtures/test_sentencepiece_with_bytefallback.model')




@require_sentencepiece
@require_tokenizers
class 			UpperCAmelCase_     (					_UpperCamelCase    ,			unittest.TestCase	):


  '''simple docstring'''







  a__       			=						GPTSwaTokenizer
  a__       			=						False
  a__       			=						True
  a__       			=						False
  def 				_lowercase				(					self     :					Any       )     ->      Dict:







    """simple docstring"""






    super().setUp()

    # We have a SentencePiece fixture for testing
    __magic_name__								=			GPTSwaTokenizer(_UpperCAmelCase     ,				eos_token="""<unk>"""     ,				bos_token="""<unk>"""     ,				pad_token="""<unk>"""       )

    tokenizer.save_pretrained(self.tmpdirname       )
  def 				_lowercase				(					self     :					Optional[Any]     ,				UpperCamelCase__     :					int       )     ->      Union[str, Any]:







    """simple docstring"""






    __magic_name__								=			'This is a test'
    __magic_name__								=			'This is a test'
    return input_text, output_text
  def 				_lowercase				(					self     :					str       )     ->      Optional[Any]:







    """simple docstring"""






    __magic_name__								=			'<s>'
    __magic_name__								=			1

    self.assertEqual(self.get_tokenizer()._convert_token_to_id(_UpperCAmelCase       )     ,				_UpperCAmelCase       )
    self.assertEqual(self.get_tokenizer()._convert_id_to_token(_UpperCAmelCase       )     ,				_UpperCAmelCase       )
  def 				_lowercase				(					self     :					Union[str, Any]       )     ->      List[str]:







    """simple docstring"""






    __magic_name__								=			list(self.get_tokenizer().get_vocab().keys()       )

    self.assertEqual(vocab_keys[0]     ,				"""<unk>"""       )
    self.assertEqual(vocab_keys[1]     ,				"""<s>"""       )
    self.assertEqual(vocab_keys[-1]     ,				"""j"""       )
    self.assertEqual(len(_UpperCAmelCase       )     ,				2000       )
  def 				_lowercase				(					self     :					Optional[int]       )     ->      List[Any]:







    """simple docstring"""






    self.assertEqual(self.get_tokenizer().vocab_size     ,				2000       )
  def 				_lowercase				(					self     :					List[Any]       )     ->      int:







    """simple docstring"""






    __magic_name__								=			GPTSwaTokenizer(_UpperCAmelCase       )

    __magic_name__								=			tokenizer.tokenize("""This is a test"""       )
    self.assertListEqual(_UpperCAmelCase     ,				["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]       )

    self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCAmelCase       )     ,				[465, 287, 265, 631, 842]       )

    __magic_name__								=			tokenizer.tokenize("""I was born in 92000, and this is falsé."""       )
    # fmt: off
    self.assertListEqual(
        _UpperCAmelCase     ,				["""▁I""", """▁was""", """▁bor""", """n""", """▁in""", """▁""", """<0x39>""", """2""", """0""", """0""", """0""", """,""", """▁and""", """▁this""", """▁is""", """▁f""", """al""", """s""", """<0xC3>""", """<0xA9>""", """."""]     ,				)
    # fmt: on

    __magic_name__								=			tokenizer.convert_tokens_to_ids(_UpperCAmelCase       )
    self.assertListEqual(
        _UpperCAmelCase     ,				[262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260]     ,				)

    __magic_name__								=			tokenizer.convert_ids_to_tokens(_UpperCAmelCase       )
    # fmt: off
    self.assertListEqual(
        _UpperCAmelCase     ,				["""▁I""", """▁was""", """▁bor""", """n""", """▁in""", """▁""", """<0x39>""", """2""", """0""", """0""", """0""", """,""", """▁and""", """▁this""", """▁is""", """▁f""", """al""", """s""", """<0xC3>""", """<0xA9>""", """."""]       )
    # fmt: on
  def 				_lowercase				(					self     :					str       )     ->      Union[str, Any]:







    """simple docstring"""






    __magic_name__								=			GPTSwaTokenizer(_UpperCAmelCase       )
    __magic_name__								=			['This is a test', 'I was born in 92000, and this is falsé.']
    __magic_name__								=			[
        [465, 287, 265, 631, 842],
        [262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260],
    ]

    # Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
    for text, expected_ids in zip(_UpperCAmelCase     ,				_UpperCAmelCase       ):
      self.assertListEqual(tokenizer.encode_fast(_UpperCAmelCase       )     ,				_UpperCAmelCase       )

    # Test that decode_fast returns the input text
    for text, token_ids in zip(_UpperCAmelCase     ,				_UpperCAmelCase       ):
      self.assertEqual(tokenizer.decode_fast(_UpperCAmelCase       )     ,				_UpperCAmelCase       )





  @slow
  def 				_lowercase				(					self     :					Optional[Any]       )     ->      List[str]:







    """simple docstring"""






    __magic_name__								=			[
        '<|python|>def fibonacci(n)\n    if n < 0:\n        print(\'Incorrect input\')',
        'Hey there, how are you doing this fine day?',
        'This is a text with a trailing spaces followed by a dot     .',
        'Häj sväjs lillebrör! =)',
        'Det är inget fel på Mr. Cool',
    ]

    # fmt: off
    __magic_name__								=			{'input_ids': [[6_3423, 5, 6811, 1_4954, 282, 816, 3821, 6_3466, 6_3425, 6_3462, 18, 6_3978, 678, 301, 1320, 6_3423, 6_3455, 6_3458, 18, 6_3982, 4246, 3940, 1901, 4_7789, 5547, 1_8994], [1_9630, 1100, 6_3446, 1342, 633, 544, 4488, 593, 5102, 2416, 6_3495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1652, 428, 268, 1936, 515, 268, 5_8593, 2_2413, 9106, 546, 268, 3_3213, 6_3979, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_5130, 6_3450, 924, 6_3449, 2249, 4062, 1558, 318, 6_3504, 2_1498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [509, 377, 2827, 2559, 332, 6575, 6_3443, 2_6801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
    # fmt: on
    self.tokenizer_integration_test_util(
        expected_encoding=_UpperCAmelCase     ,				model_name="""AI-Sweden/gpt-sw3-126m"""     ,				sequences=_UpperCAmelCase     ,				)






'''simple docstring'''

from dataclasses import asdict, dataclass
from typing import Optional

from ...configuration_utils import PretrainedConfig
from ...utils import logging


__lowerCAmelCase		     =  logging.get_logger(__name__)

# TODO Update this
__lowerCAmelCase		     =  {
    '''facebook/esm-1b''': '''https://huggingface.co/facebook/esm-1b/resolve/main/config.json''',
    # See all ESM models at https://huggingface.co/models?filter=esm
}

class       __magic_name__		(       _UpperCamelCase  ):
							lowerCAmelCase			:       Any  							=   'esm'







							def __init__(				self   :			Optional[int]      ,_UpperCAmelCase   :			Optional[Any]=None      ,_UpperCAmelCase   :			Dict=None      ,_UpperCAmelCase   :			Union[str, Any]=None      ,_UpperCAmelCase   :			List[Any]=768      ,_UpperCAmelCase   :			Union[str, Any]=12      ,_UpperCAmelCase   :			List[str]=12      ,_UpperCAmelCase   :			Tuple=3072      ,_UpperCAmelCase   :			Dict=0.1      ,_UpperCAmelCase   :			Tuple=0.1      ,_UpperCAmelCase   :			List[str]=1026      ,_UpperCAmelCase   :			List[str]=0.02      ,_UpperCAmelCase   :			Optional[int]=1E-12      ,_UpperCAmelCase   :			List[str]="absolute"      ,_UpperCAmelCase   :			Tuple=True      ,_UpperCAmelCase   :			Tuple=None      ,_UpperCAmelCase   :			List[Any]=False      ,_UpperCAmelCase   :			int=False      ,_UpperCAmelCase   :			int=None      ,_UpperCAmelCase   :			Union[str, Any]=None      ,**_UpperCAmelCase   :			List[Any]      ,):
									super().__init__(pad_token_id=_UpperCAmelCase      ,mask_token_id=_UpperCAmelCase      ,**_UpperCAmelCase						)

									_a	: Optional[Any]								=				vocab_size
									_a	: Union[str, Any]								=				hidden_size
									_a	: Dict								=				num_hidden_layers
									_a	: int								=				num_attention_heads
									_a	: Dict								=				intermediate_size
									_a	: List[Any]								=				hidden_dropout_prob
									_a	: List[Any]								=				attention_probs_dropout_prob
									_a	: Optional[Any]								=				max_position_embeddings
									_a	: Optional[int]								=				initializer_range
									_a	: List[Any]								=				layer_norm_eps
									_a	: int								=				position_embedding_type
									_a	: Optional[int]								=				use_cache
									_a	: Any								=				emb_layer_norm_before
									_a	: List[str]								=				token_dropout
									_a	: List[str]								=				is_folding_model
									if is_folding_model:
											if esmfold_config is None:
													logger.info('No esmfold_config supplied for folding model, using default values.'						)
													_a	: Dict								=				EsmFoldConfig()
											elif isinstance(_UpperCAmelCase      ,_UpperCAmelCase						):
													_a	: Dict								=				EsmFoldConfig(**_UpperCAmelCase						)
											_a	: Optional[int]								=				esmfold_config
											if vocab_list is None:
													logger.warning('No vocab_list supplied for folding model, assuming the ESM-2 vocabulary!'						)
													_a	: Optional[int]								=				get_default_vocab_list()
											else:
													_a	: Optional[int]								=				vocab_list
									else:
											_a	: Optional[Any]								=				None
											_a	: Union[str, Any]								=				None
									if self.esmfold_config is not None and getattr(self.esmfold_config      ,'use_esm_attn_map'      ,_UpperCAmelCase						):
											raise ValueError('The HuggingFace port of ESMFold does not support use_esm_attn_map at this time!'						)







							def 				__lowercase     (				self   :			Any						):
									_a	: str								=				super().to_dict()
									if isinstance(self.esmfold_config      ,_UpperCAmelCase						):
											_a	: List[str]								=				self.esmfold_config.to_dict()
									return output

@dataclass
class       __magic_name__		:
							lowerCAmelCase			:       str  							=   None
							lowerCAmelCase			:       bool  							=   True
							lowerCAmelCase			:       bool  							=   False
							lowerCAmelCase			:       bool  							=   False
							lowerCAmelCase			:       bool  							=   False
							lowerCAmelCase			:       float  							=   0

							lowerCAmelCase			:       bool  							=   True
							lowerCAmelCase			:       bool  							=   False

							lowerCAmelCase			:       int  							=   1_2_8
							lowerCAmelCase			:       "TrunkConfig"  							=   None







							def 				__lowercase     (				self   :			List[str]						):
									if self.trunk is None:
											_a	: Dict								=				TrunkConfig()
									elif isinstance(self.trunk      ,_UpperCAmelCase						):
											_a	: str								=				TrunkConfig(**self.trunk						)







							def 				__lowercase     (				self   :			List[Any]						):
									_a	: List[str]								=				asdict(self						)
									_a	: List[str]								=				self.trunk.to_dict()
									return output

@dataclass
class       __magic_name__		:
							lowerCAmelCase			:       int  							=   4_8
							lowerCAmelCase			:       int  							=   1_0_2_4
							lowerCAmelCase			:       int  							=   1_2_8
							lowerCAmelCase			:       int  							=   3_2
							lowerCAmelCase			:       int  							=   3_2
							lowerCAmelCase			:       int  							=   3_2
							lowerCAmelCase			:       float  							=   0
							lowerCAmelCase			:       float  							=   0
							lowerCAmelCase			:       bool  							=   False
							lowerCAmelCase			:       int  							=   4
							lowerCAmelCase			:       Optional[int]  							=   1_2_8
							lowerCAmelCase			:       "StructureModuleConfig"  							=   None







							def 				__lowercase     (				self   :			str						):
									if self.structure_module is None:
											_a	: Tuple								=				StructureModuleConfig()
									elif isinstance(self.structure_module      ,_UpperCAmelCase						):
											_a	: List[str]								=				StructureModuleConfig(**self.structure_module						)

									if self.max_recycles <= 0:
											raise ValueError(F"""`max_recycles` should be positive, got {self.max_recycles}."""						)
									if self.sequence_state_dim % self.sequence_state_dim != 0:
											raise ValueError(
											    '`sequence_state_dim` should be a round multiple of `sequence_state_dim`, got'
											    F""" {self.sequence_state_dim} and {self.sequence_state_dim}."""						)
									if self.pairwise_state_dim % self.pairwise_state_dim != 0:
											raise ValueError(
											    '`pairwise_state_dim` should be a round multiple of `pairwise_state_dim`, got'
											    F""" {self.pairwise_state_dim} and {self.pairwise_state_dim}."""						)

									_a	: Optional[int]								=				self.sequence_state_dim // self.sequence_head_width
									_a	: int								=				self.pairwise_state_dim // self.pairwise_head_width

									if self.sequence_state_dim != sequence_num_heads * self.sequence_head_width:
											raise ValueError(
											    '`sequence_state_dim` should be equal to `sequence_num_heads * sequence_head_width, got'
											    F""" {self.sequence_state_dim} != {sequence_num_heads} * {self.sequence_head_width}."""						)
									if self.pairwise_state_dim != pairwise_num_heads * self.pairwise_head_width:
											raise ValueError(
											    '`pairwise_state_dim` should be equal to `pairwise_num_heads * pairwise_head_width, got'
											    F""" {self.pairwise_state_dim} != {pairwise_num_heads} * {self.pairwise_head_width}."""						)
									if self.pairwise_state_dim % 2 != 0:
											raise ValueError(F"""`pairwise_state_dim` should be even, got {self.pairwise_state_dim}."""						)

									if self.dropout >= 0.4:
											raise ValueError(F"""`dropout` should not be greater than 0.4, got {self.dropout}."""						)







							def 				__lowercase     (				self   :			Optional[int]						):
									_a	: Optional[Any]								=				asdict(self						)
									_a	: Optional[Any]								=				self.structure_module.to_dict()
									return output

@dataclass
class       __magic_name__		:
							lowerCAmelCase			:       int  							=   3_8_4
							lowerCAmelCase			:       int  							=   1_2_8
							lowerCAmelCase			:       int  							=   1_6
							lowerCAmelCase			:       int  							=   1_2_8
							lowerCAmelCase			:       int  							=   1_2
							lowerCAmelCase			:       int  							=   4
							lowerCAmelCase			:       int  							=   8
							lowerCAmelCase			:       float  							=   0.1
							lowerCAmelCase			:       int  							=   8
							lowerCAmelCase			:       int  							=   1
							lowerCAmelCase			:       int  							=   2
							lowerCAmelCase			:       int  							=   7
							lowerCAmelCase			:       int  							=   1_0
							lowerCAmelCase			:       float  							=   1e-8
							lowerCAmelCase			:       float  							=   1e5







							def 				__lowercase     (				self   :			str						):
									return asdict(self						)



def 	__lowerCamelCase	( )     -> Optional[int]:
		return (
		    "<cls>",
		    "<pad>",
		    "<eos>",
		    "<unk>",
		    "L",
		    "A",
		    "G",
		    "V",
		    "S",
		    "E",
		    "R",
		    "T",
		    "I",
		    "D",
		    "P",
		    "K",
		    "Q",
		    "N",
		    "F",
		    "Y",
		    "M",
		    "H",
		    "W",
		    "C",
		    "X",
		    "B",
		    "U",
		    "Z",
		    "O",
		    ".",
		    "-",
		    "<null_1>",
		    "<mask>",
		)




