



"""simple docstring"""






import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


snake_case   					=				get_tests_dir('''fixtures/test_sentencepiece_with_bytefallback.model''')



@require_sentencepiece
@require_tokenizers
class     UpperCAmelCase    (	__SCREAMING_SNAKE_CASE,unittest.TestCase  ):
   A__		:	int           =  GPTSwaTokenizer
   A__		:	Optional[Any]           =  False
   A__		:	Any           =  True
   A__		:	List[str]           =  False



   def    __UpperCAmelCase    (       self    :	Optional[Any]				):



     """simple docstring"""
     super().setUp()

     # We have a SentencePiece fixture for testing
     _snake_case   	=      GPTSwaTokenizer(__lowerCamelCase							,       eos_token='''<unk>'''							,       bos_token='''<unk>'''							,       pad_token='''<unk>'''				)

     tokenizer.save_pretrained(self.tmpdirname				)



   def    __UpperCAmelCase    (       self    :	int							,       __lowerCamelCase    :	Any				):



     """simple docstring"""
     _snake_case   	=      '''This is a test'''
     _snake_case   	=      '''This is a test'''
     return input_text, output_text



   def    __UpperCAmelCase    (       self    :	Optional[int]				):



     """simple docstring"""
     _snake_case   	=      '''<s>'''
     _snake_case   	=      1

     self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowerCamelCase				)							,       __lowerCamelCase				)
     self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowerCamelCase				)							,       __lowerCamelCase				)



   def    __UpperCAmelCase    (       self    :	Any				):



     """simple docstring"""
     _snake_case   	=      list(self.get_tokenizer().get_vocab().keys()				)

     self.assertEqual(vocab_keys[0]							,       '''<unk>'''				)
     self.assertEqual(vocab_keys[1]							,       '''<s>'''				)
     self.assertEqual(vocab_keys[-1]							,       '''j'''				)
     self.assertEqual(len(__lowerCamelCase				)							,       2_0_0_0				)



   def    __UpperCAmelCase    (       self    :	List[Any]				):



     """simple docstring"""
     self.assertEqual(self.get_tokenizer().vocab_size							,       2_0_0_0				)



   def    __UpperCAmelCase    (       self    :	Dict				):



     """simple docstring"""
     _snake_case   	=      GPTSwaTokenizer(__lowerCamelCase				)

     _snake_case   	=      tokenizer.tokenize('''This is a test'''				)
     self.assertListEqual(__lowerCamelCase							,       ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']				)

     self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase				)							,       [4_6_5, 2_8_7, 2_6_5, 6_3_1, 8_4_2]				)

     _snake_case   	=      tokenizer.tokenize('''I was born in 92000, and this is falsé.'''				)
     # fmt: off
     self.assertListEqual(
         __lowerCamelCase							,       ['''▁I''', '''▁was''', '''▁bor''', '''n''', '''▁in''', '''▁''', '''<0x39>''', '''2''', '''0''', '''0''', '''0''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁f''', '''al''', '''s''', '''<0xC3>''', '''<0xA9>''', '''.''']							,       )
     # fmt: on

     _snake_case   	=      tokenizer.convert_tokens_to_ids(__lowerCamelCase				)
     self.assertListEqual(
         __lowerCamelCase							,       [2_6_2, 2_7_2, 1_5_2_5, 2_8_6, 2_7_1, 2_6_8, 6_0, 9_1_6, 6_3_3, 6_3_3, 6_3_3, 2_5_9, 2_6_6, 3_0_1, 2_8_7, 3_8_4, 3_6_7, 2_6_3, 1_9_8, 1_7_2, 2_6_0]							,       )

     _snake_case   	=      tokenizer.convert_ids_to_tokens(__lowerCamelCase				)
     # fmt: off
     self.assertListEqual(
         __lowerCamelCase							,       ['''▁I''', '''▁was''', '''▁bor''', '''n''', '''▁in''', '''▁''', '''<0x39>''', '''2''', '''0''', '''0''', '''0''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁f''', '''al''', '''s''', '''<0xC3>''', '''<0xA9>''', '''.''']				)
     # fmt: on



   def    __UpperCAmelCase    (       self    :	Union[str, Any]				):



     """simple docstring"""
     _snake_case   	=      GPTSwaTokenizer(__lowerCamelCase				)
     _snake_case   	=      ['''This is a test''', '''I was born in 92000, and this is falsé.''']
     _snake_case   	=      [
         [4_6_5, 2_8_7, 2_6_5, 6_3_1, 8_4_2],
         [2_6_2, 2_7_2, 1_5_2_5, 2_8_6, 2_7_1, 2_6_8, 6_0, 9_1_6, 6_3_3, 6_3_3, 6_3_3, 2_5_9, 2_6_6, 3_0_1, 2_8_7, 3_8_4, 3_6_7, 2_6_3, 1_9_8, 1_7_2, 2_6_0],
     ]

     # Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
     for text, expected_ids in zip(__lowerCamelCase							,       __lowerCamelCase				):
       self.assertListEqual(tokenizer.encode_fast(__lowerCamelCase				)							,       __lowerCamelCase				)

     # Test that decode_fast returns the input text
     for text, token_ids in zip(__lowerCamelCase							,       __lowerCamelCase				):
       self.assertEqual(tokenizer.decode_fast(__lowerCamelCase				)							,       __lowerCamelCase				)





   @slow
   def    __UpperCAmelCase    (       self    :	Tuple				):



     """simple docstring"""
     _snake_case   	=      [
         '''<|python|>def fibonacci(n)\n    if n < 0:\n        print(\'Incorrect input\')''',
         '''Hey there, how are you doing this fine day?''',
         '''This is a text with a trailing spaces followed by a dot     .''',
         '''Häj sväjs lillebrör! =)''',
         '''Det är inget fel på Mr. Cool''',
     ]

     # fmt: off
     _snake_case   	=      {'''input_ids''': [[6_3_4_2_3, 5, 6_8_1_1, 1_4_9_5_4, 2_8_2, 8_1_6, 3_8_2_1, 6_3_4_6_6, 6_3_4_2_5, 6_3_4_6_2, 1_8, 6_3_9_7_8, 6_7_8, 3_0_1, 1_3_2_0, 6_3_4_2_3, 6_3_4_5_5, 6_3_4_5_8, 1_8, 6_3_9_8_2, 4_2_4_6, 3_9_4_0, 1_9_0_1, 4_7_7_8_9, 5_5_4_7, 1_8_9_9_4], [1_9_6_3_0, 1_1_0_0, 6_3_4_4_6, 1_3_4_2, 6_3_3, 5_4_4, 4_4_8_8, 5_9_3, 5_1_0_2, 2_4_1_6, 6_3_4_9_5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1_6_5_2, 4_2_8, 2_6_8, 1_9_3_6, 5_1_5, 2_6_8, 5_8_5_9_3, 2_2_4_1_3, 9_1_0_6, 5_4_6, 2_6_8, 3_3_2_1_3, 6_3_9_7_9, 6_9_8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_5_1_3_0, 6_3_4_5_0, 9_2_4, 6_3_4_4_9, 2_2_4_9, 4_0_6_2, 1_5_5_8, 3_1_8, 6_3_5_0_4, 2_1_4_9_8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_0_9, 3_7_7, 2_8_2_7, 2_5_5_9, 3_3_2, 6_5_7_5, 6_3_4_4_3, 2_6_8_0_1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
     # fmt: on
     self.tokenizer_integration_test_util(
         expected_encoding=__lowerCamelCase							,       model_name='''AI-Sweden/gpt-sw3-126m'''							,       sequences=__lowerCamelCase							,       )










"""simple docstring"""






def 			snake_case		(		lowerCAmelCase_  ,						lowerCAmelCase_	)						->				int:
  return 1 if input_a == input_a else 0




def 			snake_case		(		)						->				None:
  assert xnor_gate(0  ,						0	) == 1
  assert xnor_gate(0  ,						1	) == 0
  assert xnor_gate(1  ,						0	) == 0
  assert xnor_gate(1  ,						1	) == 1


if __name__ == "__main__":
     print(xnor_gate(0, 0))
     print(xnor_gate(0, 1))
     print(xnor_gate(1, 0))
     print(xnor_gate(1, 1))





