

'''simple docstring'''



import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__snake_case					:      Tuple 		=     get_tests_dir('fixtures/test_sentencepiece_with_bytefallback.model')



@require_sentencepiece
@require_tokenizers
class lowerCamelCase					(    lowercase_			,		unittest.TestCase		):



			'''simple docstring'''






			__snake_case             =			GPTSwaTokenizer
			__snake_case             =			False
			__snake_case             =			True
			__snake_case             =			False


			def lowercase__		( self			:		Tuple					)     ->						Optional[int]:





									'''simple docstring'''
									super().setUp()

									# We have a SentencePiece fixture for testing
									A__	:  Optional[Any]           =GPTSwaTokenizer(__UpperCamelCase						,     eos_token="""<unk>"""						,     bos_token="""<unk>"""						,     pad_token="""<unk>"""					)

									tokenizer.save_pretrained(self.tmpdirname					)


			def lowercase__		( self			:		Any						,     lowerCAmelCase_			:		List[str]					)     ->						int:





									'''simple docstring'''
									A__	:  int           ="""This is a test"""
									A__	:  Union[str, Any]           ="""This is a test"""
									return input_text, output_text


			def lowercase__		( self			:		List[Any]					)     ->						Optional[int]:





									'''simple docstring'''
									A__	:  Optional[Any]           ="""<s>"""
									A__	:  Union[str, Any]           =1

									self.assertEqual(self.get_tokenizer()._convert_token_to_id(__UpperCamelCase					)						,     __UpperCamelCase					)
									self.assertEqual(self.get_tokenizer()._convert_id_to_token(__UpperCamelCase					)						,     __UpperCamelCase					)


			def lowercase__		( self			:		Union[str, Any]					)     ->						int:





									'''simple docstring'''
									A__	:  List[str]           =list(self.get_tokenizer().get_vocab().keys()					)

									self.assertEqual(vocab_keys[0]						,     """<unk>"""					)
									self.assertEqual(vocab_keys[1]						,     """<s>"""					)
									self.assertEqual(vocab_keys[-1]						,     """j"""					)
									self.assertEqual(len(__UpperCamelCase					)						,     20_00					)


			def lowercase__		( self			:		List[Any]					)     ->						Union[str, Any]:





									'''simple docstring'''
									self.assertEqual(self.get_tokenizer().vocab_size						,     20_00					)


			def lowercase__		( self			:		Optional[Any]					)     ->						Any:





									'''simple docstring'''
									A__	:  str           =GPTSwaTokenizer(__UpperCamelCase					)

									A__	:  str           =tokenizer.tokenize("""This is a test"""					)
									self.assertListEqual(__UpperCamelCase						,     ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]					)

									self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCamelCase					)						,     [4_65, 2_87, 2_65, 6_31, 8_42]					)

									A__	:  int           =tokenizer.tokenize("""I was born in 92000, and this is falsé."""					)
									# fmt: off
									self.assertListEqual(
									    __UpperCamelCase						,     ["""▁I""", """▁was""", """▁bor""", """n""", """▁in""", """▁""", """<0x39>""", """2""", """0""", """0""", """0""", """,""", """▁and""", """▁this""", """▁is""", """▁f""", """al""", """s""", """<0xC3>""", """<0xA9>""", """."""]						,     )
									# fmt: on

									A__	:  Dict           =tokenizer.convert_tokens_to_ids(__UpperCamelCase					)
									self.assertListEqual(
									    __UpperCamelCase						,     [2_62, 2_72, 15_25, 2_86, 2_71, 2_68, 60, 9_16, 6_33, 6_33, 6_33, 2_59, 2_66, 3_01, 2_87, 3_84, 3_67, 2_63, 1_98, 1_72, 2_60]						,     )

									A__	:  Any           =tokenizer.convert_ids_to_tokens(__UpperCamelCase					)
									# fmt: off
									self.assertListEqual(
									    __UpperCamelCase						,     ["""▁I""", """▁was""", """▁bor""", """n""", """▁in""", """▁""", """<0x39>""", """2""", """0""", """0""", """0""", """,""", """▁and""", """▁this""", """▁is""", """▁f""", """al""", """s""", """<0xC3>""", """<0xA9>""", """."""]					)
									# fmt: on


			def lowercase__		( self			:		Optional[Any]					)     ->						Optional[int]:





									'''simple docstring'''
									A__	:  List[str]           =GPTSwaTokenizer(__UpperCamelCase					)
									A__	:  List[str]           =["""This is a test""", """I was born in 92000, and this is falsé."""]
									A__	:  Union[str, Any]           =[
									    [4_65, 2_87, 2_65, 6_31, 8_42],
									    [2_62, 2_72, 15_25, 2_86, 2_71, 2_68, 60, 9_16, 6_33, 6_33, 6_33, 2_59, 2_66, 3_01, 2_87, 3_84, 3_67, 2_63, 1_98, 1_72, 2_60],
									]

									# Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
									for text, expected_ids in zip(__UpperCamelCase						,     __UpperCamelCase					):
															self.assertListEqual(tokenizer.encode_fast(__UpperCamelCase					)						,     __UpperCamelCase					)

									# Test that decode_fast returns the input text
									for text, token_ids in zip(__UpperCamelCase						,     __UpperCamelCase					):
															self.assertEqual(tokenizer.decode_fast(__UpperCamelCase					)						,     __UpperCamelCase					)



			@slow
			def lowercase__		( self			:		str					)     ->						List[Any]:





									'''simple docstring'''
									A__	:  List[str]           =[
									    """<|python|>def fibonacci(n)\n    if n < 0:\n        print('Incorrect input')""",
									    """Hey there, how are you doing this fine day?""",
									    """This is a text with a trailing spaces followed by a dot     .""",
									    """Häj sväjs lillebrör! =)""",
									    """Det är inget fel på Mr. Cool""",
									]

									# fmt: off
									A__	:  Optional[Any]           ={"""input_ids""": [[6_34_23, 5, 68_11, 1_49_54, 2_82, 8_16, 38_21, 6_34_66, 6_34_25, 6_34_62, 18, 6_39_78, 6_78, 3_01, 13_20, 6_34_23, 6_34_55, 6_34_58, 18, 6_39_82, 42_46, 39_40, 19_01, 4_77_89, 55_47, 1_89_94], [1_96_30, 11_00, 6_34_46, 13_42, 6_33, 5_44, 44_88, 5_93, 51_02, 24_16, 6_34_95, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [16_52, 4_28, 2_68, 19_36, 5_15, 2_68, 5_85_93, 2_24_13, 91_06, 5_46, 2_68, 3_32_13, 6_39_79, 6_98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_51_30, 6_34_50, 9_24, 6_34_49, 22_49, 40_62, 15_58, 3_18, 6_35_04, 2_14_98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_09, 3_77, 28_27, 25_59, 3_32, 65_75, 6_34_43, 2_68_01, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """token_type_ids""": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
									# fmt: on
									self.tokenizer_integration_test_util(
									    expected_encoding=__UpperCamelCase						,     model_name="""AI-Sweden/gpt-sw3-126m"""						,     sequences=__UpperCamelCase						,     )



import importlib.metadata
import operator
import re
import sys
from typing import Optional

from packaging import version


lowercase   =	{
    '''<''': operator.lt,
    '''<=''': operator.le,
    '''==''': operator.eq,
    '''!=''': operator.ne,
    '''>=''': operator.ge,
    '''>''': operator.gt,
}



def   __lowerCAmelCase    (				UpperCAmelCase__					:   Tuple      ,					UpperCAmelCase__					:   int      ,					UpperCAmelCase__					:   Optional[int]      ,					UpperCAmelCase__					:   Optional[int]      ,					UpperCAmelCase__					:   Optional[int]      ,					UpperCAmelCase__					:   int							)	->     Union[str, Any]:
							if got_ver is None or want_ver is None:
														raise ValueError(
														    F'''Unable to compare versions for {requirement}: need={want_ver} found={got_ver}. This is unusual. Consider'''
														    F''' reinstalling {pkg}.'''							)
							if not ops[op](version.parse(UpperCAmelCase__							)      ,					version.parse(UpperCAmelCase__							)							):
														raise ImportError(
														    F'''{requirement} is required for a normal functioning of this module, but found {pkg}=={got_ver}.{hint}'''							)



def   __lowerCAmelCase    (				UpperCAmelCase__					:   str      ,					UpperCAmelCase__					:   Optional[str] = None							)	->     None:
							lowerCamelCase_					=       F'''\n{hint}''' if hint is not None else """"""

							# non-versioned check
							if re.match(R"""^[\w_\-\d]+$"""      ,					UpperCAmelCase__							):
														lowerCamelCase_	, lowerCamelCase_	, lowerCamelCase_					=       requirement, None, None
							else:
														lowerCamelCase_					=       re.findall(R"""^([^!=<>\s]+)([\s!=<>]{1,2}.+)"""      ,					UpperCAmelCase__							)
														if not match:
																					raise ValueError(
																					    """requirement needs to be in the pip package format, .e.g., package_a==1.23, or package_b>=1.23, but"""
																					    F''' got {requirement}'''							)
														lowerCamelCase_	, lowerCamelCase_					=       match[0]
														lowerCamelCase_					=       want_full.split(""","""							)  # there could be multiple requirements
														lowerCamelCase_					=       {}
														for w in want_range:
																					lowerCamelCase_					=       re.findall(R"""^([\s!=<>]{1,2})(.+)"""      ,					UpperCAmelCase__							)
																					if not match:
																												raise ValueError(
																												    """requirement needs to be in the pip package format, .e.g., package_a==1.23, or package_b>=1.23,"""
																												    F''' but got {requirement}'''							)
																					lowerCamelCase_	, lowerCamelCase_					=       match[0]
																					lowerCamelCase_					=       want_ver
																					if op not in ops:
																												raise ValueError(F'''{requirement}: need one of {list(ops.keys()							)}, but got {op}'''							)

    # special case
							if pkg == "python":
														lowerCamelCase_					=       """.""".join([str(UpperCAmelCase__							) for x in sys.version_info[:3]]							)
														for op, want_ver in wanted.items():
																					_compare_versions(UpperCAmelCase__      ,					UpperCAmelCase__      ,					UpperCAmelCase__      ,					UpperCAmelCase__      ,					UpperCAmelCase__      ,					UpperCAmelCase__							)
														return

							# check if any version is installed
							try:
														lowerCamelCase_					=       importlib.metadata.version(UpperCAmelCase__							)
							except importlib.metadata.PackageNotFoundError:
														raise importlib.metadata.PackageNotFoundError(
														    F'''The \'{requirement}\' distribution was not found and is required by this application. {hint}'''							)

							# check that the right version is installed if version number or a range was provided
							if want_ver is not None:
														for op, want_ver in wanted.items():
																					_compare_versions(UpperCAmelCase__      ,					UpperCAmelCase__      ,					UpperCAmelCase__      ,					UpperCAmelCase__      ,					UpperCAmelCase__      ,					UpperCAmelCase__							)



def   __lowerCAmelCase    (				UpperCAmelCase__					:   List[str]							)	->     Dict:
							lowerCamelCase_					=       """Try: pip install transformers -U or pip install -e '.[dev]' if you're working with git main"""
							return require_version(UpperCAmelCase__      ,					UpperCAmelCase__							)





