



'''simple docstring'''





import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


UpperCAmelCase    					=   get_tests_dir("fixtures/test_sentencepiece.model")

@require_sentencepiece
@require_tokenizers
class 						a   (				__magic_name__				,unittest.TestCase    ):
				_snake_case		    = XLNetTokenizer
				_snake_case		    = XLNetTokenizerFast
				_snake_case		    = True
				_snake_case		    = True




				def 	__snake_case							(							self	:				str      ):
										super().setUp()

										# We have a SentencePiece fixture for testing
										snake_case       :     Optional[Any] =    XLNetTokenizer(_lowerCAmelCase,    keep_accents=_lowerCAmelCase      )
										tokenizer.sanitize_special_tokens()
										tokenizer.save_pretrained(self.tmpdirname      )




				def 	__snake_case							(							self	:				Optional[Any]      ):
										snake_case       :     int =    '''<s>'''
										snake_case       :     str =    1

										self.assertEqual(self.get_tokenizer()._convert_token_to_id(_lowerCAmelCase      ),    _lowerCAmelCase      )
										self.assertEqual(self.get_tokenizer()._convert_id_to_token(_lowerCAmelCase      ),    _lowerCAmelCase      )




				def 	__snake_case							(							self	:				Optional[Any]      ):
										snake_case       :     str =    list(self.get_tokenizer().get_vocab().keys()      )

										self.assertEqual(vocab_keys[0],    '''<unk>'''      )
										self.assertEqual(vocab_keys[1],    '''<s>'''      )
										self.assertEqual(vocab_keys[-1],    '''<eod>'''      )
										self.assertEqual(len(_lowerCAmelCase      ),    10_06      )




				def 	__snake_case							(							self	:				List[str]      ):
										self.assertEqual(self.get_tokenizer().vocab_size,    10_00      )




				def 	__snake_case							(							self	:				Any      ):
										snake_case       :     List[Any] =    XLNetTokenizer(_lowerCAmelCase,    keep_accents=_lowerCAmelCase      )

										snake_case       :     List[str] =    tokenizer.tokenize('''This is a test'''      )
										self.assertListEqual(_lowerCAmelCase,    ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']      )

										self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCAmelCase      ),    [2_85, 46, 10, 1_70, 3_82]      )

										snake_case       :     str =    tokenizer.tokenize('''I was born in 92000, and this is falsé.'''      )
										self.assertListEqual(
										    _lowerCAmelCase,    [
										        SPIECE_UNDERLINE + '''I''',
										        SPIECE_UNDERLINE + '''was''',
										        SPIECE_UNDERLINE + '''b''',
										        '''or''',
										        '''n''',
										        SPIECE_UNDERLINE + '''in''',
										        SPIECE_UNDERLINE + '''''',
										        '''9''',
										        '''2''',
										        '''0''',
										        '''0''',
										        '''0''',
										        ''',''',
										        SPIECE_UNDERLINE + '''and''',
										        SPIECE_UNDERLINE + '''this''',
										        SPIECE_UNDERLINE + '''is''',
										        SPIECE_UNDERLINE + '''f''',
										        '''al''',
										        '''s''',
										        '''é''',
										        '''.''',
										    ],    )
										snake_case       :     Tuple =    tokenizer.convert_tokens_to_ids(_lowerCAmelCase      )
										self.assertListEqual(_lowerCAmelCase,    [8, 21, 84, 55, 24, 19, 7, 0, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 0, 4]      )

										snake_case       :     List[Any] =    tokenizer.convert_ids_to_tokens(_lowerCAmelCase      )
										self.assertListEqual(
										    _lowerCAmelCase,    [
										        SPIECE_UNDERLINE + '''I''',
										        SPIECE_UNDERLINE + '''was''',
										        SPIECE_UNDERLINE + '''b''',
										        '''or''',
										        '''n''',
										        SPIECE_UNDERLINE + '''in''',
										        SPIECE_UNDERLINE + '''''',
										        '''<unk>''',
										        '''2''',
										        '''0''',
										        '''0''',
										        '''0''',
										        ''',''',
										        SPIECE_UNDERLINE + '''and''',
										        SPIECE_UNDERLINE + '''this''',
										        SPIECE_UNDERLINE + '''is''',
										        SPIECE_UNDERLINE + '''f''',
										        '''al''',
										        '''s''',
										        '''<unk>''',
										        '''.''',
										    ],    )




				def 	__snake_case							(							self	:				str      ):
										snake_case       :     int =    XLNetTokenizer(_lowerCAmelCase,    do_lower_case=_lowerCAmelCase      )
										snake_case       :     Optional[int] =    tokenizer.tokenize('''I was born in 92000, and this is falsé.'''      )
										self.assertListEqual(
										    _lowerCAmelCase,    [
										        SPIECE_UNDERLINE + '''''',
										        '''i''',
										        SPIECE_UNDERLINE + '''was''',
										        SPIECE_UNDERLINE + '''b''',
										        '''or''',
										        '''n''',
										        SPIECE_UNDERLINE + '''in''',
										        SPIECE_UNDERLINE + '''''',
										        '''9''',
										        '''2''',
										        '''0''',
										        '''0''',
										        '''0''',
										        ''',''',
										        SPIECE_UNDERLINE + '''and''',
										        SPIECE_UNDERLINE + '''this''',
										        SPIECE_UNDERLINE + '''is''',
										        SPIECE_UNDERLINE + '''f''',
										        '''al''',
										        '''se''',
										        '''.''',
										    ],    )
										self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''      ),    ['''▁he''', '''ll''', '''o''']      )




				def 	__snake_case							(							self	:				Optional[Any]      ):
										snake_case       :     Tuple =    XLNetTokenizer(_lowerCAmelCase,    do_lower_case=_lowerCAmelCase      )
										snake_case       :     Dict =    tokenizer.tokenize('''I was born in 92000, and this is falsé.'''      )
										self.assertListEqual(
										    _lowerCAmelCase,    [
										        SPIECE_UNDERLINE + '''I''',
										        SPIECE_UNDERLINE + '''was''',
										        SPIECE_UNDERLINE + '''b''',
										        '''or''',
										        '''n''',
										        SPIECE_UNDERLINE + '''in''',
										        SPIECE_UNDERLINE + '''''',
										        '''9''',
										        '''2''',
										        '''0''',
										        '''0''',
										        '''0''',
										        ''',''',
										        SPIECE_UNDERLINE + '''and''',
										        SPIECE_UNDERLINE + '''this''',
										        SPIECE_UNDERLINE + '''is''',
										        SPIECE_UNDERLINE + '''f''',
										        '''al''',
										        '''se''',
										        '''.''',
										    ],    )




				@slow
				def 	__snake_case							(							self	:				Optional[int]      ):
										snake_case       :     Optional[Any] =    XLNetTokenizer.from_pretrained('''xlnet-base-cased'''      )

										snake_case       :     Tuple =    tokenizer.encode('''sequence builders''',    add_special_tokens=_lowerCAmelCase      )
										snake_case       :     Optional[Any] =    tokenizer.encode('''multi-sequence build''',    add_special_tokens=_lowerCAmelCase      )

										snake_case       :     Dict =    tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase      )
										snake_case       :     Optional[Any] =    tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase,    _lowerCAmelCase      )

										assert encoded_sentence == text + [4, 3]
										assert encoded_pair == text + [4] + text_a + [4, 3]




				@slow
				def 	__snake_case							(							self	:				Union[str, Any]      ):
										# fmt: off
										snake_case       :     Union[str, Any] =    {'''input_ids''': [[17, 2_14_42, 2_70, 17, 10, 1_46_45, 3_18, 34, 17, 45_46, 31_45, 7_87, 13, 77_52, 2_20_18, 23, 21, 17, 45_46, 31_45, 7_87, 13, 33_52, 1_44_31, 13, 55_00, 11, 11_76, 5_80, 13, 1_68_19, 47_97, 23, 17, 10, 1_71_35, 6_58, 19, 4_57, 79_32, 13, 1_84, 19, 31_54, 1_71_35, 64_68, 19, 14_04, 1_22_69, 19, 42_29, 53_56, 1_62_64, 46, 19, 17, 2_05_45, 1_03_95, 9, 9, 9, 11, 28, 64_21, 95_31, 2_07_29, 17, 10, 3_53, 1_70_22, 11, 21, 64_21, 95_31, 1_69_49, 17, 10, 1_15_09, 7_53, 11, 33, 95, 24_21, 73_85, 9_56, 1_44_31, 26_26, 25, 8_42, 73_85, 48_36, 21, 14_29, 22_72, 98_55, 31_20, 1_61, 2_47_38, 19, 1_32_03, 6_58, 2_18, 7_87, 21, 4_30, 1_84_82, 8_47, 26_37, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3_22, 2_21_78, 27, 10_64, 22, 9_56, 13, 1_11_01, 14_29, 58_54, 2_43_13, 1_89_53, 40, 4_22, 2_43_66, 68, 17_58, 37, 1_04_83, 1_42_57, 31, 2_07, 2_63, 21, 2_03, 37_73, 25, 71, 97_35, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 20_49, 34_42, 17, 1_38_94, 33_80, 23, 95, 18, 1_76_34, 22_88, 9, 4, 3]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
										# fmt: on

										self.tokenizer_integration_test_util(
										    expected_encoding=_lowerCAmelCase,    model_name='''xlnet-base-cased''',    revision='''c841166438c31ec7ca9a106dee7bb312b73ae511''',    )



'''simple docstring'''





import inspect
import re

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_config_docstrings.py
UpperCAmelCase    					=   "src/transformers"


# This is to make sure the transformers module imported is the one in the repo.
UpperCAmelCase    					=   direct_transformers_import(PATH_TO_TRANSFORMERS)

UpperCAmelCase    					=   transformers.models.auto.configuration_auto.CONFIG_MAPPING

# Regex pattern used to find the checkpoint mentioned in the docstring of `config_class`.
# For example, `[bert-base-uncased](https://huggingface.co/bert-base-uncased)`
UpperCAmelCase    					=   re.compile(r"\[(.+?)\]\((https://huggingface\.co/.+?)\)")


UpperCAmelCase    					=   {
    "DecisionTransformerConfig",
    "EncoderDecoderConfig",
    "MusicgenConfig",
    "RagConfig",
    "SpeechEncoderDecoderConfig",
    "TimmBackboneConfig",
    "VisionEncoderDecoderConfig",
    "VisionTextDualEncoderConfig",
    "LlamaConfig",
}

def      A (       A_							:					Union[str, Any]       ):
						snake_case       :     Tuple =    None

						# source code of `config_class`
						snake_case       :     Union[str, Any] =    inspect.getsource(A_       )
						snake_case       :     List[Any] =    _re_checkpoint.findall(A_       )

						# Each `checkpoint` is a tuple of a checkpoint name and a checkpoint link.
						# For example, `('bert-base-uncased', 'https://huggingface.co/bert-base-uncased')`
						for ckpt_name, ckpt_link in checkpoints:
												# allow the link to end with `/`
												if ckpt_link.endswith('''/'''       ):
																		snake_case       :     Optional[int] =    ckpt_link[:-1]

												# verify the checkpoint name corresponds to the checkpoint link
												snake_case       :     Dict =    F"""https://huggingface.co/{ckpt_name}"""
												if ckpt_link == ckpt_link_from_name:
																		snake_case       :     Dict =    ckpt_name
																		break

						return checkpoint

def      A (       ):
						snake_case       :     List[str] =    []

						for config_class in list(CONFIG_MAPPING.values()       ):
												# Skip deprecated models
												if "models.deprecated" in config_class.__module__:
																		continue
												snake_case       :     Tuple =    get_checkpoint_from_config_class(A_       )

												snake_case       :     Optional[Any] =    config_class.__name__
												if checkpoint is None and name not in CONFIG_CLASSES_TO_IGNORE_FOR_DOCSTRING_CHECKPOINT_CHECK:
																		configs_without_checkpoint.append(A_       )

						if len(A_       ) > 0:
												snake_case       :     Optional[int] =    '''\n'''.join(sorted(A_       )       )
												raise ValueError(F"""The following configurations don't contain any valid checkpoint:\n{message}"""       )


if __name__ == "__main__":
						check_config_docstrings_have_checkpoints()

