






import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase__       		=  get_tests_dir("""fixtures/test_sentencepiece.model""")




@require_sentencepiece
@require_tokenizers
class       snake_case__ (				SCREAMING_SNAKE_CASE_						,     unittest.TestCase):

       '''simple docstring'''
       lowerCamelCase				:					Dict								=			XLNetTokenizer
       lowerCamelCase				:					List[str]								=			XLNetTokenizerFast
       lowerCamelCase				:					Tuple								=			True
       lowerCamelCase				:					int								=			True



       def        __lowercase    (				self     )			->   Union[str, Any]:






            '''simple docstring'''





            super().setUp()

            # We have a SentencePiece fixture for testing
            __snake_case      :Optional[int] 							=    XLNetTokenizer(a__      ,    keep_accents=a__     )
            tokenizer.sanitize_special_tokens()
            tokenizer.save_pretrained(self.tmpdirname     )



       def        __lowercase    (				self     )			->   Optional[Any]:






            '''simple docstring'''





            __snake_case      :Union[str, Any] 							=    '<s>'
            __snake_case      :Dict 							=    1

            self.assertEqual(self.get_tokenizer()._convert_token_to_id(a__     )      ,    a__     )
            self.assertEqual(self.get_tokenizer()._convert_id_to_token(a__     )      ,    a__     )



       def        __lowercase    (				self     )			->   Union[str, Any]:






            '''simple docstring'''





            __snake_case      :Optional[Any] 							=    list(self.get_tokenizer().get_vocab().keys()     )

            self.assertEqual(vocab_keys[0]      ,    """<unk>"""     )
            self.assertEqual(vocab_keys[1]      ,    """<s>"""     )
            self.assertEqual(vocab_keys[-1]      ,    """<eod>"""     )
            self.assertEqual(len(a__     )      ,    10_06     )



       def        __lowercase    (				self     )			->   Tuple:






            '''simple docstring'''





            self.assertEqual(self.get_tokenizer().vocab_size      ,    10_00     )



       def        __lowercase    (				self     )			->   int:






            '''simple docstring'''





            __snake_case      :str 							=    XLNetTokenizer(a__      ,    keep_accents=a__     )

            __snake_case      :str 							=    tokenizer.tokenize("""This is a test"""     )
            self.assertListEqual(a__      ,    ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]     )

            self.assertListEqual(tokenizer.convert_tokens_to_ids(a__     )      ,    [2_85, 46, 10, 1_70, 3_82]     )

            __snake_case      :Union[str, Any] 							=    tokenizer.tokenize("""I was born in 92000, and this is falsé."""     )
            self.assertListEqual(
                a__      ,    [
                    SPIECE_UNDERLINE + """I""",
                    SPIECE_UNDERLINE + """was""",
                    SPIECE_UNDERLINE + """b""",
                    """or""",
                    """n""",
                    SPIECE_UNDERLINE + """in""",
                    SPIECE_UNDERLINE + """""",
                    """9""",
                    """2""",
                    """0""",
                    """0""",
                    """0""",
                    """,""",
                    SPIECE_UNDERLINE + """and""",
                    SPIECE_UNDERLINE + """this""",
                    SPIECE_UNDERLINE + """is""",
                    SPIECE_UNDERLINE + """f""",
                    """al""",
                    """s""",
                    """é""",
                    """.""",
                ]      ,    )
            __snake_case      :Optional[Any] 							=    tokenizer.convert_tokens_to_ids(a__     )
            self.assertListEqual(a__      ,    [8, 21, 84, 55, 24, 19, 7, 0, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 0, 4]     )

            __snake_case      :Union[str, Any] 							=    tokenizer.convert_ids_to_tokens(a__     )
            self.assertListEqual(
                a__      ,    [
                    SPIECE_UNDERLINE + """I""",
                    SPIECE_UNDERLINE + """was""",
                    SPIECE_UNDERLINE + """b""",
                    """or""",
                    """n""",
                    SPIECE_UNDERLINE + """in""",
                    SPIECE_UNDERLINE + """""",
                    """<unk>""",
                    """2""",
                    """0""",
                    """0""",
                    """0""",
                    """,""",
                    SPIECE_UNDERLINE + """and""",
                    SPIECE_UNDERLINE + """this""",
                    SPIECE_UNDERLINE + """is""",
                    SPIECE_UNDERLINE + """f""",
                    """al""",
                    """s""",
                    """<unk>""",
                    """.""",
                ]      ,    )



       def        __lowercase    (				self     )			->   List[str]:






            '''simple docstring'''





            __snake_case      :Dict 							=    XLNetTokenizer(a__      ,    do_lower_case=a__     )
            __snake_case      :str 							=    tokenizer.tokenize("""I was born in 92000, and this is falsé."""     )
            self.assertListEqual(
                a__      ,    [
                    SPIECE_UNDERLINE + """""",
                    """i""",
                    SPIECE_UNDERLINE + """was""",
                    SPIECE_UNDERLINE + """b""",
                    """or""",
                    """n""",
                    SPIECE_UNDERLINE + """in""",
                    SPIECE_UNDERLINE + """""",
                    """9""",
                    """2""",
                    """0""",
                    """0""",
                    """0""",
                    """,""",
                    SPIECE_UNDERLINE + """and""",
                    SPIECE_UNDERLINE + """this""",
                    SPIECE_UNDERLINE + """is""",
                    SPIECE_UNDERLINE + """f""",
                    """al""",
                    """se""",
                    """.""",
                ]      ,    )
            self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""     )      ,    ["""▁he""", """ll""", """o"""]     )



       def        __lowercase    (				self     )			->   int:






            '''simple docstring'''





            __snake_case      :str 							=    XLNetTokenizer(a__      ,    do_lower_case=a__     )
            __snake_case      :Any 							=    tokenizer.tokenize("""I was born in 92000, and this is falsé."""     )
            self.assertListEqual(
                a__      ,    [
                    SPIECE_UNDERLINE + """I""",
                    SPIECE_UNDERLINE + """was""",
                    SPIECE_UNDERLINE + """b""",
                    """or""",
                    """n""",
                    SPIECE_UNDERLINE + """in""",
                    SPIECE_UNDERLINE + """""",
                    """9""",
                    """2""",
                    """0""",
                    """0""",
                    """0""",
                    """,""",
                    SPIECE_UNDERLINE + """and""",
                    SPIECE_UNDERLINE + """this""",
                    SPIECE_UNDERLINE + """is""",
                    SPIECE_UNDERLINE + """f""",
                    """al""",
                    """se""",
                    """.""",
                ]      ,    )



       @slow
       def        __lowercase    (				self     )			->   Optional[int]:






            '''simple docstring'''





            __snake_case      :int 							=    XLNetTokenizer.from_pretrained("""xlnet-base-cased"""     )

            __snake_case      :Any 							=    tokenizer.encode("""sequence builders"""      ,    add_special_tokens=a__     )
            __snake_case      :Optional[Any] 							=    tokenizer.encode("""multi-sequence build"""      ,    add_special_tokens=a__     )

            __snake_case      :Union[str, Any] 							=    tokenizer.build_inputs_with_special_tokens(a__     )
            __snake_case      :Any 							=    tokenizer.build_inputs_with_special_tokens(a__      ,    a__     )

            assert encoded_sentence == text + [4, 3]
            assert encoded_pair == text + [4] + text_a + [4, 3]



       @slow
       def        __lowercase    (				self     )			->   Optional[int]:






            '''simple docstring'''





            __snake_case      :Any 							=    {'input_ids': [[17, 2_14_42, 2_70, 17, 10, 1_46_45, 3_18, 34, 17, 45_46, 31_45, 7_87, 13, 77_52, 2_20_18, 23, 21, 17, 45_46, 31_45, 7_87, 13, 33_52, 1_44_31, 13, 55_00, 11, 11_76, 5_80, 13, 1_68_19, 47_97, 23, 17, 10, 1_71_35, 6_58, 19, 4_57, 79_32, 13, 1_84, 19, 31_54, 1_71_35, 64_68, 19, 14_04, 1_22_69, 19, 42_29, 53_56, 1_62_64, 46, 19, 17, 2_05_45, 1_03_95, 9, 9, 9, 11, 28, 64_21, 95_31, 2_07_29, 17, 10, 3_53, 1_70_22, 11, 21, 64_21, 95_31, 1_69_49, 17, 10, 1_15_09, 7_53, 11, 33, 95, 24_21, 73_85, 9_56, 1_44_31, 26_26, 25, 8_42, 73_85, 48_36, 21, 14_29, 22_72, 98_55, 31_20, 1_61, 2_47_38, 19, 1_32_03, 6_58, 2_18, 7_87, 21, 4_30, 1_84_82, 8_47, 26_37, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3_22, 2_21_78, 27, 10_64, 22, 9_56, 13, 1_11_01, 14_29, 58_54, 2_43_13, 1_89_53, 40, 4_22, 2_43_66, 68, 17_58, 37, 1_04_83, 1_42_57, 31, 2_07, 2_63, 21, 2_03, 37_73, 25, 71, 97_35, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 20_49, 34_42, 17, 1_38_94, 33_80, 23, 95, 18, 1_76_34, 22_88, 9, 4, 3]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
            # fmt: on

            self.tokenizer_integration_test_util(
                expected_encoding=a__      ,    model_name="""xlnet-base-cased"""      ,    revision="""c841166438c31ec7ca9a106dee7bb312b73ae511"""      ,    )







"""simple docstring"""







from typing import List, Optional, TypeVar

from .arrow_dataset import Dataset, _concatenate_map_style_datasets, _interleave_map_style_datasets
from .dataset_dict import DatasetDict, IterableDatasetDict
from .info import DatasetInfo
from .iterable_dataset import IterableDataset, _concatenate_iterable_datasets, _interleave_iterable_datasets
from .splits import NamedSplit
from .utils import logging
from .utils.py_utils import Literal


a_							=    logging.get_logger(__name__)


a_							=    TypeVar("""DatasetType""", Dataset, IterableDataset)

def     UpperCAmelCase_					(				__a   :			List[DatasetType]      ,							__a   :			Optional[List[float]] = None      ,							__a   :			Optional[int] = None      ,							__a   :			Optional[DatasetInfo] = None      ,							__a   :			Optional[NamedSplit] = None      ,							__a   :			Literal["first_exhausted", "all_exhausted"] = "first_exhausted"      ,							):




			'''simple docstring'''



			from .arrow_dataset import Dataset
			from .iterable_dataset import IterableDataset

			if not datasets:
						raise ValueError('Unable to interleave an empty list of datasets.'						)
			for i, dataset in enumerate(__a						):
						if not isinstance(__a      ,							(Dataset, IterableDataset)						):
									if isinstance(__a      ,							(DatasetDict, IterableDatasetDict)						):
												if not dataset:
															raise ValueError(
															    f"Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} "
															    'is an empty dataset dictionary.'						)
												raise ValueError(
												    f"Dataset at position {i} has at least one split: {list(__a						)}\n"
												    f"Please pick one to interleave with the other datasets, for example: dataset['{next(iter(__a						)						)}']"						)
									raise ValueError(
									    f"Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} is a {type(__a						).__name__}."						)
						if i == 0:
									_lowerCamelCase       ,							_lowerCamelCase					:					Tuple	       =    (
									    (Dataset, IterableDataset) if isinstance(__a      ,							__a						) else (IterableDataset, Dataset)
									)
						elif not isinstance(__a      ,							__a						):
									raise ValueError(
									    f"Unable to interleave a {dataset_type.__name__} (at position 0) with a {other_type.__name__} (at position {i}). Expected a list of Dataset objects or a list of IterableDataset objects."						)
			if stopping_strategy not in ["first_exhausted", "all_exhausted"]:
						raise ValueError(f"{stopping_strategy} is not supported. Please enter a valid stopping_strategy."						)
			if dataset_type is Dataset:
						return _interleave_map_style_datasets(
						    __a      ,							__a      ,							__a      ,							info=__a      ,							split=__a      ,							stopping_strategy=__a						)
			else:
						return _interleave_iterable_datasets(
						    __a      ,							__a      ,							__a      ,							info=__a      ,							split=__a      ,							stopping_strategy=__a						)



def     UpperCAmelCase_					(				__a   :			List[DatasetType]      ,							__a   :			Optional[DatasetInfo] = None      ,							__a   :			Optional[NamedSplit] = None      ,							__a   :			int = 0      ,							):




			'''simple docstring'''



			if not dsets:
						raise ValueError('Unable to concatenate an empty list of datasets.'						)
			for i, dataset in enumerate(__a						):
						if not isinstance(__a      ,							(Dataset, IterableDataset)						):
									if isinstance(__a      ,							(DatasetDict, IterableDatasetDict)						):
												if not dataset:
															raise ValueError(
															    f"Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} "
															    'is an empty dataset dictionary.'						)
												raise ValueError(
												    f"Dataset at position {i} has at least one split: {list(__a						)}\n"
												    f"Please pick one to interleave with the other datasets, for example: dataset['{next(iter(__a						)						)}']"						)
									raise ValueError(
									    f"Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} is a {type(__a						).__name__}."						)
						if i == 0:
									_lowerCamelCase       ,							_lowerCamelCase					:					Optional[Any]	       =    (
									    (Dataset, IterableDataset) if isinstance(__a      ,							__a						) else (IterableDataset, Dataset)
									)
						elif not isinstance(__a      ,							__a						):
									raise ValueError(
									    f"Unable to interleave a {dataset_type.__name__} (at position 0) with a {other_type.__name__} (at position {i}). Expected a list of Dataset objects or a list of IterableDataset objects."						)
			if dataset_type is Dataset:
						return _concatenate_map_style_datasets(__a      ,							info=__a      ,							split=__a      ,							axis=__a						)
			else:
						return _concatenate_iterable_datasets(__a      ,							info=__a      ,							split=__a      ,							axis=__a						)





