





import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


_UpperCAmelCase        =				get_tests_dir('fixtures/test_sentencepiece.model')



@require_sentencepiece
class 							_UpperCamelCase					(	lowerCAmelCase_  ,						unittest.TestCase     ):
       _UpperCamelCase				:					List[str]         =   XLMProphetNetTokenizer
       _UpperCamelCase				:					Dict         =   False
       _UpperCamelCase				:					Optional[int]         =   True

       def lowercase (		self: Any		)       -> str:




              """simple docstring"""


              super().setUp()

              # We have a SentencePiece fixture for testing
              UpperCamelCase_          =  XLMProphetNetTokenizer(_SCREAMING_SNAKE_CASE					,   keep_accents=_SCREAMING_SNAKE_CASE		)
              tokenizer.save_pretrained(self.tmpdirname		)

       def lowercase (		self: List[str]		)       -> Optional[int]:




              """simple docstring"""


              UpperCamelCase_          =  "[PAD]"
              UpperCamelCase_          =  0

              self.assertEqual(self.get_tokenizer()._convert_token_to_id(_SCREAMING_SNAKE_CASE		)					,   _SCREAMING_SNAKE_CASE		)
              self.assertEqual(self.get_tokenizer()._convert_id_to_token(_SCREAMING_SNAKE_CASE		)					,   _SCREAMING_SNAKE_CASE		)

       def lowercase (		self: Optional[Any]		)       -> List[str]:




              """simple docstring"""


              UpperCamelCase_          =  list(self.get_tokenizer().get_vocab().keys()		)

              self.assertEqual(vocab_keys[0]					,   "[PAD]"		)
              self.assertEqual(vocab_keys[1]					,   "[CLS]"		)
              self.assertEqual(vocab_keys[-1]					,   "j"		)
              self.assertEqual(len(_SCREAMING_SNAKE_CASE		)					,   1012		)

       def lowercase (		self: str		)       -> List[Any]:




              """simple docstring"""


              self.assertEqual(self.get_tokenizer().vocab_size					,   1012		)

       def lowercase (		self: List[Any]		)       -> str:




              """simple docstring"""


              UpperCamelCase_          =  XLMProphetNetTokenizer(_SCREAMING_SNAKE_CASE					,   keep_accents=_SCREAMING_SNAKE_CASE		)

              UpperCamelCase_          =  tokenizer.tokenize("This is a test"		)
              self.assertListEqual(_SCREAMING_SNAKE_CASE					,   ["▁This", "▁is", "▁a", "▁t", "est"]		)

              self.assertListEqual(
                  tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE		)					,   [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]					,   )

              UpperCamelCase_          =  tokenizer.tokenize("I was born in 92000, and this is falsé."		)
              self.assertListEqual(
                  _SCREAMING_SNAKE_CASE					,   [
                      SPIECE_UNDERLINE + "I",
                      SPIECE_UNDERLINE + "was",
                      SPIECE_UNDERLINE + "b",
                      "or",
                      "n",
                      SPIECE_UNDERLINE + "in",
                      SPIECE_UNDERLINE + "",
                      "9",
                      "2",
                      "0",
                      "0",
                      "0",
                      ",",
                      SPIECE_UNDERLINE + "and",
                      SPIECE_UNDERLINE + "this",
                      SPIECE_UNDERLINE + "is",
                      SPIECE_UNDERLINE + "f",
                      "al",
                      "s",
                      "é",
                      ".",
                  ]					,   )
              UpperCamelCase_          =  tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE		)
              self.assertListEqual(
                  _SCREAMING_SNAKE_CASE					,   [
                      value + tokenizer.fairseq_offset
                      for value in [8, 21, 84, 55, 24, 19, 7, -9, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, -9, 4]
                  ]					,   )

              UpperCamelCase_          =  tokenizer.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE		)
              self.assertListEqual(
                  _SCREAMING_SNAKE_CASE					,   [
                      SPIECE_UNDERLINE + "I",
                      SPIECE_UNDERLINE + "was",
                      SPIECE_UNDERLINE + "b",
                      "or",
                      "n",
                      SPIECE_UNDERLINE + "in",
                      SPIECE_UNDERLINE + "",
                      "[UNK]",
                      "2",
                      "0",
                      "0",
                      "0",
                      ",",
                      SPIECE_UNDERLINE + "and",
                      SPIECE_UNDERLINE + "this",
                      SPIECE_UNDERLINE + "is",
                      SPIECE_UNDERLINE + "f",
                      "al",
                      "s",
                      "[UNK]",
                      ".",
                  ]					,   )

       @cached_property
       def lowercase (		self: Dict		)       -> int:




              """simple docstring"""


              return XLMProphetNetTokenizer.from_pretrained("microsoft/xprophetnet-large-wiki100-cased"		)

       @slow
       def lowercase (		self: Union[str, Any]		)       -> int:




              """simple docstring"""


              UpperCamelCase_          =  "Hello World!"
              UpperCamelCase_          =  [35389, 6672, 49, 2]
              self.assertListEqual(_SCREAMING_SNAKE_CASE					,   self.big_tokenizer.encode(_SCREAMING_SNAKE_CASE		)		)



       @slow
       def lowercase (		self: Any		)       -> str:




              """simple docstring"""


              UpperCamelCase_          =  {"input_ids": [[11073, 82783, 18, 26, 82783, 549, 51540, 248, 17209, 1301, 217, 20, 215186, 1325, 147, 17209, 1301, 217, 20, 56370, 53, 122020, 20, 16477, 27, 87355, 4548, 20, 4728, 78392, 17, 159969, 18, 26, 24491, 629, 15, 538, 22704, 5439, 15, 2788, 24491, 9885, 15, 43534, 605, 15, 814, 18403, 33200, 29, 15, 43534, 24458, 12410, 111, 24966, 83669, 9637, 144068, 26, 850, 22346, 27, 147, 24966, 83669, 83490, 26, 39113, 735, 27, 689, 656, 2800, 1339, 4600, 53, 122020, 115785, 34, 816, 1339, 46887, 18, 147, 53905, 1951, 42238, 41170, 17732, 834, 436, 15, 27523, 98733, 217, 147, 5542, 4981, 930, 17347, 16, 2], [20091, 629, 94, 82786, 58, 490, 20, 1528, 84, 53905, 344, 80592, 110128, 18822, 5267, 1306, 62, 152537, 308, 7997, 401, 124427, 549, 35442, 225, 109, 15055, 25748, 147, 7119, 43712, 34, 767, 135366, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [592, 63784, 119466, 17, 147808, 88214, 18, 656, 81, 32, 3296, 10280, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
              # fmt: on

              self.tokenizer_integration_test_util(
                  expected_encoding=_SCREAMING_SNAKE_CASE					,   model_name="microsoft/xprophetnet-large-wiki100-cased"					,   revision="1acad1643ddd54a44df6a1b797ada8373685d90e"					,   )








from typing import Optional

import pyspark

from .. import Features, NamedSplit
from ..download import DownloadMode
from ..packaged_modules.spark.spark import Spark
from .abc import AbstractDatasetReader



class 							_UpperCamelCase					(	lowerCAmelCase_     ):

       def __init__(		self: List[Any]					,   _SCREAMING_SNAKE_CASE: pyspark.sql.DataFrame					,   _SCREAMING_SNAKE_CASE: Optional[NamedSplit] = None					,   _SCREAMING_SNAKE_CASE: Optional[Features] = None					,   _SCREAMING_SNAKE_CASE: bool = True					,   _SCREAMING_SNAKE_CASE: str = None					,   _SCREAMING_SNAKE_CASE: bool = False					,   _SCREAMING_SNAKE_CASE: str = None					,   _SCREAMING_SNAKE_CASE: bool = True					,   _SCREAMING_SNAKE_CASE: str = "arrow"					,   **_SCREAMING_SNAKE_CASE: Union[str, Any]					,   )       -> Any:




              """simple docstring"""


              super().__init__(
                  split=_SCREAMING_SNAKE_CASE					,   features=_SCREAMING_SNAKE_CASE					,   cache_dir=_SCREAMING_SNAKE_CASE					,   keep_in_memory=_SCREAMING_SNAKE_CASE					,   streaming=_SCREAMING_SNAKE_CASE					,   **_SCREAMING_SNAKE_CASE					,   )
              UpperCamelCase_          =  load_from_cache_file
              UpperCamelCase_          =  file_format
              UpperCamelCase_          =  Spark(
                  df=_SCREAMING_SNAKE_CASE					,   features=_SCREAMING_SNAKE_CASE					,   cache_dir=_SCREAMING_SNAKE_CASE					,   working_dir=_SCREAMING_SNAKE_CASE					,   **_SCREAMING_SNAKE_CASE					,   )



       def lowercase (		self: str		)       -> Optional[Any]:




              """simple docstring"""


              if self.streaming:
                     return self.builder.as_streaming_dataset(split=self.split		)
              UpperCamelCase_          =  None if self._load_from_cache_file else DownloadMode.FORCE_REDOWNLOAD
              self.builder.download_and_prepare(
                  download_mode=_SCREAMING_SNAKE_CASE					,   file_format=self._file_format					,   )
              return self.builder.as_dataset(split=self.split		)

