


"""simple docstring"""
import os
import unittest

from transformers.models.transfo_xl.tokenization_transfo_xl import VOCAB_FILES_NAMES, TransfoXLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin




class 	lowercase__				(					A_ ,unittest.TestCase				):
   __UpperCAmelCase							       =					TransfoXLTokenizer
   __UpperCAmelCase							       =					False
   __UpperCAmelCase							       =					False





   def 		UpperCamelCase_  (							self)  ->  List[Any]:
      super().setUp()

      _lowerCamelCase	:					List[Any]   =		[
          """<unk>""",
          """[CLS]""",
          """[SEP]""",
          """want""",
          """unwanted""",
          """wa""",
          """un""",
          """running""",
          """,""",
          """low""",
          """l""",
      ]
      _lowerCamelCase	:					Optional[Any]   =		os.path.join(self.tmpdirname      ,   VOCAB_FILES_NAMES["""vocab_file"""])
      with open(self.vocab_file      ,   """w"""      ,   encoding="""utf-8""") as vocab_writer:
         vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]))





   def 		UpperCamelCase_  (							self      ,   **SCREAMING_SNAKE_CASE)  ->  Optional[int]:
      _lowerCamelCase	:					Union[str, Any]   =		True
      return TransfoXLTokenizer.from_pretrained(self.tmpdirname      ,   **SCREAMING_SNAKE_CASE)





   def 		UpperCamelCase_  (							self      ,   SCREAMING_SNAKE_CASE)  ->  str:
      _lowerCamelCase	:					Optional[int]   =		"""<unk> UNwanted , running"""
      _lowerCamelCase	:					int   =		"""<unk> unwanted, running"""
      return input_text, output_text





   def 		UpperCamelCase_  (							self)  ->  int:
      _lowerCamelCase	:					Optional[Any]   =		TransfoXLTokenizer(vocab_file=self.vocab_file      ,   lower_case=SCREAMING_SNAKE_CASE)

      _lowerCamelCase	:					List[str]   =		tokenizer.tokenize("""<unk> UNwanted , running""")
      self.assertListEqual(SCREAMING_SNAKE_CASE      ,   ["""<unk>""", """unwanted""", """,""", """running"""])

      self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE)      ,   [0, 4, 8, 7])





   def 		UpperCamelCase_  (							self)  ->  Optional[Any]:
      _lowerCamelCase	:					Tuple   =		TransfoXLTokenizer(lower_case=SCREAMING_SNAKE_CASE)

      self.assertListEqual(
          tokenizer.tokenize(""" \tHeLLo ! how  \n Are yoU ?  """)      ,   ["""hello""", """!""", """how""", """are""", """you""", """?"""])





   def 		UpperCamelCase_  (							self)  ->  Union[str, Any]:
      _lowerCamelCase	:					Optional[Any]   =		TransfoXLTokenizer(lower_case=SCREAMING_SNAKE_CASE)

      self.assertListEqual(
          tokenizer.tokenize(""" \tHeLLo ! how  \n Are yoU ?  """)      ,   ["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""])





   def 		UpperCamelCase_  (							self)  ->  Optional[Any]:
      _lowerCamelCase	:					int   =		TransfoXLTokenizer(lower_case=SCREAMING_SNAKE_CASE)
      _lowerCamelCase	:					Dict   =		"""Hello (bracket) and side-scrolled [and] Henry's $5,000 with 3.34 m. What's up!?"""
      _lowerCamelCase	:					str   =		[
          """Hello""",
          """(""",
          """bracket""",
          """)""",
          """and""",
          """side""",
          """@-@""",
          """scrolled""",
          """[""",
          """and""",
          """]""",
          """Henry""",
          """'s""",
          """$""",
          """5""",
          """@,@""",
          """000""",
          """with""",
          """3""",
          """@.@""",
          """34""",
          """m""",
          """.""",
          """What""",
          """'s""",
          """up""",
          """!""",
          """?""",
      ]

      self.assertListEqual(tokenizer.tokenize(SCREAMING_SNAKE_CASE)      ,   SCREAMING_SNAKE_CASE)

      self.assertEqual(tokenizer.convert_tokens_to_string(SCREAMING_SNAKE_CASE)      ,   SCREAMING_SNAKE_CASE)





   def 		UpperCamelCase_  (							self)  ->  int:
      _lowerCamelCase	:					Optional[int]   =		self.get_tokenizer()
      _lowerCamelCase	:					Any   =		len(SCREAMING_SNAKE_CASE)

      tokenizer.add_tokens(["""new1""", """new2"""])
      tokenizer.move_added_token("""new1"""      ,   1)

      # Check that moved token is not copied (duplicate)
      self.assertEqual(len(SCREAMING_SNAKE_CASE)      ,   original_len + 2)
      # Check that token is moved to specified id
      self.assertEqual(tokenizer.encode("""new1""")      ,   [1])
      self.assertEqual(tokenizer.decode([1])      ,   """new1""")








import os
from pathlib import Path
from unittest.mock import patch

import pytest
import zstandard as zstd

from datasets.download.download_config import DownloadConfig
from datasets.utils.file_utils import (
    OfflineModeIsEnabled,
    cached_path,
    fsspec_get,
    fsspec_head,
    ftp_get,
    ftp_head,
    get_from_cache,
    http_get,
    http_head,
)


lowerCAmelCase							:       Dict					 =	"""\
    Text data.
    Second line of data."""

lowerCAmelCase							:       str					 =	"""file"""






@pytest.fixture(scope='''session'''					)
def 		lowerCAmelCase  (	UpperCamelCase__   :   Union[str, Any]					)      ->    Any:

						"""simple docstring"""





						__SCREAMING_SNAKE_CASE:				Any           =   tmp_path_factory.mktemp('''data'''					) / (FILE_PATH + '''.zstd''')
						__SCREAMING_SNAKE_CASE:				Optional[Any]           =   bytes(UpperCamelCase__       ,			'''utf-8'''					)
						with zstd.open(UpperCamelCase__       ,			'''wb'''					) as f:
												f.write(UpperCamelCase__					)
						return path






@pytest.fixture
def 		lowerCAmelCase  (	UpperCamelCase__   :   Optional[Any]					)      ->    int:

						"""simple docstring"""





						with open(os.path.join(tmpfs.local_root_dir       ,			UpperCamelCase__					)       ,			'''w'''					) as f:
												f.write(UpperCamelCase__					)
						return FILE_PATH






@pytest.mark.parametrize('''compression_format'''       ,			['''gzip''', '''xz''', '''zstd''']					)
def 		lowerCAmelCase  (	UpperCamelCase__   :   Optional[int]       ,			UpperCamelCase__   :   List[str]       ,			UpperCamelCase__   :   Dict       ,			UpperCamelCase__   :   Optional[Any]       ,			UpperCamelCase__   :   Dict       ,			UpperCamelCase__   :   List[Any]					)      ->    Any:

						"""simple docstring"""





						__SCREAMING_SNAKE_CASE:				Dict           =   {'''gzip''': gz_file, '''xz''': xz_file, '''zstd''': zstd_path}
						__SCREAMING_SNAKE_CASE:				List[str]           =   input_paths[compression_format]
						__SCREAMING_SNAKE_CASE:				List[Any]           =   tmp_path / '''cache'''
						__SCREAMING_SNAKE_CASE:				List[str]           =   DownloadConfig(cache_dir=UpperCamelCase__       ,			extract_compressed_file=UpperCamelCase__					)
						__SCREAMING_SNAKE_CASE:				Tuple           =   cached_path(UpperCamelCase__       ,			download_config=UpperCamelCase__					)
						with open(UpperCamelCase__					) as f:
												__SCREAMING_SNAKE_CASE:				Optional[Any]           =   f.read()
						with open(UpperCamelCase__					) as f:
												__SCREAMING_SNAKE_CASE:				Dict           =   f.read()
						assert extracted_file_content == expected_file_content






@pytest.mark.parametrize('''default_extracted'''       ,			[True, False]					)
@pytest.mark.parametrize('''default_cache_dir'''       ,			[True, False]					)
def 		lowerCAmelCase  (	UpperCamelCase__   :   Tuple       ,			UpperCamelCase__   :   List[str]       ,			UpperCamelCase__   :   str       ,			UpperCamelCase__   :   Optional[int]       ,			UpperCamelCase__   :   Optional[Any]					)      ->    Optional[Any]:

						"""simple docstring"""





						__SCREAMING_SNAKE_CASE:				Tuple           =   '''custom_cache'''
						__SCREAMING_SNAKE_CASE:				Optional[int]           =   '''custom_extracted_dir'''
						__SCREAMING_SNAKE_CASE:				List[Any]           =   tmp_path / '''custom_extracted_path'''
						if default_extracted:
												__SCREAMING_SNAKE_CASE:				int           =   ('''downloads''' if default_cache_dir else custom_cache_dir, '''extracted''')
						else:
												monkeypatch.setattr('''datasets.config.EXTRACTED_DATASETS_DIR'''       ,			UpperCamelCase__					)
												monkeypatch.setattr('''datasets.config.EXTRACTED_DATASETS_PATH'''       ,			str(UpperCamelCase__					)					)
												__SCREAMING_SNAKE_CASE:				Union[str, Any]           =   custom_extracted_path.parts[-2:] if default_cache_dir else (custom_cache_dir, custom_extracted_dir)

						__SCREAMING_SNAKE_CASE:				Dict           =   xz_file
						__SCREAMING_SNAKE_CASE:				Optional[Any]           =   (
						    DownloadConfig(extract_compressed_file=UpperCamelCase__					)
						    if default_cache_dir
						    else DownloadConfig(cache_dir=tmp_path / custom_cache_dir       ,			extract_compressed_file=UpperCamelCase__					)
						)
						__SCREAMING_SNAKE_CASE:				int           =   cached_path(UpperCamelCase__       ,			download_config=UpperCamelCase__					)
						assert Path(UpperCamelCase__					).parent.parts[-2:] == expected






def 		lowerCAmelCase  (	UpperCamelCase__   :   Dict					)      ->    Union[str, Any]:

						"""simple docstring"""





						__SCREAMING_SNAKE_CASE:				Optional[Any]           =   str(Path(UpperCamelCase__					).resolve()					)
						assert cached_path(UpperCamelCase__					) == text_file
						# relative path
						__SCREAMING_SNAKE_CASE:				Optional[int]           =   str(Path(UpperCamelCase__					).resolve().relative_to(Path(os.getcwd()					)					)					)
						assert cached_path(UpperCamelCase__					) == text_file






def 		lowerCAmelCase  (	UpperCamelCase__   :   Optional[Any]					)      ->    Tuple:

						"""simple docstring"""





						__SCREAMING_SNAKE_CASE:				List[Any]           =   str(tmp_path.resolve() / '''__missing_file__.txt'''					)
						with pytest.raises(UpperCamelCase__					):
												cached_path(UpperCamelCase__					)
						# relative path
						__SCREAMING_SNAKE_CASE:				int           =   '''./__missing_file__.txt'''
						with pytest.raises(UpperCamelCase__					):
												cached_path(UpperCamelCase__					)






def 		lowerCAmelCase  (	UpperCamelCase__   :   Tuple					)      ->    Union[str, Any]:

						"""simple docstring"""





						__SCREAMING_SNAKE_CASE:				Tuple           =   get_from_cache(F"""tmp://{tmpfs_file}"""					)
						with open(UpperCamelCase__					) as f:
												__SCREAMING_SNAKE_CASE:				Tuple           =   f.read()
						assert output_file_content == FILE_CONTENT






@patch('''datasets.config.HF_DATASETS_OFFLINE'''       ,			UpperCamelCase__					)
def 		lowerCAmelCase  (	)      ->    Any:

						"""simple docstring"""





						with pytest.raises(UpperCamelCase__					):
												cached_path('''https://huggingface.co'''					)






@patch('''datasets.config.HF_DATASETS_OFFLINE'''       ,			UpperCamelCase__					)
def 		lowerCAmelCase  (	UpperCamelCase__   :   int					)      ->    Union[str, Any]:

						"""simple docstring"""





						__SCREAMING_SNAKE_CASE:				Optional[int]           =   tmp_path_factory.mktemp('''data'''					) / '''file.html'''
						with pytest.raises(UpperCamelCase__					):
												http_get('''https://huggingface.co'''       ,			temp_file=UpperCamelCase__					)
						with pytest.raises(UpperCamelCase__					):
												http_head('''https://huggingface.co'''					)






@patch('''datasets.config.HF_DATASETS_OFFLINE'''       ,			UpperCamelCase__					)
def 		lowerCAmelCase  (	UpperCamelCase__   :   Dict					)      ->    List[str]:

						"""simple docstring"""





						__SCREAMING_SNAKE_CASE:				List[Any]           =   tmp_path_factory.mktemp('''data'''					) / '''file.html'''
						with pytest.raises(UpperCamelCase__					):
												ftp_get('''ftp://huggingface.co'''       ,			temp_file=UpperCamelCase__					)
						with pytest.raises(UpperCamelCase__					):
												ftp_head('''ftp://huggingface.co'''					)






@patch('''datasets.config.HF_DATASETS_OFFLINE'''       ,			UpperCamelCase__					)
def 		lowerCAmelCase  (	UpperCamelCase__   :   Optional[int]					)      ->    Tuple:

						"""simple docstring"""





						__SCREAMING_SNAKE_CASE:				Tuple           =   tmp_path_factory.mktemp('''data'''					) / '''file.html'''
						with pytest.raises(UpperCamelCase__					):
												fsspec_get('''s3://huggingface.co'''       ,			temp_file=UpperCamelCase__					)
						with pytest.raises(UpperCamelCase__					):
												fsspec_head('''s3://huggingface.co'''					)



