

import os
from pathlib import Path
from unittest.mock import patch

import pytest
import zstandard as zstd

from datasets.download.download_config import DownloadConfig
from datasets.utils.file_utils import (
    OfflineModeIsEnabled,
    cached_path,
    fsspec_get,
    fsspec_head,
    ftp_get,
    ftp_head,
    get_from_cache,
    http_get,
    http_head,
)


a__												=						'''\
    Text data.
    Second line of data.'''

a__												=						'''file'''



@pytest.fixture(scope='''session'''      )
def   __UpperCAmelCase					(      __a :     Optional[int]      )     ->	Union[str, Any]:

					"""simple docstring"""

					_a      :       int   =						tmp_path_factory.mktemp('''data'''      ) / (FILE_PATH + '''.zstd''')
					_a      :       Optional[int]   =						bytes(__a						,'''utf-8'''      )
					with zstd.open(__a						,'''wb'''      ) as f:
										f.write(__a      )
					return path



@pytest.fixture
def   __UpperCAmelCase					(      __a :     List[str]      )     ->	str:

					"""simple docstring"""

					with open(os.path.join(tmpfs.local_root_dir						,__a      )						,'''w'''      ) as f:
										f.write(__a      )
					return FILE_PATH



@pytest.mark.parametrize('''compression_format'''						,['''gzip''', '''xz''', '''zstd''']      )
def   __UpperCAmelCase					(      __a :     Union[str, Any]						,__a :     Any						,__a :     Optional[Any]						,__a :     str						,__a :     str						,__a :     Optional[int]      )     ->	str:

					"""simple docstring"""

					_a      :       Any   =						{'''gzip''': gz_file, '''xz''': xz_file, '''zstd''': zstd_path}
					_a      :       Union[str, Any]   =						input_paths[compression_format]
					_a      :       Dict   =						tmp_path / '''cache'''
					_a      :       List[str]   =						DownloadConfig(cache_dir=__a						,extract_compressed_file=__a      )
					_a      :       str   =						cached_path(__a						,download_config=__a      )
					with open(__a      ) as f:
										_a      :       List[Any]   =						f.read()
					with open(__a      ) as f:
										_a      :       Any   =						f.read()
					assert extracted_file_content == expected_file_content



@pytest.mark.parametrize('''default_extracted'''						,[True, False]      )
@pytest.mark.parametrize('''default_cache_dir'''						,[True, False]      )
def   __UpperCAmelCase					(      __a :     str						,__a :     int						,__a :     str						,__a :     Dict						,__a :     Tuple      )     ->	int:

					"""simple docstring"""

					_a      :       Union[str, Any]   =						'''custom_cache'''
					_a      :       Union[str, Any]   =						'''custom_extracted_dir'''
					_a      :       Union[str, Any]   =						tmp_path / '''custom_extracted_path'''
					if default_extracted:
										_a      :       List[Any]   =						('''downloads''' if default_cache_dir else custom_cache_dir, '''extracted''')
					else:
										monkeypatch.setattr('''datasets.config.EXTRACTED_DATASETS_DIR'''						,__a      )
										monkeypatch.setattr('''datasets.config.EXTRACTED_DATASETS_PATH'''						,str(__a      )      )
										_a      :       Tuple   =						custom_extracted_path.parts[-2:] if default_cache_dir else (custom_cache_dir, custom_extracted_dir)

					_a      :       Dict   =						xz_file
					_a      :       int   =						(
					    DownloadConfig(extract_compressed_file=__a      )
					    if default_cache_dir
					    else DownloadConfig(cache_dir=tmp_path / custom_cache_dir						,extract_compressed_file=__a      )
					)
					_a      :       int   =						cached_path(__a						,download_config=__a      )
					assert Path(__a      ).parent.parts[-2:] == expected



def   __UpperCAmelCase					(      __a :     List[Any]      )     ->	Union[str, Any]:

					"""simple docstring"""

					_a      :       Optional[Any]   =						str(Path(__a      ).resolve()      )
					assert cached_path(__a      ) == text_file
					# relative path
					_a      :       Optional[int]   =						str(Path(__a      ).resolve().relative_to(Path(os.getcwd()      )      )      )
					assert cached_path(__a      ) == text_file



def   __UpperCAmelCase					(      __a :     Optional[Any]      )     ->	Dict:

					"""simple docstring"""

					_a      :       Union[str, Any]   =						str(tmp_path.resolve() / '''__missing_file__.txt'''      )
					with pytest.raises(__a      ):
										cached_path(__a      )
					# relative path
					_a      :       Tuple   =						'''./__missing_file__.txt'''
					with pytest.raises(__a      ):
										cached_path(__a      )



def   __UpperCAmelCase					(      __a :     Union[str, Any]      )     ->	List[str]:

					"""simple docstring"""

					_a      :       Tuple   =						get_from_cache(F"""tmp://{tmpfs_file}"""      )
					with open(__a      ) as f:
										_a      :       List[Any]   =						f.read()
					assert output_file_content == FILE_CONTENT



@patch('''datasets.config.HF_DATASETS_OFFLINE'''						,__a      )
def   __UpperCAmelCase					(      )     ->	Union[str, Any]:

					"""simple docstring"""

					with pytest.raises(__a      ):
										cached_path('''https://huggingface.co'''      )



@patch('''datasets.config.HF_DATASETS_OFFLINE'''						,__a      )
def   __UpperCAmelCase					(      __a :     Any      )     ->	int:

					"""simple docstring"""

					_a      :       Optional[int]   =						tmp_path_factory.mktemp('''data'''      ) / '''file.html'''
					with pytest.raises(__a      ):
										http_get('''https://huggingface.co'''						,temp_file=__a      )
					with pytest.raises(__a      ):
										http_head('''https://huggingface.co'''      )



@patch('''datasets.config.HF_DATASETS_OFFLINE'''						,__a      )
def   __UpperCAmelCase					(      __a :     Tuple      )     ->	Tuple:

					"""simple docstring"""

					_a      :       Optional[Any]   =						tmp_path_factory.mktemp('''data'''      ) / '''file.html'''
					with pytest.raises(__a      ):
										ftp_get('''ftp://huggingface.co'''						,temp_file=__a      )
					with pytest.raises(__a      ):
										ftp_head('''ftp://huggingface.co'''      )







@patch('''datasets.config.HF_DATASETS_OFFLINE'''						,__a      )
def   __UpperCAmelCase					(      __a :     Dict      )     ->	Dict:

					"""simple docstring"""

					_a      :       Tuple   =						tmp_path_factory.mktemp('''data'''      ) / '''file.html'''
					with pytest.raises(__a      ):
										fsspec_get('''s3://huggingface.co'''						,temp_file=__a      )
					with pytest.raises(__a      ):
										fsspec_head('''s3://huggingface.co'''      )






import os
import unittest

from transformers import LxmertTokenizer, LxmertTokenizerFast
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 							UpperCAmelCase_						(     __lowercase							,			unittest.TestCase  ):
							"""simple docstring"""







							UpperCAmelCase__			:		Any             =       LxmertTokenizer
							UpperCAmelCase__			:		Optional[Any]             =       LxmertTokenizerFast
							UpperCAmelCase__			:		Any             =       True
							UpperCAmelCase__			:		Dict             =       True


							def 							__lowercase							(     self			)       ->       Union[str, Any]:
												super().setUp()

												_a      :       int   =						[
												    '''[UNK]''',
												    '''[CLS]''',
												    '''[SEP]''',
												    '''want''',
												    '''##want''',
												    '''##ed''',
												    '''wa''',
												    '''un''',
												    '''runn''',
												    '''##ing''',
												    ''',''',
												    '''low''',
												    '''lowest''',
												]
												_a      :       Any   =						os.path.join(self.tmpdirname			,				VOCAB_FILES_NAMES['''vocab_file''']			)
												with open(self.vocab_file			,				'''w'''			,				encoding='''utf-8'''			) as vocab_writer:
																	vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]			)			)


							def 							__lowercase							(     self			,				_a			)       ->       List[str]:
												_a      :       Tuple   =						'''UNwant\u00E9d,running'''
												_a      :       str   =						'''unwanted, running'''
												return input_text, output_text


							def 							__lowercase							(     self			)       ->       List[Any]:
												_a      :       str   =						self.tokenizer_class(self.vocab_file			)

												_a      :       str   =						tokenizer.tokenize('''UNwant\u00E9d,running'''			)
												self.assertListEqual(_a			,				['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']			)
												self.assertListEqual(tokenizer.convert_tokens_to_ids(_a			)			,				[7, 4, 5, 1_0, 8, 9]			)




							def 							__lowercase							(     self			)       ->       List[Any]:
												if not self.test_rust_tokenizer:
																	return

												_a      :       Optional[Any]   =						self.get_tokenizer()
												_a      :       str   =						self.get_rust_tokenizer()

												_a      :       Optional[Any]   =						'''I was born in 92000, and this is falsé.'''

												_a      :       Optional[Any]   =						tokenizer.tokenize(_a			)
												_a      :       List[Any]   =						rust_tokenizer.tokenize(_a			)
												self.assertListEqual(_a			,				_a			)

												_a      :       List[Any]   =						tokenizer.encode(_a			,				add_special_tokens=_a			)
												_a      :       Any   =						rust_tokenizer.encode(_a			,				add_special_tokens=_a			)
												self.assertListEqual(_a			,				_a			)

												_a      :       Dict   =						self.get_rust_tokenizer()
												_a      :       Optional[int]   =						tokenizer.encode(_a			)
												_a      :       Dict   =						rust_tokenizer.encode(_a			)
												self.assertListEqual(_a			,				_a			)



