




"""simple docstring"""






import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class 	a__      (							__lowercase       ,						unittest.TestCase	):
						snake_case__							   =			None
						snake_case__							   =			BloomTokenizerFast
						snake_case__							   =			BloomTokenizerFast
						snake_case__							   =			True
						snake_case__							   =			False
						snake_case__							   =			'''tokenizer_file'''
						snake_case__							   =			{'''bos_token''': '''<s>''', '''eos_token''': '''</s>''', '''unk_token''': '''<unk>''', '''pad_token''': '''<pad>'''}





						def    __UpperCamelCase  (				self   :						Optional[int])    ->							Any:




											"""simple docstring"""







											super().setUp()
											_lowerCAmelCase:List[Any]						=	BloomTokenizerFast.from_pretrained('''bigscience/tokenizer''')
											tokenizer.save_pretrained(self.tmpdirname)





						def    __UpperCamelCase  (				self   :						Tuple  ,**a__   :						List[Any])    ->							List[Any]:




											"""simple docstring"""







											kwargs.update(self.special_tokens_map)
											return BloomTokenizerFast.from_pretrained(self.tmpdirname  ,**a__)





						def    __UpperCamelCase  (				self   :						Tuple)    ->							Any:




											"""simple docstring"""







											_lowerCAmelCase:Tuple						=	self.get_rust_tokenizer()

											_lowerCAmelCase:Optional[int]						=	["""The quick brown fox</s>""", """jumps over the lazy dog</s>"""]
											_lowerCAmelCase:Any						=	[[2175, 2_3714, 7_3173, 14_4252, 2], [77, 13_2619, 3478, 368, 10_9586, 3_5433, 2]]

											_lowerCAmelCase:Optional[int]						=	tokenizer.batch_encode_plus(a__)["""input_ids"""]
											self.assertListEqual(a__  ,a__)

											_lowerCAmelCase:List[Any]						=	tokenizer.batch_decode(a__)
											self.assertListEqual(a__  ,a__)





						def    __UpperCamelCase  (				self   :						Optional[int]  ,a__   :						str=6)    ->							int:




											"""simple docstring"""







											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'):
																					_lowerCAmelCase:Optional[Any]						=	self.rust_tokenizer_class.from_pretrained(a__  ,**a__)
																					# tokenizer_r.pad_token = None # Hotfixing padding = None
																					# Simple input
																					_lowerCAmelCase:List[Any]						=	"""This is a simple input"""
																					_lowerCAmelCase:List[Any]						=	["""This is a simple input 1""", """This is a simple input 2"""]
																					_lowerCAmelCase:int						=	("""This is a simple input""", """This is a pair""")
																					_lowerCAmelCase:List[Any]						=	[
																					    ("""This is a simple input 1""", """This is a simple input 2"""),
																					    ("""This is a simple pair 1""", """This is a simple pair 2"""),
																					]

																					# Simple input tests
																					try:
																										tokenizer_r.encode(a__  ,max_length=a__)
																										tokenizer_r.encode_plus(a__  ,max_length=a__)

																										tokenizer_r.batch_encode_plus(a__  ,max_length=a__)
																										tokenizer_r.encode(a__  ,max_length=a__)
																										tokenizer_r.batch_encode_plus(a__  ,max_length=a__)
																					except ValueError:
																										self.fail('''Bloom Tokenizer should be able to deal with padding''')

																					_lowerCAmelCase:Optional[int]						=	None  # Hotfixing padding = None
																					self.assertRaises(a__  ,tokenizer_r.encode  ,a__  ,max_length=a__  ,padding='''max_length''')

																					# Simple input
																					self.assertRaises(a__  ,tokenizer_r.encode_plus  ,a__  ,max_length=a__  ,padding='''max_length''')

																					# Simple input
																					self.assertRaises(
																					    a__  ,tokenizer_r.batch_encode_plus  ,a__  ,max_length=a__  ,padding='''max_length'''  ,)

																					# Pair input
																					self.assertRaises(a__  ,tokenizer_r.encode  ,a__  ,max_length=a__  ,padding='''max_length''')

																					# Pair input
																					self.assertRaises(a__  ,tokenizer_r.encode_plus  ,a__  ,max_length=a__  ,padding='''max_length''')

																					# Pair input
																					self.assertRaises(
																					    a__  ,tokenizer_r.batch_encode_plus  ,a__  ,max_length=a__  ,padding='''max_length'''  ,)





						def    __UpperCamelCase  (				self   :						int)    ->							List[Any]:




											"""simple docstring"""







											_lowerCAmelCase:Union[str, Any]						=	self.get_rust_tokenizer()
											_lowerCAmelCase:List[Any]						=	load_dataset('''xnli'''  ,'''all_languages'''  ,split='''test'''  ,streaming=a__)

											_lowerCAmelCase:str						=	next(iter(a__))["""premise"""]  # pick up one data
											_lowerCAmelCase:Tuple						=	list(sample_data.values())

											_lowerCAmelCase:Optional[Any]						=	list(map(tokenizer.encode  ,a__))
											_lowerCAmelCase:Optional[int]						=	[tokenizer.decode(a__  ,clean_up_tokenization_spaces=a__) for x in output_tokens]
											self.assertListEqual(a__  ,a__)







						def    __UpperCamelCase  (				self   :						Optional[Any])    ->							Optional[int]:




											"""simple docstring"""







											self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map)  ,1)
											self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values())[0])  ,1)













'''simple docstring'''






import json
import os
from pathlib import Path

import pytest

from datasets.download.download_config import DownloadConfig
from datasets.download.download_manager import DownloadManager
from datasets.utils.file_utils import hash_url_to_filename


__UpperCamelCase					    =	"http://www.mocksite.com/file1.txt"
__UpperCamelCase					    =	"\"text\": [\"foo\", \"foo\"]"
__UpperCamelCase					    =	"6d8ce9aa78a471c7477201efbeabd3bb01ac2e7d100a6dc024ba1608361f90a8"




class  _A :
		lowercase__:   str				     =   200
		lowercase__:   List[str]				     =   {'''Content-Length''': '''100'''}
		lowercase__:   Union[str, Any]				     =   {}

		def 	lowercase__     (   self					:   Any  ,      **__magic_name__					:   List[Any]			)					->    Dict:


					"""simple docstring"""

					return [bytes(__magic_name__  ,      """utf-8"""			)]







def 					_a						(       *_lowerCamelCase    ,	**_lowerCamelCase ) ->				List[str]:



			"""simple docstring"""


			return MockResponse()







@pytest.mark.parametrize("""urls_type"""    ,	[str, list, dict] )
def 					_a						(       _lowerCamelCase    ,	_lowerCamelCase    ,	_lowerCamelCase ) ->				List[str]:



			"""simple docstring"""


			import requests

			monkeypatch.setattr(_lowerCamelCase    ,	"""request"""    ,	_lowerCamelCase )

			__snake_case					:      Union[str, Any]							      =     URL
			if issubclass(_lowerCamelCase    ,	_lowerCamelCase ):
						__snake_case					:      str							      =     url
			elif issubclass(_lowerCamelCase    ,	_lowerCamelCase ):
						__snake_case					:      Dict							      =     [url]
			elif issubclass(_lowerCamelCase    ,	_lowerCamelCase ):
						__snake_case					:      Union[str, Any]							      =     {"""train""": url}
			__snake_case					:      Dict							      =     """dummy"""
			__snake_case					:      List[str]							      =     """downloads"""
			__snake_case					:      List[Any]							      =     tmp_path
			__snake_case					:      List[Any]							      =     DownloadConfig(
			    cache_dir=os.path.join(_lowerCamelCase    ,	_lowerCamelCase )    ,	use_etag=_lowerCamelCase    ,	)
			__snake_case					:      List[str]							      =     DownloadManager(dataset_name=_lowerCamelCase    ,	download_config=_lowerCamelCase )
			__snake_case					:      int							      =     dl_manager.download(_lowerCamelCase )
			__snake_case					:      Tuple							      =     urls
			for downloaded_paths in [downloaded_paths]:
						if isinstance(_lowerCamelCase    ,	_lowerCamelCase ):
									__snake_case					:      Any							      =     [downloaded_paths]
									__snake_case					:      List[Any]							      =     [urls]
						elif isinstance(_lowerCamelCase    ,	_lowerCamelCase ):
									assert "train" in downloaded_paths.keys()
									__snake_case					:      Tuple							      =     downloaded_paths.values()
									__snake_case					:      Optional[int]							      =     urls.values()
						assert downloaded_paths
						for downloaded_path, input_url in zip(_lowerCamelCase    ,	_lowerCamelCase ):
									assert downloaded_path == dl_manager.downloaded_paths[input_url]
									__snake_case					:      List[str]							      =     Path(_lowerCamelCase )
									__snake_case					:      Any							      =     downloaded_path.parts
									assert parts[-1] == HASH
									assert parts[-2] == cache_subdir
									assert downloaded_path.exists()
									__snake_case					:      Union[str, Any]							      =     downloaded_path.read_text()
									assert content == CONTENT
									__snake_case					:      List[str]							      =     downloaded_path.with_suffix(""".json""" )
									assert metadata_downloaded_path.exists()
									__snake_case					:      Union[str, Any]							      =     json.loads(metadata_downloaded_path.read_text() )
									assert metadata_content == {"url": URL, "etag": None}







@pytest.mark.parametrize("""paths_type"""    ,	[str, list, dict] )
def 					_a						(       _lowerCamelCase    ,	_lowerCamelCase    ,	_lowerCamelCase ) ->				List[Any]:



			"""simple docstring"""


			__snake_case					:      Any							      =     str(_lowerCamelCase )
			if issubclass(_lowerCamelCase    ,	_lowerCamelCase ):
						__snake_case					:      Optional[int]							      =     filename
			elif issubclass(_lowerCamelCase    ,	_lowerCamelCase ):
						__snake_case					:      Tuple							      =     [filename]
			elif issubclass(_lowerCamelCase    ,	_lowerCamelCase ):
						__snake_case					:      Dict							      =     {"""train""": filename}
			__snake_case					:      Optional[Any]							      =     """dummy"""
			__snake_case					:      List[Any]							      =     xz_file.parent
			__snake_case					:      int							      =     """extracted"""
			__snake_case					:      Dict							      =     DownloadConfig(
			    cache_dir=_lowerCamelCase    ,	use_etag=_lowerCamelCase    ,	)
			__snake_case					:      List[str]							      =     DownloadManager(dataset_name=_lowerCamelCase    ,	download_config=_lowerCamelCase )
			__snake_case					:      Optional[Any]							      =     dl_manager.extract(_lowerCamelCase )
			__snake_case					:      Union[str, Any]							      =     paths
			for extracted_paths in [extracted_paths]:
						if isinstance(_lowerCamelCase    ,	_lowerCamelCase ):
									__snake_case					:      Dict							      =     [extracted_paths]
									__snake_case					:      int							      =     [paths]
						elif isinstance(_lowerCamelCase    ,	_lowerCamelCase ):
									assert "train" in extracted_paths.keys()
									__snake_case					:      int							      =     extracted_paths.values()
									__snake_case					:      int							      =     paths.values()
						assert extracted_paths
						for extracted_path, input_path in zip(_lowerCamelCase    ,	_lowerCamelCase ):
									assert extracted_path == dl_manager.extracted_paths[input_path]
									__snake_case					:      Any							      =     Path(_lowerCamelCase )
									__snake_case					:      str							      =     extracted_path.parts
									assert parts[-1] == hash_url_to_filename(_lowerCamelCase    ,	etag=_lowerCamelCase )
									assert parts[-2] == extracted_subdir
									assert extracted_path.exists()
									__snake_case					:      Optional[int]							      =     extracted_path.read_text()
									__snake_case					:      str							      =     text_file.read_text()
									assert extracted_file_content == expected_file_content







def 					_a						(       _lowerCamelCase    ,	_lowerCamelCase ) ->				Optional[int]:



			"""simple docstring"""


			assert path.endswith(""".jsonl""" )
			for num_items, line in enumerate(_lowerCamelCase    ,	start=1 ):
						__snake_case					:      Tuple							      =     json.loads(line.decode("""utf-8""" ) )
						assert item.keys() == {"col_1", "col_2", "col_3"}
			assert num_items == 4







@pytest.mark.parametrize("""archive_jsonl"""    ,	["""tar_jsonl_path""", """zip_jsonl_path"""] )
def 					_a						(       _lowerCamelCase    ,	_lowerCamelCase ) ->				Optional[int]:



			"""simple docstring"""


			__snake_case					:      Any							      =     request.getfixturevalue(_lowerCamelCase )
			__snake_case					:      str							      =     DownloadManager()
			for num_jsonl, (path, file) in enumerate(dl_manager.iter_archive(_lowerCamelCase )    ,	start=1 ):
						_test_jsonl(_lowerCamelCase    ,	_lowerCamelCase )
			assert num_jsonl == 2







@pytest.mark.parametrize("""archive_nested_jsonl"""    ,	["""tar_nested_jsonl_path""", """zip_nested_jsonl_path"""] )
def 					_a						(       _lowerCamelCase    ,	_lowerCamelCase ) ->				List[str]:



			"""simple docstring"""


			__snake_case					:      int							      =     request.getfixturevalue(_lowerCamelCase )
			__snake_case					:      List[str]							      =     DownloadManager()
			for num_tar, (path, file) in enumerate(dl_manager.iter_archive(_lowerCamelCase )    ,	start=1 ):
						for num_jsonl, (subpath, subfile) in enumerate(dl_manager.iter_archive(_lowerCamelCase )    ,	start=1 ):
									_test_jsonl(_lowerCamelCase    ,	_lowerCamelCase )
			assert num_tar == 1
			assert num_jsonl == 2







def 					_a						(       _lowerCamelCase ) ->				Any:



			"""simple docstring"""


			__snake_case					:      List[str]							      =     DownloadManager()
			for num_file, file in enumerate(dl_manager.iter_files(_lowerCamelCase )    ,	start=1 ):
						assert os.path.basename(_lowerCamelCase ) == ("test.txt" if num_file == 1 else "train.txt")
			assert num_file == 2


