




import json
import os
from pathlib import Path

import pytest

from datasets.download.download_config import DownloadConfig
from datasets.download.download_manager import DownloadManager
from datasets.utils.file_utils import hash_url_to_filename


a__		     =     '''http://www.mocksite.com/file1.txt'''
a__		     =     '''"text": ["foo", "foo"]'''
a__		     =     '''6d8ce9aa78a471c7477201efbeabd3bb01ac2e7d100a6dc024ba1608361f90a8'''


class    SCREAMING_SNAKE_CASE_				:



	"""simple docstring"""


	__magic_name__   :       Tuple      	=		200
	__magic_name__   :       Optional[Any]      	=		{'Content-Length': '100'}
	__magic_name__   :       Union[str, Any]      	=		{}






	def 					lowerCamelCase__     (					self		:						str					,     **lowerCAmelCase		:						Dict	)	->       Optional[Any]:







						"""simple docstring"""







						return [bytes(lowerCAmelCase					,     """utf-8"""	)]






def 				A__ (*snake_case   :		List[Any]		,	**snake_case   :		Dict				)     ->						Union[str, Any]:
					return MockResponse()





@pytest.mark.parametrize("""urls_type"""		,	[str, list, dict]				)
def 				A__ (snake_case   :		Any		,	snake_case   :		Optional[Any]		,	snake_case   :		int				)     ->						Optional[int]:
					import requests

					monkeypatch.setattr(snake_case		,	"""request"""		,	snake_case				)

					__UpperCamelCase					:	List[str]             =      URL
					if issubclass(snake_case		,	snake_case				):
										__UpperCamelCase					:	Tuple             =      url
					elif issubclass(snake_case		,	snake_case				):
										__UpperCamelCase					:	Optional[int]             =      [url]
					elif issubclass(snake_case		,	snake_case				):
										__UpperCamelCase					:	Optional[Any]             =      {"""train""": url}
					__UpperCamelCase					:	Optional[Any]             =      """dummy"""
					__UpperCamelCase					:	List[Any]             =      """downloads"""
					__UpperCamelCase					:	Optional[Any]             =      tmp_path
					__UpperCamelCase					:	List[str]             =      DownloadConfig(
					    cache_dir=os.path.join(snake_case		,	snake_case				)		,	use_etag=snake_case		,	)
					__UpperCamelCase					:	List[str]             =      DownloadManager(dataset_name=snake_case		,	download_config=snake_case				)
					__UpperCamelCase					:	Optional[int]             =      dl_manager.download(snake_case				)
					__UpperCamelCase					:	Any             =      urls
					for downloaded_paths in [downloaded_paths]:
										if isinstance(snake_case		,	snake_case				):
															__UpperCamelCase					:	Optional[Any]             =      [downloaded_paths]
															__UpperCamelCase					:	Any             =      [urls]
										elif isinstance(snake_case		,	snake_case				):
															assert "train" in downloaded_paths.keys()
															__UpperCamelCase					:	int             =      downloaded_paths.values()
															__UpperCamelCase					:	Tuple             =      urls.values()
										assert downloaded_paths
										for downloaded_path, input_url in zip(snake_case		,	snake_case				):
															assert downloaded_path == dl_manager.downloaded_paths[input_url]
															__UpperCamelCase					:	Any             =      Path(snake_case				)
															__UpperCamelCase					:	Dict             =      downloaded_path.parts
															assert parts[-1] == HASH
															assert parts[-2] == cache_subdir
															assert downloaded_path.exists()
															__UpperCamelCase					:	Union[str, Any]             =      downloaded_path.read_text()
															assert content == CONTENT
															__UpperCamelCase					:	List[Any]             =      downloaded_path.with_suffix(""".json"""				)
															assert metadata_downloaded_path.exists()
															__UpperCamelCase					:	Tuple             =      json.loads(metadata_downloaded_path.read_text()				)
															assert metadata_content == {"url": URL, "etag": None}





@pytest.mark.parametrize("""paths_type"""		,	[str, list, dict]				)
def 				A__ (snake_case   :		Any		,	snake_case   :		List[Any]		,	snake_case   :		Any				)     ->						Dict:
					__UpperCamelCase					:	List[Any]             =      str(snake_case				)
					if issubclass(snake_case		,	snake_case				):
										__UpperCamelCase					:	Optional[int]             =      filename
					elif issubclass(snake_case		,	snake_case				):
										__UpperCamelCase					:	Tuple             =      [filename]
					elif issubclass(snake_case		,	snake_case				):
										__UpperCamelCase					:	Optional[int]             =      {"""train""": filename}
					__UpperCamelCase					:	Dict             =      """dummy"""
					__UpperCamelCase					:	List[Any]             =      xz_file.parent
					__UpperCamelCase					:	Optional[int]             =      """extracted"""
					__UpperCamelCase					:	Optional[Any]             =      DownloadConfig(
					    cache_dir=snake_case		,	use_etag=snake_case		,	)
					__UpperCamelCase					:	Tuple             =      DownloadManager(dataset_name=snake_case		,	download_config=snake_case				)
					__UpperCamelCase					:	List[Any]             =      dl_manager.extract(snake_case				)
					__UpperCamelCase					:	Optional[int]             =      paths
					for extracted_paths in [extracted_paths]:
										if isinstance(snake_case		,	snake_case				):
															__UpperCamelCase					:	Dict             =      [extracted_paths]
															__UpperCamelCase					:	Any             =      [paths]
										elif isinstance(snake_case		,	snake_case				):
															assert "train" in extracted_paths.keys()
															__UpperCamelCase					:	Optional[int]             =      extracted_paths.values()
															__UpperCamelCase					:	List[str]             =      paths.values()
										assert extracted_paths
										for extracted_path, input_path in zip(snake_case		,	snake_case				):
															assert extracted_path == dl_manager.extracted_paths[input_path]
															__UpperCamelCase					:	Any             =      Path(snake_case				)
															__UpperCamelCase					:	int             =      extracted_path.parts
															assert parts[-1] == hash_url_to_filename(snake_case		,	etag=snake_case				)
															assert parts[-2] == extracted_subdir
															assert extracted_path.exists()
															__UpperCamelCase					:	List[Any]             =      extracted_path.read_text()
															__UpperCamelCase					:	Optional[Any]             =      text_file.read_text()
															assert extracted_file_content == expected_file_content





def 				A__ (snake_case   :		Any		,	snake_case   :		Optional[int]				)     ->						int:
					assert path.endswith(""".jsonl"""				)
					for num_items, line in enumerate(snake_case		,	start=1				):
										__UpperCamelCase					:	Dict             =      json.loads(line.decode("""utf-8"""				)				)
										assert item.keys() == {"col_1", "col_2", "col_3"}
					assert num_items == 4





@pytest.mark.parametrize("""archive_jsonl"""		,	["""tar_jsonl_path""", """zip_jsonl_path"""]				)
def 				A__ (snake_case   :		Optional[int]		,	snake_case   :		Optional[Any]				)     ->						Any:
					__UpperCamelCase					:	Optional[Any]             =      request.getfixturevalue(snake_case				)
					__UpperCamelCase					:	Dict             =      DownloadManager()
					for num_jsonl, (path, file) in enumerate(dl_manager.iter_archive(snake_case				)		,	start=1				):
										_test_jsonl(snake_case		,	snake_case				)
					assert num_jsonl == 2





@pytest.mark.parametrize("""archive_nested_jsonl"""		,	["""tar_nested_jsonl_path""", """zip_nested_jsonl_path"""]				)
def 				A__ (snake_case   :		Dict		,	snake_case   :		Optional[int]				)     ->						Union[str, Any]:
					__UpperCamelCase					:	int             =      request.getfixturevalue(snake_case				)
					__UpperCamelCase					:	List[str]             =      DownloadManager()
					for num_tar, (path, file) in enumerate(dl_manager.iter_archive(snake_case				)		,	start=1				):
										for num_jsonl, (subpath, subfile) in enumerate(dl_manager.iter_archive(snake_case				)		,	start=1				):
															_test_jsonl(snake_case		,	snake_case				)
					assert num_tar == 1
					assert num_jsonl == 2







def 				A__ (snake_case   :		Optional[Any]				)     ->						List[str]:
					__UpperCamelCase					:	Optional[int]             =      DownloadManager()
					for num_file, file in enumerate(dl_manager.iter_files(snake_case				)		,	start=1				):
										assert os.path.basename(snake_case				) == ("test.txt" if num_file == 1 else "train.txt")
					assert num_file == 2












import warnings
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


a__		     =     logging.get_logger(__name__)


class    SCREAMING_SNAKE_CASE_				(       _UpperCamelCase   ):



	"""simple docstring"""


	__magic_name__   :       int      	=		['input_ids', 'attention_mask']






	def __init__(					self		:						List[Any]					,     lowerCAmelCase		:						Union[str, Any]="</s>"					,     lowerCAmelCase		:						Tuple="<unk>"					,     lowerCAmelCase		:						Union[str, Any]="<pad>"					,     lowerCAmelCase		:						Tuple=125					,     lowerCAmelCase		:						str=None					,     **lowerCAmelCase		:						Optional[Any]					,     )	->       None:







						"""simple docstring"""







						if extra_ids > 0 and additional_special_tokens is None:
											__UpperCamelCase					:	str             =      [F'''<extra_id_{i}>''' for i in range(lowerCAmelCase	)]
						elif extra_ids > 0 and additional_special_tokens is not None:
											# Check that we have the right number of extra_id special tokens
											__UpperCamelCase					:	Tuple             =      len(set(filter(lambda lowerCAmelCase	: bool("""extra_id""" in str(lowerCAmelCase	)	)					,     lowerCAmelCase	)	)	)
											if extra_tokens != extra_ids:
																raise ValueError(
																    F'''Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are'''
																    """ provided to ByT5Tokenizer. In this case the additional_special_tokens must include the"""
																    """ extra_ids tokens"""	)

						__UpperCamelCase					:	Tuple             =      AddedToken(lowerCAmelCase					,     lstrip=lowerCAmelCase					,     rstrip=lowerCAmelCase	) if isinstance(lowerCAmelCase					,     lowerCAmelCase	) else pad_token
						__UpperCamelCase					:	Tuple             =      AddedToken(lowerCAmelCase					,     lstrip=lowerCAmelCase					,     rstrip=lowerCAmelCase	) if isinstance(lowerCAmelCase					,     lowerCAmelCase	) else eos_token
						__UpperCamelCase					:	int             =      AddedToken(lowerCAmelCase					,     lstrip=lowerCAmelCase					,     rstrip=lowerCAmelCase	) if isinstance(lowerCAmelCase					,     lowerCAmelCase	) else unk_token

						super().__init__(
						    eos_token=lowerCAmelCase					,     unk_token=lowerCAmelCase					,     pad_token=lowerCAmelCase					,     extra_ids=lowerCAmelCase					,     additional_special_tokens=lowerCAmelCase					,     **lowerCAmelCase					,     )

						__UpperCamelCase					:	Any             =      extra_ids

						__UpperCamelCase					:	List[Any]             =      2**8  # utf is 8 bits

						# define special tokens dict
						__UpperCamelCase					:	Dict[int, str]             =      {
						    self.pad_token: 0,
						    self.eos_token: 1,
						    self.unk_token: 2,
						}
						__UpperCamelCase					:	int             =      len(self.special_tokens_encoder	)
						__UpperCamelCase					:	Tuple             =      len(lowerCAmelCase	)
						for i, token in enumerate(lowerCAmelCase	):
											__UpperCamelCase					:	Optional[int]             =      self.vocab_size + i - n
						__UpperCamelCase					:	Dict[str, int]             =      {v: k for k, v in self.special_tokens_encoder.items()}






	@property
	def 					lowerCamelCase__     (					self		:						Any	)	->       Optional[int]:







						"""simple docstring"""







						return self._utf_vocab_size + self._num_special_tokens + self._extra_ids






	def 					lowerCamelCase__     (					self		:						Optional[int]					,     lowerCAmelCase		:						List[int]					,     lowerCAmelCase		:						Optional[List[int]] = None					,     lowerCAmelCase		:						bool = False	)	->       List[int]:







						"""simple docstring"""







						if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=lowerCAmelCase					,     token_ids_a=lowerCAmelCase					,     already_has_special_tokens=lowerCAmelCase	)

						# normal case: some special tokens
						if token_ids_a is None:
											return ([0] * len(lowerCAmelCase	)) + [1]
						return ([0] * len(lowerCAmelCase	)) + [1] + ([0] * len(lowerCAmelCase	)) + [1]






	def 					lowerCamelCase__     (					self		:						Tuple					,     lowerCAmelCase		:						List[int]	)	->       List[int]:







						"""simple docstring"""







						if len(lowerCAmelCase	) > 0 and token_ids[-1] == self.eos_token_id:
											warnings.warn(
											    F'''This sequence already has {self.eos_token}. In future versions this behavior may lead to duplicated'''
											    """ eos tokens being added."""	)
											return token_ids
						else:
											return token_ids + [self.eos_token_id]






	def 					lowerCamelCase__     (					self		:						List[Any]					,     lowerCAmelCase		:						List[int]					,     lowerCAmelCase		:						Optional[List[int]] = None	)	->       List[int]:







						"""simple docstring"""







						__UpperCamelCase					:	Any             =      [self.eos_token_id]

						if token_ids_a is None:
											return len(token_ids_a + eos	) * [0]
						return len(token_ids_a + eos + token_ids_a + eos	) * [0]






	def 					lowerCamelCase__     (					self		:						List[str]					,     lowerCAmelCase		:						List[int]					,     lowerCAmelCase		:						Optional[List[int]] = None	)	->       List[int]:







						"""simple docstring"""







						__UpperCamelCase					:	List[str]             =      self._add_eos_if_not_present(lowerCAmelCase	)
						if token_ids_a is None:
											return token_ids_a
						else:
											__UpperCamelCase					:	List[Any]             =      self._add_eos_if_not_present(lowerCAmelCase	)
											return token_ids_a + token_ids_a






	def 					lowerCamelCase__     (					self		:						str					,     lowerCAmelCase		:						str	)	->       List[str]:







						"""simple docstring"""







						__UpperCamelCase					:	Optional[int]             =      [chr(lowerCAmelCase	) for i in text.encode("""utf-8"""	)]
						return tokens






	def 					lowerCamelCase__     (					self		:						Union[str, Any]					,     lowerCAmelCase		:						Optional[int]	)	->       int:







						"""simple docstring"""







						if token in self.special_tokens_encoder:
											__UpperCamelCase					:	Dict             =      self.special_tokens_encoder[token]
						elif token in self.added_tokens_encoder:
											__UpperCamelCase					:	Tuple             =      self.added_tokens_encoder[token]
						elif len(lowerCAmelCase	) != 1:
											__UpperCamelCase					:	Dict             =      self.unk_token_id
						else:
											__UpperCamelCase					:	Union[str, Any]             =      ord(lowerCAmelCase	) + self._num_special_tokens
						return token_id






	def 					lowerCamelCase__     (					self		:						Union[str, Any]					,     lowerCAmelCase		:						Dict	)	->       Optional[Any]:







						"""simple docstring"""







						if index in self.special_tokens_decoder:
											__UpperCamelCase					:	str             =      self.special_tokens_decoder[index]
						else:
											__UpperCamelCase					:	Any             =      chr(index - self._num_special_tokens	)
						return token






	def 					lowerCamelCase__     (					self		:						str					,     lowerCAmelCase		:						List[Any]	)	->       List[Any]:







						"""simple docstring"""







						__UpperCamelCase					:	Union[str, Any]             =      b""""""
						for token in tokens:
											if token in self.special_tokens_decoder:
																__UpperCamelCase					:	Optional[Any]             =      self.special_tokens_decoder[token].encode("""utf-8"""	)
											elif token in self.added_tokens_decoder:
																__UpperCamelCase					:	Optional[int]             =      self.special_tokens_decoder[token].encode("""utf-8"""	)
											elif token in self.special_tokens_encoder:
																__UpperCamelCase					:	Tuple             =      token.encode("""utf-8"""	)
											elif token in self.added_tokens_encoder:
																__UpperCamelCase					:	Tuple             =      token.encode("""utf-8"""	)
											else:
																__UpperCamelCase					:	Union[str, Any]             =      bytes([ord(lowerCAmelCase	)]	)
											bstring += tok_string
						__UpperCamelCase					:	Optional[int]             =      bstring.decode("""utf-8"""					,     errors="""ignore"""	)
						return string






	def 					lowerCamelCase__     (					self		:						Optional[int]					,     lowerCAmelCase		:						str					,     lowerCAmelCase		:						Optional[str] = None	)	->       Tuple[str]:







						"""simple docstring"""







						return ()






