




# Lint as: python3
import os
import re
import urllib.parse
from pathlib import Path
from typing import Callable, List, Optional, Union
from zipfile import ZipFile

from ..utils.file_utils import cached_path, hf_github_url
from ..utils.logging import get_logger
from ..utils.version import Version


SCREAMING_SNAKE_CASE:     List[str]									= get_logger(__name__)



class    lowercase_		:
						lowerCAmelCase__			      ="dummy_data"
						lowerCAmelCase__			      ="datasets"
						lowerCAmelCase__			      =False



						def __init__(			self	:   List[str]					,							snake_case__	:   str					,							snake_case__	:   str					,							snake_case__	:   Union[Version, str]					,							snake_case__	:   Optional[str] = None					,							snake_case__	:   bool = False					,							snake_case__	:   bool = True					,							snake_case__	:   Optional[List[Callable]] = None					,							):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							0
								SCREAMING_SNAKE_CASE_					      =							dataset_name
								SCREAMING_SNAKE_CASE_					      =							cache_dir
								SCREAMING_SNAKE_CASE_					      =							use_local_dummy_data
								SCREAMING_SNAKE_CASE_					      =							config
								# download_callbacks take a single url as input
								SCREAMING_SNAKE_CASE_					      =							download_callbacks or []
								# if False, it doesn't load existing files and it returns the paths of the dummy files relative
								# to the dummy_data zip file root
								SCREAMING_SNAKE_CASE_					      =							load_existing_dummy_data

								# TODO(PVP, QL) might need to make this more general
								SCREAMING_SNAKE_CASE_					      =							str(snake_case__  )
								# to be downloaded
								SCREAMING_SNAKE_CASE_					      =							None
								SCREAMING_SNAKE_CASE_					      =							None



						@property
						def 	__a					(			self	:   str  ):

								"""simple docstring"""

								if self._dummy_file is None:
										SCREAMING_SNAKE_CASE_					      =							self.download_dummy_data()
								return self._dummy_file



						@property
						def 	__a					(			self	:   Any  ):

								"""simple docstring"""

								if self.config is not None:
										# structure is dummy / config_name / version_name
										return os.path.join('dummy'					,							self.config.name					,							self.version_name  )
								# structure is dummy / version_name
								return os.path.join('dummy'					,							self.version_name  )



						@property
						def 	__a					(			self	:   Any  ):

								"""simple docstring"""

								return os.path.join(self.dummy_data_folder					,							'dummy_data.zip'  )



						def 	__a					(			self	:   Optional[Any]  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							(
								    self.local_path_to_dummy_data if self.use_local_dummy_data is True else self.github_path_to_dummy_data
								)

								SCREAMING_SNAKE_CASE_					      =							cached_path(
								    snake_case__					,							cache_dir=self.cache_dir					,							extract_compressed_file=snake_case__					,							force_extract=snake_case__  )

								return os.path.join(snake_case__					,							self.dummy_file_name  )



						@property
						def 	__a					(			self	:   str  ):

								"""simple docstring"""

								return os.path.join(self.datasets_scripts_dir					,							self.dataset_name					,							self.dummy_zip_file  )



						@property
						def 	__a					(			self	:   List[Any]  ):

								"""simple docstring"""

								if self._bucket_url is None:
										SCREAMING_SNAKE_CASE_					      =							hf_github_url(self.dataset_name					,							self.dummy_zip_file.replace(os.sep					,							'/'  )  )
								return self._bucket_url



						@property
						def 	__a					(			self	:   Any  ):

								"""simple docstring"""

								if os.path.isdir(self.dummy_file  ):
										return self.dummy_file
								# else cut off path to file -> example `xsum`.
								return "/".join(self.dummy_file.replace(os.sep					,							'/'  ).split('/'  )[:-1]  )



						def 	__a					(			self	:   Optional[int]					,							snake_case__	:   Any					,							*snake_case__	:   Optional[int]  ):

								"""simple docstring"""

								if self.load_existing_dummy_data:
										# dummy data is downloaded and tested
										SCREAMING_SNAKE_CASE_					      =							self.dummy_file
								else:
										# dummy data cannot be downloaded and only the path to dummy file is returned
										SCREAMING_SNAKE_CASE_					      =							self.dummy_file_name

								# special case when data_url is a dict
								if isinstance(snake_case__					,							snake_case__  ):
										return self.create_dummy_data_dict(snake_case__					,							snake_case__  )
								elif isinstance(snake_case__					,							(list, tuple)  ):
										return self.create_dummy_data_list(snake_case__					,							snake_case__  )
								else:
										return self.create_dummy_data_single(snake_case__					,							snake_case__  )



						def 	__a					(			self	:   List[Any]					,							snake_case__	:   Dict					,							*snake_case__	:   List[Any]  ):

								"""simple docstring"""

								return self.download_and_extract(snake_case__  )



						def 	__a					(			self	:   int					,							snake_case__	:   List[Any]					,							snake_case__	:   int  ):

								"""simple docstring"""

								return self.download_and_extract(snake_case__  )



						def 	__a					(			self	:   List[Any]					,							snake_case__	:   List[Any]					,							*snake_case__	:   Any					,							**snake_case__	:   Tuple  ):

								"""simple docstring"""

								return path



						def 	__a					(			self	:   List[str]  ):

								"""simple docstring"""

								return {}



						def 	__a					(			self	:   Tuple					,							snake_case__	:   List[Any]					,							snake_case__	:   List[Any]  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							{}
								for key, single_urls in data_url.items():
										for download_callback in self.download_callbacks:
												if isinstance(snake_case__					,							snake_case__  ):
														for single_url in single_urls:
																download_callback(snake_case__  )
												else:
														SCREAMING_SNAKE_CASE_					      =							single_urls
														download_callback(snake_case__  )
            # we force the name of each key to be the last file / folder name of the url path
            # if the url has arguments, we need to encode them with urllib.parse.quote_plus
										if isinstance(snake_case__					,							snake_case__  ):
												SCREAMING_SNAKE_CASE_					      =							[os.path.join(snake_case__					,							urllib.parse.quote_plus(Path(snake_case__  ).name  )  ) for x in single_urls]
										else:
												SCREAMING_SNAKE_CASE_					      =							single_urls
												SCREAMING_SNAKE_CASE_					      =							os.path.join(snake_case__					,							urllib.parse.quote_plus(Path(snake_case__  ).name  )  )
										SCREAMING_SNAKE_CASE_					      =							value

								# make sure that values are unique
								if all(isinstance(snake_case__					,							snake_case__  ) for i in dummy_data_dict.values()  ) and len(set(dummy_data_dict.values()  )  ) < len(
								    dummy_data_dict.values()  ):
										# append key to value to make its name unique
										SCREAMING_SNAKE_CASE_					      =							{key: value + key for key, value in dummy_data_dict.items()}

								return dummy_data_dict



						def 	__a					(			self	:   Tuple					,							snake_case__	:   List[str]					,							snake_case__	:   Union[str, Any]  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							[]
								# trick: if there are many shards named like `data.txt-000001-of-00300`, only use the first one
								SCREAMING_SNAKE_CASE_					      =							all(bool(re.findall('[0-9]{3,}-of-[0-9]{3,}'					,							snake_case__  )  ) for url in data_url  )
								SCREAMING_SNAKE_CASE_					      =							all(
								    url.startswith('https://ftp.ncbi.nlm.nih.gov/pubmed/baseline/pubmed'  ) for url in data_url  )
								if data_url and (is_tf_records or is_pubmed_records):
										SCREAMING_SNAKE_CASE_					      =							[data_url[0]] * len(snake_case__  )
								for single_url in data_url:
										for download_callback in self.download_callbacks:
												download_callback(snake_case__  )
										# we force the name of each key to be the last file / folder name of the url path
										# if the url has arguments, we need to encode them with urllib.parse.quote_plus
										SCREAMING_SNAKE_CASE_					      =							os.path.join(snake_case__					,							urllib.parse.quote_plus(single_url.split('/'  )[-1]  )  )
										dummy_data_list.append(snake_case__  )
								return dummy_data_list



						def 	__a					(			self	:   Union[str, Any]					,							snake_case__	:   Optional[Any]					,							snake_case__	:   Tuple  ):

								"""simple docstring"""

								for download_callback in self.download_callbacks:
										download_callback(snake_case__  )
								# we force the name of each key to be the last file / folder name of the url path
								# if the url has arguments, we need to encode them with urllib.parse.quote_plus
								SCREAMING_SNAKE_CASE_					      =							os.path.join(snake_case__					,							urllib.parse.quote_plus(data_url.split('/'  )[-1]  )  )
								if os.path.exists(snake_case__  ) or not self.load_existing_dummy_data:
										return value
								else:
										# Backward compatibility, maybe deprecate at one point.
										# For many datasets with single url calls to dl_manager.download_and_extract,
										# the dummy_data.zip file is actually the zipped downloaded file
										# while now we expected the dummy_data.zip file to be a directory containing
										# the downloaded file.
										return path_to_dummy_data



						def 	__a					(			self	:   List[str]  ):

								"""simple docstring"""

								pass



						def 	__a					(			self	:   List[Any]  ):

								"""simple docstring"""

								pass



						def 	__a					(			self	:   Union[str, Any]					,							snake_case__	:   Any  ):

								"""simple docstring"""

								def _iter_archive_members(snake_case__	:   int  ):
										# this preserves the order of the members inside the ZIP archive
										SCREAMING_SNAKE_CASE_					      =							Path(self.dummy_file  ).parent
										SCREAMING_SNAKE_CASE_					      =							path.relative_to(snake_case__  )
										with ZipFile(self.local_path_to_dummy_data  ) as zip_file:
												SCREAMING_SNAKE_CASE_					      =							zip_file.namelist()
										for member in members:
												if member.startswith(relative_path.as_posix()  ):
														yield dummy_parent_path.joinpath(snake_case__  )

								SCREAMING_SNAKE_CASE_					      =							Path(snake_case__  )
								SCREAMING_SNAKE_CASE_					      =							_iter_archive_members(snake_case__  ) if self.use_local_dummy_data else path.rglob('*'  )
								for file_path in file_paths:
										if file_path.is_file() and not file_path.name.startswith(('.', '__')  ):
												yield file_path.relative_to(snake_case__  ).as_posix(), file_path.open('rb'  )






						def 	__a					(			self	:   Union[str, Any]					,							snake_case__	:   Optional[Any]  ):

								"""simple docstring"""

								if not isinstance(snake_case__					,							snake_case__  ):
										SCREAMING_SNAKE_CASE_					      =							[paths]
								for path in paths:
										if os.path.isfile(snake_case__  ):
												if os.path.basename(snake_case__  ).startswith(('.', '__')  ):
														return
												yield path
										else:
												for dirpath, dirnames, filenames in os.walk(snake_case__  ):
														if os.path.basename(snake_case__  ).startswith(('.', '__')  ):
																continue
														dirnames.sort()
														for filename in sorted(snake_case__  ):
																if filename.startswith(('.', '__')  ):
																		continue
																yield os.path.join(snake_case__					,							snake_case__  )





from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
from google.protobuf.internal import builder as _builder


# @@protoc_insertion_point(imports)

SCREAMING_SNAKE_CASE:     str									= _symbol_database.Default()


SCREAMING_SNAKE_CASE:     Dict									= _descriptor_pool.Default().AddSerializedFile(
    b'''\n\x19sentencepiece_model.proto\x12\rsentencepiece"\x80\x0c\n\x0bTrainerSpec\x12\r\n\x05input\x18\x01 \x03(\t\x12\x14\n\x0cinput_format\x18\x07 \x01(\t\x12\x14\n\x0cmodel_prefix\x18\x02 \x01(\t\x12\x41\n\nmodel_type\x18\x03 \x01(\x0e\x32$.sentencepiece.TrainerSpec.ModelType:\x07UNIGRAM\x12\x18\n\nvocab_size\x18\x04 \x01(\x05:\x04\x38\x30\x30\x30\x12\x17\n\x0f\x61\x63\x63\x65pt_language\x18\x05 \x03(\t\x12 \n\x15self_test_sample_size\x18\x06 \x01(\x05:\x01\x30\x12*\n\x1b\x65nable_differential_privacy\x18\x32 \x01(\x08:\x05\x66\x61lse\x12+\n differential_privacy_noise_level\x18\x33 \x01(\x02:\x01\x30\x12\x32\n\'differential_privacy_clipping_threshold\x18\x34 \x01(\x04:\x01\x30\x12"\n\x12\x63haracter_coverage\x18\n \x01(\x02:\x06\x30.9995\x12\x1e\n\x13input_sentence_size\x18\x0b \x01(\x04:\x01\x30\x12$\n\x16shuffle_input_sentence\x18\x13 \x01(\x08:\x04true\x12 \n\x14mining_sentence_size\x18\x0c \x01(\x05\x42\x02\x18\x01\x12"\n\x16training_sentence_size\x18\r \x01(\x05\x42\x02\x18\x01\x12(\n\x17seed_sentencepiece_size\x18\x0e \x01(\x05:\x07\x31\x30\x30\x30\x30\x30\x30\x12\x1e\n\x10shrinking_factor\x18\x0f \x01(\x02:\x04\x30.75\x12!\n\x13max_sentence_length\x18\x12 \x01(\x05:\x04\x34\x31\x39\x32\x12\x17\n\x0bnum_threads\x18\x10 \x01(\x05:\x02\x31\x36\x12\x1d\n\x12num_sub_iterations\x18\x11 \x01(\x05:\x01\x32\x12$\n\x18max_sentencepiece_length\x18\x14 \x01(\x05:\x02\x31\x36\x12%\n\x17split_by_unicode_script\x18\x15 \x01(\x08:\x04true\x12\x1d\n\x0fsplit_by_number\x18\x17 \x01(\x08:\x04true\x12!\n\x13split_by_whitespace\x18\x16 \x01(\x08:\x04true\x12)\n\x1atreat_whitespace_as_suffix\x18\x18 \x01(\x08:\x05\x66\x61lse\x12+\n\x1c\x61llow_whitespace_only_pieces\x18\x1a \x01(\x08:\x05\x66\x61lse\x12\x1b\n\x0csplit_digits\x18\x19 \x01(\x08:\x05\x66\x61lse\x12#\n\x19pretokenization_delimiter\x18\x35 \x01(\t:\x00\x12\x17\n\x0f\x63ontrol_symbols\x18\x1e \x03(\t\x12\x1c\n\x14user_defined_symbols\x18\x1f \x03(\t\x12\x16\n\x0erequired_chars\x18$ \x01(\t\x12\x1c\n\rbyte_fallback\x18# \x01(\x08:\x05\x66\x61lse\x12+\n\x1dvocabulary_output_piece_score\x18  \x01(\x08:\x04true\x12\x1e\n\x10hard_vocab_limit\x18! \x01(\x08:\x04true\x12\x1c\n\ruse_all_vocab\x18" \x01(\x08:\x05\x66\x61lse\x12\x11\n\x06unk_id\x18( \x01(\x05:\x01\x30\x12\x11\n\x06\x62os_id\x18) \x01(\x05:\x01\x31\x12\x11\n\x06\x65os_id\x18* \x01(\x05:\x01\x32\x12\x12\n\x06pad_id\x18+ \x01(\x05:\x02-1\x12\x18\n\tunk_piece\x18- \x01(\t:\x05<unk>\x12\x16\n\tbos_piece\x18. \x01(\t:\x03<s>\x12\x17\n\teos_piece\x18/ \x01(\t:\x04</s>\x12\x18\n\tpad_piece\x18\x30 \x01(\t:\x05<pad>\x12\x1a\n\x0bunk_surface\x18, \x01(\t:\x05 \xe2\x81\x87 \x12+\n\x1ctrain_extremely_large_corpus\x18\x31 \x01(\x08:\x05\x66\x61lse"5\n\tModelType\x12\x0b\n\x07UNIGRAM\x10\x01\x12\x07\n\x03\x42PE\x10\x02\x12\x08\n\x04WORD\x10\x03\x12\x08\n\x04\x43HAR\x10\x04*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02"\xd1\x01\n\x0eNormalizerSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x1c\n\x14precompiled_charsmap\x18\x02 \x01(\x0c\x12\x1e\n\x10\x61\x64\x64_dummy_prefix\x18\x03 \x01(\x08:\x04true\x12&\n\x18remove_extra_whitespaces\x18\x04 \x01(\x08:\x04true\x12 \n\x12\x65scape_whitespaces\x18\x05 \x01(\x08:\x04true\x12\x1e\n\x16normalization_rule_tsv\x18\x06 \x01(\t*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02"y\n\x0cSelfTestData\x12\x33\n\x07samples\x18\x01 \x03(\x0b\x32".sentencepiece.SelfTestData.Sample\x1a)\n\x06Sample\x12\r\n\x05input\x18\x01 \x01(\t\x12\x10\n\x08\x65xpected\x18\x02 \x01(\t*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02"\xfe\x03\n\nModelProto\x12\x37\n\x06pieces\x18\x01 \x03(\x0b\x32\'.sentencepiece.ModelProto.SentencePiece\x12\x30\n\x0ctrainer_spec\x18\x02 \x01(\x0b\x32\x1a.sentencepiece.TrainerSpec\x12\x36\n\x0fnormalizer_spec\x18\x03 \x01(\x0b\x32\x1d.sentencepiece.NormalizerSpec\x12\x33\n\x0eself_test_data\x18\x04 \x01(\x0b\x32\x1b.sentencepiece.SelfTestData\x12\x38\n\x11\x64\x65normalizer_spec\x18\x05 \x01(\x0b\x32\x1d.sentencepiece.NormalizerSpec\x1a\xd2\x01\n\rSentencePiece\x12\r\n\x05piece\x18\x01 \x01(\t\x12\r\n\x05score\x18\x02 \x01(\x02\x12\x42\n\x04type\x18\x03 \x01(\x0e\x32,.sentencepiece.ModelProto.SentencePiece.Type:\x06NORMAL"T\n\x04Type\x12\n\n\x06NORMAL\x10\x01\x12\x0b\n\x07UNKNOWN\x10\x02\x12\x0b\n\x07\x43ONTROL\x10\x03\x12\x10\n\x0cUSER_DEFINED\x10\x04\x12\x08\n\x04\x42YTE\x10\x06\x12\n\n\x06UNUSED\x10\x05*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\x42\x02H\x03'''
)

SCREAMING_SNAKE_CASE:     Optional[Any]									= globals()
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, '''sentencepiece_model_pb2''', _globals)
if _descriptor._USE_C_DESCRIPTORS is False:
						SCREAMING_SNAKE_CASE:     List[Any]									= None
						SCREAMING_SNAKE_CASE:     str									= b'''H\003'''
						# (generated by protobuf compiler, but `_TRAINERSPEC` is not defined)
						# _TRAINERSPEC.fields_by_name["mining_sentence_size"]._options = None
						# _TRAINERSPEC.fields_by_name["mining_sentence_size"]._serialized_options = b"\030\001"
						# _TRAINERSPEC.fields_by_name["training_sentence_size"]._options = None
						# _TRAINERSPEC.fields_by_name["training_sentence_size"]._serialized_options = b"\030\001"
						SCREAMING_SNAKE_CASE:     List[Any]									= 4_5
						SCREAMING_SNAKE_CASE:     List[Any]									= 1_5_8_1
						SCREAMING_SNAKE_CASE:     Union[str, Any]									= 1_5_1_7
						SCREAMING_SNAKE_CASE:     Any									= 1_5_7_0
						SCREAMING_SNAKE_CASE:     Union[str, Any]									= 1_5_8_4
						SCREAMING_SNAKE_CASE:     str									= 1_7_9_3
						SCREAMING_SNAKE_CASE:     List[Any]									= 1_7_9_5
						SCREAMING_SNAKE_CASE:     Any									= 1_9_1_6
						SCREAMING_SNAKE_CASE:     Optional[Any]									= 1_8_6_4
						SCREAMING_SNAKE_CASE:     Union[str, Any]									= 1_9_0_5
						SCREAMING_SNAKE_CASE:     str									= 1_9_1_9
						SCREAMING_SNAKE_CASE:     Any									= 2_4_2_9
						SCREAMING_SNAKE_CASE:     Dict									= 2_2_0_8
						SCREAMING_SNAKE_CASE:     Optional[Any]									= 2_4_1_8
						SCREAMING_SNAKE_CASE:     Optional[Any]									= 2_3_2_3
						SCREAMING_SNAKE_CASE:     Tuple									= 2_4_0_7
# @@protoc_insertion_point(module_scope)