
'''simple docstring'''







from unittest.mock import patch

import pyspark

from datasets.packaged_modules.spark.spark import (
    Spark,
    SparkExamplesIterable,
    _generate_iterable_examples,
)

from ..utils import (
    require_dill_gt_0_3_2,
    require_not_windows,
)
def  snake_case_		(    __SCREAMING_SNAKE_CASE :			Union[str, Any]				,				__SCREAMING_SNAKE_CASE :			Optional[int]							):



						"""simple docstring"""





						lowercase_     :					Optional[int]													=	[]
						for part_id in partition_order:
												lowercase_     :					int													=	df.where(F'''SPARK_PARTITION_ID() = {part_id}'''							).collect()
												for row_idx, row in enumerate(__SCREAMING_SNAKE_CASE							):
																		expected_row_ids_and_row_dicts.append((F'''{part_id}_{row_idx}''', row.asDict())							)
						return expected_row_ids_and_row_dicts
@require_not_windows
@require_dill_gt_0_3_2
def  snake_case_		(    ):



						"""simple docstring"""





						lowercase_     :					Optional[int]													=	pyspark.sql.SparkSession.builder.master('''local[*]'''							).appName('''pyspark'''							).getOrCreate()
						lowercase_     :					Optional[int]													=	spark.range(100							).repartition(1							)
						lowercase_     :					List[Any]													=	Spark(__SCREAMING_SNAKE_CASE							)
						# The id ints will be converted to Pyarrow int64s, so each row will be 8 bytes. Setting a max_shard_size of 16 means
						# that each partition can hold 2 rows.
						spark_builder._repartition_df_if_needed(max_shard_size=16							)
						# Given that the dataframe has 100 rows and each partition has 2 rows, we expect 50 partitions.
						assert spark_builder.df.rdd.getNumPartitions() == 50
@require_not_windows
@require_dill_gt_0_3_2
def  snake_case_		(    ):



						"""simple docstring"""





						lowercase_     :					str													=	pyspark.sql.SparkSession.builder.master('''local[*]'''							).appName('''pyspark'''							).getOrCreate()
						lowercase_     :					Optional[Any]													=	spark.range(10							).repartition(2							)
						lowercase_     :					List[Any]													=	[1, 0]
						lowercase_     :					Optional[int]													=	_generate_iterable_examples(__SCREAMING_SNAKE_CASE				,				__SCREAMING_SNAKE_CASE							)  # Reverse the partitions.
						lowercase_     :					List[str]													=	_get_expected_row_ids_and_row_dicts_for_partition_order(__SCREAMING_SNAKE_CASE				,				__SCREAMING_SNAKE_CASE							)

						for i, (row_id, row_dict) in enumerate(generate_fn()							):
												lowercase_						,			lowercase_     :					Union[str, Any]													=	expected_row_ids_and_row_dicts[i]
												assert row_id == expected_row_id
												assert row_dict == expected_row_dict
@require_not_windows
@require_dill_gt_0_3_2
def  snake_case_		(    ):



						"""simple docstring"""





						lowercase_     :					Union[str, Any]													=	pyspark.sql.SparkSession.builder.master('''local[*]'''							).appName('''pyspark'''							).getOrCreate()
						lowercase_     :					str													=	spark.range(10							).repartition(1							)
						lowercase_     :					List[str]													=	SparkExamplesIterable(__SCREAMING_SNAKE_CASE							)
						assert it.n_shards == 1
						for i, (row_id, row_dict) in enumerate(__SCREAMING_SNAKE_CASE							):
												assert row_id == F'''0_{i}'''
												assert row_dict == {"id": i}
@require_not_windows
@require_dill_gt_0_3_2
def  snake_case_		(    ):



						"""simple docstring"""





						lowercase_     :					int													=	pyspark.sql.SparkSession.builder.master('''local[*]'''							).appName('''pyspark'''							).getOrCreate()
						lowercase_     :					Any													=	spark.range(30							).repartition(3							)
						# Mock the generator so that shuffle reverses the partition indices.
						with patch('''numpy.random.Generator'''							) as generator_mock:
												lowercase_     :					Optional[int]													=	lambda __SCREAMING_SNAKE_CASE							: x.reverse()
												lowercase_     :					Optional[Any]													=	_get_expected_row_ids_and_row_dicts_for_partition_order(__SCREAMING_SNAKE_CASE				,				[2, 1, 0]							)

												lowercase_     :					Any													=	SparkExamplesIterable(__SCREAMING_SNAKE_CASE							).shuffle_data_sources(__SCREAMING_SNAKE_CASE							)
												assert shuffled_it.n_shards == 3
												for i, (row_id, row_dict) in enumerate(__SCREAMING_SNAKE_CASE							):
																		lowercase_						,			lowercase_     :					Tuple													=	expected_row_ids_and_row_dicts[i]
																		assert row_id == expected_row_id
																		assert row_dict == expected_row_dict
@require_not_windows
@require_dill_gt_0_3_2
def  snake_case_		(    ):



						"""simple docstring"""





						lowercase_     :					Tuple													=	pyspark.sql.SparkSession.builder.master('''local[*]'''							).appName('''pyspark'''							).getOrCreate()
						lowercase_     :					Optional[Any]													=	spark.range(20							).repartition(4							)

						# Partitions 0 and 2
						lowercase_     :					str													=	SparkExamplesIterable(__SCREAMING_SNAKE_CASE							).shard_data_sources(worker_id=0				,				num_workers=2							)
						assert shard_it_a.n_shards == 2
						lowercase_     :					Any													=	_get_expected_row_ids_and_row_dicts_for_partition_order(__SCREAMING_SNAKE_CASE				,				[0, 2]							)
						for i, (row_id, row_dict) in enumerate(__SCREAMING_SNAKE_CASE							):
												lowercase_						,			lowercase_     :					Tuple													=	expected_row_ids_and_row_dicts_a[i]
												assert row_id == expected_row_id
												assert row_dict == expected_row_dict

						# Partitions 1 and 3
						lowercase_     :					List[Any]													=	SparkExamplesIterable(__SCREAMING_SNAKE_CASE							).shard_data_sources(worker_id=1				,				num_workers=2							)
						assert shard_it_a.n_shards == 2
						lowercase_     :					int													=	_get_expected_row_ids_and_row_dicts_for_partition_order(__SCREAMING_SNAKE_CASE				,				[1, 3]							)
						for i, (row_id, row_dict) in enumerate(__SCREAMING_SNAKE_CASE							):
												lowercase_						,			lowercase_     :					Optional[int]													=	expected_row_ids_and_row_dicts_a[i]
												assert row_id == expected_row_id
												assert row_dict == expected_row_dict




@require_not_windows
@require_dill_gt_0_3_2
def  snake_case_		(    ):



						"""simple docstring"""





						lowercase_     :					List[str]													=	pyspark.sql.SparkSession.builder.master('''local[*]'''							).appName('''pyspark'''							).getOrCreate()
						lowercase_     :					int													=	spark.range(100							).repartition(1							)
						lowercase_     :					Union[str, Any]													=	Spark(__SCREAMING_SNAKE_CASE							)
						# Choose a small max_shard_size for maximum partitioning.
						spark_builder._repartition_df_if_needed(max_shard_size=1							)
						# The new number of partitions should not be greater than the number of rows.
						assert spark_builder.df.rdd.getNumPartitions() == 100





'''simple docstring'''







import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_lowercase			:  Optional[Any]   							=	logging.get_logger(__name__)

_lowercase			:  str   							=	"▁"

_lowercase			:  Optional[int]   							=	{"vocab_file": "sentencepiece.bpe.model"}

_lowercase			:  Dict   							=	{
    "vocab_file": {
        "facebook/xglm-564M": "https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model",
    }
}

_lowercase			:  Optional[Any]   							=	{
    "facebook/xglm-564M": 2_0_4_8,
}






class 			lowerCAmelCase__       (  lowerCamelCase_					):
					lowerCAmelCase_					       =					VOCAB_FILES_NAMES
					lowerCAmelCase_					       =					PRETRAINED_VOCAB_FILES_MAP
					lowerCAmelCase_					       =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowerCAmelCase_					       =					['''input_ids''', '''attention_mask''']






					def __init__(       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE="<s>"    ,							__SCREAMING_SNAKE_CASE="</s>"    ,							__SCREAMING_SNAKE_CASE="</s>"    ,							__SCREAMING_SNAKE_CASE="<s>"    ,							__SCREAMING_SNAKE_CASE="<unk>"    ,							__SCREAMING_SNAKE_CASE="<pad>"    ,							__SCREAMING_SNAKE_CASE = None    ,							**__SCREAMING_SNAKE_CASE    ,							):


											"""simple docstring"""





											lowercase_     :					Any													=	{} if sp_model_kwargs is None else sp_model_kwargs

											# Compatibility with the original tokenizer
											lowercase_     :					Optional[Any]													=	7
											lowercase_     :					List[Any]													=	[F'''<madeupword{i}>''' for i in range(self.num_madeup_words		)]

											lowercase_     :					Tuple													=	kwargs.get('''additional_special_tokens'''    ,							[]		)
											kwargs["additional_special_tokens"] += [
											    word for word in madeup_words if word not in kwargs["additional_special_tokens"]
											]

											super().__init__(
											    bos_token=__SCREAMING_SNAKE_CASE    ,							eos_token=__SCREAMING_SNAKE_CASE    ,							unk_token=__SCREAMING_SNAKE_CASE    ,							sep_token=__SCREAMING_SNAKE_CASE    ,							cls_token=__SCREAMING_SNAKE_CASE    ,							pad_token=__SCREAMING_SNAKE_CASE    ,							sp_model_kwargs=self.sp_model_kwargs    ,							**__SCREAMING_SNAKE_CASE    ,							)

											lowercase_     :					int													=	spm.SentencePieceProcessor(**self.sp_model_kwargs		)
											self.sp_model.Load(str(__SCREAMING_SNAKE_CASE		)		)
											lowercase_     :					Dict													=	vocab_file

											# Original fairseq vocab and spm vocab must be "aligned":
											# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
											# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
											# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
											# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

											# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
											lowercase_     :					List[Any]													=	1

											# Mimic fairseq token-to-id alignment for the first 4 token
											lowercase_     :					Optional[Any]													=	{'''<s>''': 0, '''<pad>''': 1, '''</s>''': 2, '''<unk>''': 3}

											lowercase_     :					Dict													=	len(self.sp_model		)
											lowercase_     :					int													=	{F'''<madeupword{i}>''': sp_size + i + self.fairseq_offset for i in range(self.num_madeup_words		)}
											self.fairseq_tokens_to_ids.update(__SCREAMING_SNAKE_CASE		)

											lowercase_     :					List[str]													=	{v: k for k, v in self.fairseq_tokens_to_ids.items()}






					def __getstate__(       self		):


											"""simple docstring"""





											lowercase_     :					List[Any]													=	self.__dict__.copy()
											lowercase_     :					Optional[Any]													=	None
											lowercase_     :					List[Any]													=	self.sp_model.serialized_model_proto()
											return state






					def __setstate__(       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					List[Any]													=	d

											# for backward compatibility
											if not hasattr(self    ,							'''sp_model_kwargs'''		):
																	lowercase_     :					Optional[Any]													=	{}

											lowercase_     :					int													=	spm.SentencePieceProcessor(**self.sp_model_kwargs		)
											self.sp_model.LoadFromSerializedProto(self.sp_model_proto		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE = None		):


											"""simple docstring"""





											if token_ids_a is None:
																	return [self.sep_token_id] + token_ids_a
											lowercase_     :					Optional[Any]													=	[self.sep_token_id]
											return sep + token_ids_a + sep + sep + token_ids_a






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE = None    ,							__SCREAMING_SNAKE_CASE = False		):


											"""simple docstring"""





											if already_has_special_tokens:
																	return super().get_special_tokens_mask(
																	    token_ids_a=__SCREAMING_SNAKE_CASE    ,							token_ids_a=__SCREAMING_SNAKE_CASE    ,							already_has_special_tokens=__SCREAMING_SNAKE_CASE		)

											if token_ids_a is None:
																	return [1] + ([0] * len(__SCREAMING_SNAKE_CASE		))
											return [1] + ([0] * len(__SCREAMING_SNAKE_CASE		)) + [1, 1] + ([0] * len(__SCREAMING_SNAKE_CASE		))






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE = None		):


											"""simple docstring"""





											lowercase_     :					List[Any]													=	[self.sep_token_id]

											if token_ids_a is None:
																	return len(sep + token_ids_a		) * [0]
											return len(sep + token_ids_a + sep + sep + token_ids_a		) * [0]






					@property
					def      _snake_case   (       self		):


											"""simple docstring"""





											return len(self.sp_model		) + self.fairseq_offset + self.num_madeup_words






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					Any													=	{self.convert_ids_to_tokens(__SCREAMING_SNAKE_CASE		): i for i in range(self.vocab_size		)}
											vocab.update(self.added_tokens_encoder		)
											return vocab






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											return self.sp_model.encode(__SCREAMING_SNAKE_CASE    ,							out_type=__SCREAMING_SNAKE_CASE		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											if token in self.fairseq_tokens_to_ids:
																	return self.fairseq_tokens_to_ids[token]
											lowercase_     :					str													=	self.sp_model.PieceToId(__SCREAMING_SNAKE_CASE		)

											# Need to return unknown token if the SP model returned 0
											return spm_id + self.fairseq_offset if spm_id else self.unk_token_id






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											if index in self.fairseq_ids_to_tokens:
																	return self.fairseq_ids_to_tokens[index]
											return self.sp_model.IdToPiece(index - self.fairseq_offset		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					Dict													=	''''''.join(__SCREAMING_SNAKE_CASE		).replace(__SCREAMING_SNAKE_CASE    ,							''' '''		).strip()
											return out_string






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE = None		):


											"""simple docstring"""





											if not os.path.isdir(__SCREAMING_SNAKE_CASE		):
																	logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''		)
																	return
											lowercase_     :					str													=	os.path.join(
											    __SCREAMING_SNAKE_CASE    ,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']		)

											if os.path.abspath(self.vocab_file		) != os.path.abspath(__SCREAMING_SNAKE_CASE		) and os.path.isfile(self.vocab_file		):
																	copyfile(self.vocab_file    ,							__SCREAMING_SNAKE_CASE		)
											elif not os.path.isfile(self.vocab_file		):
																	with open(__SCREAMING_SNAKE_CASE    ,							'''wb'''		) as fi:
																							lowercase_     :					Optional[int]													=	self.sp_model.serialized_model_proto()
																							fi.write(__SCREAMING_SNAKE_CASE		)

											return (out_vocab_file,)



