
'''simple docstring'''
import unittest

from transformers import CamembertTokenizer, CamembertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


lowercase__											=get_tests_dir('fixtures/test_sentencepiece.model')
lowercase__											=get_tests_dir('fixtures/test_sentencepiece_bpe.model')

lowercase__											='pt' if is_torch_available() else 'tf'







@require_sentencepiece
@require_tokenizers
class   a_ (      UpperCamelCase__		,  unittest.TestCase					):
			lowerCamelCase__	:			Tuple       =    CamembertTokenizer
			lowerCamelCase__	:			Optional[int]       =    CamembertTokenizerFast
			lowerCamelCase__	:			Union[str, Any]       =    True
			lowerCamelCase__	:			Any       =    True




			def 					lowerCAmelCase__					(       self	):
				super().setUp()

				# We have a SentencePiece fixture for testing
				a_      				=						CamembertTokenizer(UpperCAmelCase	)
				tokenizer.save_pretrained(self.tmpdirname	)




			def 					lowerCAmelCase__					(       self	):
				a_      				=						"""<pad>"""
				a_      				=						1

				self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCAmelCase	)					,   UpperCAmelCase	)
				self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCAmelCase	)					,   UpperCAmelCase	)




			def 					lowerCAmelCase__					(       self	):
				a_      				=						list(self.get_tokenizer().get_vocab().keys()	)

				self.assertEqual(vocab_keys[0]					,   """<s>NOTUSED"""	)
				self.assertEqual(vocab_keys[1]					,   """<pad>"""	)
				self.assertEqual(vocab_keys[-1]					,   """<mask>"""	)
				self.assertEqual(len(UpperCAmelCase	)					,   10_04	)




			def 					lowerCAmelCase__					(       self	):
				self.assertEqual(self.get_tokenizer().vocab_size					,   10_05	)




			def 					lowerCAmelCase__					(       self	):
				a_      				=						CamembertTokenizer(UpperCAmelCase	)
				tokenizer.save_pretrained(self.tmpdirname	)
				a_      				=						CamembertTokenizerFast.from_pretrained(self.tmpdirname	)

				a_      				=						"""I was born in 92000, and this is falsé."""

				a_      				=						tokenizer.encode(UpperCAmelCase	)
				a_      				=						rust_tokenizer.encode(UpperCAmelCase	)
				self.assertListEqual(UpperCAmelCase					,   UpperCAmelCase	)

				a_      				=						tokenizer.encode(UpperCAmelCase					,   add_special_tokens=UpperCAmelCase	)
				a_      				=						rust_tokenizer.encode(UpperCAmelCase					,   add_special_tokens=UpperCAmelCase	)
				self.assertListEqual(UpperCAmelCase					,   UpperCAmelCase	)

				# <unk> tokens are not the same for `rust` than for `slow`.
				# Because spm gives back raw token instead of `unk` in EncodeAsPieces
				# tokens = tokenizer.tokenize(sequence)
				a_      				=						tokenizer.convert_ids_to_tokens(UpperCAmelCase	)
				a_      				=						rust_tokenizer.tokenize(UpperCAmelCase	)
				self.assertListEqual(UpperCAmelCase					,   UpperCAmelCase	)




			def 					lowerCAmelCase__					(       self	):
				if not self.test_rust_tokenizer:
					return

				a_      				=						self.get_tokenizer()
				a_      				=						self.get_rust_tokenizer()

				a_      				=						"""I was born in 92000, and this is falsé."""

				a_      				=						tokenizer.tokenize(UpperCAmelCase	)
				a_      				=						rust_tokenizer.tokenize(UpperCAmelCase	)
				self.assertListEqual(UpperCAmelCase					,   UpperCAmelCase	)

				a_      				=						tokenizer.encode(UpperCAmelCase					,   add_special_tokens=UpperCAmelCase	)
				a_      				=						rust_tokenizer.encode(UpperCAmelCase					,   add_special_tokens=UpperCAmelCase	)
				self.assertListEqual(UpperCAmelCase					,   UpperCAmelCase	)

				a_      				=						self.get_rust_tokenizer()
				a_      				=						tokenizer.encode(UpperCAmelCase	)
				a_      				=						rust_tokenizer.encode(UpperCAmelCase	)
				self.assertListEqual(UpperCAmelCase					,   UpperCAmelCase	)






			@slow
			def 					lowerCAmelCase__					(       self	):
				# fmt: off
				a_      				=						{"""input_ids""": [[5, 54, 71_96, 2_97, 30, 23, 7_76, 18, 11, 32_15, 37_05, 82_52, 22, 31_64, 11_81, 21_16, 29, 16, 8_13, 25, 7_91, 33_14, 20, 34_46, 38, 2_75_75, 1_20, 6, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [5, 4_68, 17, 11, 90_88, 20, 15_17, 8, 2_28_04, 1_88_18, 10, 38, 6_29, 6_07, 6_07, 1_42, 19, 71_96, 8_67, 56, 1_03_26, 24, 22_67, 20, 4_16, 50_72, 1_56_12, 2_33, 7_34, 7, 23_99, 27, 16, 30_15, 16_49, 7, 24, 20, 43_38, 23_99, 27, 13, 34_00, 14, 13, 61_89, 8, 9_30, 9, 6]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
				# fmt: on

				# camembert is a french model. So we also use french texts.
				a_      				=						[
				    """Le transformeur est un modèle d'apprentissage profond introduit en 2017, """
				    """utilisé principalement dans le domaine du traitement automatique des langues (TAL).""",
				    """À l'instar des réseaux de neurones récurrents (RNN), les transformeurs sont conçus """
				    """pour gérer des données séquentielles, telles que le langage naturel, pour des tâches """
				    """telles que la traduction et la synthèse de texte.""",
				]

				self.tokenizer_integration_test_util(
				    expected_encoding=UpperCAmelCase					,   model_name="""camembert-base"""					,   revision="""3a0641d9a1aeb7e848a74299e7e4c4bca216b4cf"""					,   sequences=UpperCAmelCase					,   )


'''simple docstring'''
from unittest.mock import patch

import pyspark

from datasets.packaged_modules.spark.spark import (
    Spark,
    SparkExamplesIterable,
    _generate_iterable_examples,
)

from ..utils import (
    require_dill_gt_0_3_2,
    require_not_windows,
)






def      UpperCamelCase_				(      A__  ,      A__						):
	a_      				=						[]
	for part_id in partition_order:
		a_      				=						df.where(F'''SPARK_PARTITION_ID() = {part_id}'''						).collect()
		for row_idx, row in enumerate(A__						):
			expected_row_ids_and_row_dicts.append((F'''{part_id}_{row_idx}''', row.asDict())						)
	return expected_row_ids_and_row_dicts






@require_not_windows
@require_dill_gt_0_3_2
def      UpperCamelCase_				(      ):
	a_      				=						pyspark.sql.SparkSession.builder.master("""local[*]"""						).appName("""pyspark"""						).getOrCreate()
	a_      				=						spark.range(1_00						).repartition(1						)
	a_      				=						Spark(A__						)
	# The id ints will be converted to Pyarrow int64s, so each row will be 8 bytes. Setting a max_shard_size of 16 means
	# that each partition can hold 2 rows.
	spark_builder._repartition_df_if_needed(max_shard_size=16						)
	# Given that the dataframe has 100 rows and each partition has 2 rows, we expect 50 partitions.
	assert spark_builder.df.rdd.getNumPartitions() == 50






@require_not_windows
@require_dill_gt_0_3_2
def      UpperCamelCase_				(      ):
	a_      				=						pyspark.sql.SparkSession.builder.master("""local[*]"""						).appName("""pyspark"""						).getOrCreate()
	a_      				=						spark.range(10						).repartition(2						)
	a_      				=						[1, 0]
	a_      				=						_generate_iterable_examples(A__  ,      A__						)  # Reverse the partitions.
	a_      				=						_get_expected_row_ids_and_row_dicts_for_partition_order(A__  ,      A__						)

	for i, (row_id, row_dict) in enumerate(generate_fn()						):
		a_				,  a_      				=						expected_row_ids_and_row_dicts[i]
		assert row_id == expected_row_id
		assert row_dict == expected_row_dict






@require_not_windows
@require_dill_gt_0_3_2
def      UpperCamelCase_				(      ):
	a_      				=						pyspark.sql.SparkSession.builder.master("""local[*]"""						).appName("""pyspark"""						).getOrCreate()
	a_      				=						spark.range(10						).repartition(1						)
	a_      				=						SparkExamplesIterable(A__						)
	assert it.n_shards == 1
	for i, (row_id, row_dict) in enumerate(A__						):
		assert row_id == F'''0_{i}'''
		assert row_dict == {"id": i}






@require_not_windows
@require_dill_gt_0_3_2
def      UpperCamelCase_				(      ):
	a_      				=						pyspark.sql.SparkSession.builder.master("""local[*]"""						).appName("""pyspark"""						).getOrCreate()
	a_      				=						spark.range(30						).repartition(3						)
	# Mock the generator so that shuffle reverses the partition indices.
	with patch("""numpy.random.Generator"""						) as generator_mock:
		a_      				=						lambda A__						: x.reverse()
		a_      				=						_get_expected_row_ids_and_row_dicts_for_partition_order(A__  ,      [2, 1, 0]						)

		a_      				=						SparkExamplesIterable(A__						).shuffle_data_sources(A__						)
		assert shuffled_it.n_shards == 3
		for i, (row_id, row_dict) in enumerate(A__						):
			a_				,  a_      				=						expected_row_ids_and_row_dicts[i]
			assert row_id == expected_row_id
			assert row_dict == expected_row_dict






@require_not_windows
@require_dill_gt_0_3_2
def      UpperCamelCase_				(      ):
	a_      				=						pyspark.sql.SparkSession.builder.master("""local[*]"""						).appName("""pyspark"""						).getOrCreate()
	a_      				=						spark.range(20						).repartition(4						)

	# Partitions 0 and 2
	a_      				=						SparkExamplesIterable(A__						).shard_data_sources(worker_id=0  ,      num_workers=2						)
	assert shard_it_a.n_shards == 2
	a_      				=						_get_expected_row_ids_and_row_dicts_for_partition_order(A__  ,      [0, 2]						)
	for i, (row_id, row_dict) in enumerate(A__						):
		a_				,  a_      				=						expected_row_ids_and_row_dicts_a[i]
		assert row_id == expected_row_id
		assert row_dict == expected_row_dict

	# Partitions 1 and 3
	a_      				=						SparkExamplesIterable(A__						).shard_data_sources(worker_id=1  ,      num_workers=2						)
	assert shard_it_a.n_shards == 2
	a_      				=						_get_expected_row_ids_and_row_dicts_for_partition_order(A__  ,      [1, 3]						)
	for i, (row_id, row_dict) in enumerate(A__						):
		a_				,  a_      				=						expected_row_ids_and_row_dicts_a[i]
		assert row_id == expected_row_id
		assert row_dict == expected_row_dict






@require_not_windows
@require_dill_gt_0_3_2
def      UpperCamelCase_				(      ):
	a_      				=						pyspark.sql.SparkSession.builder.master("""local[*]"""						).appName("""pyspark"""						).getOrCreate()
	a_      				=						spark.range(1_00						).repartition(1						)
	a_      				=						Spark(A__						)
	# Choose a small max_shard_size for maximum partitioning.
	spark_builder._repartition_df_if_needed(max_shard_size=1						)
	# The new number of partitions should not be greater than the number of rows.
	assert spark_builder.df.rdd.getNumPartitions() == 1_00
