


'''simple docstring'''






import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_gpta import GPTaTokenizer


if TYPE_CHECKING:
		from transformers.pipelines.conversational import Conversation


__UpperCAmelCase					       =							logging.get_logger(__name__)

__UpperCAmelCase					       =							{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__UpperCAmelCase					       =							{
    '''vocab_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/vocab.json''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/vocab.json''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/vocab.json''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/vocab.json''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/merges.txt''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/merges.txt''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/merges.txt''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/merges.txt''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/tokenizer.json''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/tokenizer.json''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/tokenizer.json''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/tokenizer.json''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/tokenizer.json''',
    },
}

__UpperCAmelCase					       =							{
    '''gpt2''': 1_024,
    '''gpt2-medium''': 1_024,
    '''gpt2-large''': 1_024,
    '''gpt2-xl''': 1_024,
    '''distilgpt2''': 1_024,
}




class 	a__       (							a__						):





							'''simple docstring'''






							lowercase__    :     str					=   VOCAB_FILES_NAMES
							lowercase__    :     List[str]					=   PRETRAINED_VOCAB_FILES_MAP
							lowercase__    :     Tuple					=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowercase__    :     Optional[Any]					=   ["input_ids", "attention_mask"]
							lowercase__    :     int					=   GPTaTokenizer







							def __init__(	self      ,					lowerCamelCase_=None      ,					lowerCamelCase_=None      ,					lowerCamelCase_=None      ,					lowerCamelCase_="<|endoftext|>"      ,					lowerCamelCase_="<|endoftext|>"      ,					lowerCamelCase_="<|endoftext|>"      ,					lowerCamelCase_=False      ,					**lowerCamelCase_      ,					) ->							Any:
														super().__init__(
														    lowerCamelCase_      ,					lowerCamelCase_      ,					tokenizer_file=lowerCamelCase_      ,					unk_token=lowerCamelCase_      ,					bos_token=lowerCamelCase_      ,					eos_token=lowerCamelCase_      ,					add_prefix_space=lowerCamelCase_      ,					**lowerCamelCase_      ,					)

														lowerCAmelCase__      =			kwargs.pop('''add_bos_token'''      ,					lowerCamelCase_     )

														lowerCAmelCase__      =			json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()     )
														if pre_tok_state.get('''add_prefix_space'''      ,					lowerCamelCase_     ) != add_prefix_space:
																					lowerCAmelCase__      =			getattr(lowerCamelCase_      ,					pre_tok_state.pop('''type'''     )     )
																					lowerCAmelCase__      =			add_prefix_space
																					lowerCAmelCase__      =			pre_tok_class(**lowerCamelCase_     )

														lowerCAmelCase__      =			add_prefix_space







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					*lowerCamelCase_      ,					**lowerCamelCase_     ) ->							BatchEncoding:
														lowerCAmelCase__      =			kwargs.get('''is_split_into_words'''      ,					lowerCamelCase_     )
														assert self.add_prefix_space or not is_split_into_words, (
														    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
														    "to use it with pretokenized inputs."
														)

														return super()._batch_encode_plus(*lowerCamelCase_      ,					**lowerCamelCase_     )







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					*lowerCamelCase_      ,					**lowerCamelCase_     ) ->							BatchEncoding:
														lowerCAmelCase__      =			kwargs.get('''is_split_into_words'''      ,					lowerCamelCase_     )

														assert self.add_prefix_space or not is_split_into_words, (
														    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
														    "to use it with pretokenized inputs."
														)

														return super()._encode_plus(*lowerCamelCase_      ,					**lowerCamelCase_     )







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_      ,					lowerCamelCase_ = None     ) ->							Tuple[str]:
														lowerCAmelCase__      =			self._tokenizer.model.save(lowerCamelCase_      ,					name=lowerCamelCase_     )
														return tuple(lowerCamelCase_     )







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_     ) ->							List[int]:
														lowerCAmelCase__      =			[]
														for is_user, text in conversation.iter_texts():
																					input_ids.extend(self.encode(lowerCamelCase_      ,					add_special_tokens=lowerCamelCase_     ) + [self.eos_token_id]     )

														if len(lowerCamelCase_     ) > self.model_max_length:
																					lowerCAmelCase__      =			input_ids[-self.model_max_length :]
														return input_ids





'''simple docstring'''






from unittest.mock import patch

import pyspark

from datasets.packaged_modules.spark.spark import (
    Spark,
    SparkExamplesIterable,
    _generate_iterable_examples,
)

from ..utils import (
    require_dill_gt_0_3_2,
    require_not_windows,
)


def       _snake_case     (  A				,					A )		->					List[Any]:
							lowerCAmelCase__      =			[]
							for part_id in partition_order:
														lowerCAmelCase__      =			df.where(F"""SPARK_PARTITION_ID() = {part_id}""" ).collect()
														for row_idx, row in enumerate(A ):
																					expected_row_ids_and_row_dicts.append((F"""{part_id}_{row_idx}""", row.asDict()) )
							return expected_row_ids_and_row_dicts


@require_not_windows
@require_dill_gt_0_3_2
def       _snake_case     (  )		->					Tuple:
							lowerCAmelCase__      =			pyspark.sql.SparkSession.builder.master('''local[*]''' ).appName('''pyspark''' ).getOrCreate()
							lowerCAmelCase__      =			spark.range(100 ).repartition(1 )
							lowerCAmelCase__      =			Spark(A )
							# The id ints will be converted to Pyarrow int64s, so each row will be 8 bytes. Setting a max_shard_size of 16 means
							# that each partition can hold 2 rows.
							spark_builder._repartition_df_if_needed(max_shard_size=16 )
							# Given that the dataframe has 100 rows and each partition has 2 rows, we expect 50 partitions.
							assert spark_builder.df.rdd.getNumPartitions() == 50


@require_not_windows
@require_dill_gt_0_3_2
def       _snake_case     (  )		->					Optional[int]:
							lowerCAmelCase__      =			pyspark.sql.SparkSession.builder.master('''local[*]''' ).appName('''pyspark''' ).getOrCreate()
							lowerCAmelCase__      =			spark.range(10 ).repartition(2 )
							lowerCAmelCase__      =			[1, 0]
							lowerCAmelCase__      =			_generate_iterable_examples(A				,					A )  # Reverse the partitions.
							lowerCAmelCase__      =			_get_expected_row_ids_and_row_dicts_for_partition_order(A				,					A )

							for i, (row_id, row_dict) in enumerate(generate_fn() ):
														lowerCAmelCase__     ,      lowerCAmelCase__      =			expected_row_ids_and_row_dicts[i]
														assert row_id == expected_row_id
														assert row_dict == expected_row_dict


@require_not_windows
@require_dill_gt_0_3_2
def       _snake_case     (  )		->					Optional[Any]:
							lowerCAmelCase__      =			pyspark.sql.SparkSession.builder.master('''local[*]''' ).appName('''pyspark''' ).getOrCreate()
							lowerCAmelCase__      =			spark.range(10 ).repartition(1 )
							lowerCAmelCase__      =			SparkExamplesIterable(A )
							assert it.n_shards == 1
							for i, (row_id, row_dict) in enumerate(A ):
														assert row_id == F"""0_{i}"""
														assert row_dict == {"id": i}


@require_not_windows
@require_dill_gt_0_3_2
def       _snake_case     (  )		->					Union[str, Any]:
							lowerCAmelCase__      =			pyspark.sql.SparkSession.builder.master('''local[*]''' ).appName('''pyspark''' ).getOrCreate()
							lowerCAmelCase__      =			spark.range(30 ).repartition(3 )
							# Mock the generator so that shuffle reverses the partition indices.
							with patch('''numpy.random.Generator''' ) as generator_mock:
														lowerCAmelCase__      =			lambda A : x.reverse()
														lowerCAmelCase__      =			_get_expected_row_ids_and_row_dicts_for_partition_order(A				,					[2, 1, 0] )

														lowerCAmelCase__      =			SparkExamplesIterable(A ).shuffle_data_sources(A )
														assert shuffled_it.n_shards == 3
														for i, (row_id, row_dict) in enumerate(A ):
																					lowerCAmelCase__     ,      lowerCAmelCase__      =			expected_row_ids_and_row_dicts[i]
																					assert row_id == expected_row_id
																					assert row_dict == expected_row_dict


@require_not_windows
@require_dill_gt_0_3_2
def       _snake_case     (  )		->					Dict:
							lowerCAmelCase__      =			pyspark.sql.SparkSession.builder.master('''local[*]''' ).appName('''pyspark''' ).getOrCreate()
							lowerCAmelCase__      =			spark.range(20 ).repartition(4 )

							# Partitions 0 and 2
							lowerCAmelCase__      =			SparkExamplesIterable(A ).shard_data_sources(worker_id=0				,					num_workers=2 )
							assert shard_it_a.n_shards == 2
							lowerCAmelCase__      =			_get_expected_row_ids_and_row_dicts_for_partition_order(A				,					[0, 2] )
							for i, (row_id, row_dict) in enumerate(A ):
														lowerCAmelCase__     ,      lowerCAmelCase__      =			expected_row_ids_and_row_dicts_a[i]
														assert row_id == expected_row_id
														assert row_dict == expected_row_dict

							# Partitions 1 and 3
							lowerCAmelCase__      =			SparkExamplesIterable(A ).shard_data_sources(worker_id=1				,					num_workers=2 )
							assert shard_it_a.n_shards == 2
							lowerCAmelCase__      =			_get_expected_row_ids_and_row_dicts_for_partition_order(A				,					[1, 3] )
							for i, (row_id, row_dict) in enumerate(A ):
														lowerCAmelCase__     ,      lowerCAmelCase__      =			expected_row_ids_and_row_dicts_a[i]
														assert row_id == expected_row_id
														assert row_dict == expected_row_dict






@require_not_windows
@require_dill_gt_0_3_2
def       _snake_case     (  )		->					Dict:
							lowerCAmelCase__      =			pyspark.sql.SparkSession.builder.master('''local[*]''' ).appName('''pyspark''' ).getOrCreate()
							lowerCAmelCase__      =			spark.range(100 ).repartition(1 )
							lowerCAmelCase__      =			Spark(A )
							# Choose a small max_shard_size for maximum partitioning.
							spark_builder._repartition_df_if_needed(max_shard_size=1 )
							# The new number of partitions should not be greater than the number of rows.
							assert spark_builder.df.rdd.getNumPartitions() == 100