"""simple docstring"""





import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_gpta import GPTaTokenizer


if TYPE_CHECKING:
				from transformers.pipelines.conversational import Conversation


__A	=					logging.get_logger(__name__)

__A	=					{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

__A	=					{
    'vocab_file': {
        'gpt2': 'https://huggingface.co/gpt2/resolve/main/vocab.json',
        'gpt2-medium': 'https://huggingface.co/gpt2-medium/resolve/main/vocab.json',
        'gpt2-large': 'https://huggingface.co/gpt2-large/resolve/main/vocab.json',
        'gpt2-xl': 'https://huggingface.co/gpt2-xl/resolve/main/vocab.json',
        'distilgpt2': 'https://huggingface.co/distilgpt2/resolve/main/vocab.json',
    },
    'merges_file': {
        'gpt2': 'https://huggingface.co/gpt2/resolve/main/merges.txt',
        'gpt2-medium': 'https://huggingface.co/gpt2-medium/resolve/main/merges.txt',
        'gpt2-large': 'https://huggingface.co/gpt2-large/resolve/main/merges.txt',
        'gpt2-xl': 'https://huggingface.co/gpt2-xl/resolve/main/merges.txt',
        'distilgpt2': 'https://huggingface.co/distilgpt2/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'gpt2': 'https://huggingface.co/gpt2/resolve/main/tokenizer.json',
        'gpt2-medium': 'https://huggingface.co/gpt2-medium/resolve/main/tokenizer.json',
        'gpt2-large': 'https://huggingface.co/gpt2-large/resolve/main/tokenizer.json',
        'gpt2-xl': 'https://huggingface.co/gpt2-xl/resolve/main/tokenizer.json',
        'distilgpt2': 'https://huggingface.co/distilgpt2/resolve/main/tokenizer.json',
    },
}

__A	=					{
    'gpt2': 1024,
    'gpt2-medium': 1024,
    'gpt2-large': 1024,
    'gpt2-xl': 1024,
    'distilgpt2': 1024,
}
class 			_snake_case  (       UpperCAmelCase__		):
						snake_case__      	=      VOCAB_FILES_NAMES
						snake_case__      	=      PRETRAINED_VOCAB_FILES_MAP
						snake_case__      	=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						snake_case__      	=      ["input_ids", "attention_mask"]
						snake_case__      	=      GPTaTokenizer






						def __init__(							self   :      Union[str, Any]				,       UpperCAmelCase   :      List[Any]=None				,       UpperCAmelCase   :      List[Any]=None				,       UpperCAmelCase   :      List[Any]=None				,       UpperCAmelCase   :      str="<|endoftext|>"				,       UpperCAmelCase   :      Dict="<|endoftext|>"				,       UpperCAmelCase   :      int="<|endoftext|>"				,       UpperCAmelCase   :      str=False				,       **UpperCAmelCase   :      str				,       ):
													super().__init__(
													    _UpperCAmelCase				,       _UpperCAmelCase				,       tokenizer_file=_UpperCAmelCase				,       unk_token=_UpperCAmelCase				,       bos_token=_UpperCAmelCase				,       eos_token=_UpperCAmelCase				,       add_prefix_space=_UpperCAmelCase				,       **_UpperCAmelCase				,       )

													__lowerCamelCase       :		int		 =  kwargs.pop("add_bos_token"				,       _UpperCAmelCase     )

													__lowerCamelCase       :		Optional[Any]		 =  json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()     )
													if pre_tok_state.get("add_prefix_space"				,       _UpperCAmelCase     ) != add_prefix_space:
																				__lowerCamelCase       :		Dict		 =  getattr(_UpperCAmelCase				,       pre_tok_state.pop("type"     )     )
																				__lowerCamelCase       :		int		 =  add_prefix_space
																				__lowerCamelCase       :		Any		 =  pre_tok_class(**_UpperCAmelCase     )

													__lowerCamelCase       :		List[Any]		 =  add_prefix_space






						def    lowerCamelCase__  (							self   :      Tuple				,       *UpperCAmelCase   :      List[Any]				,       **UpperCAmelCase   :      Optional[Any]     ):
													__lowerCamelCase       :		str		 =  kwargs.get("is_split_into_words"				,       _UpperCAmelCase     )
													assert self.add_prefix_space or not is_split_into_words, (
													    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
													    "to use it with pretokenized inputs."
													)

													return super()._batch_encode_plus(*_UpperCAmelCase				,       **_UpperCAmelCase     )






						def    lowerCamelCase__  (							self   :      List[str]				,       *UpperCAmelCase   :      Optional[Any]				,       **UpperCAmelCase   :      List[Any]     ):
													__lowerCamelCase       :		int		 =  kwargs.get("is_split_into_words"				,       _UpperCAmelCase     )

													assert self.add_prefix_space or not is_split_into_words, (
													    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
													    "to use it with pretokenized inputs."
													)

													return super()._encode_plus(*_UpperCAmelCase				,       **_UpperCAmelCase     )






						def    lowerCamelCase__  (							self   :      List[str]				,       UpperCAmelCase   :      str				,       UpperCAmelCase   :      Optional[str] = None     ):
													__lowerCamelCase       :		Any		 =  self._tokenizer.model.save(_UpperCAmelCase				,       name=_UpperCAmelCase     )
													return tuple(_UpperCAmelCase     )







						def    lowerCamelCase__  (							self   :      int				,       UpperCAmelCase   :      "Conversation"     ):
													__lowerCamelCase       :		Any		 =  []
													for is_user, text in conversation.iter_texts():
																				input_ids.extend(self.encode(_UpperCAmelCase				,       add_special_tokens=_UpperCAmelCase     ) + [self.eos_token_id]     )

													if len(_UpperCAmelCase     ) > self.model_max_length:
																				__lowerCamelCase       :		Optional[int]		 =  input_ids[-self.model_max_length :]
													return input_ids



from typing import Optional

from .. import Features, NamedSplit
from ..packaged_modules.text.text import Text
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader



class 				A       (			UpperCAmelCase__				):
					'''simple docstring'''






					def __init__(self       :		Any ,   _UpperCAmelCase       :		NestedDataStructureLike[PathLike] ,   _UpperCAmelCase       :		Optional[NamedSplit] = None ,   _UpperCAmelCase       :		Optional[Features] = None ,   _UpperCAmelCase       :		str = None ,   _UpperCAmelCase       :		bool = False ,   _UpperCAmelCase       :		bool = False ,   _UpperCAmelCase       :		Optional[int] = None ,   **_UpperCAmelCase       :		Optional[int] ,   )		->  List[str]:




							"""simple docstring"""


							super().__init__(
							    _UpperCAmelCase ,   split=_UpperCAmelCase ,   features=_UpperCAmelCase ,   cache_dir=_UpperCAmelCase ,   keep_in_memory=_UpperCAmelCase ,   streaming=_UpperCAmelCase ,   num_proc=_UpperCAmelCase ,   **_UpperCAmelCase ,   )
							lowercase__	     =						path_or_paths if isinstance(_UpperCAmelCase ,   _UpperCAmelCase						) else {self.split: path_or_paths}
							lowercase__	     =						Text(
							    cache_dir=_UpperCAmelCase ,   data_files=_UpperCAmelCase ,   features=_UpperCAmelCase ,   **_UpperCAmelCase ,   )


					def     lowerCamelCase__	(self       :		Tuple						)		->  Any:




							"""simple docstring"""


							if self.streaming:
									lowercase__	     =						self.builder.as_streaming_dataset(split=self.split						)
							# Build regular (map-style) dataset
							else:
									lowercase__	     =						None
									lowercase__	     =						None
									lowercase__	     =						None
									lowercase__	     =						None

									self.builder.download_and_prepare(
									    download_config=_UpperCAmelCase ,   download_mode=_UpperCAmelCase ,   verification_mode=_UpperCAmelCase ,   base_path=_UpperCAmelCase ,   num_proc=self.num_proc ,   )
									lowercase__	     =						self.builder.as_dataset(
									    split=self.split ,   verification_mode=_UpperCAmelCase ,   in_memory=self.keep_in_memory						)
							return dataset

