
import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_gpta import GPTaTokenizer


if TYPE_CHECKING:
		from transformers.pipelines.conversational import Conversation


lowercase        =  logging.get_logger(__name__)

lowercase        =  {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

lowercase        =  {
    '''vocab_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/vocab.json''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/vocab.json''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/vocab.json''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/vocab.json''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/merges.txt''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/merges.txt''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/merges.txt''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/merges.txt''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/tokenizer.json''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/tokenizer.json''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/tokenizer.json''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/tokenizer.json''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/tokenizer.json''',
    },
}

lowercase        =  {
    '''gpt2''': 1_0_2_4,
    '''gpt2-medium''': 1_0_2_4,
    '''gpt2-large''': 1_0_2_4,
    '''gpt2-xl''': 1_0_2_4,
    '''distilgpt2''': 1_0_2_4,
}





class       __A(				lowerCamelCase_       ):
					SCREAMING_SNAKE_CASE			    =    VOCAB_FILES_NAMES
					SCREAMING_SNAKE_CASE			    =    PRETRAINED_VOCAB_FILES_MAP
					SCREAMING_SNAKE_CASE			    =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					SCREAMING_SNAKE_CASE			    =    ['''input_ids''', '''attention_mask''']
					SCREAMING_SNAKE_CASE			    =    GPTaTokenizer

					def __init__(	self					:     Tuple   ,					__UpperCamelCase					:     Optional[int]=None   ,					__UpperCamelCase					:     Optional[int]=None   ,					__UpperCamelCase					:     int=None   ,					__UpperCamelCase					:     Dict="<|endoftext|>"   ,					__UpperCamelCase					:     Any="<|endoftext|>"   ,					__UpperCamelCase					:     Optional[int]="<|endoftext|>"   ,					__UpperCamelCase					:     List[str]=False   ,					**__UpperCamelCase					:     List[str]   ,					):
												super().__init__(
												    __UpperCamelCase   ,					__UpperCamelCase   ,					tokenizer_file=__UpperCamelCase   ,					unk_token=__UpperCamelCase   ,					bos_token=__UpperCamelCase   ,					eos_token=__UpperCamelCase   ,					add_prefix_space=__UpperCamelCase   ,					**__UpperCamelCase   ,					)

												lowerCamelCase_							  =					kwargs.pop("""add_bos_token"""   ,					__UpperCamelCase   )

												lowerCamelCase_							  =					json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()   )
												if pre_tok_state.get("""add_prefix_space"""   ,					__UpperCamelCase   ) != add_prefix_space:
																			lowerCamelCase_							  =					getattr(__UpperCamelCase   ,					pre_tok_state.pop("""type"""   )   )
																			lowerCamelCase_							  =					add_prefix_space
																			lowerCamelCase_							  =					pre_tok_class(**__UpperCamelCase   )

												lowerCamelCase_							  =					add_prefix_space

					def    lowercase__      (	self					:     Union[str, Any]   ,					*__UpperCamelCase					:     List[Any]   ,					**__UpperCamelCase					:     List[Any]   ):
												lowerCamelCase_							  =					kwargs.get("""is_split_into_words"""   ,					__UpperCamelCase   )
												assert self.add_prefix_space or not is_split_into_words, (
												    F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
												    "to use it with pretokenized inputs."
												)

												return super()._batch_encode_plus(*__UpperCamelCase   ,					**__UpperCamelCase   )

					def    lowercase__      (	self					:     Any   ,					*__UpperCamelCase					:     Any   ,					**__UpperCamelCase					:     Union[str, Any]   ):
												lowerCamelCase_							  =					kwargs.get("""is_split_into_words"""   ,					__UpperCamelCase   )

												assert self.add_prefix_space or not is_split_into_words, (
												    F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
												    "to use it with pretokenized inputs."
												)

												return super()._encode_plus(*__UpperCamelCase   ,					**__UpperCamelCase   )

					def    lowercase__      (	self					:     int   ,					__UpperCamelCase					:     str   ,					__UpperCamelCase					:     Optional[str] = None   ):
												lowerCamelCase_							  =					self._tokenizer.model.save(__UpperCamelCase   ,					name=__UpperCamelCase   )
												return tuple(__UpperCamelCase   )



					def    lowercase__      (	self					:     Optional[int]   ,					__UpperCamelCase					:     "Conversation"   ):
												lowerCamelCase_							  =					[]
												for is_user, text in conversation.iter_texts():
																			input_ids.extend(self.encode(__UpperCamelCase   ,					add_special_tokens=__UpperCamelCase   ) + [self.eos_token_id]   )

												if len(__UpperCamelCase   ) > self.model_max_length:
																			lowerCamelCase_							  =					input_ids[-self.model_max_length :]
												return input_ids







import os
import socket
from contextlib import contextmanager

import torch

from ..commands.config.default import write_basic_config  # noqa: F401
from ..state import PartialState
from .dataclasses import DistributedType
from .imports import is_deepspeed_available, is_tpu_available
from .transformer_engine import convert_model
from .versions import is_torch_version


if is_deepspeed_available():
		from deepspeed import DeepSpeedEngine

if is_tpu_available(check_device=False):
		import torch_xla.core.xla_model as xm



def      __lowerCAmelCase						(							UpperCAmelCase__				:      Any       )  ->    List[str]:
							if is_torch_version("""<"""			,				"""2.0.0"""       ) or not hasattr(UpperCAmelCase__			,				"""_dynamo"""       ):
														return False
							return isinstance(UpperCAmelCase__			,				torch._dynamo.eval_frame.OptimizedModule       )



def      __lowerCAmelCase						(							UpperCAmelCase__				:      str			,				UpperCAmelCase__				:      bool = True       )  ->    str:
							lowerCamelCase_							  =					(torch.nn.parallel.DistributedDataParallel, torch.nn.DataParallel)

							lowerCamelCase_							  =					is_compiled_module(UpperCAmelCase__       )
							if is_compiled:
														lowerCamelCase_							  =					model
														lowerCamelCase_							  =					model._orig_mod

							if is_deepspeed_available():
														options += (DeepSpeedEngine,)

							while isinstance(UpperCAmelCase__			,				UpperCAmelCase__       ):
														lowerCamelCase_							  =					model.module

							if not keep_fpaa_wrapper:
														lowerCamelCase_							  =					getattr(UpperCAmelCase__			,				"""forward"""       )
														lowerCamelCase_							  =					model.__dict__.pop("""_original_forward"""			,				UpperCAmelCase__       )
														if original_forward is not None:
																					while hasattr(UpperCAmelCase__			,				"""__wrapped__"""       ):
																												lowerCamelCase_							  =					forward.__wrapped__
																												if forward == original_forward:
																																			break
																					lowerCamelCase_							  =					forward
														if getattr(UpperCAmelCase__			,				"""_converted_to_transformer_engine"""			,				UpperCAmelCase__       ):
																					convert_model(UpperCAmelCase__			,				to_transformer_engine=UpperCAmelCase__       )

							if is_compiled:
														lowerCamelCase_							  =					model
														lowerCamelCase_							  =					compiled_model

							return model



def      __lowerCAmelCase						(							)  ->    Any:
							PartialState().wait_for_everyone()



def      __lowerCAmelCase						(							UpperCAmelCase__				:      Union[str, Any]			,				UpperCAmelCase__				:      Tuple       )  ->    Dict:
							if PartialState().distributed_type == DistributedType.TPU:
														xm.save(UpperCAmelCase__			,				UpperCAmelCase__       )
							elif PartialState().local_process_index == 0:
														torch.save(UpperCAmelCase__			,				UpperCAmelCase__       )



@contextmanager
def      __lowerCAmelCase						(							**UpperCAmelCase__				:      Any       )  ->    Dict:
							for key, value in kwargs.items():
														lowerCamelCase_							  =					str(UpperCAmelCase__       )

							yield

							for key in kwargs:
														if key.upper() in os.environ:
																					del os.environ[key.upper()]



def      __lowerCAmelCase						(							UpperCAmelCase__				:      Union[str, Any]       )  ->    str:
							if not hasattr(UpperCAmelCase__			,				"""__qualname__"""       ) and not hasattr(UpperCAmelCase__			,				"""__name__"""       ):
														lowerCamelCase_							  =					getattr(UpperCAmelCase__			,				"""__class__"""			,				UpperCAmelCase__       )
							if hasattr(UpperCAmelCase__			,				"""__qualname__"""       ):
														return obj.__qualname__
							if hasattr(UpperCAmelCase__			,				"""__name__"""       ):
														return obj.__name__
							return str(UpperCAmelCase__       )



def      __lowerCAmelCase						(							UpperCAmelCase__				:      str			,				UpperCAmelCase__				:      Union[str, Any]       )  ->    Dict:
							for key, value in source.items():
														if isinstance(UpperCAmelCase__			,				UpperCAmelCase__       ):
																					lowerCamelCase_							  =					destination.setdefault(UpperCAmelCase__			,				{}       )
																					merge_dicts(UpperCAmelCase__			,				UpperCAmelCase__       )
														else:
																					lowerCamelCase_							  =					value

							return destination



def      __lowerCAmelCase						(							UpperCAmelCase__				:      int = None       )  ->    bool:
							if port is None:
														lowerCamelCase_							  =					2_9_5_0_0
							with socket.socket(socket.AF_INET			,				socket.SOCK_STREAM       ) as s:
														return s.connect_ex(("""localhost""", port)       ) == 0





