




'''simple docstring'''







import contextlib
import importlib
import io
import unittest

import transformers

# Try to import everything from transformers to ensure every object can be loaded.
from transformers import *  # noqa F406
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, require_flax, require_tf, require_torch
from transformers.utils import ContextManagers, find_labels, is_flax_available, is_tf_available, is_torch_available


if is_torch_available():
      from transformers import BertForPreTraining, BertForQuestionAnswering, BertForSequenceClassification

if is_tf_available():
      from transformers import TFBertForPreTraining, TFBertForQuestionAnswering, TFBertForSequenceClassification

if is_flax_available():
      from transformers import FlaxBertForPreTraining, FlaxBertForQuestionAnswering, FlaxBertForSequenceClassification


lowercase      :   Any      =       DUMMY_UNKNOWN_IDENTIFIER
# An actual model hosted on huggingface.co

lowercase      :   Optional[int]      =       "main"
# Default branch name
lowercase      :   Tuple      =       "f2c752cfc5c0ab6f4bdec59acea69eefbee381c2"
# One particular commit (not the top of `main`)
lowercase      :   Optional[int]      =       "aaaaaaa"
# This commit does not exist, so we should 404.

lowercase      :   int      =       "d9e9f15bc825e4b2c9249e9578f884bbcb5e3684"
# Sha-1 of config.json on the top of `main`, for checking purposes
lowercase      :   List[Any]      =       "4b243c475af8d0a7754e87d7d096c92e5199ec2fe168a2ee7998e3b8e9bcb1d3"



@contextlib.contextmanager
def        lowerCamelCase__				(      ):
     print("""Welcome!"""				)
     yield
     print("""Bye!"""				)



@contextlib.contextmanager
def        lowerCamelCase__				(      ):
     print("""Bonjour!"""				)
     yield
     print("""Au revoir!"""				)







class 					_a (unittest.TestCase	):





     '''simple docstring'''



     def 			snake_case_      (				self							)      ->				Optional[Any]:
          # If the spec is missing, importlib would not be able to import the module dynamically.
          assert transformers.__spec__ is not None
          assert importlib.util.find_spec("""transformers"""							) is not None







class 					_a (unittest.TestCase	):





     '''simple docstring'''



     @unittest.mock.patch("""sys.stdout"""							,new_callable=io.StringIO							)
     def 			snake_case_      (				self							,__a							)      ->				List[Any]:
          with ContextManagers([]							):
               print("""Transformers are awesome!"""							)
          # The print statement adds a new line at the end of the output
          self.assertEqual(mock_stdout.getvalue()							,"""Transformers are awesome!\n"""							)







     @unittest.mock.patch("""sys.stdout"""							,new_callable=io.StringIO							)
     def 			snake_case_      (				self							,__a							)      ->				List[str]:
          with ContextManagers([context_en()]							):
               print("""Transformers are awesome!"""							)
          # The output should be wrapped with an English welcome and goodbye
          self.assertEqual(mock_stdout.getvalue()							,"""Welcome!\nTransformers are awesome!\nBye!\n"""							)







     @unittest.mock.patch("""sys.stdout"""							,new_callable=io.StringIO							)
     def 			snake_case_      (				self							,__a							)      ->				List[Any]:
          with ContextManagers([context_fr(), context_en()]							):
               print("""Transformers are awesome!"""							)
          # The output should be wrapped with an English and French welcome and goodbye
          self.assertEqual(mock_stdout.getvalue()							,"""Bonjour!\nWelcome!\nTransformers are awesome!\nBye!\nAu revoir!\n"""							)







     @require_torch
     def 			snake_case_      (				self							)      ->				Any:
          self.assertEqual(find_labels(__a							)							,["""labels"""]							)
          self.assertEqual(find_labels(__a							)							,["""labels""", """next_sentence_label"""]							)
          self.assertEqual(find_labels(__a							)							,["""start_positions""", """end_positions"""]							)







          class 					_a (_UpperCamelCase	):





               '''simple docstring'''



               pass
          self.assertEqual(find_labels(__a							)							,["""labels"""]							)







     @require_tf
     def 			snake_case_      (				self							)      ->				Any:
          self.assertEqual(find_labels(__a							)							,["""labels"""]							)
          self.assertEqual(find_labels(__a							)							,["""labels""", """next_sentence_label"""]							)
          self.assertEqual(find_labels(__a							)							,["""start_positions""", """end_positions"""]							)







          class 					_a (_UpperCamelCase	):





               '''simple docstring'''



               pass
          self.assertEqual(find_labels(__a							)							,["""labels"""]							)







     @require_flax
     def 			snake_case_      (				self							)      ->				Union[str, Any]:
          # Flax models don't have labels
          self.assertEqual(find_labels(__a							)							,[]							)
          self.assertEqual(find_labels(__a							)							,[]							)
          self.assertEqual(find_labels(__a							)							,[]							)







          class 					_a (_UpperCamelCase	):





               '''simple docstring'''



               pass
          self.assertEqual(find_labels(__a							)							,[]							)








"""simple docstring"""




import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_gpta import GPTaTokenizer


if TYPE_CHECKING:
 from transformers.pipelines.conversational import Conversation


_lowerCAmelCase							:   Any				=				logging.get_logger(__name__)

_lowerCAmelCase							:   Any				=				{"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

_lowerCAmelCase							:   List[str]				=				{
    "vocab_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/vocab.json",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/vocab.json",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/vocab.json",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/vocab.json",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/vocab.json",
    },
    "merges_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/merges.txt",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/merges.txt",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/merges.txt",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/merges.txt",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/tokenizer.json",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/tokenizer.json",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/tokenizer.json",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/tokenizer.json",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/tokenizer.json",
    },
}

_lowerCAmelCase							:   List[str]				=				{
    "gpt2": 10_24,
    "gpt2-medium": 10_24,
    "gpt2-large": 10_24,
    "gpt2-xl": 10_24,
    "distilgpt2": 10_24,
}






class       UpperCAmelCase_    (							_UpperCamelCase		):
      __SCREAMING_SNAKE_CASE		:  Optional[Any]						=					VOCAB_FILES_NAMES
      __SCREAMING_SNAKE_CASE		:  int						=					PRETRAINED_VOCAB_FILES_MAP
      __SCREAMING_SNAKE_CASE		:  Optional[int]						=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __SCREAMING_SNAKE_CASE		:  Optional[int]						=					['input_ids', 'attention_mask']
      __SCREAMING_SNAKE_CASE		:  Dict						=					GPTaTokenizer



      def __init__(	self	:      Optional[Any]	,      A	:      Optional[int]=None	,      A	:      List[Any]=None	,      A	:      Optional[int]=None	,      A	:      Optional[Any]="<|endoftext|>"	,      A	:      Union[str, Any]="<|endoftext|>"	,      A	:      Optional[Any]="<|endoftext|>"	,      A	:      Optional[Any]=False	,      **A	:      str	,      ):
           super().__init__(
               A	,      A	,      tokenizer_file=A	,      unk_token=A	,      bos_token=A	,      eos_token=A	,      add_prefix_space=A	,      **A	,      )

           _UpperCAmelCase     :      List[Any]														=       kwargs.pop("add_bos_token"	,      A		)

           _UpperCAmelCase     :      List[str]														=       json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()		)
           if pre_tok_state.get("add_prefix_space"	,      A		) != add_prefix_space:
                _UpperCAmelCase     :      Dict														=       getattr(A	,      pre_tok_state.pop("type"		)		)
                _UpperCAmelCase     :      Optional[Any]														=       add_prefix_space
                _UpperCAmelCase     :      Optional[Any]														=       pre_tok_class(**A		)

           _UpperCAmelCase     :      List[Any]														=       add_prefix_space



      def 		snake_case_   (	self	:      List[Any]	,      *A	:      List[Any]	,      **A	:      List[str]		):
           _UpperCAmelCase     :      Union[str, Any]														=       kwargs.get("is_split_into_words"	,      A		)
           assert self.add_prefix_space or not is_split_into_words, (
               f'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
               "to use it with pretokenized inputs."
           )

           return super()._batch_encode_plus(*A	,      **A		)



      def 		snake_case_   (	self	:      Optional[Any]	,      *A	:      List[Any]	,      **A	:      int		):
           _UpperCAmelCase     :      Union[str, Any]														=       kwargs.get("is_split_into_words"	,      A		)

           assert self.add_prefix_space or not is_split_into_words, (
               f'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
               "to use it with pretokenized inputs."
           )

           return super()._encode_plus(*A	,      **A		)



      def 		snake_case_   (	self	:      Optional[int]	,      A	:      str	,      A	:      Optional[str] = None		):
           _UpperCAmelCase     :      List[Any]														=       self._tokenizer.model.save(A	,      name=A		)
           return tuple(A		)



      def 		snake_case_   (	self	:      int	,      A	:      "Conversation"		):
           _UpperCAmelCase     :      List[str]														=       []
           for is_user, text in conversation.iter_texts():
                input_ids.extend(self.encode(A	,      add_special_tokens=A		) + [self.eos_token_id]		)

           if len(A		) > self.model_max_length:
                _UpperCAmelCase     :      Optional[int]														=       input_ids[-self.model_max_length :]
           return input_ids
