






import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roberta import RobertaTokenizer


lowerCamelCase__							=     logging.get_logger(__name__)

lowerCamelCase__							=     {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

lowerCamelCase__							=     {
    """vocab_file""": {
        """roberta-base""": """https://huggingface.co/roberta-base/resolve/main/vocab.json""",
        """roberta-large""": """https://huggingface.co/roberta-large/resolve/main/vocab.json""",
        """roberta-large-mnli""": """https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json""",
        """distilroberta-base""": """https://huggingface.co/distilroberta-base/resolve/main/vocab.json""",
        """roberta-base-openai-detector""": """https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json""",
        """roberta-large-openai-detector""": (
            """https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """roberta-base""": """https://huggingface.co/roberta-base/resolve/main/merges.txt""",
        """roberta-large""": """https://huggingface.co/roberta-large/resolve/main/merges.txt""",
        """roberta-large-mnli""": """https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt""",
        """distilroberta-base""": """https://huggingface.co/distilroberta-base/resolve/main/merges.txt""",
        """roberta-base-openai-detector""": """https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt""",
        """roberta-large-openai-detector""": (
            """https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt"""
        ),
    },
    """tokenizer_file""": {
        """roberta-base""": """https://huggingface.co/roberta-base/resolve/main/tokenizer.json""",
        """roberta-large""": """https://huggingface.co/roberta-large/resolve/main/tokenizer.json""",
        """roberta-large-mnli""": """https://huggingface.co/roberta-large-mnli/resolve/main/tokenizer.json""",
        """distilroberta-base""": """https://huggingface.co/distilroberta-base/resolve/main/tokenizer.json""",
        """roberta-base-openai-detector""": (
            """https://huggingface.co/roberta-base-openai-detector/resolve/main/tokenizer.json"""
        ),
        """roberta-large-openai-detector""": (
            """https://huggingface.co/roberta-large-openai-detector/resolve/main/tokenizer.json"""
        ),
    },
}

lowerCamelCase__							=     {
    """roberta-base""": 512,
    """roberta-large""": 512,
    """roberta-large-mnli""": 512,
    """distilroberta-base""": 512,
    """roberta-base-openai-detector""": 512,
    """roberta-large-openai-detector""": 512,
}




class 			snake_case__   (							lowercase_):

       '''simple docstring'''
       lowerCamelCase							:				str 				= VOCAB_FILES_NAMES
       lowerCamelCase							:				Tuple 				= PRETRAINED_VOCAB_FILES_MAP
       lowerCamelCase							:				Tuple 				= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       lowerCamelCase							:				Tuple 				= ["input_ids", "attention_mask"]
       lowerCamelCase							:				Any 				= RobertaTokenizer



       def __init__(							self			,						a__=None			,						a__=None			,						a__=None			,						a__="replace"			,						a__="<s>"			,						a__="</s>"			,						a__="</s>"			,						a__="<s>"			,						a__="<unk>"			,						a__="<pad>"			,						a__="<mask>"			,						a__=False			,						a__=True			,						**a__			,						) ->      Dict:






            '''simple docstring'''





            super().__init__(
                a__			,						a__			,						tokenizer_file=a__			,						errors=a__			,						bos_token=a__			,						eos_token=a__			,						sep_token=a__			,						cls_token=a__			,						unk_token=a__			,						pad_token=a__			,						mask_token=a__			,						add_prefix_space=a__			,						trim_offsets=a__			,						**a__			,						)

            __snake_case			:Optional[int]          =						json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()    )
            if pre_tok_state.get("""add_prefix_space"""			,						a__    ) != add_prefix_space:
                 __snake_case			:str          =						getattr(a__			,						pre_tok_state.pop("""type"""    )    )
                 __snake_case			:Optional[Any]          =						add_prefix_space
                 __snake_case			:str          =						pre_tok_class(**a__    )

            __snake_case			:List[Any]          =						add_prefix_space

            __snake_case			:Optional[Any]          =						"""post_processor"""
            __snake_case			:Dict          =						getattr(self.backend_tokenizer			,						a__			,						a__    )
            if tokenizer_component_instance:
                 __snake_case			:Dict          =						json.loads(tokenizer_component_instance.__getstate__()    )

                 # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
                 if "sep" in state:
                      __snake_case			:Tuple          =						tuple(state["""sep"""]    )
                 if "cls" in state:
                      __snake_case			:Optional[Any]          =						tuple(state["""cls"""]    )

                 __snake_case			:int          =						False

                 if state.get("""add_prefix_space"""			,						a__    ) != add_prefix_space:
                      __snake_case			:Tuple          =						add_prefix_space
                      __snake_case			:Dict          =						True

                 if state.get("""trim_offsets"""			,						a__    ) != trim_offsets:
                      __snake_case			:Optional[Any]          =						trim_offsets
                      __snake_case			:Optional[int]          =						True

                 if changes_to_apply:
                      __snake_case			:List[str]          =						getattr(a__			,						state.pop("""type"""    )    )
                      __snake_case			:Optional[Any]          =						component_class(**a__    )
                      setattr(self.backend_tokenizer			,						a__			,						a__    )



       @property
       def 					__lowercase						(							self    ) ->      str:






            '''simple docstring'''





            if self._mask_token is None:
                 if self.verbose:
                      logger.error("""Using mask_token, but it is not set yet."""    )
                 return None
            return str(self._mask_token    )



       @mask_token.setter
       def 					__lowercase						(							self			,						a__    ) ->      List[str]:






            '''simple docstring'''





            __snake_case			:Optional[Any]          =						AddedToken(a__			,						lstrip=a__			,						rstrip=a__    ) if isinstance(a__			,						a__    ) else value
            __snake_case			:Tuple          =						value



       def 					__lowercase						(							self			,						*a__			,						**a__    ) ->      BatchEncoding:






            '''simple docstring'''





            __snake_case			:List[Any]          =						kwargs.get("""is_split_into_words"""			,						a__    )
            assert self.add_prefix_space or not is_split_into_words, (
                F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
                "to use it with pretokenized inputs."
            )

            return super()._batch_encode_plus(*a__			,						**a__    )



       def 					__lowercase						(							self			,						*a__			,						**a__    ) ->      BatchEncoding:






            '''simple docstring'''





            __snake_case			:Any          =						kwargs.get("""is_split_into_words"""			,						a__    )

            assert self.add_prefix_space or not is_split_into_words, (
                F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
                "to use it with pretokenized inputs."
            )

            return super()._encode_plus(*a__			,						**a__    )



       def 					__lowercase						(							self			,						a__			,						a__ = None    ) ->      Tuple[str]:






            '''simple docstring'''





            __snake_case			:List[str]          =						self._tokenizer.model.save(a__			,						name=a__    )
            return tuple(a__    )



       def 					__lowercase						(							self			,						a__			,						a__=None    ) ->      Optional[int]:






            '''simple docstring'''





            __snake_case			:str          =						[self.bos_token_id] + token_ids_a + [self.eos_token_id]
            if token_ids_a is None:
                 return output

            return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]



       def 					__lowercase						(							self			,						a__			,						a__ = None    ) ->      List[int]:






            '''simple docstring'''





            __snake_case			:str          =						[self.sep_token_id]
            __snake_case			:Dict          =						[self.cls_token_id]

            if token_ids_a is None:
                 return len(cls + token_ids_a + sep    ) * [0]
            return len(cls + token_ids_a + sep + sep + token_ids_a + sep    ) * [0]










import importlib
import json
import os
import sys
import tempfile
import unittest
from pathlib import Path

import transformers
import transformers.models.auto
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.bert.configuration_bert import BertConfig
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, get_tests_dir


sys.path.append(str(Path(__file__).parent.parent.parent.parent / """utils"""))

from test_module.custom_configuration import CustomConfig  # noqa E402


lowerCamelCase__							=     get_tests_dir("""fixtures/dummy-config.json""")




class 			snake_case__   (							unittest.TestCase):

       '''simple docstring'''



       def 					__lowercase						(							self    ) ->      Union[str, Any]:






            '''simple docstring'''





            __snake_case			:Optional[int]          =						0



       def 					__lowercase						(							self    ) ->      int:






            '''simple docstring'''





            self.assertIsNotNone(transformers.models.auto.__spec__    )
            self.assertIsNotNone(importlib.util.find_spec("""transformers.models.auto"""    )    )



       def 					__lowercase						(							self    ) ->      List[str]:






            '''simple docstring'''





            __snake_case			:Optional[Any]          =						AutoConfig.from_pretrained("""bert-base-uncased"""    )
            self.assertIsInstance(a__			,						a__    )



       def 					__lowercase						(							self    ) ->      Optional[int]:






            '''simple docstring'''





            __snake_case			:List[str]          =						AutoConfig.from_pretrained(a__    )
            self.assertIsInstance(a__			,						a__    )



       def 					__lowercase						(							self    ) ->      List[Any]:






            '''simple docstring'''





            __snake_case			:Tuple          =						AutoConfig.from_pretrained(a__    )
            self.assertIsInstance(a__			,						a__    )



       def 					__lowercase						(							self    ) ->      str:






            '''simple docstring'''





            __snake_case			:Optional[int]          =						AutoConfig.for_model("""roberta"""    )
            self.assertIsInstance(a__			,						a__    )



       def 					__lowercase						(							self    ) ->      List[Any]:






            '''simple docstring'''





            with tempfile.TemporaryDirectory() as tmp_dir:
                 # This model name contains bert and roberta, but roberta ends up being picked.
                 __snake_case			:List[Any]          =						os.path.join(a__			,						"""fake-roberta"""    )
                 os.makedirs(a__			,						exist_ok=a__    )
                 with open(os.path.join(a__			,						"""config.json"""    )			,						"""w"""    ) as f:
                      f.write(json.dumps({}    )    )
                 __snake_case			:Optional[Any]          =						AutoConfig.from_pretrained(a__    )
                 self.assertEqual(type(a__    )			,						a__    )



       def 					__lowercase						(							self    ) ->      Dict:






            '''simple docstring'''





            try:
                 AutoConfig.register("""custom"""			,						a__    )
                 # Wrong model type will raise an error
                 with self.assertRaises(a__    ):
                      AutoConfig.register("""model"""			,						a__    )
                 # Trying to register something existing in the Transformers library will raise an error
                 with self.assertRaises(a__    ):
                      AutoConfig.register("""bert"""			,						a__    )

                 # Now that the config is registered, it can be used as any other config with the auto-API
                 __snake_case			:Optional[int]          =						CustomConfig()
                 with tempfile.TemporaryDirectory() as tmp_dir:
                      config.save_pretrained(a__    )
                      __snake_case			:Tuple          =						AutoConfig.from_pretrained(a__    )
                      self.assertIsInstance(a__			,						a__    )

            finally:
                 if "custom" in CONFIG_MAPPING._extra_content:
                      del CONFIG_MAPPING._extra_content["custom"]



       def 					__lowercase						(							self    ) ->      int:






            '''simple docstring'''





            with self.assertRaisesRegex(
                a__			,						"""bert-base is not a local folder and is not a valid model identifier"""    ):
                 __snake_case			:str          =						AutoConfig.from_pretrained("""bert-base"""    )



       def 					__lowercase						(							self    ) ->      str:






            '''simple docstring'''





            with self.assertRaisesRegex(
                a__			,						R"""aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"""    ):
                 __snake_case			:Dict          =						AutoConfig.from_pretrained(a__			,						revision="""aaaaaa"""    )



       def 					__lowercase						(							self    ) ->      Optional[Any]:






            '''simple docstring'''





            with self.assertRaisesRegex(
                a__			,						"""hf-internal-testing/no-config-test-repo does not appear to have a file named config.json."""			,						):
                 __snake_case			:Optional[int]          =						AutoConfig.from_pretrained("""hf-internal-testing/no-config-test-repo"""    )



       def 					__lowercase						(							self    ) ->      int:






            '''simple docstring'''





            with self.assertRaises(a__    ):
                 __snake_case			:Dict          =						AutoConfig.from_pretrained("""hf-internal-testing/test_dynamic_model"""    )
            # If remote code is disabled, we can't load this config.
            with self.assertRaises(a__    ):
                 __snake_case			:int          =						AutoConfig.from_pretrained("""hf-internal-testing/test_dynamic_model"""			,						trust_remote_code=a__    )

            __snake_case			:Tuple          =						AutoConfig.from_pretrained("""hf-internal-testing/test_dynamic_model"""			,						trust_remote_code=a__    )
            self.assertEqual(config.__class__.__name__			,						"""NewModelConfig"""    )

            # Test config can be reloaded.
            with tempfile.TemporaryDirectory() as tmp_dir:
                 config.save_pretrained(a__    )
                 __snake_case			:Union[str, Any]          =						AutoConfig.from_pretrained(a__			,						trust_remote_code=a__    )
            self.assertEqual(reloaded_config.__class__.__name__			,						"""NewModelConfig"""    )



       def 					__lowercase						(							self    ) ->      Dict:






            '''simple docstring'''





            class 			snake_case__   (							lowercase_):

                   '''simple docstring'''
                   lowerCamelCase							:				Optional[Any] 				= "new-model"

            try:
                 AutoConfig.register("""new-model"""			,						a__    )
                 # If remote code is not set, the default is to use local
                 __snake_case			:List[Any]          =						AutoConfig.from_pretrained("""hf-internal-testing/test_dynamic_model"""    )
                 self.assertEqual(config.__class__.__name__			,						"""NewModelConfigLocal"""    )

                 # If remote code is disabled, we load the local one.
                 __snake_case			:Tuple          =						AutoConfig.from_pretrained("""hf-internal-testing/test_dynamic_model"""			,						trust_remote_code=a__    )
                 self.assertEqual(config.__class__.__name__			,						"""NewModelConfigLocal"""    )

                 # If remote is enabled, we load from the Hub
                 __snake_case			:List[Any]          =						AutoConfig.from_pretrained("""hf-internal-testing/test_dynamic_model"""			,						trust_remote_code=a__    )
                 self.assertEqual(config.__class__.__name__			,						"""NewModelConfig"""    )

            finally:
                 if "new-model" in CONFIG_MAPPING._extra_content:
                      del CONFIG_MAPPING._extra_content["new-model"]


