





"""simple docstring"""





import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
							from .tokenization_albert import AlbertTokenizer
else:
							_UpperCamelCase					      =   None

_UpperCamelCase					      =   logging.get_logger(__name__)
_UpperCamelCase					      =   {'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

_UpperCamelCase					      =   {
    'vocab_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/spiece.model',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/spiece.model',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/spiece.model',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/spiece.model',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model',
    },
    'tokenizer_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/tokenizer.json',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/tokenizer.json',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/tokenizer.json',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/tokenizer.json',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/tokenizer.json',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/tokenizer.json',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/tokenizer.json',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/tokenizer.json',
    },
}

_UpperCamelCase					      =   {
    'albert-base-v1': 512,
    'albert-large-v1': 512,
    'albert-xlarge-v1': 512,
    'albert-xxlarge-v1': 512,
    'albert-base-v2': 512,
    'albert-large-v2': 512,
    'albert-xlarge-v2': 512,
    'albert-xxlarge-v2': 512,
}

_UpperCamelCase					      =   '▁'
class 		SCREAMING_SNAKE_CASE_				(     snake_case__ ):
				"""simple docstring"""
				__snake_case :							Optional[Any]									=		VOCAB_FILES_NAMES
				__snake_case :							Dict									=		PRETRAINED_VOCAB_FILES_MAP
				__snake_case :							Union[str, Any]									=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__snake_case :							int									=		AlbertTokenizer




				def __init__(  self	:Optional[Any]    ,     __lowercase	:int=None    ,     __lowercase	:Any=None    ,     __lowercase	:Union[str, Any]=True    ,     __lowercase	:Optional[Any]=True    ,     __lowercase	:Optional[int]=False    ,     __lowercase	:Tuple="[CLS]"    ,     __lowercase	:str="[SEP]"    ,     __lowercase	:Optional[int]="<unk>"    ,     __lowercase	:Union[str, Any]="[SEP]"    ,     __lowercase	:List[Any]="<pad>"    ,     __lowercase	:List[str]="[CLS]"    ,     __lowercase	:int="[MASK]"    ,     **__lowercase	:int    ,     ):
											# Mask token behave like a normal word, i.e. include the space before it and
											# is included in the raw text, there should be a match in a non-normalized sentence.
											__lowerCamelCase   :   Dict       =(
											    AddedToken(__lowercase    ,     lstrip=__lowercase    ,     rstrip=__lowercase    ,     normalized=__lowercase      )
											    if isinstance(__lowercase    ,     __lowercase      )
											    else mask_token
											)

											super().__init__(
											    __lowercase    ,     tokenizer_file=__lowercase    ,     do_lower_case=__lowercase    ,     remove_space=__lowercase    ,     keep_accents=__lowercase    ,     bos_token=__lowercase    ,     eos_token=__lowercase    ,     unk_token=__lowercase    ,     sep_token=__lowercase    ,     pad_token=__lowercase    ,     cls_token=__lowercase    ,     mask_token=__lowercase    ,     **__lowercase    ,     )

											__lowerCamelCase   :   List[Any]       =do_lower_case
											__lowerCamelCase   :   Optional[int]       =remove_space
											__lowerCamelCase   :   Optional[Any]       =keep_accents
											__lowerCamelCase   :   Optional[Any]       =vocab_file
											__lowerCamelCase   :   Dict       =False if not self.vocab_file else True




				def        __lowercase	(  self	:str    ,     __lowercase	:List[int]    ,     __lowercase	:Optional[List[int]] = None      ):
											__lowerCamelCase   :   Any       =[self.sep_token_id]
											__lowerCamelCase   :   Dict       =[self.cls_token_id]
											if token_ids_a is None:
																		return cls + token_ids_a + sep
											return cls + token_ids_a + sep + token_ids_a + sep




				def        __lowercase	(  self	:Union[str, Any]    ,     __lowercase	:List[int]    ,     __lowercase	:Optional[List[int]] = None      ):
											__lowerCamelCase   :   str       =[self.sep_token_id]
											__lowerCamelCase   :   str       =[self.cls_token_id]

											if token_ids_a is None:
																		return len(cls + token_ids_a + sep      ) * [0]
											return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]




				def        __lowercase	(  self	:int    ,     __lowercase	:str    ,     __lowercase	:Optional[str] = None      ):
											if not self.can_save_slow_tokenizer:
																		raise ValueError(
																		    '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
																		    '''tokenizer.'''      )

											if not os.path.isdir(__lowercase      ):
																		logger.error(f'Vocabulary path ({save_directory}) should be a directory'      )
																		return
											__lowerCamelCase   :   Optional[int]       =os.path.join(
											    __lowercase    ,     (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']      )

											if os.path.abspath(self.vocab_file      ) != os.path.abspath(__lowercase      ):
																		copyfile(self.vocab_file    ,     __lowercase      )

											return (out_vocab_file,)











"""simple docstring"""





import contextlib
import importlib
import io
import unittest

import transformers

# Try to import everything from transformers to ensure every object can be loaded.
from transformers import *  # noqa F406
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, require_flax, require_tf, require_torch
from transformers.utils import ContextManagers, find_labels, is_flax_available, is_tf_available, is_torch_available


if is_torch_available():
							from transformers import BertForPreTraining, BertForQuestionAnswering, BertForSequenceClassification

if is_tf_available():
							from transformers import TFBertForPreTraining, TFBertForQuestionAnswering, TFBertForSequenceClassification

if is_flax_available():
							from transformers import FlaxBertForPreTraining, FlaxBertForQuestionAnswering, FlaxBertForSequenceClassification


_UpperCamelCase					      =   DUMMY_UNKNOWN_IDENTIFIER
# An actual model hosted on huggingface.co

_UpperCamelCase					      =   'main'
# Default branch name
_UpperCamelCase					      =   'f2c752cfc5c0ab6f4bdec59acea69eefbee381c2'
# One particular commit (not the top of `main`)
_UpperCamelCase					      =   'aaaaaaa'
# This commit does not exist, so we should 404.

_UpperCamelCase					      =   'd9e9f15bc825e4b2c9249e9578f884bbcb5e3684'
# Sha-1 of config.json on the top of `main`, for checking purposes
_UpperCamelCase					      =   '4b243c475af8d0a7754e87d7d096c92e5199ec2fe168a2ee7998e3b8e9bcb1d3'



@contextlib.contextmanager
def 					lowerCAmelCase_      (   ):


							'''simple docstring'''







							print('''Welcome!''' )
							yield
							print('''Bye!''' )



@contextlib.contextmanager
def 					lowerCAmelCase_      (   ):


							'''simple docstring'''







							print('''Bonjour!''' )
							yield
							print('''Au revoir!''' )
class 		SCREAMING_SNAKE_CASE_				(     unittest.TestCase ):
				"""simple docstring"""




				def        __lowercase	(  self	:str      ):
											# If the spec is missing, importlib would not be able to import the module dynamically.
											assert transformers.__spec__ is not None
											assert importlib.util.find_spec('''transformers'''      ) is not None

class 		SCREAMING_SNAKE_CASE_				(     unittest.TestCase ):
				"""simple docstring"""




				@unittest.mock.patch('''sys.stdout'''    ,     new_callable=io.StringIO      )
				def        __lowercase	(  self	:str    ,     __lowercase	:str      ):
											with ContextManagers([]      ):
																		print('''Transformers are awesome!'''      )
											# The print statement adds a new line at the end of the output
											self.assertEqual(mock_stdout.getvalue()    ,     '''Transformers are awesome!\n'''      )




				@unittest.mock.patch('''sys.stdout'''    ,     new_callable=io.StringIO      )
				def        __lowercase	(  self	:Any    ,     __lowercase	:Tuple      ):
											with ContextManagers([context_en()]      ):
																		print('''Transformers are awesome!'''      )
											# The output should be wrapped with an English welcome and goodbye
											self.assertEqual(mock_stdout.getvalue()    ,     '''Welcome!\nTransformers are awesome!\nBye!\n'''      )




				@unittest.mock.patch('''sys.stdout'''    ,     new_callable=io.StringIO      )
				def        __lowercase	(  self	:List[Any]    ,     __lowercase	:Optional[Any]      ):
											with ContextManagers([context_fr(), context_en()]      ):
																		print('''Transformers are awesome!'''      )
											# The output should be wrapped with an English and French welcome and goodbye
											self.assertEqual(mock_stdout.getvalue()    ,     '''Bonjour!\nWelcome!\nTransformers are awesome!\nBye!\nAu revoir!\n'''      )




				@require_torch
				def        __lowercase	(  self	:Tuple      ):
											self.assertEqual(find_labels(__lowercase      )    ,     ['''labels''']      )
											self.assertEqual(find_labels(__lowercase      )    ,     ['''labels''', '''next_sentence_label''']      )
											self.assertEqual(find_labels(__lowercase      )    ,     ['''start_positions''', '''end_positions''']      )
											class 		SCREAMING_SNAKE_CASE_				(     snake_case__ ):
															"""simple docstring"""
															pass

											self.assertEqual(find_labels(__lowercase      )    ,     ['''labels''']      )




				@require_tf
				def        __lowercase	(  self	:str      ):
											self.assertEqual(find_labels(__lowercase      )    ,     ['''labels''']      )
											self.assertEqual(find_labels(__lowercase      )    ,     ['''labels''', '''next_sentence_label''']      )
											self.assertEqual(find_labels(__lowercase      )    ,     ['''start_positions''', '''end_positions''']      )
											class 		SCREAMING_SNAKE_CASE_				(     snake_case__ ):
															"""simple docstring"""
															pass

											self.assertEqual(find_labels(__lowercase      )    ,     ['''labels''']      )




				@require_flax
				def        __lowercase	(  self	:str      ):
											# Flax models don't have labels
											self.assertEqual(find_labels(__lowercase      )    ,     []      )
											self.assertEqual(find_labels(__lowercase      )    ,     []      )
											self.assertEqual(find_labels(__lowercase      )    ,     []      )
											class 		SCREAMING_SNAKE_CASE_				(     snake_case__ ):
															"""simple docstring"""
															pass

											self.assertEqual(find_labels(__lowercase      )    ,     []      )




