






from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_herbert import HerbertTokenizer


_UpperCamelCase			:	List[Any]								=logging.get_logger(__name__)

_UpperCamelCase			:	List[str]								={'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

_UpperCamelCase			:	Union[str, Any]								={
    'vocab_file': {
        'allegro/herbert-base-cased': 'https://huggingface.co/allegro/herbert-base-cased/resolve/main/vocab.json'
    },
    'merges_file': {
        'allegro/herbert-base-cased': 'https://huggingface.co/allegro/herbert-base-cased/resolve/main/merges.txt'
    },
}

_UpperCamelCase			:	Any								={'allegro/herbert-base-cased': 514}
_UpperCamelCase			:	List[str]								={}






class 						UpperCAmelCase__ (							__snake_case							):
		__snake_case      :       Optional[Any]     =   VOCAB_FILES_NAMES
		__snake_case      :       Any     =   PRETRAINED_VOCAB_FILES_MAP
		__snake_case      :       Any     =   PRETRAINED_INIT_CONFIGURATION
		__snake_case      :       Union[str, Any]     =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		__snake_case      :       Any     =   HerbertTokenizer







		def __init__(					self       ,A__=None       ,A__=None       ,A__=None       ,A__="<s>"       ,A__="<unk>"       ,A__="<pad>"       ,A__="<mask>"       ,A__="</s>"       ,**A__       ,):
						super().__init__(
						    A__       ,A__       ,tokenizer_file=A__       ,cls_token=A__       ,unk_token=A__       ,pad_token=A__       ,mask_token=A__       ,sep_token=A__       ,**A__       ,)







		def  A__      (					self       ,A__       ,A__ = None		):
						_A	:				List[str]   =      [self.cls_token_id]
						_A	:				Optional[int]   =      [self.sep_token_id]
						if token_ids_a is None:
										return cls + token_ids_a + sep

						return cls + token_ids_a + sep + token_ids_a + sep







		def  A__      (					self       ,A__       ,A__ = None       ,A__ = False		):
						if already_has_special_tokens:
										return super().get_special_tokens_mask(
										    token_ids_a=A__       ,token_ids_a=A__       ,already_has_special_tokens=A__		)

						if token_ids_a is None:
										return [1] + ([0] * len(A__		)) + [1]
						return [1] + ([0] * len(A__		)) + [1] + ([0] * len(A__		)) + [1]







		def  A__      (					self       ,A__       ,A__ = None		):
						_A	:				Union[str, Any]   =      [self.sep_token_id]
						_A	:				List[str]   =      [self.cls_token_id]

						if token_ids_a is None:
										return len(cls + token_ids_a + sep		) * [0]
						return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]







		def  A__      (					self       ,A__       ,A__ = None		):
						_A	:				List[Any]   =      self._tokenizer.model.save(A__       ,name=A__		)
						return tuple(A__		)











from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


_UpperCamelCase			:	Dict								={
    'configuration_albert': ['ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'AlbertConfig', 'AlbertOnnxConfig'],
}

try:
	if not is_sentencepiece_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	_UpperCamelCase			:	List[Any]								=['AlbertTokenizer']

try:
	if not is_tokenizers_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	_UpperCamelCase			:	List[Any]								=['AlbertTokenizerFast']

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	_UpperCamelCase			:	Optional[int]								=[
	    'ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST',
	    'AlbertForMaskedLM',
	    'AlbertForMultipleChoice',
	    'AlbertForPreTraining',
	    'AlbertForQuestionAnswering',
	    'AlbertForSequenceClassification',
	    'AlbertForTokenClassification',
	    'AlbertModel',
	    'AlbertPreTrainedModel',
	    'load_tf_weights_in_albert',
	]

try:
	if not is_tf_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	_UpperCamelCase			:	List[Any]								=[
	    'TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST',
	    'TFAlbertForMaskedLM',
	    'TFAlbertForMultipleChoice',
	    'TFAlbertForPreTraining',
	    'TFAlbertForQuestionAnswering',
	    'TFAlbertForSequenceClassification',
	    'TFAlbertForTokenClassification',
	    'TFAlbertMainLayer',
	    'TFAlbertModel',
	    'TFAlbertPreTrainedModel',
	]

try:
	if not is_flax_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	_UpperCamelCase			:	Any								=[
	    'FlaxAlbertForMaskedLM',
	    'FlaxAlbertForMultipleChoice',
	    'FlaxAlbertForPreTraining',
	    'FlaxAlbertForQuestionAnswering',
	    'FlaxAlbertForSequenceClassification',
	    'FlaxAlbertForTokenClassification',
	    'FlaxAlbertModel',
	    'FlaxAlbertPreTrainedModel',
	]

if TYPE_CHECKING:
	from .configuration_albert import ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, AlbertConfig, AlbertOnnxConfig

	try:
		if not is_sentencepiece_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .tokenization_albert import AlbertTokenizer

	try:
		if not is_tokenizers_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .tokenization_albert_fast import AlbertTokenizerFast

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_albert import (
		    ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
		    AlbertForMaskedLM,
		    AlbertForMultipleChoice,
		    AlbertForPreTraining,
		    AlbertForQuestionAnswering,
		    AlbertForSequenceClassification,
		    AlbertForTokenClassification,
		    AlbertModel,
		    AlbertPreTrainedModel,
		    load_tf_weights_in_albert,
		)

	try:
		if not is_tf_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_tf_albert import (
		    TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
		    TFAlbertForMaskedLM,
		    TFAlbertForMultipleChoice,
		    TFAlbertForPreTraining,
		    TFAlbertForQuestionAnswering,
		    TFAlbertForSequenceClassification,
		    TFAlbertForTokenClassification,
		    TFAlbertMainLayer,
		    TFAlbertModel,
		    TFAlbertPreTrainedModel,
		)

	try:
		if not is_flax_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_flax_albert import (
		    FlaxAlbertForMaskedLM,
		    FlaxAlbertForMultipleChoice,
		    FlaxAlbertForPreTraining,
		    FlaxAlbertForQuestionAnswering,
		    FlaxAlbertForSequenceClassification,
		    FlaxAlbertForTokenClassification,
		    FlaxAlbertModel,
		    FlaxAlbertPreTrainedModel,
		)
else:
	import sys

	_UpperCamelCase			:	str								=_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)



