



'''simple docstring'''

import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bart import BartTokenizer


snake_case_			=     logging.get_logger(__name__)


snake_case_			=     {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

# See all BART models at https://huggingface.co/models?filter=bart
snake_case_			=     {
    """vocab_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/vocab.json""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/vocab.json""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/merges.txt""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/merges.txt""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/tokenizer.json""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/tokenizer.json""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/tokenizer.json""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/tokenizer.json""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/tokenizer.json""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/tokenizer.json""",
    },
}

snake_case_			=     {
    """facebook/bart-base""": 1024,
    """facebook/bart-large""": 1024,
    """facebook/bart-large-mnli""": 1024,
    """facebook/bart-large-cnn""": 1024,
    """facebook/bart-large-xsum""": 1024,
    """yjernite/bart_eli5""": 1024,
}


class 	a__     (				_lowercase	):
	__magic_name__		:    Any	    =     VOCAB_FILES_NAMES
	__magic_name__		:    Tuple	    =     PRETRAINED_VOCAB_FILES_MAP
	__magic_name__		:    str	    =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	__magic_name__		:    Any	    =     ["input_ids", "attention_mask"]
	__magic_name__		:    Optional[Any]	    =     BartTokenizer


	def __init__(self : List[str],					__UpperCAmelCase : Optional[int]=None,					__UpperCAmelCase : List[Any]=None,					__UpperCAmelCase : Optional[int]=None,					__UpperCAmelCase : int="replace",					__UpperCAmelCase : List[str]="<s>",					__UpperCAmelCase : Tuple="</s>",					__UpperCAmelCase : int="</s>",					__UpperCAmelCase : Dict="<s>",					__UpperCAmelCase : int="<unk>",					__UpperCAmelCase : Dict="<pad>",					__UpperCAmelCase : Optional[Any]="<mask>",					__UpperCAmelCase : Optional[int]=False,					__UpperCAmelCase : Tuple=True,					**__UpperCAmelCase : Any,					)     ->			str:

					"""simple docstring"""

					super().__init__(
					    __UpperCAmelCase,					__UpperCAmelCase,					tokenizer_file=__UpperCAmelCase,					errors=__UpperCAmelCase,					bos_token=__UpperCAmelCase,					eos_token=__UpperCAmelCase,					sep_token=__UpperCAmelCase,					cls_token=__UpperCAmelCase,					unk_token=__UpperCAmelCase,					pad_token=__UpperCAmelCase,					mask_token=__UpperCAmelCase,					add_prefix_space=__UpperCAmelCase,					trim_offsets=__UpperCAmelCase,					**__UpperCAmelCase,					)

					SCREAMING_SNAKE_CASE  :						int      					= json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()      )
					if pre_tok_state.get('''add_prefix_space''',					__UpperCAmelCase      ) != add_prefix_space:
									SCREAMING_SNAKE_CASE  :						Dict      					= getattr(__UpperCAmelCase,					pre_tok_state.pop('''type'''      )      )
									SCREAMING_SNAKE_CASE  :						str      					= add_prefix_space
									SCREAMING_SNAKE_CASE  :						Optional[Any]      					= pre_tok_class(**__UpperCAmelCase      )

					SCREAMING_SNAKE_CASE  :						Tuple      					= add_prefix_space

					# the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
					SCREAMING_SNAKE_CASE  :						str      					= '''post_processor'''
					SCREAMING_SNAKE_CASE  :						int      					= getattr(self.backend_tokenizer,					__UpperCAmelCase,					__UpperCAmelCase      )
					if tokenizer_component_instance:
									SCREAMING_SNAKE_CASE  :						List[str]      					= json.loads(tokenizer_component_instance.__getstate__()      )

									# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
									if "sep" in state:
													SCREAMING_SNAKE_CASE  :						int      					= tuple(state['''sep''']      )
									if "cls" in state:
													SCREAMING_SNAKE_CASE  :						List[str]      					= tuple(state['''cls''']      )

									SCREAMING_SNAKE_CASE  :						List[str]      					= False

									if state.get('''add_prefix_space''',					__UpperCAmelCase      ) != add_prefix_space:
													SCREAMING_SNAKE_CASE  :						Any      					= add_prefix_space
													SCREAMING_SNAKE_CASE  :						List[Any]      					= True

									if state.get('''trim_offsets''',					__UpperCAmelCase      ) != trim_offsets:
													SCREAMING_SNAKE_CASE  :						List[Any]      					= trim_offsets
													SCREAMING_SNAKE_CASE  :						List[str]      					= True

									if changes_to_apply:
													SCREAMING_SNAKE_CASE  :						Union[str, Any]      					= getattr(__UpperCAmelCase,					state.pop('''type'''      )      )
													SCREAMING_SNAKE_CASE  :						Optional[int]      					= component_class(**__UpperCAmelCase      )
													setattr(self.backend_tokenizer,					__UpperCAmelCase,					__UpperCAmelCase      )


	@property
	def 				lowercase__  (self : Dict      )     ->			str:

					"""simple docstring"""

					if self._mask_token is None:
									if self.verbose:
													logger.error('''Using mask_token, but it is not set yet.'''      )
									return None
					return str(self._mask_token      )


	@mask_token.setter
	def 				lowercase__  (self : Union[str, Any],					__UpperCAmelCase : List[Any]      )     ->			Any:

					"""simple docstring"""

					SCREAMING_SNAKE_CASE  :						Tuple      					= AddedToken(__UpperCAmelCase,					lstrip=__UpperCAmelCase,					rstrip=__UpperCAmelCase      ) if isinstance(__UpperCAmelCase,					__UpperCAmelCase      ) else value
					SCREAMING_SNAKE_CASE  :						Union[str, Any]      					= value


	def 				lowercase__  (self : int,					*__UpperCAmelCase : Optional[int],					**__UpperCAmelCase : Union[str, Any]      )     ->			BatchEncoding:

					"""simple docstring"""

					SCREAMING_SNAKE_CASE  :						List[Any]      					= kwargs.get('''is_split_into_words''',					__UpperCAmelCase      )

					if is_split_into_words and not self.add_prefix_space:
									raise ValueError(
									    F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
									    '''to use it with pretokenized inputs.'''      )

					return super()._batch_encode_plus(*__UpperCAmelCase,					**__UpperCAmelCase      )


	def 				lowercase__  (self : Tuple,					*__UpperCAmelCase : Union[str, Any],					**__UpperCAmelCase : Dict      )     ->			BatchEncoding:

					"""simple docstring"""

					SCREAMING_SNAKE_CASE  :						List[Any]      					= kwargs.get('''is_split_into_words''',					__UpperCAmelCase      )

					if is_split_into_words and not self.add_prefix_space:
									raise ValueError(
									    F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
									    '''to use it with pretokenized inputs.'''      )

					return super()._encode_plus(*__UpperCAmelCase,					**__UpperCAmelCase      )


	def 				lowercase__  (self : Optional[Any],					__UpperCAmelCase : str,					__UpperCAmelCase : Optional[str] = None      )     ->			Tuple[str]:

					"""simple docstring"""

					SCREAMING_SNAKE_CASE  :						Dict      					= self._tokenizer.model.save(__UpperCAmelCase,					name=__UpperCAmelCase      )
					return tuple(__UpperCAmelCase      )


	def 				lowercase__  (self : Tuple,					__UpperCAmelCase : Optional[int],					__UpperCAmelCase : Optional[Any]=None      )     ->			Dict:

					"""simple docstring"""

					SCREAMING_SNAKE_CASE  :						List[Any]      					= [self.bos_token_id] + token_ids_a + [self.eos_token_id]
					if token_ids_a is None:
									return output

					return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]






	def 				lowercase__  (self : int,					__UpperCAmelCase : List[int],					__UpperCAmelCase : Optional[List[int]] = None      )     ->			List[int]:

					"""simple docstring"""

					SCREAMING_SNAKE_CASE  :						List[Any]      					= [self.sep_token_id]
					SCREAMING_SNAKE_CASE  :						str      					= [self.cls_token_id]

					if token_ids_a is None:
									return len(cls + token_ids_a + sep      ) * [0]
					return len(cls + token_ids_a + sep + sep + token_ids_a + sep      ) * [0]









'''simple docstring'''

from ...configuration_utils import PretrainedConfig
from ...utils import logging


snake_case_			=     logging.get_logger(__name__)

snake_case_			=     {
    """microsoft/markuplm-base""": """https://huggingface.co/microsoft/markuplm-base/resolve/main/config.json""",
    """microsoft/markuplm-large""": """https://huggingface.co/microsoft/markuplm-large/resolve/main/config.json""",
}


class 	a__     (				_lowercase	):
	__magic_name__		:    List[Any]	    =     "markuplm"


	def __init__(self : List[Any],					__UpperCAmelCase : int=30522,					__UpperCAmelCase : Tuple=768,					__UpperCAmelCase : str=12,					__UpperCAmelCase : List[Any]=12,					__UpperCAmelCase : Tuple=3072,					__UpperCAmelCase : Union[str, Any]="gelu",					__UpperCAmelCase : Union[str, Any]=0.1,					__UpperCAmelCase : str=0.1,					__UpperCAmelCase : Union[str, Any]=512,					__UpperCAmelCase : Dict=2,					__UpperCAmelCase : int=0.02,					__UpperCAmelCase : Optional[Any]=1e-12,					__UpperCAmelCase : Union[str, Any]=0,					__UpperCAmelCase : Optional[Any]=0,					__UpperCAmelCase : Union[str, Any]=2,					__UpperCAmelCase : int=256,					__UpperCAmelCase : Tuple=1024,					__UpperCAmelCase : Optional[Any]=216,					__UpperCAmelCase : str=1001,					__UpperCAmelCase : Optional[Any]=32,					__UpperCAmelCase : Dict=50,					__UpperCAmelCase : Union[str, Any]="absolute",					__UpperCAmelCase : Tuple=True,					__UpperCAmelCase : List[str]=None,					**__UpperCAmelCase : Union[str, Any],					)     ->			Union[str, Any]:

					"""simple docstring"""

					super().__init__(
					    pad_token_id=__UpperCAmelCase,					bos_token_id=__UpperCAmelCase,					eos_token_id=__UpperCAmelCase,					**__UpperCAmelCase,					)
					SCREAMING_SNAKE_CASE  :						List[Any]      					= vocab_size
					SCREAMING_SNAKE_CASE  :						Dict      					= hidden_size
					SCREAMING_SNAKE_CASE  :						Optional[Any]      					= num_hidden_layers
					SCREAMING_SNAKE_CASE  :						Dict      					= num_attention_heads
					SCREAMING_SNAKE_CASE  :						Any      					= hidden_act
					SCREAMING_SNAKE_CASE  :						Tuple      					= intermediate_size
					SCREAMING_SNAKE_CASE  :						Tuple      					= hidden_dropout_prob
					SCREAMING_SNAKE_CASE  :						List[str]      					= attention_probs_dropout_prob
					SCREAMING_SNAKE_CASE  :						List[str]      					= max_position_embeddings
					SCREAMING_SNAKE_CASE  :						Dict      					= type_vocab_size
					SCREAMING_SNAKE_CASE  :						Tuple      					= initializer_range
					SCREAMING_SNAKE_CASE  :						Optional[Any]      					= layer_norm_eps
					SCREAMING_SNAKE_CASE  :						int      					= position_embedding_type
					SCREAMING_SNAKE_CASE  :						List[Any]      					= use_cache
					SCREAMING_SNAKE_CASE  :						int      					= classifier_dropout
					# additional properties
					SCREAMING_SNAKE_CASE  :						List[Any]      					= max_depth
					SCREAMING_SNAKE_CASE  :						Any      					= max_xpath_tag_unit_embeddings
					SCREAMING_SNAKE_CASE  :						int      					= max_xpath_subs_unit_embeddings
					SCREAMING_SNAKE_CASE  :						Dict      					= tag_pad_id
					SCREAMING_SNAKE_CASE  :						Optional[int]      					= subs_pad_id
					SCREAMING_SNAKE_CASE  :						int      					= xpath_unit_hidden_size




