



"""simple docstring"""
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_lxmert import LxmertTokenizer


__SCREAMING_SNAKE_CASE	:    Union[str, Any]     				=   {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__SCREAMING_SNAKE_CASE	:    Union[str, Any]     				=   {
    '''vocab_file''': {
        '''unc-nlp/lxmert-base-uncased''': '''https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/vocab.txt''',
    },
    '''tokenizer_file''': {
        '''unc-nlp/lxmert-base-uncased''': (
            '''https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/tokenizer.json'''
        ),
    },
}

__SCREAMING_SNAKE_CASE	:    List[str]     				=   {
    '''unc-nlp/lxmert-base-uncased''': 5_1_2,
}

__SCREAMING_SNAKE_CASE	:    Optional[Any]     				=   {
    '''unc-nlp/lxmert-base-uncased''': {'''do_lower_case''': True},
}


class 				lowerCamelCase_( A__     ):





					'''simple docstring'''



					lowercase__ :							List[str]              =			VOCAB_FILES_NAMES
					lowercase__ :							Tuple              =			PRETRAINED_VOCAB_FILES_MAP
					lowercase__ :							Union[str, Any]              =			PRETRAINED_INIT_CONFIGURATION
					lowercase__ :							Union[str, Any]              =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowercase__ :							Union[str, Any]              =			LxmertTokenizer

					def __init__(     self					,							lowerCamelCase__=None					,							lowerCamelCase__=None					,							lowerCamelCase__=True					,							lowerCamelCase__="[UNK]"					,							lowerCamelCase__="[SEP]"					,							lowerCamelCase__="[PAD]"					,							lowerCamelCase__="[CLS]"					,							lowerCamelCase__="[MASK]"					,							lowerCamelCase__=True					,							lowerCamelCase__=None					,							**lowerCamelCase__					,							):
									super().__init__(
									    UpperCamelCase_					,							tokenizer_file=UpperCamelCase_					,							do_lower_case=UpperCamelCase_					,							unk_token=UpperCamelCase_					,							sep_token=UpperCamelCase_					,							pad_token=UpperCamelCase_					,							cls_token=UpperCamelCase_					,							mask_token=UpperCamelCase_					,							tokenize_chinese_chars=UpperCamelCase_					,							strip_accents=UpperCamelCase_					,							**UpperCamelCase_					,							)

									_lowerCamelCase   	=					json.loads(self.backend_tokenizer.normalizer.__getstate__() )
									if (
									    normalizer_state.get('''lowercase'''					,							UpperCamelCase_ ) != do_lower_case
									    or normalizer_state.get('''strip_accents'''					,							UpperCamelCase_ ) != strip_accents
									    or normalizer_state.get('''handle_chinese_chars'''					,							UpperCamelCase_ ) != tokenize_chinese_chars
									):
													_lowerCamelCase   	=					getattr(UpperCamelCase_					,							normalizer_state.pop('''type''' ) )
													_lowerCamelCase   	=					do_lower_case
													_lowerCamelCase   	=					strip_accents
													_lowerCamelCase   	=					tokenize_chinese_chars
													_lowerCamelCase   	=					normalizer_class(**UpperCamelCase_ )

									_lowerCamelCase   	=					do_lower_case

					def       snake_case__   (     self					,							lowerCamelCase__					,							lowerCamelCase__=None ):
									_lowerCamelCase   	=					[self.cls_token_id] + token_ids_a + [self.sep_token_id]

									if token_ids_a:
													output += token_ids_a + [self.sep_token_id]

									return output

					def       snake_case__   (     self					,							lowerCamelCase__					,							lowerCamelCase__ = None ):
									_lowerCamelCase   	=					[self.sep_token_id]
									_lowerCamelCase   	=					[self.cls_token_id]
									if token_ids_a is None:
													return len(cls + token_ids_a + sep ) * [0]
									return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]





					def       snake_case__   (     self					,							lowerCamelCase__					,							lowerCamelCase__ = None ):
									_lowerCamelCase   	=					self._tokenizer.model.save(UpperCamelCase_					,							name=UpperCamelCase_ )
									return tuple(UpperCamelCase_ )




'''simple docstring'''
from .glue import glue_convert_examples_to_features, glue_output_modes, glue_processors, glue_tasks_num_labels
from .squad import SquadExample, SquadFeatures, SquadVaProcessor, SquadVaProcessor, squad_convert_examples_to_features
from .utils import DataProcessor, InputExample, InputFeatures, SingleSentenceClassificationProcessor
from .xnli import xnli_output_modes, xnli_processors, xnli_tasks_num_labels