import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


lowerCAmelCase__							= logging.get_logger(__name__)

lowerCAmelCase__							= {'vocab_file': 'vocab.txt'}

lowerCAmelCase__							= {
    'vocab_file': {
        'tau/splinter-base': 'https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt',
        'tau/splinter-base-qass': 'https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt',
        'tau/splinter-large': 'https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt',
        'tau/splinter-large-qass': 'https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt',
    }
}

lowerCAmelCase__							= {
    'tau/splinter-base': 512,
    'tau/splinter-base-qass': 512,
    'tau/splinter-large': 512,
    'tau/splinter-large-qass': 512,
}

lowerCAmelCase__							= {
    'tau/splinter-base': {'do_lower_case': False},
    'tau/splinter-base-qass': {'do_lower_case': False},
    'tau/splinter-large': {'do_lower_case': False},
    'tau/splinter-large-qass': {'do_lower_case': False},
}
class 				a_ (						SCREAMING_SNAKE_CASE						):






 '''simple docstring'''






 UpperCAmelCase_	 =					VOCAB_FILES_NAMES
 UpperCAmelCase_	 =					PRETRAINED_VOCAB_FILES_MAP
 UpperCAmelCase_	 =					PRETRAINED_INIT_CONFIGURATION
 UpperCAmelCase_	 =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 UpperCAmelCase_	 =					SplinterTokenizer




 def __init__(			self					:  Dict			,    lowercase__					:  List[Any]=None			,    lowercase__					:  List[str]=None			,    lowercase__					:  str=True			,    lowercase__					:  List[Any]="[UNK]"			,    lowercase__					:  Union[str, Any]="[SEP]"			,    lowercase__					:  Optional[int]="[PAD]"			,    lowercase__					:  str="[CLS]"			,    lowercase__					:  List[str]="[MASK]"			,    lowercase__					:  Dict="[QUESTION]"			,    lowercase__					:  Union[str, Any]=True			,    lowercase__					:  Union[str, Any]=None			,    **lowercase__					:  Any			,    ):







        '''simple docstring'''



        super().__init__(
            lowercase__			,    tokenizer_file=lowercase__			,    do_lower_case=lowercase__			,    unk_token=lowercase__			,    sep_token=lowercase__			,    pad_token=lowercase__			,    cls_token=lowercase__			,    mask_token=lowercase__			,    tokenize_chinese_chars=lowercase__			,    strip_accents=lowercase__			,    additional_special_tokens=(question_token,)			,    **lowercase__			,    )

        lowerCAmelCase__     	=					json.loads(self.backend_tokenizer.normalizer.__getstate__())
        if (
            pre_tok_state.get('lowercase'			,    lowercase__) != do_lower_case
            or pre_tok_state.get('strip_accents'			,    lowercase__) != strip_accents
        ):
               lowerCAmelCase__     	=					getattr(lowercase__			,    pre_tok_state.pop('type'))
               lowerCAmelCase__     	=					do_lower_case
               lowerCAmelCase__     	=					strip_accents
               lowerCAmelCase__     	=					pre_tok_class(**lowercase__)

        lowerCAmelCase__     	=					do_lower_case




 @property
 def 		__snake_case						(			self					:  str):







        '''simple docstring'''



        return self.convert_tokens_to_ids(self.question_token)




 def 		__snake_case						(			self					:  Optional[int]			,    lowercase__					:  List[int]			,    lowercase__					:  Optional[List[int]] = None):







        '''simple docstring'''



        if token_ids_a is None:
               return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

        lowerCAmelCase__     	=					[self.cls_token_id]
        lowerCAmelCase__     	=					[self.sep_token_id]
        lowerCAmelCase__     	=					[self.question_token_id] + [self.convert_tokens_to_ids('.')]
        if self.padding_side == "right":
               # Input is question-then-context
               return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
        else:
               # Input is context-then-question
               return cls + token_ids_a + sep + token_ids_a + question_suffix + sep




 def 		__snake_case						(			self					:  Optional[int]			,    lowercase__					:  List[int]			,    lowercase__					:  Optional[List[int]] = None):







        '''simple docstring'''



        lowerCAmelCase__     	=					[self.sep_token_id]
        lowerCAmelCase__     	=					[self.cls_token_id]
        lowerCAmelCase__     	=					[self.question_token_id] + [self.convert_tokens_to_ids('.')]
        if token_ids_a is None:
               return len(cls + token_ids_a + sep) * [0]

        if self.padding_side == "right":
               # Input is question-then-context
               return len(cls + token_ids_a + question_suffix + sep) * [0] + len(token_ids_a + sep) * [1]
        else:
               # Input is context-then-question
               return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + question_suffix + sep) * [1]




 def 		__snake_case						(			self					:  Dict			,    lowercase__					:  str			,    lowercase__					:  Optional[str] = None):







        '''simple docstring'''



        lowerCAmelCase__     	=					self._tokenizer.model.save(lowercase__			,    name=lowercase__)
        return tuple(lowercase__)

from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_clip import CLIPTokenizer


lowerCAmelCase__							= logging.get_logger(__name__)

lowerCAmelCase__							= {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

lowerCAmelCase__							= {
    'vocab_file': {
        'openai/clip-vit-base-patch32': 'https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/vocab.json',
    },
    'merges_file': {
        'openai/clip-vit-base-patch32': 'https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'openai/clip-vit-base-patch32': (
            'https://huggingface.co/openai/clip-vit-base-patch32/resolve/main/tokenizer.json'
        ),
    },
}

lowerCAmelCase__							= {
    'openai/clip-vit-base-patch32': 77,
}
class 				a_ (						SCREAMING_SNAKE_CASE						):






 '''simple docstring'''






 UpperCAmelCase_	 =					VOCAB_FILES_NAMES
 UpperCAmelCase_	 =					PRETRAINED_VOCAB_FILES_MAP
 UpperCAmelCase_	 =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 UpperCAmelCase_	 =					['input_ids', 'attention_mask']
 UpperCAmelCase_	 =					CLIPTokenizer




 def __init__(			self					:  Optional[int]			,    lowercase__					:  Optional[int]=None			,    lowercase__					:  Tuple=None			,    lowercase__					:  Tuple=None			,    lowercase__					:  str="<|endoftext|>"			,    lowercase__					:  List[str]="<|startoftext|>"			,    lowercase__					:  Union[str, Any]="<|endoftext|>"			,    lowercase__					:  Any="<|endoftext|>"			,    **lowercase__					:  Any			,    ):







        '''simple docstring'''



        super().__init__(
            lowercase__			,    lowercase__			,    tokenizer_file=lowercase__			,    unk_token=lowercase__			,    bos_token=lowercase__			,    eos_token=lowercase__			,    pad_token=lowercase__			,    **lowercase__			,    )

        if not isinstance(self.backend_tokenizer.pre_tokenizer			,    pre_tokenizers.Sequence):
               raise ValueError(
                   'The `backend_tokenizer` provided does not match the expected format. The CLIP tokenizer has been'
                   ' heavily modified from transformers version 4.17.0. You need to convert the tokenizer you are using'
                   ' to be compatible with this version.The easiest way to do so is'
                   ' `CLIPTokenizerFast.from_pretrained("path_to_local_folder_or_hub_repo, from_slow=True)`. If you want'
                   ' to use your existing tokenizer, you will have to revert to a version prior to 4.17.0 of'
                   ' transformers.')

        self._wrap_decode_method_backend_tokenizer()




 def 		__snake_case						(			self					:  Optional[int]):







        '''simple docstring'''



        lowerCAmelCase__     	=					self.backend_tokenizer.decode

        def new_decode_method(*lowercase__					:  str			,    **lowercase__					:  int):
               lowerCAmelCase__     	=					orig_decode_method(*lowercase__			,    **lowercase__)
               lowerCAmelCase__     	=					text.replace(self.backend_tokenizer.model.end_of_word_suffix			,    ' ').strip()
               return text

        lowerCAmelCase__     	=					new_decode_method




 def 		__snake_case						(			self					:  Tuple			,    lowercase__					:  List[int]			,    lowercase__					:  Optional[List[int]] = None):







        '''simple docstring'''



        lowerCAmelCase__     	=					[self.bos_token_id]
        lowerCAmelCase__     	=					[self.eos_token_id]

        if token_ids_a is None:
               return bos_token + token_ids_a + eos_token
        return bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token




 def 		__snake_case						(			self					:  Any			,    lowercase__					:  List[int]			,    lowercase__					:  Optional[List[int]] = None):







        '''simple docstring'''



        lowerCAmelCase__     	=					[self.bos_token_id]
        lowerCAmelCase__     	=					[self.eos_token_id]

        if token_ids_a is None:
               return len(bos_token + token_ids_a + eos_token) * [0]
        return len(bos_token + token_ids_a + eos_token + eos_token + token_ids_a + eos_token) * [0]




 def 		__snake_case						(			self					:  Dict			,    lowercase__					:  str			,    lowercase__					:  Optional[str] = None):







        '''simple docstring'''



        lowerCAmelCase__     	=					self._tokenizer.model.save(lowercase__			,    name=lowercase__)
        return tuple(lowercase__)
