





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


UpperCamelCase__          =logging.get_logger(__name__)

UpperCamelCase__          ={'vocab_file': 'vocab.txt'}

UpperCamelCase__          ={
    'vocab_file': {
        'tau/splinter-base': 'https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt',
        'tau/splinter-base-qass': 'https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt',
        'tau/splinter-large': 'https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt',
        'tau/splinter-large-qass': 'https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt',
    }
}

UpperCamelCase__          ={
    'tau/splinter-base': 512,
    'tau/splinter-base-qass': 512,
    'tau/splinter-large': 512,
    'tau/splinter-large-qass': 512,
}

UpperCamelCase__          ={
    'tau/splinter-base': {'do_lower_case': False},
    'tau/splinter-base-qass': {'do_lower_case': False},
    'tau/splinter-large': {'do_lower_case': False},
    'tau/splinter-large-qass': {'do_lower_case': False},
}





class 	lowerCAmelCase__(       __lowercase			):






       '''simple docstring'''


       __snake_case       	=			VOCAB_FILES_NAMES
       __snake_case       	=			PRETRAINED_VOCAB_FILES_MAP
       __snake_case       	=			PRETRAINED_INIT_CONFIGURATION
       __snake_case       	=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       __snake_case       	=			SplinterTokenizer




       def __init__(					self			,  __lowerCamelCase=None			,  __lowerCamelCase=None			,  __lowerCamelCase=True			,  __lowerCamelCase="[UNK]"			,  __lowerCamelCase="[SEP]"			,  __lowerCamelCase="[PAD]"			,  __lowerCamelCase="[CLS]"			,  __lowerCamelCase="[MASK]"			,  __lowerCamelCase="[QUESTION]"			,  __lowerCamelCase=True			,  __lowerCamelCase=None			,  **__lowerCamelCase			,  )	->   List[Any]:
            super().__init__(
                __lowerCamelCase			,  tokenizer_file=__lowerCamelCase			,  do_lower_case=__lowerCamelCase			,  unk_token=__lowerCamelCase			,  sep_token=__lowerCamelCase			,  pad_token=__lowerCamelCase			,  cls_token=__lowerCamelCase			,  mask_token=__lowerCamelCase			,  tokenize_chinese_chars=__lowerCamelCase			,  strip_accents=__lowerCamelCase			,  additional_special_tokens=(question_token,)			,  **__lowerCamelCase			,  )

            _SCREAMING_SNAKE_CASE				:		Tuple  			=							json.loads(self.backend_tokenizer.normalizer.__getstate__()							)
            if (
                pre_tok_state.get("lowercase"			,  __lowerCamelCase							) != do_lower_case
                or pre_tok_state.get("strip_accents"			,  __lowerCamelCase							) != strip_accents
            ):
                 _SCREAMING_SNAKE_CASE				:		List[str]  			=							getattr(__lowerCamelCase			,  pre_tok_state.pop("type"							)							)
                 _SCREAMING_SNAKE_CASE				:		Tuple  			=							do_lower_case
                 _SCREAMING_SNAKE_CASE				:		Union[str, Any]  			=							strip_accents
                 _SCREAMING_SNAKE_CASE				:		List[Any]  			=							pre_tok_class(**__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE				:		Optional[int]  			=							do_lower_case




       @property
       def 			UpperCamelCase_	(					self							)	->   Tuple:
            return self.convert_tokens_to_ids(self.question_token							)




       def 			UpperCamelCase_	(					self			,  __lowerCamelCase			,  __lowerCamelCase = None							)	->   List[int]:
            if token_ids_a is None:
                 return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

            _SCREAMING_SNAKE_CASE				:		int  			=							[self.cls_token_id]
            _SCREAMING_SNAKE_CASE				:		Optional[Any]  			=							[self.sep_token_id]
            _SCREAMING_SNAKE_CASE				:		Optional[int]  			=							[self.question_token_id] + [self.convert_tokens_to_ids("."							)]
            if self.padding_side == "right":
                 # Input is question-then-context
                 return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
            else:
                 # Input is context-then-question
                 return cls + token_ids_a + sep + token_ids_a + question_suffix + sep




       def 			UpperCamelCase_	(					self			,  __lowerCamelCase			,  __lowerCamelCase = None							)	->   List[int]:
            _SCREAMING_SNAKE_CASE				:		int  			=							[self.sep_token_id]
            _SCREAMING_SNAKE_CASE				:		Tuple  			=							[self.cls_token_id]
            _SCREAMING_SNAKE_CASE				:		Dict  			=							[self.question_token_id] + [self.convert_tokens_to_ids("."							)]
            if token_ids_a is None:
                 return len(cls + token_ids_a + sep							) * [0]

            if self.padding_side == "right":
                 # Input is question-then-context
                 return len(cls + token_ids_a + question_suffix + sep							) * [0] + len(token_ids_a + sep							) * [1]
            else:
                 # Input is context-then-question
                 return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + question_suffix + sep							) * [1]







       def 			UpperCamelCase_	(					self			,  __lowerCamelCase			,  __lowerCamelCase = None							)	->   Tuple[str]:
            _SCREAMING_SNAKE_CASE				:		Any  			=							self._tokenizer.model.save(__lowerCamelCase			,  name=__lowerCamelCase							)
            return tuple(__lowerCamelCase							)







"""simple docstring"""







import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
  from .tokenization_xglm import XGLMTokenizer
else:
  _UpperCamelCase     			=  None


_UpperCamelCase     			=  logging.get_logger(__name__)

_UpperCamelCase     			=  {"""vocab_file""": """sentencepiece.bpe.model""", """tokenizer_file""": """tokenizer.json"""}

_UpperCamelCase     			=  {
    """vocab_file""": {
        """facebook/xglm-564M""": """https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model""",
    },
    """tokenizer_file""": {
        """facebook/xglm-564M""": """https://huggingface.co/facebook/xglm-564M/resolve/main/tokenizer.json""",
    },
}

_UpperCamelCase     			=  {
    """facebook/xglm-564M""": 2048,
}







class 					lowerCamelCase__					(       snake_case ):
    SCREAMING_SNAKE_CASE		=			VOCAB_FILES_NAMES
    SCREAMING_SNAKE_CASE		=			PRETRAINED_VOCAB_FILES_MAP
    SCREAMING_SNAKE_CASE		=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    SCREAMING_SNAKE_CASE		=			['''input_ids''', '''attention_mask''']
    SCREAMING_SNAKE_CASE		=			XGLMTokenizer






    def __init__( self	,A=None	,A=None	,A="<s>"	,A="</s>"	,A="</s>"	,A="<s>"	,A="<unk>"	,A="<pad>"	,**A	,):
          # Compatibility with the original tokenizer
          UpperCAmelCase							=       7
          UpperCAmelCase							=       [F'''<madeupword{i}>''' for i in range(self.num_madeup_words					)]

          UpperCAmelCase							=       kwargs.get("""additional_special_tokens"""	,[]					)
          kwargs["additional_special_tokens"] += [
              word for word in madeup_words if word not in kwargs["additional_special_tokens"]
          ]

          super().__init__(
              A	,tokenizer_file=A	,bos_token=A	,eos_token=A	,sep_token=A	,cls_token=A	,unk_token=A	,pad_token=A	,**A	,)

          UpperCAmelCase							=       vocab_file
          UpperCAmelCase							=       False if not self.vocab_file else True






    def 				_UpperCamelCase							( self	,A	,A = None					):

          if token_ids_a is None:
                return [self.sep_token_id] + token_ids_a
          UpperCAmelCase							=       [self.sep_token_id]
          return sep + token_ids_a + sep + sep + token_ids_a






    def 				_UpperCamelCase							( self	,A	,A = None					):
          UpperCAmelCase							=       [self.sep_token_id]

          if token_ids_a is None:
                return len(sep + token_ids_a					) * [0]
          return len(sep + token_ids_a + sep + sep + token_ids_a					) * [0]






    def 				_UpperCamelCase							( self	,A	,A = None					):
          if not self.can_save_slow_tokenizer:
                raise ValueError(
                    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
                    """tokenizer."""					)

          if not os.path.isdir(A					):
                logger.error(F'''Vocabulary path ({save_directory}) should be a directory.'''					)
                return
          UpperCAmelCase							=       os.path.join(
              A	,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]					)

          if os.path.abspath(self.vocab_file					) != os.path.abspath(A					):
                copyfile(self.vocab_file	,A					)

          return (out_vocab_file,)



