


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


lowerCamelCase__							:						Optional[int]							     =   logging.get_logger(__name__)

lowerCamelCase__							:						Optional[int]							     =   {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

lowerCamelCase__							:						Dict							     =   {
    """vocab_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"""
        ),
        """google/realm-orqa-nq-openqa""": """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-nq-reader""": """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-openqa""": """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-reader""": """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-openqa""": (
            """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-reader""": (
            """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-openqa""": (
            """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-reader""": (
            """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"""
        ),
    },
}

lowerCamelCase__							:						Optional[int]							     =   {
    """google/realm-cc-news-pretrained-embedder""": 5_1_2,
    """google/realm-cc-news-pretrained-encoder""": 5_1_2,
    """google/realm-cc-news-pretrained-scorer""": 5_1_2,
    """google/realm-cc-news-pretrained-openqa""": 5_1_2,
    """google/realm-orqa-nq-openqa""": 5_1_2,
    """google/realm-orqa-nq-reader""": 5_1_2,
    """google/realm-orqa-wq-openqa""": 5_1_2,
    """google/realm-orqa-wq-reader""": 5_1_2,
}

lowerCamelCase__							:						Optional[Any]							     =   {
    """google/realm-cc-news-pretrained-embedder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-encoder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-scorer""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-reader""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-reader""": {"""do_lower_case""": True},
}
class 		__magic_name__						(snake_case_				):





     '''simple docstring'''



     __lowercase				:							Optional[Any]	   =       VOCAB_FILES_NAMES
     __lowercase				:							Dict	   =       PRETRAINED_VOCAB_FILES_MAP
     __lowercase				:							List[Any]	   =       PRETRAINED_INIT_CONFIGURATION
     __lowercase				:							int	   =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __lowercase				:							Union[str, Any]	   =       RealmTokenizer

     def __init__(					self:Dict     ,		_a:List[str]=None     ,		_a:Union[str, Any]=None     ,		_a:Dict=True     ,		_a:List[Any]="[UNK]"     ,		_a:Any="[SEP]"     ,		_a:Union[str, Any]="[PAD]"     ,		_a:int="[CLS]"     ,		_a:Tuple="[MASK]"     ,		_a:List[Any]=True     ,		_a:str=None     ,		**_a:Any     ,		):
       super().__init__(
           _a     ,		tokenizer_file=_a     ,		do_lower_case=_a     ,		unk_token=_a     ,		sep_token=_a     ,		pad_token=_a     ,		cls_token=_a     ,		mask_token=_a     ,		tokenize_chinese_chars=_a     ,		strip_accents=_a     ,		**_a     ,		)

       snake_case__						= json.loads(self.backend_tokenizer.normalizer.__getstate__()   )
       if (
           normalizer_state.get('''lowercase'''     ,		_a   ) != do_lower_case
           or normalizer_state.get('''strip_accents'''     ,		_a   ) != strip_accents
           or normalizer_state.get('''handle_chinese_chars'''     ,		_a   ) != tokenize_chinese_chars
       ):
         snake_case__						= getattr(_a     ,		normalizer_state.pop('''type'''   )   )
         snake_case__						= do_lower_case
         snake_case__						= strip_accents
         snake_case__						= tokenize_chinese_chars
         snake_case__						= normalizer_class(**_a   )

       snake_case__						= do_lower_case

     def 	SCREAMING_SNAKE_CASE__						(					self:Tuple     ,		_a:int     ,		**_a:int   ):
       snake_case__						= PaddingStrategy.MAX_LENGTH

       snake_case__						= text
       snake_case__						= kwargs.pop('''text_pair'''     ,		_a   )
       snake_case__						= kwargs.pop('''return_tensors'''     ,		_a   )

       snake_case__						= {
           '''input_ids''': [],
           '''attention_mask''': [],
           '''token_type_ids''': [],
       }

       for idx, candidate_text in enumerate(_a   ):
         if batch_text_pair is not None:
           snake_case__						= batch_text_pair[idx]
         else:
           snake_case__						= None

         snake_case__						= super().__call__(_a     ,		_a     ,		return_tensors=_a     ,		**_a   )

         snake_case__						= encoded_candidates.get('''input_ids'''   )
         snake_case__						= encoded_candidates.get('''attention_mask'''   )
         snake_case__						= encoded_candidates.get('''token_type_ids'''   )

         if encoded_input_ids is not None:
           output_data["input_ids"].append(_a   )
         if encoded_attention_mask is not None:
           output_data["attention_mask"].append(_a   )
         if encoded_token_type_ids is not None:
           output_data["token_type_ids"].append(_a   )

       snake_case__						= {key: item for key, item in output_data.items() if len(_a   ) != 0}

       return BatchEncoding(_a     ,		tensor_type=_a   )

     def 	SCREAMING_SNAKE_CASE__						(					self:Tuple     ,		_a:int     ,		_a:int=None   ):
       snake_case__						= [self.cls_token_id] + token_ids_a + [self.sep_token_id]

       if token_ids_a:
         output += token_ids_a + [self.sep_token_id]

       return output

     def 	SCREAMING_SNAKE_CASE__						(					self:Optional[Any]     ,		_a:List[int]     ,		_a:Optional[List[int]] = None   ):
       snake_case__						= [self.sep_token_id]
       snake_case__						= [self.cls_token_id]
       if token_ids_a is None:
         return len(cls + token_ids_a + sep   ) * [0]
       return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]







     def 	SCREAMING_SNAKE_CASE__						(					self:Any     ,		_a:str     ,		_a:Optional[str] = None   ):
       snake_case__						= self._tokenizer.model.save(_a     ,		name=_a   )
       return tuple(_a   )




"""simple docstring"""

import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_UpperCamelCase		     =    logging.get_logger(__name__)

_UpperCamelCase		     =    """▁"""

_UpperCamelCase		     =    {"""vocab_file""": """sentencepiece.bpe.model""", """monolingual_vocab_file""": """dict.txt"""}

_UpperCamelCase		     =    {
    """vocab_file""": {
        """vinai/bartpho-syllable""": """https://huggingface.co/vinai/bartpho-syllable/resolve/main/sentencepiece.bpe.model""",
    },
    """monolingual_vocab_file""": {
        """vinai/bartpho-syllable""": """https://huggingface.co/vinai/bartpho-syllable/resolve/main/dict.txt""",
    },
}

_UpperCamelCase		     =    {"""vinai/bartpho-syllable""": 1_0_2_4}






class 			__a					(						__magic_name__       ):





       """simple docstring"""


       __UpperCamelCase					:      int    =						VOCAB_FILES_NAMES
       __UpperCamelCase					:      List[str]    =						PRETRAINED_VOCAB_FILES_MAP
       __UpperCamelCase					:      List[Any]    =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       __UpperCamelCase					:      int    =						['input_ids', 'attention_mask']






       def __init__(   self			,					snake_case			,					snake_case			,					snake_case="<s>"			,					snake_case="</s>"			,					snake_case="</s>"			,					snake_case="<s>"			,					snake_case="<unk>"			,					snake_case="<pad>"			,					snake_case="<mask>"			,					snake_case = None			,					**snake_case			,					):





        """simple docstring"""

        lowerCAmelCase__   :				Union[str, Any] 			=				AddedToken(snake_case			,					lstrip=snake_case			,					rstrip=snake_case   ) if isinstance(snake_case			,					snake_case   ) else mask_token

        lowerCAmelCase__   :				Union[str, Any] 			=				{} if sp_model_kwargs is None else sp_model_kwargs

        super().__init__(
            bos_token=snake_case			,					eos_token=snake_case			,					unk_token=snake_case			,					sep_token=snake_case			,					cls_token=snake_case			,					pad_token=snake_case			,					mask_token=snake_case			,					sp_model_kwargs=self.sp_model_kwargs			,					**snake_case			,					)

        lowerCAmelCase__   :				Union[str, Any] 			=				vocab_file
        lowerCAmelCase__   :				Optional[Any] 			=				monolingual_vocab_file
        lowerCAmelCase__   :				Optional[Any] 			=				spm.SentencePieceProcessor(**self.sp_model_kwargs   )
        self.sp_model.Load(str(snake_case   )   )

        # Load the reduced vocab

        # Keep order of special tokens for backward compatibility
        lowerCAmelCase__   :				Optional[int] 			=				{}
        lowerCAmelCase__   :				int 			=				0
        for token in [bos_token, pad_token, eos_token, unk_token, sep_token, cls_token]:
         if str(snake_case   ) not in self.fairseq_tokens_to_ids:
          lowerCAmelCase__   :				Optional[int] 			=				cnt
          cnt += 1
        with open(snake_case			,					"r"			,					encoding="utf-8"   ) as f:
         for line in f.readlines():
          lowerCAmelCase__   :				Union[str, Any] 			=				line.strip().split()[0]
          lowerCAmelCase__   :				List[str] 			=				len(self.fairseq_tokens_to_ids   )
        if str(snake_case   ) not in self.fairseq_tokens_to_ids:
         lowerCAmelCase__   :				int 			=				len(self.fairseq_tokens_to_ids   )

        lowerCAmelCase__   :				Union[str, Any] 			=				{v: k for k, v in self.fairseq_tokens_to_ids.items()}






       def __getstate__(   self   ):





        """simple docstring"""

        lowerCAmelCase__   :				List[Any] 			=				self.__dict__.copy()
        lowerCAmelCase__   :				str 			=				None
        lowerCAmelCase__   :				Union[str, Any] 			=				self.sp_model.serialized_model_proto()
        return state






       def __setstate__(   self			,					snake_case   ):





        """simple docstring"""

        lowerCAmelCase__   :				Any 			=				d

        # for backward compatibility
        if not hasattr(self			,					"sp_model_kwargs"   ):
         lowerCAmelCase__   :				Any 			=				{}

        lowerCAmelCase__   :				Dict 			=				spm.SentencePieceProcessor(**self.sp_model_kwargs   )
        self.sp_model.LoadFromSerializedProto(self.sp_model_proto   )






       def 			SCREAMING_SNAKE_CASE_	(   self			,					snake_case			,					snake_case = None   ):





        """simple docstring"""

        if token_ids_a is None:
         return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
        lowerCAmelCase__   :				Any 			=				[self.cls_token_id]
        lowerCAmelCase__   :				Dict 			=				[self.sep_token_id]
        return cls + token_ids_a + sep + sep + token_ids_a + sep






       def 			SCREAMING_SNAKE_CASE_	(   self			,					snake_case			,					snake_case = None			,					snake_case = False   ):





        """simple docstring"""

        if already_has_special_tokens:
         return super().get_special_tokens_mask(
             token_ids_a=snake_case			,					token_ids_a=snake_case			,					already_has_special_tokens=snake_case   )

        if token_ids_a is None:
         return [1] + ([0] * len(snake_case   )) + [1]
        return [1] + ([0] * len(snake_case   )) + [1, 1] + ([0] * len(snake_case   )) + [1]






       def 			SCREAMING_SNAKE_CASE_	(   self			,					snake_case			,					snake_case = None   ):





        """simple docstring"""

        lowerCAmelCase__   :				Optional[Any] 			=				[self.sep_token_id]
        lowerCAmelCase__   :				Optional[int] 			=				[self.cls_token_id]

        if token_ids_a is None:
         return len(cls + token_ids_a + sep   ) * [0]
        return len(cls + token_ids_a + sep + sep + token_ids_a + sep   ) * [0]






       @property
       def 			SCREAMING_SNAKE_CASE_	(   self   ):





        """simple docstring"""

        return len(self.fairseq_ids_to_tokens   )






       def 			SCREAMING_SNAKE_CASE_	(   self   ):





        """simple docstring"""

        lowerCAmelCase__   :				List[str] 			=				{self.convert_ids_to_tokens(snake_case   ): i for i in range(self.vocab_size   )}
        vocab.update(self.added_tokens_encoder   )
        return vocab






       def 			SCREAMING_SNAKE_CASE_	(   self			,					snake_case   ):





        """simple docstring"""

        return self.sp_model.encode(snake_case			,					out_type=snake_case   )






       def 			SCREAMING_SNAKE_CASE_	(   self			,					snake_case   ):





        """simple docstring"""

        if token in self.fairseq_tokens_to_ids:
         return self.fairseq_tokens_to_ids[token]
        else:
         return self.unk_token_id






       def 			SCREAMING_SNAKE_CASE_	(   self			,					snake_case   ):





        """simple docstring"""

        return self.fairseq_ids_to_tokens[index]






       def 			SCREAMING_SNAKE_CASE_	(   self			,					snake_case   ):





        """simple docstring"""

        lowerCAmelCase__   :				List[str] 			=				"".join(snake_case   ).replace(snake_case			,					" "   ).strip()
        return out_string






       def 			SCREAMING_SNAKE_CASE_	(   self			,					snake_case			,					snake_case = None   ):





        """simple docstring"""

        if not os.path.isdir(snake_case   ):
         logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""   )
         return
        lowerCAmelCase__   :				List[str] 			=				os.path.join(
            snake_case			,					(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]   )
        lowerCAmelCase__   :				Optional[Any] 			=				os.path.join(
            snake_case			,					(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["monolingual_vocab_file"]			,					)

        if os.path.abspath(self.vocab_file   ) != os.path.abspath(snake_case   ) and os.path.isfile(self.vocab_file   ):
         copyfile(self.vocab_file			,					snake_case   )
        elif not os.path.isfile(self.vocab_file   ):
         with open(snake_case			,					"wb"   ) as fi:
          lowerCAmelCase__   :				str 			=				self.sp_model.serialized_model_proto()
          fi.write(snake_case   )

        if os.path.abspath(self.monolingual_vocab_file   ) != os.path.abspath(
            snake_case   ) and os.path.isfile(self.monolingual_vocab_file   ):
         copyfile(self.monolingual_vocab_file			,					snake_case   )
        elif not os.path.isfile(self.monolingual_vocab_file   ):
         with open(snake_case			,					"w"			,					encoding="utf-8"   ) as fp:
          for token in self.fairseq_tokens_to_ids:
           if token not in self.all_special_tokens:
            fp.write(F"""{str(snake_case   )} \n"""   )

        return out_vocab_file, out_monolingual_vocab_file






