
import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCamelCase_    :			Union[str, Any]    =	logging.get_logger(__name__)
lowerCamelCase_    :			str    =	{"""vocab_file""": """spiece.model"""}

lowerCamelCase_    :			List[str]    =	{
    """vocab_file""": {
        """albert-base-v1""": """https://huggingface.co/albert-base-v1/resolve/main/spiece.model""",
        """albert-large-v1""": """https://huggingface.co/albert-large-v1/resolve/main/spiece.model""",
        """albert-xlarge-v1""": """https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model""",
        """albert-xxlarge-v1""": """https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model""",
        """albert-base-v2""": """https://huggingface.co/albert-base-v2/resolve/main/spiece.model""",
        """albert-large-v2""": """https://huggingface.co/albert-large-v2/resolve/main/spiece.model""",
        """albert-xlarge-v2""": """https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model""",
        """albert-xxlarge-v2""": """https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model""",
    }
}

lowerCamelCase_    :			Optional[int]    =	{
    """albert-base-v1""": 5_12,
    """albert-large-v1""": 5_12,
    """albert-xlarge-v1""": 5_12,
    """albert-xxlarge-v1""": 5_12,
    """albert-base-v2""": 5_12,
    """albert-large-v2""": 5_12,
    """albert-xlarge-v2""": 5_12,
    """albert-xxlarge-v2""": 5_12,
}

lowerCamelCase_    :			Any    =	"""▁"""

class 							_UpperCamelCase					(       _A	):






			'''simple docstring'''

			__UpperCamelCase   :    Dict          =			VOCAB_FILES_NAMES
			__UpperCamelCase   :    str          =			PRETRAINED_VOCAB_FILES_MAP
			__UpperCamelCase   :    Optional[int]          =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES

			def __init__(							self			:							Dict    ,						snake_case_			:							Union[str, Any]    ,						snake_case_			:							Dict=True    ,						snake_case_			:							int=True    ,						snake_case_			:							Optional[Any]=False    ,						snake_case_			:							Dict="[CLS]"    ,						snake_case_			:							Optional[int]="[SEP]"    ,						snake_case_			:							Any="<unk>"    ,						snake_case_			:							Tuple="[SEP]"    ,						snake_case_			:							Optional[int]="<pad>"    ,						snake_case_			:							Union[str, Any]="[CLS]"    ,						snake_case_			:							Any="[MASK]"    ,						snake_case_			:							Optional[Dict[str, Any]] = None    ,						**snake_case_			:							List[Any]    ,						):
							# Mask token behave like a normal word, i.e. include the space before it and
							# is included in the raw text, there should be a match in a non-normalized sentence.
							UpperCamelCase_:				Union[str, Any]						=    (
							    AddedToken(snake_case_    ,						lstrip=snake_case_    ,						rstrip=snake_case_    ,						normalized=snake_case_   )
							    if isinstance(snake_case_    ,						snake_case_   )
							    else mask_token
							)

							UpperCamelCase_:				List[str]						=    {} if sp_model_kwargs is None else sp_model_kwargs

							super().__init__(
							    do_lower_case=snake_case_    ,						remove_space=snake_case_    ,						keep_accents=snake_case_    ,						bos_token=snake_case_    ,						eos_token=snake_case_    ,						unk_token=snake_case_    ,						sep_token=snake_case_    ,						pad_token=snake_case_    ,						cls_token=snake_case_    ,						mask_token=snake_case_    ,						sp_model_kwargs=self.sp_model_kwargs    ,						**snake_case_    ,						)

							UpperCamelCase_:				int						=    do_lower_case
							UpperCamelCase_:				List[str]						=    remove_space
							UpperCamelCase_:				str						=    keep_accents
							UpperCamelCase_:				List[str]						=    vocab_file

							UpperCamelCase_:				Dict						=    spm.SentencePieceProcessor(**self.sp_model_kwargs   )
							self.sp_model.Load(snake_case_   )

			@property
			def    lowerCAmelCase__						(							self			:							Union[str, Any]   ):
							return len(self.sp_model   )

			def    lowerCAmelCase__						(							self			:							int   ):
							UpperCamelCase_:				List[str]						=    {self.convert_ids_to_tokens(snake_case_   ): i for i in range(self.vocab_size   )}
							vocab.update(self.added_tokens_encoder   )
							return vocab

			def __getstate__(							self			:							Union[str, Any]   ):
							UpperCamelCase_:				Tuple						=    self.__dict__.copy()
							UpperCamelCase_:				Any						=    None
							return state

			def __setstate__(							self			:							Dict    ,						snake_case_			:							int   ):
							UpperCamelCase_:				List[str]						=    d

							# for backward compatibility
							if not hasattr(self    ,						"""sp_model_kwargs"""   ):
											UpperCamelCase_:				str						=    {}

							UpperCamelCase_:				List[str]						=    spm.SentencePieceProcessor(**self.sp_model_kwargs   )
							self.sp_model.Load(self.vocab_file   )

			def    lowerCAmelCase__						(							self			:							Union[str, Any]    ,						snake_case_			:							Union[str, Any]   ):
							if self.remove_space:
											UpperCamelCase_:				Optional[int]						=    """ """.join(inputs.strip().split()   )
							else:
											UpperCamelCase_:				List[str]						=    inputs
							UpperCamelCase_:				List[Any]						=    outputs.replace("""``"""    ,						"""\""""   ).replace("""''"""    ,						"""\""""   )

							if not self.keep_accents:
											UpperCamelCase_:				Any						=    unicodedata.normalize("""NFKD"""    ,						snake_case_   )
											UpperCamelCase_:				Optional[int]						=    """""".join([c for c in outputs if not unicodedata.combining(snake_case_   )]   )
							if self.do_lower_case:
											UpperCamelCase_:				Dict						=    outputs.lower()

							return outputs

			def    lowerCAmelCase__						(							self			:							str    ,						snake_case_			:							str   ):
							UpperCamelCase_:				Tuple						=    self.preprocess_text(snake_case_   )
							UpperCamelCase_:				List[Any]						=    self.sp_model.encode(snake_case_    ,						out_type=snake_case_   )
							UpperCamelCase_:				Optional[int]						=    []
							for piece in pieces:
											if len(snake_case_   ) > 1 and piece[-1] == str(""","""   ) and piece[-2].isdigit():
															UpperCamelCase_:				List[Any]						=    self.sp_model.EncodeAsPieces(piece[:-1].replace(snake_case_    ,						""""""   )   )
															if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																			if len(cur_pieces[0]   ) == 1:
																							UpperCamelCase_:				List[str]						=    cur_pieces[1:]
																			else:
																							UpperCamelCase_:				Tuple						=    cur_pieces[0][1:]
															cur_pieces.append(piece[-1]   )
															new_pieces.extend(snake_case_   )
											else:
															new_pieces.append(snake_case_   )

							return new_pieces

			def    lowerCAmelCase__						(							self			:							Tuple    ,						snake_case_			:							Dict   ):
							return self.sp_model.PieceToId(snake_case_   )

			def    lowerCAmelCase__						(							self			:							Tuple    ,						snake_case_			:							Any   ):
							return self.sp_model.IdToPiece(snake_case_   )

			def    lowerCAmelCase__						(							self			:							Tuple    ,						snake_case_			:							Union[str, Any]   ):
							UpperCamelCase_:				Union[str, Any]						=    []
							UpperCamelCase_:				Optional[int]						=    """"""
							UpperCamelCase_:				Optional[int]						=    False
							for token in tokens:
											# make sure that special tokens are not decoded using sentencepiece model
											if token in self.all_special_tokens:
															if not prev_is_special:
																			out_string += " "
															out_string += self.sp_model.decode(snake_case_   ) + token
															UpperCamelCase_:				Tuple						=    True
															UpperCamelCase_:				str						=    []
											else:
															current_sub_tokens.append(snake_case_   )
															UpperCamelCase_:				Optional[Any]						=    False
							out_string += self.sp_model.decode(snake_case_   )
							return out_string.strip()

			def    lowerCAmelCase__						(							self			:							Optional[int]    ,						snake_case_			:							List[int]    ,						snake_case_			:							Optional[List[int]] = None   ):
							UpperCamelCase_:				Any						=    [self.sep_token_id]
							UpperCamelCase_:				Union[str, Any]						=    [self.cls_token_id]
							if token_ids_a is None:
											return cls + token_ids_a + sep
							return cls + token_ids_a + sep + token_ids_a + sep

			def    lowerCAmelCase__						(							self			:							List[str]    ,						snake_case_			:							List[int]    ,						snake_case_			:							Optional[List[int]] = None    ,						snake_case_			:							bool = False   ):

							if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=snake_case_    ,						token_ids_a=snake_case_    ,						already_has_special_tokens=snake_case_   )

							if token_ids_a is not None:
											return [1] + ([0] * len(snake_case_   )) + [1] + ([0] * len(snake_case_   )) + [1]
							return [1] + ([0] * len(snake_case_   )) + [1]

			def    lowerCAmelCase__						(							self			:							int    ,						snake_case_			:							List[int]    ,						snake_case_			:							Optional[List[int]] = None   ):
							UpperCamelCase_:				str						=    [self.sep_token_id]
							UpperCamelCase_:				Any						=    [self.cls_token_id]

							if token_ids_a is None:
											return len(cls + token_ids_a + sep   ) * [0]
							return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]




			def    lowerCAmelCase__						(							self			:							Optional[int]    ,						snake_case_			:							str    ,						snake_case_			:							Optional[str] = None   ):
							if not os.path.isdir(snake_case_   ):
											logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''   )
											return
							UpperCamelCase_:				Union[str, Any]						=    os.path.join(
							    snake_case_    ,						(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]   )

							if os.path.abspath(self.vocab_file   ) != os.path.abspath(snake_case_   ) and os.path.isfile(self.vocab_file   ):
											copyfile(self.vocab_file    ,						snake_case_   )
							elif not os.path.isfile(self.vocab_file   ):
											with open(snake_case_    ,						"""wb"""   ) as fi:
															UpperCamelCase_:				int						=    self.sp_model.serialized_model_proto()
															fi.write(snake_case_   )

							return (out_vocab_file,)








from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase_    :			Any    =	logging.get_logger(__name__)

lowerCamelCase_    :			Dict    =	{
    """facebook/dpr-ctx_encoder-single-nq-base""": (
        """https://huggingface.co/facebook/dpr-ctx_encoder-single-nq-base/resolve/main/config.json"""
    ),
    """facebook/dpr-question_encoder-single-nq-base""": (
        """https://huggingface.co/facebook/dpr-question_encoder-single-nq-base/resolve/main/config.json"""
    ),
    """facebook/dpr-reader-single-nq-base""": (
        """https://huggingface.co/facebook/dpr-reader-single-nq-base/resolve/main/config.json"""
    ),
    """facebook/dpr-ctx_encoder-multiset-base""": (
        """https://huggingface.co/facebook/dpr-ctx_encoder-multiset-base/resolve/main/config.json"""
    ),
    """facebook/dpr-question_encoder-multiset-base""": (
        """https://huggingface.co/facebook/dpr-question_encoder-multiset-base/resolve/main/config.json"""
    ),
    """facebook/dpr-reader-multiset-base""": (
        """https://huggingface.co/facebook/dpr-reader-multiset-base/resolve/main/config.json"""
    ),
}

class 							_UpperCamelCase					(       _A	):






			'''simple docstring'''

			__UpperCamelCase   :    Any          =			"""dpr"""

			def __init__(							self			:							str    ,						snake_case_			:							Any=3_0522    ,						snake_case_			:							List[str]=768    ,						snake_case_			:							List[str]=12    ,						snake_case_			:							Union[str, Any]=12    ,						snake_case_			:							Tuple=3072    ,						snake_case_			:							Any="gelu"    ,						snake_case_			:							Optional[Any]=0.1    ,						snake_case_			:							str=0.1    ,						snake_case_			:							Dict=512    ,						snake_case_			:							int=2    ,						snake_case_			:							Union[str, Any]=0.02    ,						snake_case_			:							str=1e-12    ,						snake_case_			:							List[str]=0    ,						snake_case_			:							Any="absolute"    ,						snake_case_			:							int = 0    ,						**snake_case_			:							int    ,						):
							super().__init__(pad_token_id=snake_case_    ,						**snake_case_   )

							UpperCamelCase_:				str						=    vocab_size
							UpperCamelCase_:				Optional[int]						=    hidden_size
							UpperCamelCase_:				int						=    num_hidden_layers
							UpperCamelCase_:				Any						=    num_attention_heads
							UpperCamelCase_:				int						=    hidden_act
							UpperCamelCase_:				Optional[Any]						=    intermediate_size
							UpperCamelCase_:				Tuple						=    hidden_dropout_prob
							UpperCamelCase_:				Tuple						=    attention_probs_dropout_prob
							UpperCamelCase_:				str						=    max_position_embeddings
							UpperCamelCase_:				Any						=    type_vocab_size
							UpperCamelCase_:				Tuple						=    initializer_range
							UpperCamelCase_:				List[Any]						=    layer_norm_eps
							UpperCamelCase_:				Tuple						=    projection_dim
							UpperCamelCase_:				Dict						=    position_embedding_type






