"""simple docstring"""



import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__lowercase		:		Dict														=							logging.get_logger(__name__)

__lowercase		:		Optional[int]														=							"▁"

__lowercase		:		str														=							{"vocab_file": "sentencepiece.bpe.model"}

__lowercase		:		List[str]														=							{
    "vocab_file": {
        "xlm-roberta-base": "https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model",
        "xlm-roberta-large": "https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model",
        "xlm-roberta-large-finetuned-conll02-dutch": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model"
        ),
        "xlm-roberta-large-finetuned-conll02-spanish": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model"
        ),
        "xlm-roberta-large-finetuned-conll03-english": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model"
        ),
        "xlm-roberta-large-finetuned-conll03-german": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model"
        ),
    }
}

__lowercase		:		Tuple														=							{
    "xlm-roberta-base": 512,
    "xlm-roberta-large": 512,
    "xlm-roberta-large-finetuned-conll02-dutch": 512,
    "xlm-roberta-large-finetuned-conll02-spanish": 512,
    "xlm-roberta-large-finetuned-conll03-english": 512,
    "xlm-roberta-large-finetuned-conll03-german": 512,
}



class      _A	(						_UpperCAmelCase					):




    """simple docstring"""




    UpperCamelCase_	:					Union[str, Any]							=					VOCAB_FILES_NAMES
    UpperCamelCase_	:					Tuple							=					PRETRAINED_VOCAB_FILES_MAP
    UpperCamelCase_	:					Tuple							=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    UpperCamelCase_	:					Optional[Any]							=					['''input_ids''', '''attention_mask''']
    def __init__(		self							:     List[str]	,      A_							:     List[str]	,      A_							:     Tuple="<s>"	,      A_							:     Tuple="</s>"	,      A_							:     int="</s>"	,      A_							:     Dict="<s>"	,      A_							:     Optional[Any]="<unk>"	,      A_							:     Dict="<pad>"	,      A_							:     Tuple="<mask>"	,      A_							:     Optional[Dict[str, Any]] = None	,      **A_							:     List[str]	,      )    ->       None:
         # Mask token behave like a normal word, i.e. include the space before it
         __snake_case				=							AddedToken(A_	,      lstrip=A_	,      rstrip=A_    ) if isinstance(A_	,      A_    ) else mask_token

         __snake_case				=							{} if sp_model_kwargs is None else sp_model_kwargs

         super().__init__(
             bos_token=A_	,      eos_token=A_	,      unk_token=A_	,      sep_token=A_	,      cls_token=A_	,      pad_token=A_	,      mask_token=A_	,      sp_model_kwargs=self.sp_model_kwargs	,      **A_	,      )

         __snake_case				=							spm.SentencePieceProcessor(**self.sp_model_kwargs    )
         self.sp_model.Load(str(A_    )    )
         __snake_case				=							vocab_file

         # Original fairseq vocab and spm vocab must be "aligned":
         # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
         # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
         # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
         # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

         # Mimic fairseq token-to-id alignment for the first 4 token
         __snake_case				=							{'''<s>''': 0, '''<pad>''': 1, '''</s>''': 2, '''<unk>''': 3}

         # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
         __snake_case				=							1

         __snake_case				=							len(self.sp_model    ) + self.fairseq_offset
         __snake_case				=							{v: k for k, v in self.fairseq_tokens_to_ids.items()}
    def __getstate__(		self							:     Optional[int]    )    ->       List[str]:
         __snake_case				=							self.__dict__.copy()
         __snake_case				=							None
         __snake_case				=							self.sp_model.serialized_model_proto()
         return state
    def __setstate__(		self							:     Dict	,      A_							:     Any    )    ->       Optional[Any]:
         __snake_case				=							d

         # for backward compatibility
         if not hasattr(self	,      '''sp_model_kwargs'''    ):
              __snake_case				=							{}

         __snake_case				=							spm.SentencePieceProcessor(**self.sp_model_kwargs    )
         self.sp_model.LoadFromSerializedProto(self.sp_model_proto    )
    def  lowercase					(		self							:     Optional[Any]	,      A_							:     List[int]	,      A_							:     Optional[List[int]] = None    )    ->       List[int]:

         if token_ids_a is None:
              return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
         __snake_case				=							[self.cls_token_id]
         __snake_case				=							[self.sep_token_id]
         return cls + token_ids_a + sep + sep + token_ids_a + sep
    def  lowercase					(		self							:     Dict	,      A_							:     List[int]	,      A_							:     Optional[List[int]] = None	,      A_							:     bool = False    )    ->       List[int]:

         if already_has_special_tokens:
              return super().get_special_tokens_mask(
                  token_ids_a=A_	,      token_ids_a=A_	,      already_has_special_tokens=A_    )

         if token_ids_a is None:
              return [1] + ([0] * len(A_    )) + [1]
         return [1] + ([0] * len(A_    )) + [1, 1] + ([0] * len(A_    )) + [1]
    def  lowercase					(		self							:     List[Any]	,      A_							:     List[int]	,      A_							:     Optional[List[int]] = None    )    ->       List[int]:
         __snake_case				=							[self.sep_token_id]
         __snake_case				=							[self.cls_token_id]

         if token_ids_a is None:
              return len(cls + token_ids_a + sep    ) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep    ) * [0]
    @property
    def  lowercase					(		self							:     Union[str, Any]    )    ->       List[Any]:
         return len(self.sp_model    ) + self.fairseq_offset + 1  # Add the <mask> token
    def  lowercase					(		self							:     Tuple    )    ->       str:
         __snake_case				=							{self.convert_ids_to_tokens(A_    ): i for i in range(self.vocab_size    )}
         vocab.update(self.added_tokens_encoder    )
         return vocab
    def  lowercase					(		self							:     int	,      A_							:     str    )    ->       List[str]:
         return self.sp_model.encode(A_	,      out_type=A_    )
    def  lowercase					(		self							:     Optional[Any]	,      A_							:     Union[str, Any]    )    ->       Dict:
         if token in self.fairseq_tokens_to_ids:
              return self.fairseq_tokens_to_ids[token]
         __snake_case				=							self.sp_model.PieceToId(A_    )

         # Need to return unknown token if the SP model returned 0
         return spm_id + self.fairseq_offset if spm_id else self.unk_token_id
    def  lowercase					(		self							:     Union[str, Any]	,      A_							:     List[str]    )    ->       Tuple:
         if index in self.fairseq_ids_to_tokens:
              return self.fairseq_ids_to_tokens[index]
         return self.sp_model.IdToPiece(index - self.fairseq_offset    )
    def  lowercase					(		self							:     str	,      A_							:     List[Any]    )    ->       Optional[Any]:
         __snake_case				=							''''''.join(A_    ).replace(A_	,      ''' '''    ).strip()
         return out_string







    def  lowercase					(		self							:     Any	,      A_							:     str	,      A_							:     Optional[str] = None    )    ->       Tuple[str]:
         if not os.path.isdir(A_    ):
              logger.error(f"Vocabulary path ({save_directory}) should be a directory"    )
              return
         __snake_case				=							os.path.join(
             A_	,      (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']    )

         if os.path.abspath(self.vocab_file    ) != os.path.abspath(A_    ) and os.path.isfile(self.vocab_file    ):
              copyfile(self.vocab_file	,      A_    )
         elif not os.path.isfile(self.vocab_file    ):
              with open(A_	,      '''wb'''    ) as fi:
                   __snake_case				=							self.sp_model.serialized_model_proto()
                   fi.write(A_    )

         return (out_vocab_file,)
"""simple docstring"""



from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


__lowercase		:		Optional[Any]														=							{
    "configuration_falcon": ["FALCON_PRETRAINED_CONFIG_ARCHIVE_MAP", "FalconConfig"],
}

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       __lowercase		:		str														=							[
           "FALCON_PRETRAINED_MODEL_ARCHIVE_LIST",
           "FalconForCausalLM",
           "FalconModel",
           "FalconPreTrainedModel",
           "FalconForSequenceClassification",
           "FalconForTokenClassification",
           "FalconForQuestionAnswering",
       ]


if TYPE_CHECKING:
       from .configuration_falcon import FALCON_PRETRAINED_CONFIG_ARCHIVE_MAP, FalconConfig

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_falcon import (
                  FALCON_PRETRAINED_MODEL_ARCHIVE_LIST,
                  FalconForCausalLM,
                  FalconForQuestionAnswering,
                  FalconForSequenceClassification,
                  FalconForTokenClassification,
                  FalconModel,
                  FalconPreTrainedModel,
              )


else:
       import sys

       __lowercase		:		List[Any]														=							_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)