





"""simple docstring"""


import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCamelCase__						= logging.get_logger(__name__)

lowerCamelCase__						= {"""vocab_file""": """sentencepiece.bpe.model"""}

lowerCamelCase__						= {
    """vocab_file""": {
        """camembert-base""": """https://huggingface.co/camembert-base/resolve/main/sentencepiece.bpe.model""",
    }
}

lowerCamelCase__						= {
    """camembert-base""": 512,
}

lowerCamelCase__						= """▁"""



class   __SCREAMING_SNAKE_CASE     (    UpperCAmelCase_ ):







  '''simple docstring'''


  SCREAMING_SNAKE_CASE__				:List[Any]						  =       VOCAB_FILES_NAMES
  SCREAMING_SNAKE_CASE__				:Optional[Any]						  =       PRETRAINED_VOCAB_FILES_MAP
  SCREAMING_SNAKE_CASE__				:Optional[Any]						  =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  SCREAMING_SNAKE_CASE__				:str						  =       ['input_ids', 'attention_mask']

  def __init__(		self :							int       , __a :							int       , __a :							Tuple="<s>"       , __a :							List[str]="</s>"       , __a :							int="</s>"       , __a :							Any="<s>"       , __a :							Union[str, Any]="<unk>"       , __a :							List[Any]="<pad>"       , __a :							Optional[Any]="<mask>"       , __a :							Any=["<s>NOTUSED", "</s>NOTUSED"]       , __a :							Any = None       , **__a :							Any       , )		->					Dict:
        _UpperCamelCase   :						Dict    =							AddedToken(SCREAMING_SNAKE_CASE_       , lstrip=SCREAMING_SNAKE_CASE_       , rstrip=SCREAMING_SNAKE_CASE_						) if isinstance(SCREAMING_SNAKE_CASE_       , SCREAMING_SNAKE_CASE_						) else mask_token

        _UpperCamelCase   :						int    =							{} if sp_model_kwargs is None else sp_model_kwargs

        super().__init__(
            bos_token=SCREAMING_SNAKE_CASE_       , eos_token=SCREAMING_SNAKE_CASE_       , unk_token=SCREAMING_SNAKE_CASE_       , sep_token=SCREAMING_SNAKE_CASE_       , cls_token=SCREAMING_SNAKE_CASE_       , pad_token=SCREAMING_SNAKE_CASE_       , mask_token=SCREAMING_SNAKE_CASE_       , additional_special_tokens=SCREAMING_SNAKE_CASE_       , sp_model_kwargs=self.sp_model_kwargs       , **SCREAMING_SNAKE_CASE_       , )
        _UpperCamelCase   :						Tuple    =							spm.SentencePieceProcessor(**self.sp_model_kwargs						)
        self.sp_model.Load(str(SCREAMING_SNAKE_CASE_						)						)
        _UpperCamelCase   :						Optional[Any]    =							vocab_file
        # HACK: These tokens were added by fairseq but don't seem to be actually used when duplicated in the actual
        # sentencepiece vocabulary (this is the case for <s> and </s>
        _UpperCamelCase   :						int    =							{"""<s>NOTUSED""": 0, """<pad>""": 1, """</s>NOTUSED""": 2, """<unk>""": 3}
        _UpperCamelCase   :						Union[str, Any]    =							len(self.fairseq_tokens_to_ids						)
        _UpperCamelCase   :						Any    =							len(self.sp_model						) + len(self.fairseq_tokens_to_ids						)
        _UpperCamelCase   :						Union[str, Any]    =							{v: k for k, v in self.fairseq_tokens_to_ids.items()}

  def  __SCREAMING_SNAKE_CASE					(		self :							str       , __a :							Any       , __a :							Tuple = None						)		->					Dict:




        if token_ids_a is None:
              return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
        _UpperCamelCase   :						Tuple    =							[self.cls_token_id]
        _UpperCamelCase   :						Dict    =							[self.sep_token_id]
        return cls + token_ids_a + sep + sep + token_ids_a + sep

  def  __SCREAMING_SNAKE_CASE					(		self :							str       , __a :							Union[str, Any]       , __a :							List[Any] = None       , __a :							Optional[int] = False						)		->					Optional[Any]:




        if already_has_special_tokens:
              return super().get_special_tokens_mask(
                  token_ids_a=SCREAMING_SNAKE_CASE_       , token_ids_a=SCREAMING_SNAKE_CASE_       , already_has_special_tokens=SCREAMING_SNAKE_CASE_						)

        if token_ids_a is None:
              return [1] + ([0] * len(SCREAMING_SNAKE_CASE_						)) + [1]
        return [1] + ([0] * len(SCREAMING_SNAKE_CASE_						)) + [1, 1] + ([0] * len(SCREAMING_SNAKE_CASE_						)) + [1]

  def  __SCREAMING_SNAKE_CASE					(		self :							Any       , __a :							Dict       , __a :							Any = None						)		->					Dict:
        _UpperCamelCase   :						List[Any]    =							[self.sep_token_id]
        _UpperCamelCase   :						Dict    =							[self.cls_token_id]

        if token_ids_a is None:
              return len(cls + token_ids_a + sep						) * [0]
        return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]

  @property
  def  __SCREAMING_SNAKE_CASE					(		self :							Tuple						)		->					Tuple:
        return len(self.fairseq_tokens_to_ids						) + len(self.sp_model						)

  def  __SCREAMING_SNAKE_CASE					(		self :							Optional[Any]						)		->					List[Any]:
        _UpperCamelCase   :						Any    =							{self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_						): i for i in range(self.vocab_size						)}
        vocab.update(self.added_tokens_encoder						)
        return vocab

  def  __SCREAMING_SNAKE_CASE					(		self :							List[Any]       , __a :							Optional[int]						)		->					Optional[Any]:
        return self.sp_model.encode(SCREAMING_SNAKE_CASE_       , out_type=SCREAMING_SNAKE_CASE_						)

  def  __SCREAMING_SNAKE_CASE					(		self :							Tuple       , __a :							Any						)		->					Tuple:




        if token in self.fairseq_tokens_to_ids:
              return self.fairseq_tokens_to_ids[token]
        elif self.sp_model.PieceToId(SCREAMING_SNAKE_CASE_						) == 0:
              # Convert sentence piece unk token to fairseq unk token index
              return self.unk_token_id
        return self.fairseq_offset + self.sp_model.PieceToId(SCREAMING_SNAKE_CASE_						)

  def  __SCREAMING_SNAKE_CASE					(		self :							Any       , __a :							Union[str, Any]						)		->					Any:




        if index in self.fairseq_ids_to_tokens:
              return self.fairseq_ids_to_tokens[index]
        return self.sp_model.IdToPiece(index - self.fairseq_offset						)

  def  __SCREAMING_SNAKE_CASE					(		self :							str       , __a :							Optional[int]						)		->					Any:
        _UpperCamelCase   :						Tuple    =							[]
        _UpperCamelCase   :						Optional[Any]    =							""""""
        _UpperCamelCase   :						List[Any]    =							False
        for token in tokens:
              # make sure that special tokens are not decoded using sentencepiece model
              if token in self.all_special_tokens:
                    if not prev_is_special:
                          out_string += " "
                    out_string += self.sp_model.decode(SCREAMING_SNAKE_CASE_						) + token
                    _UpperCamelCase   :						List[str]    =							True
                    _UpperCamelCase   :						List[Any]    =							[]
              else:
                    current_sub_tokens.append(SCREAMING_SNAKE_CASE_						)
                    _UpperCamelCase   :						Optional[int]    =							False
        out_string += self.sp_model.decode(SCREAMING_SNAKE_CASE_						)
        return out_string.strip()

  def __getstate__(		self :							Optional[Any]						)		->					Any:
        _UpperCamelCase   :						List[str]    =							self.__dict__.copy()
        _UpperCamelCase   :						int    =							None
        return state

  def __setstate__(		self :							Tuple       , __a :							Union[str, Any]						)		->					Optional[int]:
        _UpperCamelCase   :						Optional[int]    =							d

        # for backward compatibility
        if not hasattr(self       , "sp_model_kwargs"						):
              _UpperCamelCase   :						Union[str, Any]    =							{}

        _UpperCamelCase   :						Optional[int]    =							spm.SentencePieceProcessor(**self.sp_model_kwargs						)
        self.sp_model.Load(self.vocab_file						)


  def  __SCREAMING_SNAKE_CASE					(		self :							str       , __a :							List[str]       , __a :							Tuple = None						)		->					List[Any]:




        if not os.path.isdir(SCREAMING_SNAKE_CASE_						):
              logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''						)
              return
        _UpperCamelCase   :						Optional[int]    =							os.path.join(
            SCREAMING_SNAKE_CASE_       , (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]						)

        if os.path.abspath(self.vocab_file						) != os.path.abspath(SCREAMING_SNAKE_CASE_						) and os.path.isfile(self.vocab_file						):
              copyfile(self.vocab_file       , SCREAMING_SNAKE_CASE_						)
        elif not os.path.isfile(self.vocab_file						):
              with open(SCREAMING_SNAKE_CASE_       , "wb"						) as fi:
                    _UpperCamelCase   :						Tuple    =							self.sp_model.serialized_model_proto()
                    fi.write(SCREAMING_SNAKE_CASE_						)

        return (out_vocab_file,)


# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


lowerCamelCase__     :		Tuple        = {
    """configuration_mgp_str""": ["""MGP_STR_PRETRAINED_CONFIG_ARCHIVE_MAP""", """MgpstrConfig"""],
    """processing_mgp_str""": ["""MgpstrProcessor"""],
    """tokenization_mgp_str""": ["""MgpstrTokenizer"""],
}

try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowerCamelCase__     :		Optional[int]        = [
         """MGP_STR_PRETRAINED_MODEL_ARCHIVE_LIST""",
         """MgpstrModel""",
         """MgpstrPreTrainedModel""",
         """MgpstrForSceneTextRecognition""",
     ]

if TYPE_CHECKING:
     from .configuration_mgp_str import MGP_STR_PRETRAINED_CONFIG_ARCHIVE_MAP, MgpstrConfig
     from .processing_mgp_str import MgpstrProcessor
     from .tokenization_mgp_str import MgpstrTokenizer

     try:
          if not is_torch_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_mgp_str import (
              MGP_STR_PRETRAINED_MODEL_ARCHIVE_LIST,
              MgpstrForSceneTextRecognition,
              MgpstrModel,
              MgpstrPreTrainedModel,
          )
else:
     import sys

     lowerCamelCase__     :		List[Any]        = _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)






