

# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
import importlib.metadata
import json
import os
from dataclasses import dataclass
from typing import Any, Dict, Union

from packaging import version

from ..utils import is_torch_available, logging


if is_torch_available():
      import torch


a_     :    Optional[int]     		=   logging.get_logger(__name__)




@dataclass
class   UpperCamelCase			:

    def __init__(      self			:   int				,     snake_case__			:   Tuple=False				,     snake_case__			:   str=False				,     snake_case__			:   Any=6.0				,     snake_case__			:   str=None				,     snake_case__			:   Optional[Any]=False				,     snake_case__			:   Tuple=False				,     snake_case__			:   Any=None				,     snake_case__			:   List[Any]="fp4"				,     snake_case__			:   List[Any]=False				,     **snake_case__			:   Optional[Any]				,     ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE		    =      load_in_abit
     SCREAMING_SNAKE_CASE		    =      load_in_abit
     SCREAMING_SNAKE_CASE		    =      llm_inta_threshold
     SCREAMING_SNAKE_CASE		    =      llm_inta_skip_modules
     SCREAMING_SNAKE_CASE		    =      llm_inta_enable_fpaa_cpu_offload
     SCREAMING_SNAKE_CASE		    =      llm_inta_has_fpaa_weight
     SCREAMING_SNAKE_CASE		    =      bnb_abit_quant_type
     SCREAMING_SNAKE_CASE		    =      bnb_abit_use_double_quant

     if bnb_abit_compute_dtype is None:
      SCREAMING_SNAKE_CASE		    =      torch.floataa
     elif isinstance(snake_case__				,     snake_case__     ):
      SCREAMING_SNAKE_CASE		    =      getattr(snake_case__				,     snake_case__     )
     elif isinstance(snake_case__				,     torch.dtype     ):
      SCREAMING_SNAKE_CASE		    =      bnb_abit_compute_dtype
     else:
      raise ValueError('bnb_4bit_compute_dtype must be a string or a torch.dtype'     )

     self.post_init()

    def UpperCamelCase      (      self			:   int     ):

     """simple docstring"""






     if not isinstance(self.llm_inta_threshold				,     snake_case__     ):
      raise ValueError('llm_int8_threshold must be a float'     )

     if self.llm_inta_skip_modules is not None and not isinstance(self.llm_inta_skip_modules				,     snake_case__     ):
      raise ValueError('llm_int8_skip_modules must be a list of strings'     )
     if not isinstance(self.llm_inta_enable_fpaa_cpu_offload				,     snake_case__     ):
      raise ValueError('llm_int8_enable_fp32_cpu_offload must be a boolean'     )

     if not isinstance(self.llm_inta_has_fpaa_weight				,     snake_case__     ):
      raise ValueError('llm_int8_has_fp16_weight must be a boolean'     )

     if self.bnb_abit_compute_dtype is not None and not isinstance(self.bnb_abit_compute_dtype				,     torch.dtype     ):
      raise ValueError('bnb_4bit_compute_dtype must be torch.dtype'     )

     if not isinstance(self.bnb_abit_quant_type				,     snake_case__     ):
      raise ValueError('bnb_4bit_quant_type must be a string'     )

     if not isinstance(self.bnb_abit_use_double_quant				,     snake_case__     ):
      raise ValueError('bnb_4bit_use_double_quant must be a boolean'     )

     if self.load_in_abit and not version.parse(importlib.metadata.version('bitsandbytes'     )     ) >= version.parse(
         '0.39.0'     ):
      raise ValueError(
          '4 bit quantization requires bitsandbytes>=0.39.0 - please upgrade your bitsandbytes version'     )

    def UpperCamelCase      (      self			:   Any     ):

     """simple docstring"""






     return self.load_in_abit or self.load_in_abit

    def UpperCamelCase      (      self			:   List[Any]     ):

     """simple docstring"""






     if self.load_in_abit:
      return "llm_int8"
     elif self.load_in_abit and self.bnb_abit_quant_type == "fp4":
      return "fp4"
     elif self.load_in_abit and self.bnb_abit_quant_type == "nf4":
      return "nf4"
     else:
      return None

    @classmethod
    def UpperCamelCase      (      cls			:   Union[str, Any]				,     snake_case__			:   int				,     snake_case__			:   Optional[Any]				,     **snake_case__			:   int     ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE		    =      cls(**snake_case__     )

     SCREAMING_SNAKE_CASE		    =      []
     for key, value in kwargs.items():
      if hasattr(snake_case__				,     snake_case__     ):
       setattr(snake_case__				,     snake_case__				,     snake_case__     )
       to_remove.append(snake_case__     )
     for key in to_remove:
      kwargs.pop(snake_case__				,     snake_case__     )

     if return_unused_kwargs:
      return config, kwargs
     else:
      return config

    def UpperCamelCase      (      self			:   Any				,     snake_case__			:   Union[str, os.PathLike]     ):

     """simple docstring"""






     with open(snake_case__				,     'w'				,     encoding='utf-8'     ) as writer:
      SCREAMING_SNAKE_CASE		    =      self.to_dict()
      SCREAMING_SNAKE_CASE		    =      json.dumps(snake_case__				,     indent=2				,     sort_keys=snake_case__     ) + '\n'

      writer.write(snake_case__     )

    def UpperCamelCase      (      self			:   Any     ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE		    =      copy.deepcopy(self.__dict__     )
     SCREAMING_SNAKE_CASE		    =      str(output['bnb_4bit_compute_dtype']     ).split('.'     )[1]

     return output

    def __repr__(      self			:   Optional[Any]     ):

     """simple docstring"""






     return F"""{self.__class__.__name__} {self.to_json_string()}"""

    def UpperCamelCase      (      self			:   int				,     snake_case__			:   bool = True     ):

     """simple docstring"""






     if use_diff is True:
      SCREAMING_SNAKE_CASE		    =      self.to_diff_dict()
     else:
      SCREAMING_SNAKE_CASE		    =      self.to_dict()
     return json.dumps(snake_case__				,     indent=2				,     sort_keys=snake_case__     ) + "\n"




    def UpperCamelCase      (      self			:   Optional[int]     ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE		    =      self.to_dict()

     # get the default config dict
     SCREAMING_SNAKE_CASE		    =      BitsAndBytesConfig().to_dict()

     SCREAMING_SNAKE_CASE		    =      {}

     # only serialize values that differ from the default config
     for key, value in config_dict.items():
      if value != default_config_dict[key]:
       SCREAMING_SNAKE_CASE		    =      value

     return serializable_config_dict




import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
      from .tokenization_rembert import RemBertTokenizer
else:
      a_     :    int     		=   None

a_     :    List[Any]     		=   logging.get_logger(__name__)
a_     :    List[Any]     		=   {"vocab_file": "sentencepiece.model", "tokenizer_file": "tokenizer.json"}

a_     :    Optional[int]     		=   {
    "vocab_file": {
        "google/rembert": "https://huggingface.co/google/rembert/resolve/main/sentencepiece.model",
    },
    "tokenizer_file": {
        "google/rembert": "https://huggingface.co/google/rembert/resolve/main/tokenizer.json",
    },
}

a_     :    Union[str, Any]     		=   {
    "google/rembert": 256,
}

a_     :    List[Any]     		=   "▁"




class   UpperCamelCase			(  SCREAMING_SNAKE_CASE ):
    __UpperCamelCase    =VOCAB_FILES_NAMES
    __UpperCamelCase    =PRETRAINED_VOCAB_FILES_MAP
    __UpperCamelCase    =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    __UpperCamelCase    =RemBertTokenizer

    def __init__(      self			:   int				,     snake_case__			:   Any=None				,     snake_case__			:   Optional[Any]=None				,     snake_case__			:   Optional[int]=True				,     snake_case__			:   Any=True				,     snake_case__			:   Any=False				,     snake_case__			:   Dict="[CLS]"				,     snake_case__			:   List[str]="[SEP]"				,     snake_case__			:   List[str]="<unk>"				,     snake_case__			:   Union[str, Any]="[SEP]"				,     snake_case__			:   List[Any]="<pad>"				,     snake_case__			:   Tuple="[CLS]"				,     snake_case__			:   str="[MASK]"				,     **snake_case__			:   Union[str, Any]				,     ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE		    =      AddedToken(snake_case__				,     lstrip=snake_case__				,     rstrip=snake_case__     ) if isinstance(snake_case__				,     snake_case__     ) else mask_token

     super().__init__(
         snake_case__				,     tokenizer_file=snake_case__				,     do_lower_case=snake_case__				,     remove_space=snake_case__				,     keep_accents=snake_case__				,     bos_token=snake_case__				,     eos_token=snake_case__				,     unk_token=snake_case__				,     sep_token=snake_case__				,     pad_token=snake_case__				,     cls_token=snake_case__				,     mask_token=snake_case__				,     **snake_case__				,     )

     SCREAMING_SNAKE_CASE		    =      do_lower_case
     SCREAMING_SNAKE_CASE		    =      remove_space
     SCREAMING_SNAKE_CASE		    =      keep_accents
     SCREAMING_SNAKE_CASE		    =      vocab_file
     SCREAMING_SNAKE_CASE		    =      False if not self.vocab_file else True

    def UpperCamelCase      (      self			:   Tuple				,     snake_case__			:   List[int]				,     snake_case__			:   Optional[List[int]] = None     ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE		    =      [self.sep_token_id]
     SCREAMING_SNAKE_CASE		    =      [self.cls_token_id]
     if token_ids_a is None:
      return cls + token_ids_a + sep
     return cls + token_ids_a + sep + token_ids_a + sep

    def UpperCamelCase      (      self			:   Tuple				,     snake_case__			:   List[int]				,     snake_case__			:   Optional[List[int]] = None				,     snake_case__			:   bool = False     ):

     """simple docstring"""






     if already_has_special_tokens:
      if token_ids_a is not None:
       raise ValueError(
           'You should not supply a second sequence if the provided sequence of '
           'ids is already formatted with special tokens for the model.'     )
      return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

     if token_ids_a is not None:
      return [1] + ([0] * len(snake_case__     )) + [1] + ([0] * len(snake_case__     )) + [1]
     return [1] + ([0] * len(snake_case__     )) + [1]

    def UpperCamelCase      (      self			:   int				,     snake_case__			:   List[int]				,     snake_case__			:   Optional[List[int]] = None     ):

     """simple docstring"""






     SCREAMING_SNAKE_CASE		    =      [self.sep_token_id]
     SCREAMING_SNAKE_CASE		    =      [self.cls_token_id]

     if token_ids_a is None:
      return len(cls + token_ids_a + sep     ) * [0]
     return len(cls + token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1]




    def UpperCamelCase      (      self			:   Dict				,     snake_case__			:   str				,     snake_case__			:   Optional[str] = None     ):

     """simple docstring"""






     if not os.path.isdir(snake_case__     ):
      logger.error('Vocabulary path ({}) should be a directory'.format(snake_case__     )     )
      return
     SCREAMING_SNAKE_CASE		    =      os.path.join(
         snake_case__				,     (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']     )

     if os.path.abspath(self.vocab_file     ) != os.path.abspath(snake_case__     ):
      copyfile(self.vocab_file				,     snake_case__     )

     return (out_vocab_file,)

