

import json
from typing import Dict, List, Optional, Tuple, Union

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding, EncodedInput
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_led import LEDTokenizer


lowerCAmelCase     :   Dict						     =  logging.get_logger(__name__)


lowerCAmelCase     :   Tuple						     =  {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

lowerCAmelCase     :   Any						     =  {
    'vocab_file': {
        'allenai/led-base-16384': 'https://huggingface.co/allenai/led-base-16384/resolve/main/vocab.json',
    },
    'merges_file': {
        'allenai/led-base-16384': 'https://huggingface.co/allenai/led-base-16384/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'allenai/led-base-16384': 'https://huggingface.co/allenai/led-base-16384/resolve/main/tokenizer.json',
    },
}

lowerCAmelCase     :   Any						     =  {
    'allenai/led-base-16384': 1_63_84,
}
class  _A		(  __magic_name__):
  SCREAMING_SNAKE_CASE    :      str       =  VOCAB_FILES_NAMES
  SCREAMING_SNAKE_CASE    :      Union[str, Any]       =  PRETRAINED_VOCAB_FILES_MAP
  SCREAMING_SNAKE_CASE    :      Tuple       =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  SCREAMING_SNAKE_CASE    :      int       =  LEDTokenizer
  SCREAMING_SNAKE_CASE    :      Tuple       =  ['''input_ids''', '''attention_mask''']

  def __init__(				self ,					_SCREAMING_SNAKE_CASE=None ,					_SCREAMING_SNAKE_CASE=None ,					_SCREAMING_SNAKE_CASE=None ,					_SCREAMING_SNAKE_CASE="replace" ,					_SCREAMING_SNAKE_CASE="<s>" ,					_SCREAMING_SNAKE_CASE="</s>" ,					_SCREAMING_SNAKE_CASE="</s>" ,					_SCREAMING_SNAKE_CASE="<s>" ,					_SCREAMING_SNAKE_CASE="<unk>" ,					_SCREAMING_SNAKE_CASE="<pad>" ,					_SCREAMING_SNAKE_CASE="<mask>" ,					_SCREAMING_SNAKE_CASE=False ,					_SCREAMING_SNAKE_CASE=True ,					**_SCREAMING_SNAKE_CASE ,					):





        """simple docstring"""







        super().__init__(
            _SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE ,					tokenizer_file=_SCREAMING_SNAKE_CASE ,					errors=_SCREAMING_SNAKE_CASE ,					bos_token=_SCREAMING_SNAKE_CASE ,					eos_token=_SCREAMING_SNAKE_CASE ,					sep_token=_SCREAMING_SNAKE_CASE ,					cls_token=_SCREAMING_SNAKE_CASE ,					unk_token=_SCREAMING_SNAKE_CASE ,					pad_token=_SCREAMING_SNAKE_CASE ,					mask_token=_SCREAMING_SNAKE_CASE ,					add_prefix_space=_SCREAMING_SNAKE_CASE ,					trim_offsets=_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE ,					)

        SCREAMING_SNAKE_CASE_				:  str           =     json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()					)
        if pre_tok_state.get('add_prefix_space' ,					_SCREAMING_SNAKE_CASE					) != add_prefix_space:
              SCREAMING_SNAKE_CASE_				:  Any           =     getattr(_SCREAMING_SNAKE_CASE ,					pre_tok_state.pop('type'					)					)
              SCREAMING_SNAKE_CASE_				:  Union[str, Any]           =     add_prefix_space
              SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     pre_tok_class(**_SCREAMING_SNAKE_CASE					)

        SCREAMING_SNAKE_CASE_				:  str           =     add_prefix_space

        # the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
        SCREAMING_SNAKE_CASE_				:  Any           =     'post_processor'
        SCREAMING_SNAKE_CASE_				:  Tuple           =     getattr(self.backend_tokenizer ,					_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE					)
        if tokenizer_component_instance:
              SCREAMING_SNAKE_CASE_				:  Union[str, Any]           =     json.loads(tokenizer_component_instance.__getstate__()					)

              # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
              if "sep" in state:
                    SCREAMING_SNAKE_CASE_				:  Dict           =     tuple(state['sep']					)
              if "cls" in state:
                    SCREAMING_SNAKE_CASE_				:  Any           =     tuple(state['cls']					)

              SCREAMING_SNAKE_CASE_				:  List[Any]           =     False

              if state.get('add_prefix_space' ,					_SCREAMING_SNAKE_CASE					) != add_prefix_space:
                    SCREAMING_SNAKE_CASE_				:  Tuple           =     add_prefix_space
                    SCREAMING_SNAKE_CASE_				:  Tuple           =     True

              if state.get('trim_offsets' ,					_SCREAMING_SNAKE_CASE					) != trim_offsets:
                    SCREAMING_SNAKE_CASE_				:  Dict           =     trim_offsets
                    SCREAMING_SNAKE_CASE_				:  Dict           =     True

              if changes_to_apply:
                    SCREAMING_SNAKE_CASE_				:  List[str]           =     getattr(_SCREAMING_SNAKE_CASE ,					state.pop('type'					)					)
                    SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     component_class(**_SCREAMING_SNAKE_CASE					)
                    setattr(self.backend_tokenizer ,					_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE					)

  @property
  # Copied from transformers.models.bart.tokenization_bart_fast.BartTokenizerFast.mask_token with BART->LED
  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        if self._mask_token is None:
              if self.verbose:
                    logger.error('Using mask_token, but it is not set yet.'					)
              return None
        return str(self._mask_token					)

  @mask_token.setter
  def  UpperCAmelCase    (				self ,					_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  List[str]           =     AddedToken(_SCREAMING_SNAKE_CASE ,					lstrip=_SCREAMING_SNAKE_CASE ,					rstrip=_SCREAMING_SNAKE_CASE					) if isinstance(_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE					) else value
        SCREAMING_SNAKE_CASE_				:  str           =     value

  def  UpperCAmelCase    (				self ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  Any           =     kwargs.get('is_split_into_words' ,					_SCREAMING_SNAKE_CASE					)

        if is_split_into_words and not self.add_prefix_space:
              raise ValueError(
                  f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
                  'to use it with pretokenized inputs.'					)

        return super()._batch_encode_plus(*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					)

  def  UpperCAmelCase    (				self ,					*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  List[Any]           =     kwargs.get('is_split_into_words' ,					_SCREAMING_SNAKE_CASE					)

        if is_split_into_words and not self.add_prefix_space:
              raise ValueError(
                  f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
                  'to use it with pretokenized inputs.'					)

        return super()._encode_plus(*_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					)

  def  UpperCAmelCase    (				self ,					_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE = None					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  int           =     self._tokenizer.model.save(_SCREAMING_SNAKE_CASE ,					name=_SCREAMING_SNAKE_CASE					)
        return tuple(_SCREAMING_SNAKE_CASE					)

  def  UpperCAmelCase    (				self ,					_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE=None					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  Optional[int]           =     [self.bos_token_id] + token_ids_a + [self.eos_token_id]
        if token_ids_a is None:
              return output

        return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]

  def  UpperCAmelCase    (				self ,					_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE = None					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  str           =     [self.sep_token_id]
        SCREAMING_SNAKE_CASE_				:  Tuple           =     [self.cls_token_id]

        if token_ids_a is None:
              return len(cls + token_ids_a + sep					) * [0]
        return len(cls + token_ids_a + sep + sep + token_ids_a + sep					) * [0]



  def  UpperCAmelCase    (				self ,					_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE = None ,					_SCREAMING_SNAKE_CASE = PaddingStrategy.DO_NOT_PAD ,					_SCREAMING_SNAKE_CASE = None ,					_SCREAMING_SNAKE_CASE = None ,					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  Dict           =     super()._pad(
            encoded_inputs=_SCREAMING_SNAKE_CASE ,					max_length=_SCREAMING_SNAKE_CASE ,					padding_strategy=_SCREAMING_SNAKE_CASE ,					pad_to_multiple_of=_SCREAMING_SNAKE_CASE ,					return_attention_mask=_SCREAMING_SNAKE_CASE ,					)

        # Load from model defaults
        if return_attention_mask is None:
              SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     'attention_mask' in self.model_input_names

        if return_attention_mask and "global_attention_mask" in encoded_inputs:
              SCREAMING_SNAKE_CASE_				:  Optional[int]           =     encoded_inputs[self.model_input_names[0]]
              # `global_attention_mask` need to have the same length as other (sequential) inputs.
              SCREAMING_SNAKE_CASE_				:  str           =     len(encoded_inputs['global_attention_mask']					) != len(_SCREAMING_SNAKE_CASE					)

              if needs_to_be_padded:
                    SCREAMING_SNAKE_CASE_				:  Union[str, Any]           =     len(_SCREAMING_SNAKE_CASE					) - len(encoded_inputs['global_attention_mask']					)

                    if self.padding_side == "right":
                          # Use `-1` since `0` in `global_attention_mask` means `local attention` instead of `not to attend`
                          SCREAMING_SNAKE_CASE_				:  Dict           =     (
                              encoded_inputs['global_attention_mask'] + [-1] * difference
                          )
                    elif self.padding_side == "left":
                          SCREAMING_SNAKE_CASE_				:  Any           =     [-1] * difference + encoded_inputs[
                              'global_attention_mask'
                          ]
                    else:
                          raise ValueError('Invalid padding strategy:' + str(self.padding_side					)					)

        return encoded_inputs









# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
import importlib.metadata
import json
import os
from dataclasses import dataclass
from typing import Any, Dict, Union

from packaging import version

from ..utils import is_torch_available, logging


if is_torch_available():
    import torch


lowerCAmelCase     :   Optional[int]						     =  logging.get_logger(__name__)
@dataclass
class  _A		:

  def __init__(				self ,					_SCREAMING_SNAKE_CASE=False ,					_SCREAMING_SNAKE_CASE=False ,					_SCREAMING_SNAKE_CASE=6.0 ,					_SCREAMING_SNAKE_CASE=None ,					_SCREAMING_SNAKE_CASE=False ,					_SCREAMING_SNAKE_CASE=False ,					_SCREAMING_SNAKE_CASE=None ,					_SCREAMING_SNAKE_CASE="fp4" ,					_SCREAMING_SNAKE_CASE=False ,					**_SCREAMING_SNAKE_CASE ,					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  str           =     load_in_abit
        SCREAMING_SNAKE_CASE_				:  Tuple           =     load_in_abit
        SCREAMING_SNAKE_CASE_				:  Dict           =     llm_inta_threshold
        SCREAMING_SNAKE_CASE_				:  Tuple           =     llm_inta_skip_modules
        SCREAMING_SNAKE_CASE_				:  Optional[int]           =     llm_inta_enable_fpaa_cpu_offload
        SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     llm_inta_has_fpaa_weight
        SCREAMING_SNAKE_CASE_				:  List[str]           =     bnb_abit_quant_type
        SCREAMING_SNAKE_CASE_				:  int           =     bnb_abit_use_double_quant

        if bnb_abit_compute_dtype is None:
              SCREAMING_SNAKE_CASE_				:  List[Any]           =     torch.floataa
        elif isinstance(_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE					):
              SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     getattr(_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE					)
        elif isinstance(_SCREAMING_SNAKE_CASE ,					torch.dtype					):
              SCREAMING_SNAKE_CASE_				:  Optional[int]           =     bnb_abit_compute_dtype
        else:
              raise ValueError('bnb_4bit_compute_dtype must be a string or a torch.dtype'					)

        self.post_init()

  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        if not isinstance(self.llm_inta_threshold ,					_SCREAMING_SNAKE_CASE					):
              raise ValueError('llm_int8_threshold must be a float'					)

        if self.llm_inta_skip_modules is not None and not isinstance(self.llm_inta_skip_modules ,					_SCREAMING_SNAKE_CASE					):
              raise ValueError('llm_int8_skip_modules must be a list of strings'					)
        if not isinstance(self.llm_inta_enable_fpaa_cpu_offload ,					_SCREAMING_SNAKE_CASE					):
              raise ValueError('llm_int8_enable_fp32_cpu_offload must be a boolean'					)

        if not isinstance(self.llm_inta_has_fpaa_weight ,					_SCREAMING_SNAKE_CASE					):
              raise ValueError('llm_int8_has_fp16_weight must be a boolean'					)

        if self.bnb_abit_compute_dtype is not None and not isinstance(self.bnb_abit_compute_dtype ,					torch.dtype					):
              raise ValueError('bnb_4bit_compute_dtype must be torch.dtype'					)

        if not isinstance(self.bnb_abit_quant_type ,					_SCREAMING_SNAKE_CASE					):
              raise ValueError('bnb_4bit_quant_type must be a string'					)

        if not isinstance(self.bnb_abit_use_double_quant ,					_SCREAMING_SNAKE_CASE					):
              raise ValueError('bnb_4bit_use_double_quant must be a boolean'					)

        if self.load_in_abit and not version.parse(importlib.metadata.version('bitsandbytes'					)					) >= version.parse(
            '0.39.0'					):
              raise ValueError(
                  '4 bit quantization requires bitsandbytes>=0.39.0 - please upgrade your bitsandbytes version'					)

  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        return self.load_in_abit or self.load_in_abit

  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        if self.load_in_abit:
              return "llm_int8"
        elif self.load_in_abit and self.bnb_abit_quant_type == "fp4":
              return "fp4"
        elif self.load_in_abit and self.bnb_abit_quant_type == "nf4":
              return "nf4"
        else:
              return None

  @classmethod
  def  UpperCAmelCase    (				cls ,					_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  Optional[int]           =     cls(**_SCREAMING_SNAKE_CASE					)

        SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     []
        for key, value in kwargs.items():
              if hasattr(_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE					):
                    setattr(_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE					)
                    to_remove.append(_SCREAMING_SNAKE_CASE					)
        for key in to_remove:
              kwargs.pop(_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE					)

        if return_unused_kwargs:
              return config, kwargs
        else:
              return config

  def  UpperCAmelCase    (				self ,					_SCREAMING_SNAKE_CASE					):





        """simple docstring"""







        with open(_SCREAMING_SNAKE_CASE ,					'w' ,					encoding='utf-8'					) as writer:
              SCREAMING_SNAKE_CASE_				:  Optional[int]           =     self.to_dict()
              SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     json.dumps(_SCREAMING_SNAKE_CASE ,					indent=2 ,					sort_keys=_SCREAMING_SNAKE_CASE					) + '\n'

              writer.write(_SCREAMING_SNAKE_CASE					)

  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  str           =     copy.deepcopy(self.__dict__					)
        SCREAMING_SNAKE_CASE_				:  str           =     str(output['bnb_4bit_compute_dtype']					).split('.'					)[1]

        return output

  def __repr__(				self					):





        """simple docstring"""







        return f"{self.__class__.__name__} {self.to_json_string()}"

  def  UpperCAmelCase    (				self ,					_SCREAMING_SNAKE_CASE = True					):





        """simple docstring"""







        if use_diff is True:
              SCREAMING_SNAKE_CASE_				:  int           =     self.to_diff_dict()
        else:
              SCREAMING_SNAKE_CASE_				:  List[str]           =     self.to_dict()
        return json.dumps(_SCREAMING_SNAKE_CASE ,					indent=2 ,					sort_keys=_SCREAMING_SNAKE_CASE					) + "\n"



  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  int           =     self.to_dict()

        # get the default config dict
        SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     BitsAndBytesConfig().to_dict()

        SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     {}

        # only serialize values that differ from the default config
        for key, value in config_dict.items():
              if value != default_config_dict[key]:
                    SCREAMING_SNAKE_CASE_				:  int           =     value

        return serializable_config_dict






