




# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
import importlib.metadata
import json
import os
from dataclasses import dataclass
from typing import Any, Dict, Union

from packaging import version

from ..utils import is_torch_available, logging


if is_torch_available():
							import torch


UpperCAmelCase_		:				Dict									=     logging.get_logger(__name__)




@dataclass
class 		__A      :

						def __init__(		self   :Union[str, Any]    , __snake_case   :List[Any]=False    , __snake_case   :Dict=False    , __snake_case   :Optional[Any]=6.0    , __snake_case   :Any=None    , __snake_case   :str=False    , __snake_case   :str=False    , __snake_case   :Union[str, Any]=None    , __snake_case   :List[str]="fp4"    , __snake_case   :str=False    , **__snake_case   :str    , ):
										'''simple docstring'''

										__magic_name__	:	List[str]								=load_in_abit
										__magic_name__	:	int								=load_in_abit
										__magic_name__	:	Any								=llm_inta_threshold
										__magic_name__	:	Optional[Any]								=llm_inta_skip_modules
										__magic_name__	:	Union[str, Any]								=llm_inta_enable_fpaa_cpu_offload
										__magic_name__	:	List[Any]								=llm_inta_has_fpaa_weight
										__magic_name__	:	Optional[Any]								=bnb_abit_quant_type
										__magic_name__	:	int								=bnb_abit_use_double_quant

										if bnb_abit_compute_dtype is None:
														__magic_name__	:	Optional[Any]								=torch.floataa
										elif isinstance(__snake_case    , __snake_case ):
														__magic_name__	:	Optional[Any]								=getattr(__snake_case    , __snake_case )
										elif isinstance(__snake_case    , torch.dtype ):
														__magic_name__	:	Union[str, Any]								=bnb_abit_compute_dtype
										else:
														raise ValueError("""bnb_4bit_compute_dtype must be a string or a torch.dtype""" )

										self.post_init()

						def     A__						(		self   :str ):
										'''simple docstring'''

										if not isinstance(self.llm_inta_threshold    , __snake_case ):
														raise ValueError("""llm_int8_threshold must be a float""" )

										if self.llm_inta_skip_modules is not None and not isinstance(self.llm_inta_skip_modules    , __snake_case ):
														raise ValueError("""llm_int8_skip_modules must be a list of strings""" )
										if not isinstance(self.llm_inta_enable_fpaa_cpu_offload    , __snake_case ):
														raise ValueError("""llm_int8_enable_fp32_cpu_offload must be a boolean""" )

										if not isinstance(self.llm_inta_has_fpaa_weight    , __snake_case ):
														raise ValueError("""llm_int8_has_fp16_weight must be a boolean""" )

										if self.bnb_abit_compute_dtype is not None and not isinstance(self.bnb_abit_compute_dtype    , torch.dtype ):
														raise ValueError("""bnb_4bit_compute_dtype must be torch.dtype""" )

										if not isinstance(self.bnb_abit_quant_type    , __snake_case ):
														raise ValueError("""bnb_4bit_quant_type must be a string""" )

										if not isinstance(self.bnb_abit_use_double_quant    , __snake_case ):
														raise ValueError("""bnb_4bit_use_double_quant must be a boolean""" )

										if self.load_in_abit and not version.parse(importlib.metadata.version("""bitsandbytes""" ) ) >= version.parse(
										    """0.39.0""" ):
														raise ValueError(
														    """4 bit quantization requires bitsandbytes>=0.39.0 - please upgrade your bitsandbytes version""" )

						def     A__						(		self   :Union[str, Any] ):
										'''simple docstring'''

										return self.load_in_abit or self.load_in_abit

						def     A__						(		self   :List[Any] ):
										'''simple docstring'''

										if self.load_in_abit:
														return "llm_int8"
										elif self.load_in_abit and self.bnb_abit_quant_type == "fp4":
														return "fp4"
										elif self.load_in_abit and self.bnb_abit_quant_type == "nf4":
														return "nf4"
										else:
														return None

						@classmethod
						def     A__						(		cls   :List[Any]    , __snake_case   :Tuple    , __snake_case   :Union[str, Any]    , **__snake_case   :Optional[Any] ):
										'''simple docstring'''

										__magic_name__	:	List[Any]								=cls(**__snake_case )

										__magic_name__	:	int								=[]
										for key, value in kwargs.items():
														if hasattr(__snake_case    , __snake_case ):
																		setattr(__snake_case    , __snake_case    , __snake_case )
																		to_remove.append(__snake_case )
										for key in to_remove:
														kwargs.pop(__snake_case    , __snake_case )

										if return_unused_kwargs:
														return config, kwargs
										else:
														return config

						def     A__						(		self   :Any    , __snake_case   :Union[str, os.PathLike] ):
										'''simple docstring'''

										with open(__snake_case    , """w"""    , encoding="""utf-8""" ) as writer:
														__magic_name__	:	Optional[int]								=self.to_dict()
														__magic_name__	:	Optional[int]								=json.dumps(__snake_case    , indent=2    , sort_keys=__snake_case ) + """\n"""

														writer.write(__snake_case )

						def     A__						(		self   :List[str] ):
										'''simple docstring'''

										__magic_name__	:	int								=copy.deepcopy(self.__dict__ )
										__magic_name__	:	int								=str(output["""bnb_4bit_compute_dtype"""] ).split(""".""" )[1]

										return output

						def __repr__(		self   :Union[str, Any] ):
										'''simple docstring'''

										return f"{self.__class__.__name__} {self.to_json_string()}"

						def     A__						(		self   :int    , __snake_case   :bool = True ):
										'''simple docstring'''

										if use_diff is True:
														__magic_name__	:	List[str]								=self.to_diff_dict()
										else:
														__magic_name__	:	List[str]								=self.to_dict()
										return json.dumps(__snake_case    , indent=2    , sort_keys=__snake_case ) + "\n"


						def     A__						(		self   :Dict ):
										'''simple docstring'''

										__magic_name__	:	List[Any]								=self.to_dict()

										# get the default config dict
										__magic_name__	:	Optional[Any]								=BitsAndBytesConfig().to_dict()

										__magic_name__	:	str								={}

										# only serialize values that differ from the default config
										for key, value in config_dict.items():
														if value != default_config_dict[key]:
																		__magic_name__	:	str								=value

										return serializable_config_dict










import json
import os
import unittest

from transformers.models.blenderbot_small.tokenization_blenderbot_small import (
    VOCAB_FILES_NAMES,
    BlenderbotSmallTokenizer,
)

from ...test_tokenization_common import TokenizerTesterMixin




class 		__A      (			UpperCamelCase__					,						unittest.TestCase			):
						UpperCamelCase     						=					BlenderbotSmallTokenizer
						UpperCamelCase     						=					False

						def     A__						(		self   :Dict ):
										'''simple docstring'''

										super().setUp()

										__magic_name__	:	Tuple								=["""__start__""", """adapt""", """act""", """ap@@""", """te""", """__end__""", """__unk__"""]
										__magic_name__	:	Optional[Any]								=dict(zip(__snake_case    , range(len(__snake_case ) ) ) )

										__magic_name__	:	List[Any]								=["""#version: 0.2""", """a p""", """t e</w>""", """ap t</w>""", """a d""", """ad apt</w>""", """a c""", """ac t</w>""", """"""]
										__magic_name__	:	int								={"""unk_token""": """__unk__""", """bos_token""": """__start__""", """eos_token""": """__end__"""}

										__magic_name__	:	Dict								=os.path.join(self.tmpdirname    , VOCAB_FILES_NAMES["""vocab_file"""] )
										__magic_name__	:	int								=os.path.join(self.tmpdirname    , VOCAB_FILES_NAMES["""merges_file"""] )
										with open(self.vocab_file    , """w"""    , encoding="""utf-8""" ) as fp:
														fp.write(json.dumps(__snake_case ) + """\n""" )
										with open(self.merges_file    , """w"""    , encoding="""utf-8""" ) as fp:
														fp.write("""\n""".join(__snake_case ) )

						def     A__						(		self   :Union[str, Any]    , **__snake_case   :Optional[int] ):
										'''simple docstring'''

										kwargs.update(self.special_tokens_map )
										return BlenderbotSmallTokenizer.from_pretrained(self.tmpdirname    , **__snake_case )

						def     A__						(		self   :str    , __snake_case   :List[Any] ):
										'''simple docstring'''

										__magic_name__	:	Any								="""adapt act apte"""
										__magic_name__	:	str								="""adapt act apte"""
										return input_text, output_text

						def     A__						(		self   :List[str] ):
										'''simple docstring'''

										__magic_name__	:	Tuple								=BlenderbotSmallTokenizer(self.vocab_file    , self.merges_file    , **self.special_tokens_map )
										__magic_name__	:	List[str]								="""adapt act apte"""
										__magic_name__	:	Optional[int]								=["""adapt""", """act""", """ap@@""", """te"""]
										__magic_name__	:	List[Any]								=tokenizer.tokenize(__snake_case )
										self.assertListEqual(__snake_case    , __snake_case )

										__magic_name__	:	int								=[tokenizer.bos_token] + tokens + [tokenizer.eos_token]

										__magic_name__	:	Any								=[0, 1, 2, 3, 4, 5]
										self.assertListEqual(tokenizer.convert_tokens_to_ids(__snake_case )    , __snake_case )

						def     A__						(		self   :Optional[int] ):
										'''simple docstring'''

										__magic_name__	:	Any								=BlenderbotSmallTokenizer.from_pretrained("""facebook/blenderbot-90M""" )
										assert tok("""sam""" ).input_ids == [13_84]
										__magic_name__	:	Tuple								="""I am a small frog."""
										__magic_name__	:	Tuple								=tok([src_text]    , padding=__snake_case    , truncation=__snake_case )["""input_ids"""]
										__magic_name__	:	Optional[Any]								=tok.batch_decode(__snake_case    , skip_special_tokens=__snake_case    , clean_up_tokenization_spaces=__snake_case )[0]
										assert src_text != decoded  # I wish it did!
										assert decoded == "i am a small frog ."


						def     A__						(		self   :Optional[Any] ):
										'''simple docstring'''

										__magic_name__	:	List[Any]								=BlenderbotSmallTokenizer.from_pretrained("""facebook/blenderbot-90M""" )
										__magic_name__	:	int								="""I am a small frog ."""
										__magic_name__	:	List[Any]								="""."""
										__magic_name__	:	List[str]								=tok(__snake_case )["""input_ids"""]
										__magic_name__	:	List[str]								=tok(__snake_case )["""input_ids"""]

										assert encoded[-1] == encoded_dot[0]




