
import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_snake_case      :							Dict     	=    logging.get_logger(__name__)

_snake_case      :							int     	=    {
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

_snake_case      :							Any     	=    {
    "vocab_file": {"ctrl": "https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json"},
    "merges_file": {"ctrl": "https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt"},
}

_snake_case      :							int     	=    {
    "ctrl": 2_56,
}

_snake_case      :							str     	=    {
    "Pregnancy": 16_86_29,
    "Christianity": 76_75,
    "Explain": 10_64_23,
    "Fitness": 6_34_40,
    "Saving": 6_31_63,
    "Ask": 2_71_71,
    "Ass": 9_59_85,
    "Joke": 16_35_09,
    "Questions": 4_56_22,
    "Thoughts": 4_96_05,
    "Retail": 5_23_42,
    "Feminism": 16_43_38,
    "Writing": 1_19_92,
    "Atheism": 19_22_63,
    "Netflix": 4_86_16,
    "Computing": 3_96_39,
    "Opinion": 4_32_13,
    "Alone": 4_49_67,
    "Funny": 5_89_17,
    "Gaming": 4_03_58,
    "Human": 40_88,
    "India": 13_31,
    "Joker": 7_71_38,
    "Diet": 3_62_06,
    "Legal": 1_18_59,
    "Norman": 49_39,
    "Tip": 7_26_89,
    "Weight": 5_23_43,
    "Movies": 4_62_73,
    "Running": 2_34_25,
    "Science": 20_90,
    "Horror": 3_77_93,
    "Confession": 6_05_72,
    "Finance": 1_22_50,
    "Politics": 1_63_60,
    "Scary": 19_19_85,
    "Support": 1_26_54,
    "Technologies": 3_25_16,
    "Teenage": 6_61_60,
    "Event": 3_27_69,
    "Learned": 6_74_60,
    "Notion": 18_27_70,
    "Wikipedia": 3_75_83,
    "Books": 66_65,
    "Extract": 7_60_50,
    "Confessions": 10_27_01,
    "Conspiracy": 7_59_32,
    "Links": 6_36_74,
    "Narcissus": 15_04_25,
    "Relationship": 5_47_66,
    "Relationships": 13_47_96,
    "Reviews": 4_16_71,
    "News": 42_56,
    "Translation": 2_68_20,
    "multilingual": 12_84_06,
}

def  __snake_case							(	__magic_name__						):







				'''simple docstring'''


				lowercase         =      set()
				lowercase         =      word[0]
				for char in word[1:]:
								pairs.add((prev_char, char)						)
								lowercase         =      char

				lowercase         =      set(__magic_name__						)
				return pairs





class   UpperCamelCase_      (      __a		):







						'''simple docstring'''



						UpperCamelCase      :  Any				   =					VOCAB_FILES_NAMES
						UpperCamelCase      :  List[str]				   =					PRETRAINED_VOCAB_FILES_MAP
						UpperCamelCase      :  Dict				   =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						UpperCamelCase      :  Optional[Any]				   =					CONTROL_CODES





						def __init__(			self							:Union[str, Any]		,    lowerCAmelCase__							:Tuple		,    lowerCAmelCase__							:Optional[Any]		,    lowerCAmelCase__							:Dict="<unk>"		,    **lowerCAmelCase__							:Optional[Any]     )  ->List[str]:
										super().__init__(unk_token=lowerCAmelCase__		,    **lowerCAmelCase__     )

										with open(lowerCAmelCase__		,    encoding="utf-8"     ) as vocab_handle:
														lowercase         =      json.load(lowerCAmelCase__     )
										lowercase         =      {v: k for k, v in self.encoder.items()}
										with open(lowerCAmelCase__		,    encoding="utf-8"     ) as merges_handle:
														lowercase         =      merges_handle.read().split("\n"     )[1:-1]
										lowercase         =      [tuple(merge.split()     ) for merge in merges]
										lowercase         =      dict(zip(lowerCAmelCase__		,    range(len(lowerCAmelCase__     )     )     )     )
										lowercase         =      {}





						@property
						def        SCREAMING_SNAKE_CASE(			self							:int     )  ->Union[str, Any]:
										return len(self.encoder     )





						def        SCREAMING_SNAKE_CASE(			self							:Optional[int]     )  ->Any:
										return dict(self.encoder		,    **self.added_tokens_encoder     )





						def        SCREAMING_SNAKE_CASE(			self							:List[str]		,    lowerCAmelCase__							:Optional[Any]     )  ->Dict:
										if token in self.cache:
														return self.cache[token]
										lowercase         =      tuple(lowerCAmelCase__     )
										lowercase         =      tuple(list(word[:-1]     ) + [word[-1] + "</w>"]     )
										lowercase         =      get_pairs(lowerCAmelCase__     )

										if not pairs:
														return token

										while True:
														lowercase         =      min(lowerCAmelCase__		,    key=lambda lowerCAmelCase__     : self.bpe_ranks.get(lowerCAmelCase__		,    float("inf"     )     )     )
														if bigram not in self.bpe_ranks:
																		break
														lowercase     ,	lowercase         =      bigram
														lowercase         =      []
														lowercase         =      0
														while i < len(lowerCAmelCase__     ):
																		try:
																						lowercase         =      word.index(lowerCAmelCase__		,    lowerCAmelCase__     )
																		except ValueError:
																						new_word.extend(word[i:]     )
																						break
																		else:
																						new_word.extend(word[i:j]     )
																						lowercase         =      j

																		if word[i] == first and i < len(lowerCAmelCase__     ) - 1 and word[i + 1] == second:
																						new_word.append(first + second     )
																						i += 2
																		else:
																						new_word.append(word[i]     )
																						i += 1
														lowercase         =      tuple(lowerCAmelCase__     )
														lowercase         =      new_word
														if len(lowerCAmelCase__     ) == 1:
																		break
														else:
																		lowercase         =      get_pairs(lowerCAmelCase__     )
										lowercase         =      "@@ ".join(lowerCAmelCase__     )
										lowercase         =      word[:-4]
										lowercase         =      word
										return word





						def        SCREAMING_SNAKE_CASE(			self							:int		,    lowerCAmelCase__							:Any     )  ->str:
										lowercase         =      []

										lowercase         =      re.findall(r"\S+\n?"		,    lowerCAmelCase__     )

										for token in words:
														split_tokens.extend(list(self.bpe(lowerCAmelCase__     ).split(" "     )     )     )
										return split_tokens





						def        SCREAMING_SNAKE_CASE(			self							:Tuple		,    lowerCAmelCase__							:Union[str, Any]     )  ->Optional[int]:
										return self.encoder.get(lowerCAmelCase__		,    self.encoder.get(self.unk_token     )     )





						def        SCREAMING_SNAKE_CASE(			self							:Union[str, Any]		,    lowerCAmelCase__							:Optional[int]     )  ->Dict:
										return self.decoder.get(lowerCAmelCase__		,    self.unk_token     )





						def        SCREAMING_SNAKE_CASE(			self							:int		,    lowerCAmelCase__							:List[str]     )  ->List[str]:
										lowercase         =      " ".join(lowerCAmelCase__     ).replace("@@ "		,    ""     ).strip()
										return out_string





						def        SCREAMING_SNAKE_CASE(			self							:Dict		,    lowerCAmelCase__							:str		,    lowerCAmelCase__							:Optional[str] = None     )  ->Tuple[str]:
										if not os.path.isdir(lowerCAmelCase__     ):
														logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''     )
														return
										lowercase         =      os.path.join(
										    lowerCAmelCase__		,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]     )
										lowercase         =      os.path.join(
										    lowerCAmelCase__		,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]     )

										with open(lowerCAmelCase__		,    "w"		,    encoding="utf-8"     ) as f:
														f.write(json.dumps(self.encoder		,    indent=2		,    sort_keys=lowerCAmelCase__		,    ensure_ascii=lowerCAmelCase__     ) + "\n"     )

										lowercase         =      0
										with open(lowerCAmelCase__		,    "w"		,    encoding="utf-8"     ) as writer:
														writer.write("#version: 0.2\n"     )
														for bpe_tokens, token_index in sorted(self.bpe_ranks.items()		,    key=lambda lowerCAmelCase__     : kv[1]     ):
																		if index != token_index:
																						logger.warning(
																						    F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																						    " Please check that the tokenizer is not corrupted!"     )
																						lowercase         =      token_index
																		writer.write(" ".join(lowerCAmelCase__     ) + "\n"     )
																		index += 1

										return vocab_file, merge_file

						# def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
						#     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
						#     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
						#     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
						#     return ''.join(tokens_generated_so_far)


# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
import importlib.metadata
import json
import os
from dataclasses import dataclass
from typing import Any, Dict, Union

from packaging import version

from ..utils import is_torch_available, logging


if is_torch_available():
						import torch


_snake_case      :							Optional[Any]     	=    logging.get_logger(__name__)





@dataclass
class   UpperCamelCase_      :







						'''simple docstring'''



						def __init__(			self							:str		,    lowerCAmelCase__							:Any=False		,    lowerCAmelCase__							:Tuple=False		,    lowerCAmelCase__							:List[str]=6.0		,    lowerCAmelCase__							:int=None		,    lowerCAmelCase__							:Optional[Any]=False		,    lowerCAmelCase__							:List[Any]=False		,    lowerCAmelCase__							:int=None		,    lowerCAmelCase__							:Tuple="fp4"		,    lowerCAmelCase__							:Union[str, Any]=False		,    **lowerCAmelCase__							:Optional[Any]		,    )  ->Any:
										lowercase         =      load_in_abit
										lowercase         =      load_in_abit
										lowercase         =      llm_inta_threshold
										lowercase         =      llm_inta_skip_modules
										lowercase         =      llm_inta_enable_fpaa_cpu_offload
										lowercase         =      llm_inta_has_fpaa_weight
										lowercase         =      bnb_abit_quant_type
										lowercase         =      bnb_abit_use_double_quant

										if bnb_abit_compute_dtype is None:
														lowercase         =      torch.floataa
										elif isinstance(lowerCAmelCase__		,    lowerCAmelCase__     ):
														lowercase         =      getattr(lowerCAmelCase__		,    lowerCAmelCase__     )
										elif isinstance(lowerCAmelCase__		,    torch.dtype     ):
														lowercase         =      bnb_abit_compute_dtype
										else:
														raise ValueError("bnb_4bit_compute_dtype must be a string or a torch.dtype"     )

										self.post_init()





						def        SCREAMING_SNAKE_CASE(			self							:Any     )  ->str:
										if not isinstance(self.llm_inta_threshold		,    lowerCAmelCase__     ):
														raise ValueError("llm_int8_threshold must be a float"     )

										if self.llm_inta_skip_modules is not None and not isinstance(self.llm_inta_skip_modules		,    lowerCAmelCase__     ):
														raise ValueError("llm_int8_skip_modules must be a list of strings"     )
										if not isinstance(self.llm_inta_enable_fpaa_cpu_offload		,    lowerCAmelCase__     ):
														raise ValueError("llm_int8_enable_fp32_cpu_offload must be a boolean"     )

										if not isinstance(self.llm_inta_has_fpaa_weight		,    lowerCAmelCase__     ):
														raise ValueError("llm_int8_has_fp16_weight must be a boolean"     )

										if self.bnb_abit_compute_dtype is not None and not isinstance(self.bnb_abit_compute_dtype		,    torch.dtype     ):
														raise ValueError("bnb_4bit_compute_dtype must be torch.dtype"     )

										if not isinstance(self.bnb_abit_quant_type		,    lowerCAmelCase__     ):
														raise ValueError("bnb_4bit_quant_type must be a string"     )

										if not isinstance(self.bnb_abit_use_double_quant		,    lowerCAmelCase__     ):
														raise ValueError("bnb_4bit_use_double_quant must be a boolean"     )

										if self.load_in_abit and not version.parse(importlib.metadata.version("bitsandbytes"     )     ) >= version.parse(
										    "0.39.0"     ):
														raise ValueError(
														    "4 bit quantization requires bitsandbytes>=0.39.0 - please upgrade your bitsandbytes version"     )





						def        SCREAMING_SNAKE_CASE(			self							:List[str]     )  ->Tuple:
										return self.load_in_abit or self.load_in_abit





						def        SCREAMING_SNAKE_CASE(			self							:List[str]     )  ->int:
										if self.load_in_abit:
														return "llm_int8"
										elif self.load_in_abit and self.bnb_abit_quant_type == "fp4":
														return "fp4"
										elif self.load_in_abit and self.bnb_abit_quant_type == "nf4":
														return "nf4"
										else:
														return None





						@classmethod
						def        SCREAMING_SNAKE_CASE(			cls							:Union[str, Any]		,    lowerCAmelCase__							:List[Any]		,    lowerCAmelCase__							:Union[str, Any]		,    **lowerCAmelCase__							:Any     )  ->Optional[Any]:
										lowercase         =      cls(**lowerCAmelCase__     )

										lowercase         =      []
										for key, value in kwargs.items():
														if hasattr(lowerCAmelCase__		,    lowerCAmelCase__     ):
																		setattr(lowerCAmelCase__		,    lowerCAmelCase__		,    lowerCAmelCase__     )
																		to_remove.append(lowerCAmelCase__     )
										for key in to_remove:
														kwargs.pop(lowerCAmelCase__		,    lowerCAmelCase__     )

										if return_unused_kwargs:
														return config, kwargs
										else:
														return config





						def        SCREAMING_SNAKE_CASE(			self							:Dict		,    lowerCAmelCase__							:Union[str, os.PathLike]     )  ->Optional[int]:
										with open(lowerCAmelCase__		,    "w"		,    encoding="utf-8"     ) as writer:
														lowercase         =      self.to_dict()
														lowercase         =      json.dumps(lowerCAmelCase__		,    indent=2		,    sort_keys=lowerCAmelCase__     ) + "\n"

														writer.write(lowerCAmelCase__     )





						def        SCREAMING_SNAKE_CASE(			self							:Any     )  ->Dict[str, Any]:
										lowercase         =      copy.deepcopy(self.__dict__     )
										lowercase         =      str(output["bnb_4bit_compute_dtype"]     ).split("."     )[1]

										return output





						def __repr__(			self							:int     )  ->Optional[Any]:
										return F'''{self.__class__.__name__} {self.to_json_string()}'''





						def        SCREAMING_SNAKE_CASE(			self							:Optional[Any]		,    lowerCAmelCase__							:bool = True     )  ->str:
										if use_diff is True:
														lowercase         =      self.to_diff_dict()
										else:
														lowercase         =      self.to_dict()
										return json.dumps(lowerCAmelCase__		,    indent=2		,    sort_keys=lowerCAmelCase__     ) + "\n"





						def        SCREAMING_SNAKE_CASE(			self							:int     )  ->Dict[str, Any]:
										lowercase         =      self.to_dict()

										# get the default config dict
										lowercase         =      BitsAndBytesConfig().to_dict()

										lowercase         =      {}

										# only serialize values that differ from the default config
										for key, value in config_dict.items():
														if value != default_config_dict[key]:
																		lowercase         =      value

										return serializable_config_dict
