



import os
from shutil import copyfile
from typing import TYPE_CHECKING, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging
from ...utils.versions import require_version


if TYPE_CHECKING:
	from transformers.pipelines.conversational import Conversation

require_version('''tokenizers>=0.13.3''')

if is_sentencepiece_available():
	from .tokenization_llama import LlamaTokenizer
else:
	__A				    =						None

__A				    =						logging.get_logger(__name__)
__A				    =						{'''vocab_file''': '''tokenizer.model''', '''tokenizer_file''': '''tokenizer.json'''}

__A    ,			__A				    =						'''[INST]''', '''[/INST]'''
__A    ,			__A				    =						'''<<SYS>>\n''', '''\n<</SYS>>\n\n'''

# fmt: off
__A				    =						'''You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your\
answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure\
that your responses are socially unbiased and positive in nature.

If a question does not make any sense, or is not factually coherent, explain why instead of answering something not\
correct. If you don\'t know the answer to a question, please don\'t share false information.'''






class 		lowercase						( lowerCAmelCase__):






	"""simple docstring"""


	a__      :					Tuple		   =							VOCAB_FILES_NAMES
	a__      :					List[Any]		   =							LlamaTokenizer
	a__      :					List[Any]		   =							"left"
	a__      :					str		   =							["input_ids", "attention_mask"]




	def __init__(	self	:						Optional[Any]						,  __UpperCAmelCase	:						Optional[int]=None						,  __UpperCAmelCase	:						Union[str, Any]=None						,  __UpperCAmelCase	:						Tuple=False						,  __UpperCAmelCase	:						Optional[int]="<unk>"						,  __UpperCAmelCase	:						str="<s>"						,  __UpperCAmelCase	:						List[Any]="</s>"						,  __UpperCAmelCase	:						int=True						,  __UpperCAmelCase	:						Optional[int]=False						,  **__UpperCAmelCase	:						Dict						,  )			-> Union[str, Any]:
							super().__init__(
							    vocab_file=_lowerCamelCase						,  tokenizer_file=_lowerCamelCase						,  clean_up_tokenization_spaces=_lowerCamelCase						,  unk_token=_lowerCamelCase						,  bos_token=_lowerCamelCase						,  eos_token=_lowerCamelCase						,  **_lowerCamelCase						,  )
							UpperCAmelCase_=   add_bos_token
							UpperCAmelCase_=   add_eos_token
							self.update_post_processor()

							UpperCAmelCase_=   vocab_file
							UpperCAmelCase_=   False if not self.vocab_file else True




	def   _SCREAMING_SNAKE_CASE		(	self	:						Union[str, Any]     )			-> List[str]:
							UpperCAmelCase_=   self.bos_token
							UpperCAmelCase_=   self.bos_token_id

							UpperCAmelCase_=   self.eos_token
							UpperCAmelCase_=   self.eos_token_id

							UpperCAmelCase_=   F"""{(bos+":0 ") * self.add_bos_token}$A:0{(" "+eos+":0") * self.add_eos_token}"""
							UpperCAmelCase_=   F"""{single}{(" "+bos+":1") * self.add_bos_token} $B:1{(" "+eos+":1") * self.add_eos_token}"""

							UpperCAmelCase_=   []
							if self.add_bos_token:
													special_tokens.append((bos, bos_token_id)     )
							if self.add_eos_token:
													special_tokens.append((eos, eos_token_id)     )
							UpperCAmelCase_=   processors.TemplateProcessing(
							    single=_lowerCamelCase						,  pair=_lowerCamelCase						,  special_tokens=_lowerCamelCase     )




	@property
	def   _SCREAMING_SNAKE_CASE		(	self	:						str     )			-> Union[str, Any]:
							return self._add_eos_token




	@property
	def   _SCREAMING_SNAKE_CASE		(	self	:						List[Any]     )			-> int:
							return self._add_bos_token




	@add_eos_token.setter
	def   _SCREAMING_SNAKE_CASE		(	self	:						str						,  __UpperCAmelCase	:						Union[str, Any]     )			-> Union[str, Any]:
							UpperCAmelCase_=   value
							self.update_post_processor()




	@add_bos_token.setter
	def   _SCREAMING_SNAKE_CASE		(	self	:						List[Any]						,  __UpperCAmelCase	:						Union[str, Any]     )			-> Any:
							UpperCAmelCase_=   value
							self.update_post_processor()




	def   _SCREAMING_SNAKE_CASE		(	self	:						int						,  __UpperCAmelCase	:						List[str]						,  __UpperCAmelCase	:						Any = None     )			-> Tuple[str]:





							if not self.can_save_slow_tokenizer:
													raise ValueError(
													    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
													    """tokenizer."""     )

							if not os.path.isdir(_lowerCamelCase     ):
													logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""     )
													return
							UpperCAmelCase_=   os.path.join(
							    _lowerCamelCase						,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]     )

							if os.path.abspath(self.vocab_file     ) != os.path.abspath(_lowerCamelCase     ):
													copyfile(self.vocab_file						,  _lowerCamelCase     )

							return (out_vocab_file,)




	def   _SCREAMING_SNAKE_CASE		(	self	:						Any						,  __UpperCAmelCase	:						Tuple     )			-> List[Any]:
							UpperCAmelCase_=   list(conversation.iter_texts()     )
							if not all([is_user for is_user, msg in dialogue[::2]]     ) or not all(
							    [not is_user for is_user, msg in dialogue[1::2]]     ):
													raise ValueError(
													    """The model only supports \'user\' and \'assistant\' roles, starting with user and alternating (u/a/u/a/u...)"""     )

							UpperCAmelCase_=   []
							if len(conversation.past_user_inputs     ) > 0:
													if not conversation.past_user_inputs[0].startswith(_lowerCamelCase     ) or E_SYS not in conversation.past_user_inputs[0]:
																			UpperCAmelCase_=   (
																			    B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + conversation.past_user_inputs[0]
																			)
							elif not dialogue[0][1].startswith(_lowerCamelCase     ) or E_SYS not in dialogue[0][1]:
													UpperCAmelCase_=   (dialogue[0][0], B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + dialogue[0][1])

							dialog_tokens += sum(
							    [
							        [self.bos_token_id]
							        + self.encode(
							            F"""{B_INST} {(prompt[1]).strip()} {E_INST} {(answer[1]).strip()} """						,  add_special_tokens=_lowerCamelCase     )
							        + [self.eos_token_id]
							        for prompt, answer in zip(dialogue[::2]						,  dialogue[1::2]     )
							    ]						,  []						,  )
							if not (dialogue[-1][0]):
													raise ValueError(F"""Last message must be from user, got {dialogue[-1]["role"]}"""     )
							dialog_tokens += [self.bos_token_id] + self.encode(
							    F"""{B_INST} {(dialogue[-1][1]).strip()} {E_INST}"""						,  add_special_tokens=_lowerCamelCase     )
							return dialog_tokens











'''simple docstring'''



import importlib
import inspect
import json
import os
from collections import OrderedDict

# Build the list of all feature extractors
from ...configuration_utils import PretrainedConfig
from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code
from ...feature_extraction_utils import FeatureExtractionMixin
from ...image_processing_utils import ImageProcessingMixin
from ...tokenization_utils import TOKENIZER_CONFIG_FILE
from ...utils import FEATURE_EXTRACTOR_NAME, get_file_from_repo, logging
from .auto_factory import _LazyAutoMapping
from .configuration_auto import (
    CONFIG_MAPPING_NAMES,
    AutoConfig,
    model_type_to_module_name,
    replace_list_option_in_docstrings,
)
from .feature_extraction_auto import AutoFeatureExtractor
from .image_processing_auto import AutoImageProcessor
from .tokenization_auto import AutoTokenizer


_SCREAMING_SNAKE_CASE    =			logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE    =			OrderedDict(
    [
        ('''align''', '''AlignProcessor'''),
        ('''altclip''', '''AltCLIPProcessor'''),
        ('''bark''', '''BarkProcessor'''),
        ('''blip''', '''BlipProcessor'''),
        ('''blip-2''', '''Blip2Processor'''),
        ('''bridgetower''', '''BridgeTowerProcessor'''),
        ('''chinese_clip''', '''ChineseCLIPProcessor'''),
        ('''clap''', '''ClapProcessor'''),
        ('''clip''', '''CLIPProcessor'''),
        ('''clipseg''', '''CLIPSegProcessor'''),
        ('''flava''', '''FlavaProcessor'''),
        ('''git''', '''GitProcessor'''),
        ('''groupvit''', '''CLIPProcessor'''),
        ('''hubert''', '''Wav2Vec2Processor'''),
        ('''instructblip''', '''InstructBlipProcessor'''),
        ('''layoutlmv2''', '''LayoutLMv2Processor'''),
        ('''layoutlmv3''', '''LayoutLMv3Processor'''),
        ('''markuplm''', '''MarkupLMProcessor'''),
        ('''mctct''', '''MCTCTProcessor'''),
        ('''mgp-str''', '''MgpstrProcessor'''),
        ('''oneformer''', '''OneFormerProcessor'''),
        ('''owlvit''', '''OwlViTProcessor'''),
        ('''pix2struct''', '''Pix2StructProcessor'''),
        ('''sam''', '''SamProcessor'''),
        ('''sew''', '''Wav2Vec2Processor'''),
        ('''sew-d''', '''Wav2Vec2Processor'''),
        ('''speech_to_text''', '''Speech2TextProcessor'''),
        ('''speech_to_text_2''', '''Speech2Text2Processor'''),
        ('''speecht5''', '''SpeechT5Processor'''),
        ('''trocr''', '''TrOCRProcessor'''),
        ('''tvlt''', '''TvltProcessor'''),
        ('''unispeech''', '''Wav2Vec2Processor'''),
        ('''unispeech-sat''', '''Wav2Vec2Processor'''),
        ('''vilt''', '''ViltProcessor'''),
        ('''vision-text-dual-encoder''', '''VisionTextDualEncoderProcessor'''),
        ('''wav2vec2''', '''Wav2Vec2Processor'''),
        ('''wav2vec2-conformer''', '''Wav2Vec2Processor'''),
        ('''wavlm''', '''Wav2Vec2Processor'''),
        ('''whisper''', '''WhisperProcessor'''),
        ('''xclip''', '''XCLIPProcessor'''),
    ]
)

_SCREAMING_SNAKE_CASE    =			_LazyAutoMapping(CONFIG_MAPPING_NAMES, PROCESSOR_MAPPING_NAMES)




def       _lowerCAmelCase  ( lowerCamelCase_	:     str ):
			for module_name, processors in PROCESSOR_MAPPING_NAMES.items():
						if class_name in processors:
									__lowercase   =							model_type_to_module_name(lowerCamelCase_ )

									__lowercase   =							importlib.import_module(f".{module_name}"						,				'''transformers.models''' )
									try:
												return getattr(lowerCamelCase_						,				lowerCamelCase_ )
									except AttributeError:
												continue

			for processor in PROCESSOR_MAPPING._extra_content.values():
						if getattr(lowerCamelCase_						,				'''__name__'''						,				lowerCamelCase_ ) == class_name:
									return processor

    # We did not fine the class, but maybe it's because a dep is missing. In that case, the class will be in the main
    # init and we return the proper dummy to get an appropriate error message.
			__lowercase   =							importlib.import_module('''transformers''' )
			if hasattr(lowerCamelCase_						,				lowerCamelCase_ ):
						return getattr(lowerCamelCase_						,				lowerCamelCase_ )

			return None






class 							__lowercase				:







	'''simple docstring'''

	def __init__(self       )							->							List[Any]:







				'''simple docstring'''





				raise EnvironmentError(
				    '''AutoProcessor is designed to be instantiated '''
				    '''using the `AutoProcessor.from_pretrained(pretrained_model_name_or_path)` method.'''       )


	@classmethod
	@replace_list_option_in_docstrings(_lowerCamelCase       )
	def   _UpperCAmelCase   (cls   ,_lowerCamelCase   ,**_lowerCamelCase       )							->							Optional[Any]:







				'''simple docstring'''





				__lowercase   =							kwargs.pop('''config'''   ,_lowerCamelCase       )
				__lowercase   =							kwargs.pop('''trust_remote_code'''   ,_lowerCamelCase       )
				__lowercase   =							True

				__lowercase   =							None
				__lowercase   =							None

				# First, let's see if we have a preprocessor config.
				# Filter the kwargs for `get_file_from_repo`.
				__lowercase   =							{
				    key: kwargs[key] for key in inspect.signature(_lowerCamelCase       ).parameters.keys() if key in kwargs
				}
				# Let's start by checking whether the processor class is saved in an image processor
				__lowercase   =							get_file_from_repo(
				    _lowerCamelCase   ,_lowerCamelCase   ,**_lowerCamelCase       )
				if preprocessor_config_file is not None:
							__lowercase  ,						__lowercase   =							ImageProcessingMixin.get_image_processor_dict(_lowerCamelCase   ,**_lowerCamelCase       )
							__lowercase   =							config_dict.get('''processor_class'''   ,_lowerCamelCase       )
							if "AutoProcessor" in config_dict.get('''auto_map'''   ,{}       ):
										__lowercase   =							config_dict['''auto_map''']['''AutoProcessor''']

        # If not found, let's check whether the processor class is saved in a feature extractor config
				if preprocessor_config_file is not None and processor_class is None:
							__lowercase  ,						__lowercase   =							FeatureExtractionMixin.get_feature_extractor_dict(_lowerCamelCase   ,**_lowerCamelCase       )
							__lowercase   =							config_dict.get('''processor_class'''   ,_lowerCamelCase       )
							if "AutoProcessor" in config_dict.get('''auto_map'''   ,{}       ):
										__lowercase   =							config_dict['''auto_map''']['''AutoProcessor''']

				if processor_class is None:
							# Next, let's check whether the processor class is saved in a tokenizer
							__lowercase   =							get_file_from_repo(
							    _lowerCamelCase   ,_lowerCamelCase   ,**_lowerCamelCase       )
							if tokenizer_config_file is not None:
										with open(_lowerCamelCase   ,encoding='''utf-8'''       ) as reader:
													__lowercase   =							json.load(_lowerCamelCase       )

										__lowercase   =							config_dict.get('''processor_class'''   ,_lowerCamelCase       )
										if "AutoProcessor" in config_dict.get('''auto_map'''   ,{}       ):
													__lowercase   =							config_dict['''auto_map''']['''AutoProcessor''']

				if processor_class is None:
							# Otherwise, load config, if it can be loaded.
							if not isinstance(_lowerCamelCase   ,_lowerCamelCase       ):
										__lowercase   =							AutoConfig.from_pretrained(
										    _lowerCamelCase   ,trust_remote_code=_lowerCamelCase   ,**_lowerCamelCase       )

							# And check if the config contains the processor class.
							__lowercase   =							getattr(_lowerCamelCase   ,'''processor_class'''   ,_lowerCamelCase       )
							if hasattr(_lowerCamelCase   ,'''auto_map'''       ) and "AutoProcessor" in config.auto_map:
										__lowercase   =							config.auto_map['''AutoProcessor''']

				if processor_class is not None:
							__lowercase   =							processor_class_from_name(_lowerCamelCase       )

				__lowercase   =							processor_auto_map is not None
				__lowercase   =							processor_class is not None or type(_lowerCamelCase       ) in PROCESSOR_MAPPING
				__lowercase   =							resolve_trust_remote_code(
				    _lowerCamelCase   ,_lowerCamelCase   ,_lowerCamelCase   ,_lowerCamelCase       )

				if has_remote_code and trust_remote_code:
							__lowercase   =							get_class_from_dynamic_module(
							    _lowerCamelCase   ,_lowerCamelCase   ,**_lowerCamelCase       )
							__lowercase   =							kwargs.pop('''code_revision'''   ,_lowerCamelCase       )
							if os.path.isdir(_lowerCamelCase       ):
										processor_class.register_for_auto_class()
							return processor_class.from_pretrained(
							    _lowerCamelCase   ,trust_remote_code=_lowerCamelCase   ,**_lowerCamelCase       )
				elif processor_class is not None:
							return processor_class.from_pretrained(
							    _lowerCamelCase   ,trust_remote_code=_lowerCamelCase   ,**_lowerCamelCase       )
				# Last try: we use the PROCESSOR_MAPPING.
				elif type(_lowerCamelCase       ) in PROCESSOR_MAPPING:
							return PROCESSOR_MAPPING[type(_lowerCamelCase       )].from_pretrained(_lowerCamelCase   ,**_lowerCamelCase       )

				# At this stage, there doesn't seem to be a `Processor` class available for this model, so let's try a
				# tokenizer.
				try:
							return AutoTokenizer.from_pretrained(
							    _lowerCamelCase   ,trust_remote_code=_lowerCamelCase   ,**_lowerCamelCase       )
				except Exception:
							try:
										return AutoImageProcessor.from_pretrained(
										    _lowerCamelCase   ,trust_remote_code=_lowerCamelCase   ,**_lowerCamelCase       )
							except Exception:
										pass

							try:
										return AutoFeatureExtractor.from_pretrained(
										    _lowerCamelCase   ,trust_remote_code=_lowerCamelCase   ,**_lowerCamelCase       )
							except Exception:
										pass

				raise ValueError(
				    f"Unrecognized processing class in {pretrained_model_name_or_path}. Can't instantiate a processor, a "
				    '''tokenizer, an image processor or a feature extractor for this model. Make sure the repository contains'''
				    '''the files of at least one of those processing classes.'''       )


	@staticmethod
	def   _UpperCAmelCase   (_lowerCamelCase   ,_lowerCamelCase       )							->							int:







				'''simple docstring'''





				PROCESSOR_MAPPING.register(_lowerCamelCase   ,_lowerCamelCase       )





