


import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
							from transformers.pipelines.conversational import Conversation


snake_case_						:			Tuple													=							logging.get_logger(__name__)

snake_case_						:			Dict													=							{'''vocab_file''': '''vocab.txt''', '''emoji_file''': '''emoji.json'''}

snake_case_						:			Tuple													=							{
    '''vocab_file''': {
        '''abeja/gpt-neox-japanese-2.7b''': '''https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt''',
    },
    '''emoji_file''': {
        '''abeja/gpt-neox-japanese-2.7b''': '''https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json''',
    },
}

snake_case_						:			int													=							{
    '''abeja/gpt-neox-japanese-2.7b''': 20_48,
}







def 						SCREAMING_SNAKE_CASE__(   a__	,a__):
				with open(lowerCamelCase__	,'''r'''	,encoding='''utf-8''') as f:
								_SCREAMING_SNAKE_CASE			  =json.loads(f.read())

				_SCREAMING_SNAKE_CASE			  =collections.OrderedDict()
				_SCREAMING_SNAKE_CASE			  =collections.OrderedDict()
				_SCREAMING_SNAKE_CASE			  =collections.OrderedDict()
				with open(lowerCamelCase__	,'''r'''	,encoding='''utf-8''') as f:
								_SCREAMING_SNAKE_CASE			  =f.readlines()
				_SCREAMING_SNAKE_CASE			  =[[t.rstrip('''\n''')] if (t == ''',''' or ''',''' not in t) else t.rstrip('''\n''').split(''',''') for t in token]
				for idx, b in enumerate(lowerCamelCase__):
								_SCREAMING_SNAKE_CASE			  =b
								_SCREAMING_SNAKE_CASE			  =idx
								for wd in b:
												_SCREAMING_SNAKE_CASE			  =idx

				return vocab, raw_vocab, ids_to_tokens, emoji

class 						A__							(		UpperCamelCase__  ):
						UpperCAmelCase             =						VOCAB_FILES_NAMES
						UpperCAmelCase             =						PRETRAINED_VOCAB_FILES_MAP
						UpperCAmelCase             =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						UpperCAmelCase             =						["""input_ids""", """attention_mask"""]




						def __init__(       self					:      Optional[Any]				,    _a					:      Tuple				,    _a					:      Optional[Any]				,    _a					:      Optional[Any]="<|endoftext|>"				,    _a					:      List[str]="<|endoftext|>"				,    _a					:      Any="<|startoftext|>"				,    _a					:      Any="<|endoftext|>"				,    _a					:      Tuple=False				,    **_a					:      List[str]				,    )							->     Optional[Any]:


										"""simple docstring"""





										super().__init__(
										    unk_token=_a				,    pad_token=_a				,    bos_token=_a				,    eos_token=_a				,    do_clean_text=_a				,    **_a				,    )
										if not os.path.isfile(_a  ):
														raise ValueError(
														    f"Can't find a vocabulary file at path '{vocab_file}'. To load the vocabulary from a Google pretrained"
														    ''' model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'''  )
										if not os.path.isfile(_a  ):
														raise ValueError(
														    f"Can't find a emoji file at path '{emoji_file}'. To load the emoji information from a Google"
														    ''' pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`'''  )
										_SCREAMING_SNAKE_CASE			  =do_clean_text
										_SCREAMING_SNAKE_CASE  ,				_SCREAMING_SNAKE_CASE  ,				_SCREAMING_SNAKE_CASE  ,				_SCREAMING_SNAKE_CASE			  =load_vocab_and_emoji(_a				,    _a  )
										_SCREAMING_SNAKE_CASE			  =SubWordJapaneseTokenizer(
										    vocab=self.vocab				,    ids_to_tokens=self.ids_to_tokens				,    emoji=self.emoji  )




						@property
						def 							__UpperCamelCase				(       self					:      Any  )							->     List[str]:


										"""simple docstring"""





										return len(self.raw_vocab  )




						def 							__UpperCamelCase				(       self					:      str  )							->     str:


										"""simple docstring"""





										return dict(self.raw_vocab				,    **self.added_tokens_encoder  )




						def 							__UpperCamelCase				(       self					:      List[str]				,    _a					:      Tuple  )							->     Optional[Any]:


										"""simple docstring"""





										return self.subword_tokenizer.tokenize(_a				,    clean=self.do_clean_text  )




						def 							__UpperCamelCase				(       self					:      int				,    _a					:      Optional[int]  )							->     Dict:


										"""simple docstring"""





										return self.vocab.get(_a				,    self.vocab.get(self.unk_token  )  )




						def 							__UpperCamelCase				(       self					:      int				,    _a					:      List[Any]  )							->     Tuple:


										"""simple docstring"""





										return self.subword_tokenizer.convert_id_to_token(_a  )




						def 							__UpperCamelCase				(       self					:      str				,    _a					:      int  )							->     Optional[Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =''''''.join(_a  ).strip()
										return out_string




						def 							__UpperCamelCase				(       self					:      Optional[int]				,    _a					:      "Conversation"  )							->     List[int]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =[]
										for is_user, text in conversation.iter_texts():
														input_ids.extend(self.encode(_a				,    add_special_tokens=_a  ) + [self.eos_token_id]  )

										if len(_a  ) > self.model_max_length:
														_SCREAMING_SNAKE_CASE			  =input_ids[-self.model_max_length :]
										return input_ids




						def 							__UpperCamelCase				(       self					:      int				,    _a					:      str				,    _a					:      Optional[str] = None  )							->     Tuple[str]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =0
										if os.path.isdir(_a  ):
														_SCREAMING_SNAKE_CASE			  =os.path.join(
														    _a				,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']  )
														_SCREAMING_SNAKE_CASE			  =os.path.join(
														    _a				,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''emoji_file''']  )
										else:
														_SCREAMING_SNAKE_CASE			  =(
														    (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory + VOCAB_FILES_NAMES['''vocab_file''']
														)
														_SCREAMING_SNAKE_CASE			  =(
														    (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory + VOCAB_FILES_NAMES['''emoji_file''']
														)
										with open(_a				,    '''w'''				,    encoding='''utf-8'''  ) as writer:
														for token_index, token in self.ids_to_tokens.items():
																		if index != token_index:
																						logger.warning(
																						    f"Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."
																						    ''' Please check that the vocabulary is not corrupted!'''  )
																						_SCREAMING_SNAKE_CASE			  =token_index
																		writer.write(''','''.join(_a  ) + '''\n'''  )
																		index += 1
										with open(_a				,    '''w'''				,    encoding='''utf-8'''  ) as writer:
														json.dump(self.emoji				,    _a  )
										return vocab_file, emoji_file

class 						A__							(		UpperCamelCase__  ):




						def __init__(       self					:      str				,    _a					:      Dict				,    _a					:      Optional[Any]				,    _a					:      List[Any]  )							->     int:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =vocab  # same as swe
										_SCREAMING_SNAKE_CASE			  =ids_to_tokens  # same as bpe
										_SCREAMING_SNAKE_CASE			  =emoji
										_SCREAMING_SNAKE_CASE			  =np.max([len(_a  ) for w in self.vocab.keys()]  )
										_SCREAMING_SNAKE_CASE			  =re.compile(R'''(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)'''  )
										_SCREAMING_SNAKE_CASE			  =re.compile(R'''[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*'''  )
										_SCREAMING_SNAKE_CASE			  =re.compile(R'''[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}'''  )
										_SCREAMING_SNAKE_CASE			  =re.compile(
										    R'''([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'''  )
										_SCREAMING_SNAKE_CASE			  =re.compile(
										    R'''(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*'''  )
										_SCREAMING_SNAKE_CASE			  =re.compile(
										    R'''((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*'''  )
										_SCREAMING_SNAKE_CASE			  ='''─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿'''
										_SCREAMING_SNAKE_CASE			  ='''▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟'''
										_SCREAMING_SNAKE_CASE			  =str.maketrans({k: '''<BLOCK>''' for k in keisen + blocks}  )




						def __len__(       self					:      List[str]  )							->     List[str]:


										"""simple docstring"""





										return len(self.ids_to_tokens  )




						def 							__UpperCamelCase				(       self					:      Tuple				,    _a					:      Any  )							->     Optional[int]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.content_repattera.sub('''<URL>'''				,    _a  )
										_SCREAMING_SNAKE_CASE			  =self.content_repattera.sub('''<EMAIL>'''				,    _a  )
										_SCREAMING_SNAKE_CASE			  =self.content_repattera.sub('''<TEL>'''				,    _a  )
										_SCREAMING_SNAKE_CASE			  =self.content_repattera.sub('''<DATE>'''				,    _a  )
										_SCREAMING_SNAKE_CASE			  =self.content_repattera.sub('''<DATE>'''				,    _a  )
										_SCREAMING_SNAKE_CASE			  =self.content_repattera.sub('''<PRICE>'''				,    _a  )
										_SCREAMING_SNAKE_CASE			  =content.translate(self.content_transa  )
										while "<BLOCK><BLOCK>" in content:
														_SCREAMING_SNAKE_CASE			  =content.replace('''<BLOCK><BLOCK>'''				,    '''<BLOCK>'''  )
										return content




						def 							__UpperCamelCase				(       self					:      Any				,    _a					:      int				,    _a					:      Any=False  )							->     Any:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =text.replace(''' '''				,    '''<SP>'''  )
										_SCREAMING_SNAKE_CASE			  =text.replace('''　'''				,    '''<SP>'''  )
										_SCREAMING_SNAKE_CASE			  =text.replace('''\r\n'''				,    '''<BR>'''  )
										_SCREAMING_SNAKE_CASE			  =text.replace('''\n'''				,    '''<BR>'''  )
										_SCREAMING_SNAKE_CASE			  =text.replace('''\r'''				,    '''<BR>'''  )
										_SCREAMING_SNAKE_CASE			  =text.replace('''\t'''				,    '''<TAB>'''  )
										_SCREAMING_SNAKE_CASE			  =text.replace('''—'''				,    '''ー'''  )
										_SCREAMING_SNAKE_CASE			  =text.replace('''−'''				,    '''ー'''  )
										for k, v in self.emoji["emoji"].items():
														if k in text:
																		_SCREAMING_SNAKE_CASE			  =text.replace(_a				,    _a  )
										if clean:
														_SCREAMING_SNAKE_CASE			  =self.clean_text(_a  )

										def check_simbol(_a					:      Union[str, Any]  ):
														_SCREAMING_SNAKE_CASE			  =x.encode()
														if len(_a  ) == 1 and len(_a  ) == 2:
																		_SCREAMING_SNAKE_CASE			  =(int(e[0]  ) << 8) + int(e[1]  )
																		if (
																		    (c >= 0xC2A1 and c <= 0xC2BF)
																		    or (c >= 0xC780 and c <= 0xC783)
																		    or (c >= 0xCAB9 and c <= 0xCBBF)
																		    or (c >= 0xCC80 and c <= 0xCDA2)
																		):
																						return True
														return False

										def checkuae(_a					:      List[Any]  ):
														_SCREAMING_SNAKE_CASE			  =x.encode()
														if len(_a  ) == 1 and len(_a  ) == 3:
																		_SCREAMING_SNAKE_CASE			  =(int(e[0]  ) << 16) + (int(e[1]  ) << 8) + int(e[2]  )
																		if c >= 0xE28080 and c <= 0xE2B07F:
																						return True
														return False

										_SCREAMING_SNAKE_CASE			  =0
										_SCREAMING_SNAKE_CASE			  =[]
										while pos < len(_a  ):
														_SCREAMING_SNAKE_CASE			  =min(len(_a  )				,    pos + self.maxlen + 1  ) if text[pos] == '''<''' else pos + 3
														_SCREAMING_SNAKE_CASE			  =[]  # (token_id, token, pos)
														for e in range(_a				,    _a				,    -1  ):
																		_SCREAMING_SNAKE_CASE			  =text[pos:e]
																		if wd in self.vocab:
																						if wd[0] == "<" and len(_a  ) > 2:
																										_SCREAMING_SNAKE_CASE			  =[(self.vocab[wd], wd, e)]
																										break
																						else:
																										candidates.append((self.vocab[wd], wd, e)  )
														if len(_a  ) > 0:
																		# the smallest token_id is adopted
																		_SCREAMING_SNAKE_CASE  ,				_SCREAMING_SNAKE_CASE  ,				_SCREAMING_SNAKE_CASE			  =sorted(_a				,    key=lambda _a  : x[0]  )[0]
																		result.append(_a  )
																		_SCREAMING_SNAKE_CASE			  =e
														else:
																		_SCREAMING_SNAKE_CASE			  =pos + 1
																		_SCREAMING_SNAKE_CASE			  =text[pos:end]
																		if check_simbol(_a  ):
																						result.append('''<KIGOU>'''  )
																		elif checkuae(_a  ):
																						result.append('''<U2000U2BFF>'''  )
																		else:
																						for i in wd.encode('''utf-8'''  ):
																										result.append('''<|byte%d|>''' % i  )
																		_SCREAMING_SNAKE_CASE			  =end
										return result




						def 							__UpperCamelCase				(       self					:      List[str]				,    _a					:      Optional[Any]				,    _a					:      Optional[Any]="\n"  )							->     List[Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =[]
										_SCREAMING_SNAKE_CASE			  =[]
										_SCREAMING_SNAKE_CASE			  =self.ids_to_tokens[index][0]
										if word[:6] == "<|byte" and word[-2:] == "|>":
														byte_tokens.append(int(word[6:-2]  )  )
										else:
														if len(_a  ) > 0:
																		words.append(bytearray(_a  ).decode('''utf-8'''				,    errors='''replace'''  )  )
																		_SCREAMING_SNAKE_CASE			  =[]
														if word[:7] == "<|emoji" and word[-2:] == "|>":
																		words.append(self.emoji['''emoji_inv'''][word]  )
														elif word == "<SP>":
																		words.append(''' '''  )
														elif word == "<BR>":
																		words.append(_a  )
														elif word == "<TAB>":
																		words.append('''\t'''  )
														elif word == "<BLOCK>":
																		words.append('''▀'''  )
														elif word == "<KIGOU>":
																		words.append('''ǀ'''  )
														elif word == "<U2000U2BFF>":
																		words.append('''‖'''  )
														else:
																		words.append(_a  )
										if len(_a  ) > 0:
														words.append(bytearray(_a  ).decode('''utf-8'''				,    errors='''replace'''  )  )
										_SCREAMING_SNAKE_CASE			  =''''''.join(_a  )
										return text



# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import re

from ..models.auto import AutoProcessor
from ..models.vision_encoder_decoder import VisionEncoderDecoderModel
from ..utils import is_vision_available
from .base import PipelineTool


if is_vision_available():
							from PIL import Image

class 						A__							(		UpperCamelCase__  ):
						UpperCAmelCase             =						"naver-clova-ix/donut-base-finetuned-docvqa"
						UpperCAmelCase             =						(
						    "This is a tool that answers a question about an document (pdf). It takes an input named `document` which "
						    "should be the document containing the information, as well as a `question` that is the question about the "
						    "document. It returns a text that contains the answer to the question."
						)
						UpperCAmelCase             =						"document_qa"
						UpperCAmelCase             =						AutoProcessor
						UpperCAmelCase             =						VisionEncoderDecoderModel

						UpperCAmelCase             =						["image", "text"]
						UpperCAmelCase             =						["text"]




						def __init__(       self					:      Any				,    *_a					:      int				,    **_a					:      Dict  )							->     int:


										"""simple docstring"""





										if not is_vision_available():
														raise ValueError('''Pillow must be installed to use the DocumentQuestionAnsweringTool.'''  )

										super().__init__(*_a				,    **_a  )




						def 							__UpperCamelCase				(       self					:      Optional[Any]				,    _a					:      "Image"				,    _a					:      str  )							->     Tuple:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  ='''<s_docvqa><s_question>{user_input}</s_question><s_answer>'''
										_SCREAMING_SNAKE_CASE			  =task_prompt.replace('''{user_input}'''				,    _a  )
										_SCREAMING_SNAKE_CASE			  =self.pre_processor.tokenizer(
										    _a				,    add_special_tokens=_a				,    return_tensors='''pt'''  ).input_ids
										_SCREAMING_SNAKE_CASE			  =self.pre_processor(_a				,    return_tensors='''pt'''  ).pixel_values

										return {"decoder_input_ids": decoder_input_ids, "pixel_values": pixel_values}




						def 							__UpperCamelCase				(       self					:      List[Any]				,    _a					:      Optional[Any]  )							->     int:


										"""simple docstring"""





										return self.model.generate(
										    inputs['''pixel_values'''].to(self.device  )				,    decoder_input_ids=inputs['''decoder_input_ids'''].to(self.device  )				,    max_length=self.model.decoder.config.max_position_embeddings				,    early_stopping=_a				,    pad_token_id=self.pre_processor.tokenizer.pad_token_id				,    eos_token_id=self.pre_processor.tokenizer.eos_token_id				,    use_cache=_a				,    num_beams=1				,    bad_words_ids=[[self.pre_processor.tokenizer.unk_token_id]]				,    return_dict_in_generate=_a				,    ).sequences




						def 							__UpperCamelCase				(       self					:      Any				,    _a					:      int  )							->     Any:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.pre_processor.batch_decode(_a  )[0]
										_SCREAMING_SNAKE_CASE			  =sequence.replace(self.pre_processor.tokenizer.eos_token				,    ''''''  )
										_SCREAMING_SNAKE_CASE			  =sequence.replace(self.pre_processor.tokenizer.pad_token				,    ''''''  )
										_SCREAMING_SNAKE_CASE			  =re.sub(R'''<.*?>'''				,    ''''''				,    _a				,    count=1  ).strip()  # remove first task start token
										_SCREAMING_SNAKE_CASE			  =self.pre_processor.tokenajson(_a  )

										return sequence["answer"]