



'''simple docstring'''






import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase:   List[str]					=		logging.get_logger(__name__)

lowerCAmelCase:   List[str]					=		'▁'

lowerCAmelCase:   int					=		{'vocab_file': 'vocab.txt', 'sentencepiece_model_ckpt': 'sentencepiece.bpe.model'}

lowerCAmelCase:   Optional[Any]					=		{
    'sentencepiece_model_file': 'sentencepiece.bpe.model',
    'vocab_file': 'vocab.txt',
}

lowerCAmelCase:   List[str]					=		{
    'vocab_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
    },
    'sentencepiece_model_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
    },
}

lowerCAmelCase:   Union[str, Any]					=		{
    'ernie-m-base': 5_1_4,
    'ernie-m-large': 5_1_4,
}

lowerCAmelCase:   Optional[int]					=		{
    'ernie-m-base': {'do_lower_case': False},
    'ernie-m-large': {'do_lower_case': False},
}

class 				a__(   lowerCamelCase__      ):
    lowercase__					=       ["input_ids"]

    lowercase__					=       VOCAB_FILES_NAMES
    lowercase__					=       PRETRAINED_INIT_CONFIGURATION
    lowercase__					=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowercase__					=       PRETRAINED_VOCAB_FILES_MAP
    lowercase__					=       RESOURCE_FILES_NAMES







    def __init__( self					:    int		,     __snake_case					:    List[str]		,     __snake_case					:    List[Any]=None		,     __snake_case					:    List[str]=False		,     __snake_case					:    Dict="utf8"		,     __snake_case					:    Any="[UNK]"		,     __snake_case					:    List[Any]="[SEP]"		,     __snake_case					:    List[str]="[PAD]"		,     __snake_case					:    Optional[Any]="[CLS]"		,     __snake_case					:    Tuple="[MASK]"		,     __snake_case					:    Optional[Dict[str, Any]] = None		,     **__snake_case					:    List[Any]		,     ):
      # Mask token behave like a normal word, i.e. include the space before it and
      # is included in the raw text, there should be a match in a non-normalized sentence.

      a		:   Dict	  =	{} if sp_model_kwargs is None else sp_model_kwargs
      super().__init__(
          do_lower_case=__snake_case		,     unk_token=__snake_case		,     sep_token=__snake_case		,     pad_token=__snake_case		,     cls_token=__snake_case		,     mask_token=__snake_case		,     vocab_file=__snake_case		,     encoding=__snake_case		,     sp_model_kwargs=self.sp_model_kwargs		,     **__snake_case		,     )
      a		:   Optional[int]	  =	do_lower_case
      a		:   Union[str, Any]	  =	sentencepiece_model_ckpt
      a		:   List[Any]	  =	spm.SentencePieceProcessor(**self.sp_model_kwargs      )
      self.sp_model.Load(__snake_case      )

      # to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
      if vocab_file is not None:
        a		:   Dict	  =	self.load_vocab(filepath=__snake_case      )
      else:
        a		:   Union[str, Any]	  =	{self.sp_model.id_to_piece(__snake_case      ): id for id in range(self.sp_model.get_piece_size()      )}
      a		:   Any	  =	{v: k for k, v in self.vocab.items()}







    def      lowercase_			( self					:    Dict		,     __snake_case					:    Union[str, Any]      ):
      if text is None:
        return None

      a		:   List[Any]	  =	self.tokenize(__snake_case      )
      a  ,      a		:   Any	  =	'', []

      for i, ch in enumerate(__snake_case      ):
        if ch in self.SP_CHAR_MAPPING:
          a		:   int	  =	self.SP_CHAR_MAPPING.get(__snake_case      )
        else:
          a		:   Tuple	  =	unicodedata.normalize('NFKC'		,     __snake_case      )
        if self.is_whitespace(__snake_case      ):
          continue
        normalized_text += ch
        char_mapping.extend([i] * len(__snake_case      )      )

      a  ,      a  ,      a		:   List[str]	  =	normalized_text, [], 0

      if self.do_lower_case:
        a		:   Optional[int]	  =	text.lower()

      for token in split_tokens:
        if token[:1] == "▁":
          a		:   Optional[int]	  =	token[1:]
        a		:   List[Any]	  =	text[offset:].index(__snake_case      ) + offset
        a		:   Union[str, Any]	  =	start + len(__snake_case      )

        token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)      )
        a		:   List[Any]	  =	end
      return token_mapping







    @property
    def      lowercase_			( self					:    Tuple      ):
      return len(self.vocab      )







    def      lowercase_			( self					:    Optional[Any]      ):
      return dict(self.vocab		,     **self.added_tokens_encoder      )







    def __getstate__( self					:    Union[str, Any]      ):
      a		:   str	  =	self.__dict__.copy()
      a		:   List[Any]	  =	None
      return state







    def __setstate__( self					:    List[str]		,     __snake_case					:    str      ):
      a		:   Optional[int]	  =	d

      # for backward compatibility
      if not hasattr(self		,     'sp_model_kwargs'      ):
        a		:   List[Any]	  =	{}

      a		:   str	  =	spm.SentencePieceProcessor(**self.sp_model_kwargs      )
      self.sp_model.Load(self.sentencepiece_model_ckpt      )







    def      lowercase_			( self					:    str		,     __snake_case					:    Optional[Any]      ):
      return "".join((self.SP_CHAR_MAPPING.get(__snake_case		,     __snake_case      ) for c in text)      )







    def      lowercase_			( self					:    int		,     __snake_case					:    Any		,     __snake_case					:    List[str]=False		,     __snake_case					:    Optional[Any]=64		,     __snake_case					:    Union[str, Any]=0.1      ):

      if self.sp_model_kwargs.get('enable_sampling'      ) is True:
        a		:   Union[str, Any]	  =	True
      if self.sp_model_kwargs.get('alpha'      ) is not None:
        a		:   List[Any]	  =	self.sp_model_kwargs.get('alpha'      )
      if self.sp_model_kwargs.get('nbest_size'      ) is not None:
        a		:   Union[str, Any]	  =	self.sp_model_kwargs.get('nbest_size'      )

      if not enable_sampling:
        a		:   Optional[Any]	  =	self.sp_model.EncodeAsPieces(__snake_case      )
      else:
        a		:   List[Any]	  =	self.sp_model.SampleEncodeAsPieces(__snake_case		,     __snake_case		,     __snake_case      )
      a		:   Any	  =	[]
      for pi, piece in enumerate(__snake_case      ):
        if piece == SPIECE_UNDERLINE:
          if not pieces[pi + 1].startswith(__snake_case      ) and pi != 0:
            new_pieces.append(__snake_case      )
            continue
          else:
            continue
        a		:   List[str]	  =	0
        for i, chunk in enumerate(__snake_case      ):
          if chunk == SPIECE_UNDERLINE:
            continue
          if self.is_ch_char(__snake_case      ) or self.is_punct(__snake_case      ):
            if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
              new_pieces.append(piece[lst_i:i]      )
            new_pieces.append(__snake_case      )
            a		:   Optional[Any]	  =	i + 1
          elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
            if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
              new_pieces.append(piece[lst_i:i]      )
            a		:   Union[str, Any]	  =	i
          elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
            if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
              new_pieces.append(piece[lst_i:i]      )
            a		:   str	  =	i
        if len(__snake_case      ) > lst_i:
          new_pieces.append(piece[lst_i:]      )
      return new_pieces







    def      lowercase_			( self					:    Union[str, Any]		,     __snake_case					:    Any      ):
      a		:   List[Any]	  =	''.join(__snake_case      ).replace(__snake_case		,     ' '      ).strip()
      return out_string







    def      lowercase_			( self					:    List[str]		,     __snake_case					:    Dict      ):
      a		:   Union[str, Any]	  =	self.convert_ids_to_tokens(__snake_case      )
      a		:   List[str]	  =	''.join(__snake_case      ).replace(__snake_case		,     ' '      ).strip()
      return out_string







    def      lowercase_			( self					:    str		,     __snake_case					:    List[str]      ):
      return self.vocab.get(__snake_case		,     self.vocab.get(self.unk_token      )      )







    def      lowercase_			( self					:    Optional[Any]		,     __snake_case					:    str      ):
      return self.reverse_vocab.get(__snake_case		,     self.unk_token      )







    def      lowercase_			( self					:    str		,     __snake_case					:    Any		,     __snake_case					:    List[str]=None      ):
      if token_ids_a is None:
        return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
      a		:   Optional[int]	  =	[self.cls_token_id]
      a		:   Optional[int]	  =	[self.sep_token_id]
      return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep







    def      lowercase_			( self					:    Dict		,     __snake_case					:    Tuple		,     __snake_case					:    Optional[int]=None      ):
      if offset_mapping_a is None:
        return [(0, 0)] + offset_mapping_a + [(0, 0)]

      return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]







    def      lowercase_			( self					:    Dict		,     __snake_case					:    Optional[int]		,     __snake_case					:    Union[str, Any]=None		,     __snake_case					:    str=False      ):

      if already_has_special_tokens:
        if token_ids_a is not None:
          raise ValueError(
              'You should not supply a second sequence if the provided sequence of '
              'ids is already formatted with special tokens for the model.'      )
        return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

      if token_ids_a is not None:
        return [1] + ([0] * len(__snake_case      )) + [1, 1] + ([0] * len(__snake_case      )) + [1]
      return [1] + ([0] * len(__snake_case      )) + [1]







    def      lowercase_			( self					:    str		,     __snake_case					:    List[int]		,     __snake_case					:    Optional[List[int]] = None      ):
      # called when `add_special_tokens` is True, so align with `build_inputs_with_special_tokens` method
      if token_ids_a is None:
        # [CLS] X [SEP]
        return (len(__snake_case      ) + 2) * [0]

      # [CLS] A [SEP] [SEP] B [SEP]
      return [0] * (len(__snake_case      ) + 1) + [1] * (len(__snake_case      ) + 3)







    def      lowercase_			( self					:    List[str]		,     __snake_case					:    List[Any]      ):
      if "\u4e00" <= char <= "\u9fff":
        return True
      return False







    def      lowercase_			( self					:    Any		,     __snake_case					:    Optional[Any]      ):
      if ("a" <= char <= "z") or ("A" <= char <= "Z"):
        return True
      return False







    def      lowercase_			( self					:    Any		,     __snake_case					:    Optional[int]      ):
      if char in ",;:.?!~，；：。？！《》【】":
        return True
      return False







    def      lowercase_			( self					:    Optional[Any]		,     __snake_case					:    List[Any]      ):
      if char == " " or char == "\t" or char == "\n" or char == "\r":
        return True
      if len(__snake_case      ) == 1:
        a		:   Optional[Any]	  =	unicodedata.category(__snake_case      )
        if cat == "Zs":
          return True
      return False







    def      lowercase_			( self					:    Any		,     __snake_case					:    List[str]      ):
      a		:   str	  =	{}
      with io.open(__snake_case		,     'r'		,     encoding='utf-8'      ) as f:
        for index, line in enumerate(__snake_case      ):
          a		:   Any	  =	line.rstrip('\n'      )
          a		:   str	  =	int(__snake_case      )

      return token_to_idx







    def      lowercase_			( self					:    str		,     __snake_case					:    str		,     __snake_case					:    Optional[str] = None      ):
      a		:   Optional[int]	  =	0
      if os.path.isdir(__snake_case      ):
        a		:   Tuple	  =	os.path.join(
            __snake_case		,     (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']      )
      else:
        a		:   Optional[int]	  =	(filename_prefix + '-' if filename_prefix else '') + save_directory
      with open(__snake_case		,     'w'		,     encoding='utf-8'      ) as writer:
        for token, token_index in sorted(self.vocab.items()		,     key=lambda __snake_case      : kv[1]      ):
          if index != token_index:
            logger.warning(
                F"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
                ' Please check that the vocabulary is not corrupted!'      )
            a		:   Tuple	  =	token_index
          writer.write(token + '\n'      )
          index += 1

      a		:   int	  =	os.path.join(__snake_case		,     'sentencepiece.bpe.model'      )
      with open(__snake_case		,     'wb'      ) as fi:
        a		:   Optional[int]	  =	self.sp_model.serialized_model_proto()
        fi.write(__snake_case      )

      return (vocab_file,)




'''simple docstring'''






# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import re

from ..models.auto import AutoProcessor
from ..models.vision_encoder_decoder import VisionEncoderDecoderModel
from ..utils import is_vision_available
from .base import PipelineTool


if is_vision_available():
     from PIL import Image

class 				a__(   lowerCamelCase__      ):
    lowercase__					=       """naver-clova-ix/donut-base-finetuned-docvqa"""
    lowercase__					=       (
        """This is a tool that answers a question about an document (pdf). It takes an input named `document` which """
        """should be the document containing the information, as well as a `question` that is the question about the """
        """document. It returns a text that contains the answer to the question."""
    )
    lowercase__					=       """document_qa"""
    lowercase__					=       AutoProcessor
    lowercase__					=       VisionEncoderDecoderModel

    lowercase__					=       ["""image""", """text"""]
    lowercase__					=       ["""text"""]







    def __init__( self					:    str		,     *__snake_case					:    Any		,     **__snake_case					:    List[Any]      ):
      if not is_vision_available():
        raise ValueError('Pillow must be installed to use the DocumentQuestionAnsweringTool.'      )

      super().__init__(*__snake_case		,     **__snake_case      )







    def      lowercase_			( self					:    Union[str, Any]		,     __snake_case					:    "Image"		,     __snake_case					:    str      ):
      a		:   Union[str, Any]	  =	'<s_docvqa><s_question>{user_input}</s_question><s_answer>'
      a		:   Any	  =	task_prompt.replace('{user_input}'		,     __snake_case      )
      a		:   Dict	  =	self.pre_processor.tokenizer(
          __snake_case		,     add_special_tokens=__snake_case		,     return_tensors='pt'      ).input_ids
      a		:   Tuple	  =	self.pre_processor(__snake_case		,     return_tensors='pt'      ).pixel_values

      return {"decoder_input_ids": decoder_input_ids, "pixel_values": pixel_values}







    def      lowercase_			( self					:    Any		,     __snake_case					:    int      ):
      return self.model.generate(
          inputs['pixel_values'].to(self.device      )		,     decoder_input_ids=inputs['decoder_input_ids'].to(self.device      )		,     max_length=self.model.decoder.config.max_position_embeddings		,     early_stopping=__snake_case		,     pad_token_id=self.pre_processor.tokenizer.pad_token_id		,     eos_token_id=self.pre_processor.tokenizer.eos_token_id		,     use_cache=__snake_case		,     num_beams=1		,     bad_words_ids=[[self.pre_processor.tokenizer.unk_token_id]]		,     return_dict_in_generate=__snake_case		,     ).sequences







    def      lowercase_			( self					:    List[str]		,     __snake_case					:    Tuple      ):
      a		:   str	  =	self.pre_processor.batch_decode(__snake_case      )[0]
      a		:   str	  =	sequence.replace(self.pre_processor.tokenizer.eos_token		,     ''      )
      a		:   Union[str, Any]	  =	sequence.replace(self.pre_processor.tokenizer.pad_token		,     ''      )
      a		:   List[str]	  =	re.sub(r'<.*?>'		,     ''		,     __snake_case		,     count=1      ).strip()  # remove first task start token
      a		:   Dict	  =	self.pre_processor.tokenajson(__snake_case      )

      return sequence["answer"]