


"""simple docstring"""






from typing import Dict, Iterable, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import normalize, rescale, resize, to_channel_dimension_format, to_pil_image
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_pytesseract_available, is_vision_available, logging, requires_backends


if is_vision_available():
   import PIL

# soft dependency
if is_pytesseract_available():
   import pytesseract

__snake_case		=       logging.get_logger(__name__)
def       A_   (   _lowerCAmelCase     :					Optional[int],     _lowerCAmelCase     :					Tuple,     _lowerCAmelCase     :					Tuple       ):






     """simple docstring"""

     return [
         int(10_00 * (box[0] / width)       ),
         int(10_00 * (box[1] / height)       ),
         int(10_00 * (box[2] / width)       ),
         int(10_00 * (box[3] / height)       ),
     ]
def       A_   (   _lowerCAmelCase     :					np.ndarray,     _lowerCAmelCase     :					Optional[str],     _lowerCAmelCase     :					Optional[str]       ):






     """simple docstring"""

     _a    	=				to_pil_image(_lowerCAmelCase       )
     _a	,      _a    	=				pil_image.size
     _a    	=				pytesseract.image_to_data(_lowerCAmelCase,     lang=_lowerCAmelCase,     output_type='''dict''',     config=_lowerCAmelCase       )
     _a	,      _a	,      _a	,      _a	,      _a    	=				data['''text'''], data['''left'''], data['''top'''], data['''width'''], data['''height''']

     # filter empty words and corresponding coordinates
     _a    	=				[idx for idx, word in enumerate(_lowerCAmelCase       ) if not word.strip()]
     _a    	=				[word for idx, word in enumerate(_lowerCAmelCase       ) if idx not in irrelevant_indices]
     _a    	=				[coord for idx, coord in enumerate(_lowerCAmelCase       ) if idx not in irrelevant_indices]
     _a    	=				[coord for idx, coord in enumerate(_lowerCAmelCase       ) if idx not in irrelevant_indices]
     _a    	=				[coord for idx, coord in enumerate(_lowerCAmelCase       ) if idx not in irrelevant_indices]
     _a    	=				[coord for idx, coord in enumerate(_lowerCAmelCase       ) if idx not in irrelevant_indices]

     # turn coordinates into (left, top, left+width, top+height) format
     _a    	=				[]
     for x, y, w, h in zip(_lowerCAmelCase,     _lowerCAmelCase,     _lowerCAmelCase,     _lowerCAmelCase       ):
          _a    	=				[x, y, x + w, y + h]
          actual_boxes.append(_lowerCAmelCase       )

     # finally, normalize the bounding boxes
     _a    	=				[]
     for box in actual_boxes:
          normalized_boxes.append(normalize_box(_lowerCAmelCase,     _lowerCAmelCase,     _lowerCAmelCase       )       )

     assert len(_lowerCAmelCase       ) == len(_lowerCAmelCase       ), "Not as many words as there are bounding boxes"

     return words, normalized_boxes


class     __lowerCamelCase  (					a__				):

     '''simple docstring'''
     A_       :	List[str]    =		['pixel_values']


     def __init__(      self    ,				__UpperCAmelCase = True    ,				__UpperCAmelCase = None    ,				__UpperCAmelCase = PILImageResampling.BILINEAR    ,				__UpperCAmelCase = True    ,				__UpperCAmelCase = 1 / 255    ,				__UpperCAmelCase = True    ,				__UpperCAmelCase = None    ,				__UpperCAmelCase = None    ,				__UpperCAmelCase = True    ,				__UpperCAmelCase = None    ,				__UpperCAmelCase = ""    ,				**__UpperCAmelCase    ,				)       ->							None:
          super().__init__(**__UpperCAmelCase							)
          _a    	=				size if size is not None else {'''height''': 224, '''width''': 224}
          _a    	=				get_size_dict(__UpperCAmelCase							)

          _a    	=				do_resize
          _a    	=				size
          _a    	=				resample
          _a    	=				do_rescale
          _a    	=				rescale_value
          _a    	=				do_normalize
          _a    	=				image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
          _a    	=				image_std if image_std is not None else IMAGENET_STANDARD_STD
          _a    	=				apply_ocr
          _a    	=				ocr_lang
          _a    	=				tesseract_config


     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase    ,				__UpperCAmelCase    ,				__UpperCAmelCase = PILImageResampling.BILINEAR    ,				__UpperCAmelCase = None    ,				**__UpperCAmelCase    ,				)       ->							np.ndarray:
          _a    	=				get_size_dict(__UpperCAmelCase							)
          if "height" not in size or "width" not in size:
               raise ValueError(F'The size dictionary must contain the keys \'height\' and \'width\'. Got {size.keys()}'							)
          _a    	=				(size['''height'''], size['''width'''])
          return resize(__UpperCAmelCase    ,				size=__UpperCAmelCase    ,				resample=__UpperCAmelCase    ,				data_format=__UpperCAmelCase    ,				**__UpperCAmelCase							)


     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase    ,				__UpperCAmelCase    ,				__UpperCAmelCase = None    ,				**__UpperCAmelCase    ,				)       ->							np.ndarray:
          return rescale(__UpperCAmelCase    ,				scale=__UpperCAmelCase    ,				data_format=__UpperCAmelCase    ,				**__UpperCAmelCase							)


     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase    ,				__UpperCAmelCase    ,				__UpperCAmelCase    ,				__UpperCAmelCase = None    ,				**__UpperCAmelCase    ,				)       ->							np.ndarray:
          return normalize(__UpperCAmelCase    ,				mean=__UpperCAmelCase    ,				std=__UpperCAmelCase    ,				data_format=__UpperCAmelCase    ,				**__UpperCAmelCase							)





     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase    ,				__UpperCAmelCase = None    ,				__UpperCAmelCase = None    ,				__UpperCAmelCase=None    ,				__UpperCAmelCase = None    ,				__UpperCAmelCase = None    ,				__UpperCAmelCase = None    ,				__UpperCAmelCase = None    ,				__UpperCAmelCase = None    ,				__UpperCAmelCase = None    ,				__UpperCAmelCase = None    ,				__UpperCAmelCase = None    ,				__UpperCAmelCase = None    ,				__UpperCAmelCase = ChannelDimension.FIRST    ,				**__UpperCAmelCase    ,				)       ->							PIL.Image.Image:
          _a    	=				do_resize if do_resize is not None else self.do_resize
          _a    	=				size if size is not None else self.size
          _a    	=				get_size_dict(__UpperCAmelCase							)
          _a    	=				resample if resample is not None else self.resample
          _a    	=				do_rescale if do_rescale is not None else self.do_rescale
          _a    	=				rescale_factor if rescale_factor is not None else self.rescale_factor
          _a    	=				do_normalize if do_normalize is not None else self.do_normalize
          _a    	=				image_mean if image_mean is not None else self.image_mean
          _a    	=				image_std if image_std is not None else self.image_std
          _a    	=				apply_ocr if apply_ocr is not None else self.apply_ocr
          _a    	=				ocr_lang if ocr_lang is not None else self.ocr_lang
          _a    	=				tesseract_config if tesseract_config is not None else self.tesseract_config

          _a    	=				make_list_of_images(__UpperCAmelCase							)

          if not valid_images(__UpperCAmelCase							):
               raise ValueError(
                   '''Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '''
                   '''torch.Tensor, tf.Tensor or jax.ndarray.'''							)

          if do_resize and size is None:
               raise ValueError('''Size must be specified if do_resize is True.'''							)

          if do_rescale and rescale_factor is None:
               raise ValueError('''Rescale factor must be specified if do_rescale is True.'''							)

          if do_normalize and (image_mean is None or image_std is None):
               raise ValueError('''If do_normalize is True, image_mean and image_std must be specified.'''							)

          # All transformations expect numpy arrays.
          _a    	=				[to_numpy_array(__UpperCAmelCase							) for image in images]

          # Tesseract OCR to get words + normalized bounding boxes
          if apply_ocr:
               requires_backends(self    ,				'''pytesseract'''							)
               _a    	=				[]
               _a    	=				[]
               for image in images:
                    _a	,      _a    	=				apply_tesseract(__UpperCAmelCase    ,				__UpperCAmelCase    ,				__UpperCAmelCase							)
                    words_batch.append(__UpperCAmelCase							)
                    boxes_batch.append(__UpperCAmelCase							)

          if do_resize:
               _a    	=				[self.resize(image=__UpperCAmelCase    ,				size=__UpperCAmelCase    ,				resample=__UpperCAmelCase							) for image in images]

          if do_rescale:
               _a    	=				[self.rescale(image=__UpperCAmelCase    ,				scale=__UpperCAmelCase							) for image in images]

          if do_normalize:
               _a    	=				[self.normalize(image=__UpperCAmelCase    ,				mean=__UpperCAmelCase    ,				std=__UpperCAmelCase							) for image in images]

          _a    	=				[to_channel_dimension_format(__UpperCAmelCase    ,				__UpperCAmelCase							) for image in images]

          _a    	=				BatchFeature(data={'''pixel_values''': images}    ,				tensor_type=__UpperCAmelCase							)

          if apply_ocr:
               _a    	=				words_batch
               _a    	=				boxes_batch
          return data



"""simple docstring"""






import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__snake_case		=       logging.get_logger(__name__)
__snake_case		=       {'''vocab_file''': '''spiece.model'''}

__snake_case		=       {
    '''vocab_file''': {
        '''albert-base-v1''': '''https://huggingface.co/albert-base-v1/resolve/main/spiece.model''',
        '''albert-large-v1''': '''https://huggingface.co/albert-large-v1/resolve/main/spiece.model''',
        '''albert-xlarge-v1''': '''https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model''',
        '''albert-xxlarge-v1''': '''https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model''',
        '''albert-base-v2''': '''https://huggingface.co/albert-base-v2/resolve/main/spiece.model''',
        '''albert-large-v2''': '''https://huggingface.co/albert-large-v2/resolve/main/spiece.model''',
        '''albert-xlarge-v2''': '''https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model''',
        '''albert-xxlarge-v2''': '''https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model''',
    }
}

__snake_case		=       {
    '''albert-base-v1''': 512,
    '''albert-large-v1''': 512,
    '''albert-xlarge-v1''': 512,
    '''albert-xxlarge-v1''': 512,
    '''albert-base-v2''': 512,
    '''albert-large-v2''': 512,
    '''albert-xlarge-v2''': 512,
    '''albert-xxlarge-v2''': 512,
}

__snake_case		=       '''▁'''


class     __lowerCamelCase  (					a__				):

     '''simple docstring'''
     A_       :	Optional[int]    =		VOCAB_FILES_NAMES
     A_       :	int    =		PRETRAINED_VOCAB_FILES_MAP
     A_       :	Optional[int]    =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES


     def __init__(      self    ,				__UpperCAmelCase    ,				__UpperCAmelCase=True    ,				__UpperCAmelCase=True    ,				__UpperCAmelCase=False    ,				__UpperCAmelCase="[CLS]"    ,				__UpperCAmelCase="[SEP]"    ,				__UpperCAmelCase="<unk>"    ,				__UpperCAmelCase="[SEP]"    ,				__UpperCAmelCase="<pad>"    ,				__UpperCAmelCase="[CLS]"    ,				__UpperCAmelCase="[MASK]"    ,				__UpperCAmelCase = None    ,				**__UpperCAmelCase    ,				)       ->							None:
          # Mask token behave like a normal word, i.e. include the space before it and
          # is included in the raw text, there should be a match in a non-normalized sentence.
          _a    	=				(
              AddedToken(__UpperCAmelCase    ,				lstrip=__UpperCAmelCase    ,				rstrip=__UpperCAmelCase    ,				normalized=__UpperCAmelCase							)
              if isinstance(__UpperCAmelCase    ,				__UpperCAmelCase							)
              else mask_token
          )

          _a    	=				{} if sp_model_kwargs is None else sp_model_kwargs

          super().__init__(
              do_lower_case=__UpperCAmelCase    ,				remove_space=__UpperCAmelCase    ,				keep_accents=__UpperCAmelCase    ,				bos_token=__UpperCAmelCase    ,				eos_token=__UpperCAmelCase    ,				unk_token=__UpperCAmelCase    ,				sep_token=__UpperCAmelCase    ,				pad_token=__UpperCAmelCase    ,				cls_token=__UpperCAmelCase    ,				mask_token=__UpperCAmelCase    ,				sp_model_kwargs=self.sp_model_kwargs    ,				**__UpperCAmelCase    ,				)

          _a    	=				do_lower_case
          _a    	=				remove_space
          _a    	=				keep_accents
          _a    	=				vocab_file

          _a    	=				spm.SentencePieceProcessor(**self.sp_model_kwargs							)
          self.sp_model.Load(__UpperCAmelCase							)


     @property
     def 							_UpperCAmelCase      (      self							)       ->							str:
          return len(self.sp_model							)


     def 							_UpperCAmelCase      (      self							)       ->							Optional[Any]:
          _a    	=				{self.convert_ids_to_tokens(__UpperCAmelCase							): i for i in range(self.vocab_size							)}
          vocab.update(self.added_tokens_encoder							)
          return vocab


     def __getstate__(      self							)       ->							int:
          _a    	=				self.__dict__.copy()
          _a    	=				None
          return state


     def __setstate__(      self    ,				__UpperCAmelCase							)       ->							Tuple:
          _a    	=				d

          # for backward compatibility
          if not hasattr(self    ,				'''sp_model_kwargs'''							):
               _a    	=				{}

          _a    	=				spm.SentencePieceProcessor(**self.sp_model_kwargs							)
          self.sp_model.Load(self.vocab_file							)


     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase							)       ->							List[str]:
          if self.remove_space:
               _a    	=				''' '''.join(inputs.strip().split()							)
          else:
               _a    	=				inputs
          _a    	=				outputs.replace('''``'''    ,				'''"'''							).replace('''\'\''''    ,				'''"'''							)

          if not self.keep_accents:
               _a    	=				unicodedata.normalize('''NFKD'''    ,				__UpperCAmelCase							)
               _a    	=				''''''.join([c for c in outputs if not unicodedata.combining(__UpperCAmelCase							)]							)
          if self.do_lower_case:
               _a    	=				outputs.lower()

          return outputs


     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase							)       ->							List[str]:
          _a    	=				self.preprocess_text(__UpperCAmelCase							)
          _a    	=				self.sp_model.encode(__UpperCAmelCase    ,				out_type=__UpperCAmelCase							)
          _a    	=				[]
          for piece in pieces:
               if len(__UpperCAmelCase							) > 1 and piece[-1] == str(''','''							) and piece[-2].isdigit():
                    _a    	=				self.sp_model.EncodeAsPieces(piece[:-1].replace(__UpperCAmelCase    ,				''''''							)							)
                    if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                         if len(cur_pieces[0]							) == 1:
                              _a    	=				cur_pieces[1:]
                         else:
                              _a    	=				cur_pieces[0][1:]
                    cur_pieces.append(piece[-1]							)
                    new_pieces.extend(__UpperCAmelCase							)
               else:
                    new_pieces.append(__UpperCAmelCase							)

          return new_pieces


     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase							)       ->							str:
          return self.sp_model.PieceToId(__UpperCAmelCase							)


     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase							)       ->							int:
          return self.sp_model.IdToPiece(__UpperCAmelCase							)


     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase							)       ->							Optional[Any]:
          _a    	=				[]
          _a    	=				''''''
          _a    	=				False
          for token in tokens:
               # make sure that special tokens are not decoded using sentencepiece model
               if token in self.all_special_tokens:
                    if not prev_is_special:
                         out_string += " "
                    out_string += self.sp_model.decode(__UpperCAmelCase							) + token
                    _a    	=				True
                    _a    	=				[]
               else:
                    current_sub_tokens.append(__UpperCAmelCase							)
                    _a    	=				False
          out_string += self.sp_model.decode(__UpperCAmelCase							)
          return out_string.strip()


     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase    ,				__UpperCAmelCase = None							)       ->							List[int]:
          _a    	=				[self.sep_token_id]
          _a    	=				[self.cls_token_id]
          if token_ids_a is None:
               return cls + token_ids_a + sep
          return cls + token_ids_a + sep + token_ids_a + sep


     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase    ,				__UpperCAmelCase = None    ,				__UpperCAmelCase = False							)       ->							List[int]:

          if already_has_special_tokens:
               return super().get_special_tokens_mask(
                   token_ids_a=__UpperCAmelCase    ,				token_ids_a=__UpperCAmelCase    ,				already_has_special_tokens=__UpperCAmelCase							)

          if token_ids_a is not None:
               return [1] + ([0] * len(__UpperCAmelCase							)) + [1] + ([0] * len(__UpperCAmelCase							)) + [1]
          return [1] + ([0] * len(__UpperCAmelCase							)) + [1]


     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase    ,				__UpperCAmelCase = None							)       ->							List[int]:
          _a    	=				[self.sep_token_id]
          _a    	=				[self.cls_token_id]

          if token_ids_a is None:
               return len(cls + token_ids_a + sep							) * [0]
          return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]





     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase    ,				__UpperCAmelCase = None							)       ->							Tuple[str]:
          if not os.path.isdir(__UpperCAmelCase							):
               logger.error(F'Vocabulary path ({save_directory}) should be a directory'							)
               return
          _a    	=				os.path.join(
              __UpperCAmelCase    ,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']							)

          if os.path.abspath(self.vocab_file							) != os.path.abspath(__UpperCAmelCase							) and os.path.isfile(self.vocab_file							):
               copyfile(self.vocab_file    ,				__UpperCAmelCase							)
          elif not os.path.isfile(self.vocab_file							):
               with open(__UpperCAmelCase    ,				'''wb'''							) as fi:
                    _a    	=				self.sp_model.serialized_model_proto()
                    fi.write(__UpperCAmelCase							)

          return (out_vocab_file,)