




'''simple docstring'''

import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / '''utils'''))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
						from test_module.custom_tokenization_fast import CustomTokenizerFast



class 		SCREAMING_SNAKE_CASE__  (			unittest.TestCase	):





			def 							A (      self						:    Optional[Any]							):



						"""simple docstring"""




						__snake_case			=		mock.Mock()
						__snake_case			=		500
						__snake_case			=		{}
						__snake_case			=		HTTPError
						__snake_case			=		{}

						# Download this model to make sure it's in the cache.
						__snake_case			=		BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"							)

						# Under the mock environment we get a 500 error when trying to reach the tokenizer.
						with mock.patch("requests.Session.request"      ,		return_value=a_							) as mock_head:
									__snake_case			=		BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"							)
									# This check we did call the fake head request
									mock_head.assert_called()





			@require_tokenizers
			def 							A (      self						:    Optional[Any]							):



						"""simple docstring"""




						__snake_case			=		mock.Mock()
						__snake_case			=		500
						__snake_case			=		{}
						__snake_case			=		HTTPError
						__snake_case			=		{}

						# Download this model to make sure it's in the cache.
						__snake_case			=		GPTaTokenizerFast.from_pretrained("gpt2"							)

						# Under the mock environment we get a 500 error when trying to reach the tokenizer.
						with mock.patch("requests.Session.request"      ,		return_value=a_							) as mock_head:
									__snake_case			=		GPTaTokenizerFast.from_pretrained("gpt2"							)
									# This check we did call the fake head request
									mock_head.assert_called()





			def 							A (      self						:    Optional[Any]							):



						"""simple docstring"""




						try:
									__snake_case			=		tempfile.mktemp()
									with open(a_      ,		"wb"							) as f:
												http_get("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"      ,		a_							)

									__snake_case			=		AlbertTokenizer.from_pretrained(a_							)
						finally:
									os.remove(a_							)

						# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
						# the current folder and have the right name.
						if os.path.isfile("tokenizer.json"							):
									# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
									return
						try:
									with open("tokenizer.json"      ,		"wb"							) as f:
												http_get("https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"      ,		a_							)
									__snake_case			=		AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2"							)
									# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
									self.assertEqual(tokenizer.vocab_size      ,		1_000							)
									# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

						finally:
									os.remove("tokenizer.json"							)






			def 							A (      self						:    str							):



						"""simple docstring"""




						__snake_case			=		AlbertTokenizer.from_pretrained("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"							)







@is_staging_test
class 		SCREAMING_SNAKE_CASE__  (			unittest.TestCase	):
			__SCREAMING_SNAKE_CASE    =			["""[UNK]""", """[CLS]""", """[SEP]""", """[PAD]""", """[MASK]""", """bla""", """blou"""]





			@classmethod
			def 							A (      cls						:    List[Any]							):



						"""simple docstring"""




						__snake_case			=		TOKEN
						HfFolder.save_token(a_							)





			@classmethod
			def 							A (      cls						:    List[Any]							):



						"""simple docstring"""




						try:
									delete_repo(token=cls._token      ,		repo_id="test-tokenizer"							)
						except HTTPError:
									pass

						try:
									delete_repo(token=cls._token      ,		repo_id="valid_org/test-tokenizer-org"							)
						except HTTPError:
									pass

						try:
									delete_repo(token=cls._token      ,		repo_id="test-dynamic-tokenizer"							)
						except HTTPError:
									pass





			def 							A (      self						:    int							):



						"""simple docstring"""




						with tempfile.TemporaryDirectory() as tmp_dir:
									__snake_case			=		os.path.join(a_      ,		"vocab.txt"							)
									with open(a_      ,		"w"      ,		encoding="utf-8"							) as vocab_writer:
												vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]							)							)
									__snake_case			=		BertTokenizer(a_							)

						tokenizer.push_to_hub("test-tokenizer"      ,		use_auth_token=self._token							)
						__snake_case			=		BertTokenizer.from_pretrained(f'''{USER}/test-tokenizer'''							)
						self.assertDictEqual(new_tokenizer.vocab      ,		tokenizer.vocab							)

						# Reset repo
						delete_repo(token=self._token      ,		repo_id="test-tokenizer"							)

						# Push to hub via save_pretrained
						with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(a_      ,		repo_id="test-tokenizer"      ,		push_to_hub=a_      ,		use_auth_token=self._token							)

						__snake_case			=		BertTokenizer.from_pretrained(f'''{USER}/test-tokenizer'''							)
						self.assertDictEqual(new_tokenizer.vocab      ,		tokenizer.vocab							)





			def 							A (      self						:    int							):



						"""simple docstring"""




						with tempfile.TemporaryDirectory() as tmp_dir:
									__snake_case			=		os.path.join(a_      ,		"vocab.txt"							)
									with open(a_      ,		"w"      ,		encoding="utf-8"							) as vocab_writer:
												vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]							)							)
									__snake_case			=		BertTokenizer(a_							)

						tokenizer.push_to_hub("valid_org/test-tokenizer-org"      ,		use_auth_token=self._token							)
						__snake_case			=		BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"							)
						self.assertDictEqual(new_tokenizer.vocab      ,		tokenizer.vocab							)

						# Reset repo
						delete_repo(token=self._token      ,		repo_id="valid_org/test-tokenizer-org"							)

						# Push to hub via save_pretrained
						with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(
									    a_      ,		repo_id="valid_org/test-tokenizer-org"      ,		push_to_hub=a_      ,		use_auth_token=self._token							)

						__snake_case			=		BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"							)
						self.assertDictEqual(new_tokenizer.vocab      ,		tokenizer.vocab							)






			@require_tokenizers
			def 							A (      self						:    List[str]							):



						"""simple docstring"""




						CustomTokenizer.register_for_auto_class()
						with tempfile.TemporaryDirectory() as tmp_dir:
									__snake_case			=		os.path.join(a_      ,		"vocab.txt"							)
									with open(a_      ,		"w"      ,		encoding="utf-8"							) as vocab_writer:
												vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]							)							)
									__snake_case			=		CustomTokenizer(a_							)

						# No fast custom tokenizer
						tokenizer.push_to_hub("test-dynamic-tokenizer"      ,		use_auth_token=self._token							)

						__snake_case			=		AutoTokenizer.from_pretrained(f'''{USER}/test-dynamic-tokenizer'''      ,		trust_remote_code=a_							)
						# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
						self.assertEqual(tokenizer.__class__.__name__      ,		"CustomTokenizer"							)

						# Fast and slow custom tokenizer
						CustomTokenizerFast.register_for_auto_class()
						with tempfile.TemporaryDirectory() as tmp_dir:
									__snake_case			=		os.path.join(a_      ,		"vocab.txt"							)
									with open(a_      ,		"w"      ,		encoding="utf-8"							) as vocab_writer:
												vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]							)							)

									__snake_case			=		BertTokenizerFast.from_pretrained(a_							)
									bert_tokenizer.save_pretrained(a_							)
									__snake_case			=		CustomTokenizerFast.from_pretrained(a_							)

						tokenizer.push_to_hub("test-dynamic-tokenizer"      ,		use_auth_token=self._token							)

						__snake_case			=		AutoTokenizer.from_pretrained(f'''{USER}/test-dynamic-tokenizer'''      ,		trust_remote_code=a_							)
						# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
						self.assertEqual(tokenizer.__class__.__name__      ,		"CustomTokenizerFast"							)
						__snake_case			=		AutoTokenizer.from_pretrained(
						    f'''{USER}/test-dynamic-tokenizer'''      ,		use_fast=a_      ,		trust_remote_code=a_							)
						# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
						self.assertEqual(tokenizer.__class__.__name__      ,		"CustomTokenizer"							)







class 		SCREAMING_SNAKE_CASE__  (			unittest.TestCase	):





			def 							A (      self						:    Optional[int]							):



						"""simple docstring"""




						__snake_case			=		Trie()
						trie.add("Hello 友達"							)
						self.assertEqual(trie.data      ,		{"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}}							)
						trie.add("Hello"							)
						trie.data
						self.assertEqual(trie.data      ,		{"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}}							)





			def 							A (      self						:    str							):



						"""simple docstring"""




						__snake_case			=		Trie()
						self.assertEqual(trie.split("[CLS] This is a extra_id_100"							)      ,		["[CLS] This is a extra_id_100"]							)
						trie.add("[CLS]"							)
						trie.add("extra_id_1"							)
						trie.add("extra_id_100"							)
						self.assertEqual(trie.split("[CLS] This is a extra_id_100"							)      ,		["[CLS]", " This is a ", "extra_id_100"]							)





			def 							A (      self						:    Optional[Any]							):



						"""simple docstring"""




						__snake_case			=		Trie()
						trie.add("A"							)
						self.assertEqual(trie.split("ABC"							)      ,		["A", "BC"]							)
						self.assertEqual(trie.split("BCA"							)      ,		["BC", "A"]							)





			def 							A (      self						:    List[Any]							):



						"""simple docstring"""




						__snake_case			=		Trie()
						trie.add("TOKEN]"							)
						trie.add("[SPECIAL_TOKEN]"							)
						self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"							)      ,		["This is something ", "[SPECIAL_TOKEN]"]							)





			def 							A (      self						:    str							):



						"""simple docstring"""




						__snake_case			=		Trie()
						trie.add("A"							)
						trie.add("P"							)
						trie.add("[SPECIAL_TOKEN]"							)
						self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"							)      ,		["This is something ", "[SPECIAL_TOKEN]"]							)





			def 							A (      self						:    Optional[int]							):



						"""simple docstring"""




						__snake_case			=		Trie()
						trie.add("AB"							)
						trie.add("B"							)
						trie.add("C"							)
						self.assertEqual(trie.split("ABC"							)      ,		["AB", "C"]							)





			def 							A (      self						:    Tuple							):



						"""simple docstring"""




						__snake_case			=		Trie()
						trie.add("ABC"							)
						trie.add("B"							)
						trie.add("CD"							)
						self.assertEqual(trie.split("ABCD"							)      ,		["ABC", "D"]							)






			def 							A (      self						:    Any							):



						"""simple docstring"""




						__snake_case			=		Trie()
						__snake_case			=		trie.cut_text("ABC"      ,		[0, 0, 2, 1, 2, 3]							)
						self.assertEqual(a_      ,		["AB", "C"]							)








from typing import Dict, Iterable, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import normalize, rescale, resize, to_channel_dimension_format, to_pil_image
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_pytesseract_available, is_vision_available, logging, requires_backends


if is_vision_available():
    import PIL

# soft dependency
if is_pytesseract_available():
    import pytesseract

UpperCamelCase__		 =     logging.get_logger(__name__)







def 						lowerCamelCase__      ( __A    :Optional[int]	,__A    :Any	,__A    :str		):


       """simple docstring"""




       return [
           int(1_0_0_0 * (box[0] / width)		),
           int(1_0_0_0 * (box[1] / height)		),
           int(1_0_0_0 * (box[2] / width)		),
           int(1_0_0_0 * (box[3] / height)		),
       ]







def 						lowerCamelCase__      ( __A    :np.ndarray	,__A    :Optional[str]	,__A    :Optional[str]		):


       """simple docstring"""




       __snake_case       	=							to_pil_image(__A		)
       __snake_case	,    __snake_case       	=							pil_image.size
       __snake_case       	=							pytesseract.image_to_data(__A	,lang=__A	,output_type="""dict"""	,config=__A		)
       __snake_case	,    __snake_case	,    __snake_case	,    __snake_case	,    __snake_case       	=							data["""text"""], data["""left"""], data["""top"""], data["""width"""], data["""height"""]

       # filter empty words and corresponding coordinates
       __snake_case       	=							[idx for idx, word in enumerate(__A		) if not word.strip()]
       __snake_case       	=							[word for idx, word in enumerate(__A		) if idx not in irrelevant_indices]
       __snake_case       	=							[coord for idx, coord in enumerate(__A		) if idx not in irrelevant_indices]
       __snake_case       	=							[coord for idx, coord in enumerate(__A		) if idx not in irrelevant_indices]
       __snake_case       	=							[coord for idx, coord in enumerate(__A		) if idx not in irrelevant_indices]
       __snake_case       	=							[coord for idx, coord in enumerate(__A		) if idx not in irrelevant_indices]

       # turn coordinates into (left, top, left+width, top+height) format
       __snake_case       	=							[]
       for x, y, w, h in zip(__A	,__A	,__A	,__A		):
              __snake_case       	=							[x, y, x + w, y + h]
              actual_boxes.append(__A		)

       # finally, normalize the bounding boxes
       __snake_case       	=							[]
       for box in actual_boxes:
              normalized_boxes.append(normalize_box(__A	,__A	,__A		)		)

       assert len(__A		) == len(__A		), "Not as many words as there are bounding boxes"

       return words, normalized_boxes




class     __snake_case    (				snake_case__  ):
       """simple docstring"""







       __SCREAMING_SNAKE_CASE				    =							["pixel_values"]


       def __init__(		self      , _UpperCamelCase = True      , _UpperCamelCase = None      , _UpperCamelCase = PILImageResampling.BILINEAR      , _UpperCamelCase = True      , _UpperCamelCase = 1 / 2_55      , _UpperCamelCase = True      , _UpperCamelCase = None      , _UpperCamelCase = None      , _UpperCamelCase = True      , _UpperCamelCase = None      , _UpperCamelCase = ""      , **_UpperCamelCase      , )      ->							None:






              """simple docstring"""


              super().__init__(**_UpperCamelCase   )
              __snake_case       	=							size if size is not None else {"""height""": 2_24, """width""": 2_24}
              __snake_case       	=							get_size_dict(_UpperCamelCase   )

              __snake_case       	=							do_resize
              __snake_case       	=							size
              __snake_case       	=							resample
              __snake_case       	=							do_rescale
              __snake_case       	=							rescale_value
              __snake_case       	=							do_normalize
              __snake_case       	=							image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
              __snake_case       	=							image_std if image_std is not None else IMAGENET_STANDARD_STD
              __snake_case       	=							apply_ocr
              __snake_case       	=							ocr_lang
              __snake_case       	=							tesseract_config


       def 			a		(		self      , _UpperCamelCase      , _UpperCamelCase      , _UpperCamelCase = PILImageResampling.BILINEAR      , _UpperCamelCase = None      , **_UpperCamelCase      , )      ->							np.ndarray:






              """simple docstring"""


              __snake_case       	=							get_size_dict(_UpperCamelCase   )
              if "height" not in size or "width" not in size:
                     raise ValueError(F'The size dictionary must contain the keys \'height\' and \'width\'. Got {size.keys()}'   )
              __snake_case       	=							(size["""height"""], size["""width"""])
              return resize(_UpperCamelCase      , size=_UpperCamelCase      , resample=_UpperCamelCase      , data_format=_UpperCamelCase      , **_UpperCamelCase   )


       def 			a		(		self      , _UpperCamelCase      , _UpperCamelCase      , _UpperCamelCase = None      , **_UpperCamelCase      , )      ->							np.ndarray:






              """simple docstring"""


              return rescale(_UpperCamelCase      , scale=_UpperCamelCase      , data_format=_UpperCamelCase      , **_UpperCamelCase   )


       def 			a		(		self      , _UpperCamelCase      , _UpperCamelCase      , _UpperCamelCase      , _UpperCamelCase = None      , **_UpperCamelCase      , )      ->							np.ndarray:






              """simple docstring"""


              return normalize(_UpperCamelCase      , mean=_UpperCamelCase      , std=_UpperCamelCase      , data_format=_UpperCamelCase      , **_UpperCamelCase   )






       def 			a		(		self      , _UpperCamelCase      , _UpperCamelCase = None      , _UpperCamelCase = None      , _UpperCamelCase=None      , _UpperCamelCase = None      , _UpperCamelCase = None      , _UpperCamelCase = None      , _UpperCamelCase = None      , _UpperCamelCase = None      , _UpperCamelCase = None      , _UpperCamelCase = None      , _UpperCamelCase = None      , _UpperCamelCase = None      , _UpperCamelCase = ChannelDimension.FIRST      , **_UpperCamelCase      , )      ->							PIL.Image.Image:






              """simple docstring"""


              __snake_case       	=							do_resize if do_resize is not None else self.do_resize
              __snake_case       	=							size if size is not None else self.size
              __snake_case       	=							get_size_dict(_UpperCamelCase   )
              __snake_case       	=							resample if resample is not None else self.resample
              __snake_case       	=							do_rescale if do_rescale is not None else self.do_rescale
              __snake_case       	=							rescale_factor if rescale_factor is not None else self.rescale_factor
              __snake_case       	=							do_normalize if do_normalize is not None else self.do_normalize
              __snake_case       	=							image_mean if image_mean is not None else self.image_mean
              __snake_case       	=							image_std if image_std is not None else self.image_std
              __snake_case       	=							apply_ocr if apply_ocr is not None else self.apply_ocr
              __snake_case       	=							ocr_lang if ocr_lang is not None else self.ocr_lang
              __snake_case       	=							tesseract_config if tesseract_config is not None else self.tesseract_config

              __snake_case       	=							make_list_of_images(_UpperCamelCase   )

              if not valid_images(_UpperCamelCase   ):
                     raise ValueError(
                         """Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, """
                         """torch.Tensor, tf.Tensor or jax.ndarray."""   )

              if do_resize and size is None:
                     raise ValueError("""Size must be specified if do_resize is True."""   )

              if do_rescale and rescale_factor is None:
                     raise ValueError("""Rescale factor must be specified if do_rescale is True."""   )

              if do_normalize and (image_mean is None or image_std is None):
                     raise ValueError("""If do_normalize is True, image_mean and image_std must be specified."""   )

              # All transformations expect numpy arrays.
              __snake_case       	=							[to_numpy_array(_UpperCamelCase   ) for image in images]

              # Tesseract OCR to get words + normalized bounding boxes
              if apply_ocr:
                     requires_backends(self      , """pytesseract"""   )
                     __snake_case       	=							[]
                     __snake_case       	=							[]
                     for image in images:
                            __snake_case	,    __snake_case       	=							apply_tesseract(_UpperCamelCase      , _UpperCamelCase      , _UpperCamelCase   )
                            words_batch.append(_UpperCamelCase   )
                            boxes_batch.append(_UpperCamelCase   )

              if do_resize:
                     __snake_case       	=							[self.resize(image=_UpperCamelCase      , size=_UpperCamelCase      , resample=_UpperCamelCase   ) for image in images]

              if do_rescale:
                     __snake_case       	=							[self.rescale(image=_UpperCamelCase      , scale=_UpperCamelCase   ) for image in images]

              if do_normalize:
                     __snake_case       	=							[self.normalize(image=_UpperCamelCase      , mean=_UpperCamelCase      , std=_UpperCamelCase   ) for image in images]

              __snake_case       	=							[to_channel_dimension_format(_UpperCamelCase      , _UpperCamelCase   ) for image in images]

              __snake_case       	=							BatchFeature(data={"""pixel_values""": images}      , tensor_type=_UpperCamelCase   )

              if apply_ocr:
                     __snake_case       	=							words_batch
                     __snake_case       	=							boxes_batch
              return data

