




import collections
import os
from typing import List, Optional, Tuple

from transformers.utils import is_jieba_available, requires_backends


if is_jieba_available():
			import jieba

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__SCREAMING_SNAKE_CASE 						=logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE 						={"""vocab_file""": """vocab.txt"""}

__SCREAMING_SNAKE_CASE 						={
    """vocab_file""": {
        """openbmb/cpm-ant-10b""": """https://huggingface.co/openbmb/cpm-ant-10b/blob/main/vocab.txt""",
    },
}

__SCREAMING_SNAKE_CASE 						={
    """openbmb/cpm-ant-10b""": 1_024,
}

def 			a   (_lowerCAmelCase     ):
							SCREAMING_SNAKE_CASE_   	=    collections.OrderedDict()
							with open(_lowerCAmelCase  ,    '''r'''  ,    encoding='''utf-8'''     ) as reader:
														SCREAMING_SNAKE_CASE_   	=    reader.readlines()
							for index, token in enumerate(_lowerCAmelCase     ):
														SCREAMING_SNAKE_CASE_   	=    token.rstrip('''\n'''     )
														SCREAMING_SNAKE_CASE_   	=    index
							return vocab






class  __magic_name__						(  __UpperCAmelCase):
				'''simple docstring'''







				def __init__(							self:   str		,					_lowerCamelCase:   List[Any]		,					_lowerCamelCase:   List[Any]="<unk>"		,					_lowerCamelCase:   str=2_00		):
											SCREAMING_SNAKE_CASE_   	=    vocab
											SCREAMING_SNAKE_CASE_   	=    unk_token
											SCREAMING_SNAKE_CASE_   	=    max_input_chars_per_word





				def     _A							(							self:   str		,					_lowerCamelCase:   List[str]		):
											SCREAMING_SNAKE_CASE_   	=    list(_lowerCamelCase		)
											if len(_lowerCamelCase		) > self.max_input_chars_per_word:
																		return [self.unk_token]

											SCREAMING_SNAKE_CASE_   	=    0
											SCREAMING_SNAKE_CASE_   	=    []
											while start < len(_lowerCamelCase		):
																		SCREAMING_SNAKE_CASE_   	=    len(_lowerCamelCase		)
																		SCREAMING_SNAKE_CASE_   	=    None
																		while start < end:
																									SCREAMING_SNAKE_CASE_   	=    ''''''.join(chars[start:end]		)
																									if substr in self.vocab:
																																SCREAMING_SNAKE_CASE_   	=    substr
																																break
																									end -= 1
																		if cur_substr is None:
																									sub_tokens.append(self.unk_token		)
																									start += 1
																		else:
																									sub_tokens.append(_lowerCamelCase		)
																									SCREAMING_SNAKE_CASE_   	=    end

											return sub_tokens






class  __magic_name__						(  __UpperCAmelCase):
				'''simple docstring'''







				SCREAMING_SNAKE_CASE__					:							int	      =					VOCAB_FILES_NAMES
				SCREAMING_SNAKE_CASE__					:							Dict	      =					PRETRAINED_VOCAB_FILES_MAP
				SCREAMING_SNAKE_CASE__					:							Dict	      =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				SCREAMING_SNAKE_CASE__					:							List[Any]	      =					["input_ids", "attention_mask"]
				SCREAMING_SNAKE_CASE__					:							Optional[Any]	      =					False


				def __init__(							self:   Dict		,					_lowerCamelCase:   str		,					_lowerCamelCase:   List[Any]="<d>"		,					_lowerCamelCase:   List[Any]="</d>"		,					_lowerCamelCase:   Dict="<s>"		,					_lowerCamelCase:   Optional[int]="</s>"		,					_lowerCamelCase:   Dict="<pad>"		,					_lowerCamelCase:   Optional[Any]="<unk>"		,					_lowerCamelCase:   Dict="</n>"		,					_lowerCamelCase:   Any="</_>"		,					_lowerCamelCase:   int="left"		,					**_lowerCamelCase:   List[str]		,					):
											requires_backends(self		,					['''jieba''']		)
											super().__init__(
											    bod_token=_lowerCamelCase		,					eod_token=_lowerCamelCase		,					bos_token=_lowerCamelCase		,					eos_token=_lowerCamelCase		,					pad_token=_lowerCamelCase		,					unk_token=_lowerCamelCase		,					line_token=_lowerCamelCase		,					space_token=_lowerCamelCase		,					padding_side=_lowerCamelCase		,					**_lowerCamelCase		,					)
											SCREAMING_SNAKE_CASE_   	=    bod_token
											SCREAMING_SNAKE_CASE_   	=    eod_token
											SCREAMING_SNAKE_CASE_   	=    load_vocab(_lowerCamelCase		)
											SCREAMING_SNAKE_CASE_   	=    self.encoder[space_token]
											SCREAMING_SNAKE_CASE_   	=    self.encoder[line_token]

											del self.encoder[space_token]
											del self.encoder[line_token]

											SCREAMING_SNAKE_CASE_   	=    collections.OrderedDict(sorted(self.encoder.items()		,					key=lambda _lowerCamelCase		: x[1]		)		)
											SCREAMING_SNAKE_CASE_   	=    {v: k for k, v in self.encoder.items()}

											SCREAMING_SNAKE_CASE_   	=    WordpieceTokenizer(vocab=self.encoder		,					unk_token=self.unk_token		)


				@property
				def     _A							(							self:   Optional[int]		):
											return self.encoder[self.bod_token]


				@property
				def     _A							(							self:   List[Any]		):
											return self.encoder[self.eod_token]


				@property
				def     _A							(							self:   Union[str, Any]		):
											return self.encoder["\n"]


				@property
				def     _A							(							self:   List[str]		):
											return len(self.encoder		)


				def     _A							(							self:   Optional[int]		):
											return dict(self.encoder		,					**self.added_tokens_encoder		)


				def     _A							(							self:   List[str]		,					_lowerCamelCase:   str		):
											SCREAMING_SNAKE_CASE_   	=    []
											for x in jieba.cut(_lowerCamelCase		,					cut_all=_lowerCamelCase		):
																		output_tokens.extend(self.wordpiece_tokenizer.tokenize(_lowerCamelCase		)		)
											return output_tokens


				def     _A							(							self:   Tuple		,					_lowerCamelCase:   int		,					**_lowerCamelCase:   Optional[Any]		):
											SCREAMING_SNAKE_CASE_   	=    [i for i in token_ids if i >= 0]
											SCREAMING_SNAKE_CASE_   	=    [
											    x for x in token_ids if x != self.pad_token_id and x != self.eos_token_id and x != self.bos_token_id
											]
											return super()._decode(_lowerCamelCase		,					**_lowerCamelCase		)


				def     _A							(							self:   Optional[int]		,					_lowerCamelCase:   Dict		):
											return token in self.encoder


				def     _A							(							self:   List[str]		,					_lowerCamelCase:   List[str]		):
											return "".join(_lowerCamelCase		)


				def     _A							(							self:   Optional[int]		,					_lowerCamelCase:   str		):
											return self.encoder.get(_lowerCamelCase		,					self.encoder.get(self.unk_token		)		)


				def     _A							(							self:   Optional[int]		,					_lowerCamelCase:   Dict		):
											return self.decoder.get(_lowerCamelCase		,					self.unk_token		)


				def     _A							(							self:   Union[str, Any]		,					_lowerCamelCase:   str		,					_lowerCamelCase:   Optional[str] = None		):
											if os.path.isdir(_lowerCamelCase		):
																		SCREAMING_SNAKE_CASE_   	=    os.path.join(
																		    _lowerCamelCase		,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']		)
											else:
																		SCREAMING_SNAKE_CASE_   	=    (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory
											SCREAMING_SNAKE_CASE_   	=    0
											if " " in self.encoder:
																		SCREAMING_SNAKE_CASE_   	=    self.encoder[''' ''']
																		del self.encoder[" "]
											if "\n" in self.encoder:
																		SCREAMING_SNAKE_CASE_   	=    self.encoder['''\n''']
																		del self.encoder["\n"]
											SCREAMING_SNAKE_CASE_   	=    collections.OrderedDict(sorted(self.encoder.items()		,					key=lambda _lowerCamelCase		: x[1]		)		)
											with open(_lowerCamelCase		,					'''w'''		,					encoding='''utf-8'''		) as writer:
																		for token, token_index in self.encoder.items():
																									if index != token_index:
																																logger.warning(
																																    f"Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."
																																    ''' Please check that the vocabulary is not corrupted!'''		)
																																SCREAMING_SNAKE_CASE_   	=    token_index
																									writer.write(token + '''\n'''		)
																									index += 1
											return (vocab_file,)


				def     _A							(							self:   Union[str, Any]		,					_lowerCamelCase:   List[int]		,					_lowerCamelCase:   List[int] = None		):
											if token_ids_a is None:
																		return [self.bos_token_id] + token_ids_a
											return [self.bos_token_id] + token_ids_a + [self.bos_token_id] + token_ids_a





				def     _A							(							self:   List[Any]		,					_lowerCamelCase:   List[int]		,					_lowerCamelCase:   Optional[List[int]] = None		,					_lowerCamelCase:   bool = False		):

											if already_has_special_tokens:
																		return super().get_special_tokens_mask(
																		    token_ids_a=_lowerCamelCase		,					token_ids_a=_lowerCamelCase		,					already_has_special_tokens=_lowerCamelCase		)

											if token_ids_a is not None:
																		return [1] + ([0] * len(_lowerCamelCase		)) + [1] + ([0] * len(_lowerCamelCase		))
											return [1] + ([0] * len(_lowerCamelCase		))








import unittest

import numpy as np

from transformers.testing_utils import is_flaky, require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
			import torch

if is_vision_available():
			from PIL import Image

			from transformers import DonutImageProcessor






class  __magic_name__						(  unittest.TestCase):
				'''simple docstring'''







				def __init__(							self:   Union[str, Any]		,					_lowerCamelCase:   List[str]		,					_lowerCamelCase:   Union[str, Any]=7		,					_lowerCamelCase:   int=3		,					_lowerCamelCase:   Optional[int]=18		,					_lowerCamelCase:   Optional[Any]=30		,					_lowerCamelCase:   Any=4_00		,					_lowerCamelCase:   List[str]=True		,					_lowerCamelCase:   str=None		,					_lowerCamelCase:   Union[str, Any]=True		,					_lowerCamelCase:   Union[str, Any]=False		,					_lowerCamelCase:   int=True		,					_lowerCamelCase:   Any=True		,					_lowerCamelCase:   List[str]=[0.5, 0.5, 0.5]		,					_lowerCamelCase:   Dict=[0.5, 0.5, 0.5]		,					):
											SCREAMING_SNAKE_CASE_   	=    parent
											SCREAMING_SNAKE_CASE_   	=    batch_size
											SCREAMING_SNAKE_CASE_   	=    num_channels
											SCREAMING_SNAKE_CASE_   	=    image_size
											SCREAMING_SNAKE_CASE_   	=    min_resolution
											SCREAMING_SNAKE_CASE_   	=    max_resolution
											SCREAMING_SNAKE_CASE_   	=    do_resize
											SCREAMING_SNAKE_CASE_   	=    size if size is not None else {'''height''': 18, '''width''': 20}
											SCREAMING_SNAKE_CASE_   	=    do_thumbnail
											SCREAMING_SNAKE_CASE_   	=    do_align_axis
											SCREAMING_SNAKE_CASE_   	=    do_pad
											SCREAMING_SNAKE_CASE_   	=    do_normalize
											SCREAMING_SNAKE_CASE_   	=    image_mean
											SCREAMING_SNAKE_CASE_   	=    image_std





				def     _A							(							self:   str		):
											return {
											    "do_resize": self.do_resize,
											    "size": self.size,
											    "do_thumbnail": self.do_thumbnail,
											    "do_align_long_axis": self.do_align_axis,
											    "do_pad": self.do_pad,
											    "do_normalize": self.do_normalize,
											    "image_mean": self.image_mean,
											    "image_std": self.image_std,
											}






@require_torch
@require_vision
class  __magic_name__						(  __UpperCAmelCase    ,  unittest.TestCase):
				'''simple docstring'''







				SCREAMING_SNAKE_CASE__					:							Dict	      =					DonutImageProcessor if is_vision_available() else None


				def     _A							(							self:   Optional[int]		):
											SCREAMING_SNAKE_CASE_   	=    DonutImageProcessingTester(self		)


				@property
				def     _A							(							self:   List[str]		):
											return self.image_processor_tester.prepare_image_processor_dict()


				def     _A							(							self:   Union[str, Any]		):
											SCREAMING_SNAKE_CASE_   	=    self.image_processing_class(**self.image_processor_dict		)
											self.assertTrue(hasattr(_lowerCamelCase		,					'''do_resize'''		)		)
											self.assertTrue(hasattr(_lowerCamelCase		,					'''size'''		)		)
											self.assertTrue(hasattr(_lowerCamelCase		,					'''do_thumbnail'''		)		)
											self.assertTrue(hasattr(_lowerCamelCase		,					'''do_align_long_axis'''		)		)
											self.assertTrue(hasattr(_lowerCamelCase		,					'''do_pad'''		)		)
											self.assertTrue(hasattr(_lowerCamelCase		,					'''do_normalize'''		)		)
											self.assertTrue(hasattr(_lowerCamelCase		,					'''image_mean'''		)		)
											self.assertTrue(hasattr(_lowerCamelCase		,					'''image_std'''		)		)


				def     _A							(							self:   List[str]		):
											SCREAMING_SNAKE_CASE_   	=    self.image_processing_class.from_dict(self.image_processor_dict		)
											self.assertEqual(image_processor.size		,					{'''height''': 18, '''width''': 20}		)

											SCREAMING_SNAKE_CASE_   	=    self.image_processing_class.from_dict(self.image_processor_dict		,					size=42		)
											self.assertEqual(image_processor.size		,					{'''height''': 42, '''width''': 42}		)

											# Previous config had dimensions in (width, height) order
											SCREAMING_SNAKE_CASE_   	=    self.image_processing_class.from_dict(self.image_processor_dict		,					size=(42, 84)		)
											self.assertEqual(image_processor.size		,					{'''height''': 84, '''width''': 42}		)


				def     _A							(							self:   List[Any]		):
											pass


				@is_flaky()
				def     _A							(							self:   int		):
											# Initialize image_processing
											SCREAMING_SNAKE_CASE_   	=    self.image_processing_class(**self.image_processor_dict		)
											# create random PIL images
											SCREAMING_SNAKE_CASE_   	=    prepare_image_inputs(self.image_processor_tester		,					equal_resolution=_lowerCamelCase		)
											for image in image_inputs:
																		self.assertIsInstance(_lowerCamelCase		,					Image.Image		)

											# Test not batched input
											SCREAMING_SNAKE_CASE_   	=    image_processing(image_inputs[0]		,					return_tensors='''pt'''		).pixel_values
											self.assertEqual(
											    encoded_images.shape		,					(
											        1,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.size['''height'''],
											        self.image_processor_tester.size['''width'''],
											    )		,					)

											# Test batched
											SCREAMING_SNAKE_CASE_   	=    image_processing(_lowerCamelCase		,					return_tensors='''pt'''		).pixel_values
											self.assertEqual(
											    encoded_images.shape		,					(
											        self.image_processor_tester.batch_size,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.size['''height'''],
											        self.image_processor_tester.size['''width'''],
											    )		,					)


				@is_flaky()
				def     _A							(							self:   Optional[Any]		):
											# Initialize image_processing
											SCREAMING_SNAKE_CASE_   	=    self.image_processing_class(**self.image_processor_dict		)
											# create random numpy tensors
											SCREAMING_SNAKE_CASE_   	=    prepare_image_inputs(self.image_processor_tester		,					equal_resolution=_lowerCamelCase		,					numpify=_lowerCamelCase		)
											for image in image_inputs:
																		self.assertIsInstance(_lowerCamelCase		,					np.ndarray		)

											# Test not batched input
											SCREAMING_SNAKE_CASE_   	=    image_processing(image_inputs[0]		,					return_tensors='''pt'''		).pixel_values
											self.assertEqual(
											    encoded_images.shape		,					(
											        1,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.size['''height'''],
											        self.image_processor_tester.size['''width'''],
											    )		,					)

											# Test batched
											SCREAMING_SNAKE_CASE_   	=    image_processing(_lowerCamelCase		,					return_tensors='''pt'''		).pixel_values
											self.assertEqual(
											    encoded_images.shape		,					(
											        self.image_processor_tester.batch_size,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.size['''height'''],
											        self.image_processor_tester.size['''width'''],
											    )		,					)





				@is_flaky()
				def     _A							(							self:   List[str]		):
											# Initialize image_processing
											SCREAMING_SNAKE_CASE_   	=    self.image_processing_class(**self.image_processor_dict		)
											# create random PyTorch tensors
											SCREAMING_SNAKE_CASE_   	=    prepare_image_inputs(self.image_processor_tester		,					equal_resolution=_lowerCamelCase		,					torchify=_lowerCamelCase		)
											for image in image_inputs:
																		self.assertIsInstance(_lowerCamelCase		,					torch.Tensor		)

											# Test not batched input
											SCREAMING_SNAKE_CASE_   	=    image_processing(image_inputs[0]		,					return_tensors='''pt'''		).pixel_values
											self.assertEqual(
											    encoded_images.shape		,					(
											        1,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.size['''height'''],
											        self.image_processor_tester.size['''width'''],
											    )		,					)

											# Test batched
											SCREAMING_SNAKE_CASE_   	=    image_processing(_lowerCamelCase		,					return_tensors='''pt'''		).pixel_values
											self.assertEqual(
											    encoded_images.shape		,					(
											        self.image_processor_tester.batch_size,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.size['''height'''],
											        self.image_processor_tester.size['''width'''],
											    )		,					)


