





import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
		from PIL import Image

		from transformers import CLIPImageProcessor, CLIPProcessor




@require_vision
class 				__lowercase			(			unittest.TestCase							):







					'''simple docstring'''




					def  _lowerCamelCase		(						self     ):
									__a :						Optional[Any]			       =  tempfile.mkdtemp()

									# fmt: off
									__a :						Tuple			       =  ['''l''', '''o''', '''w''', '''e''', '''r''', '''s''', '''t''', '''i''', '''d''', '''n''', '''lo''', '''l</w>''', '''w</w>''', '''r</w>''', '''t</w>''', '''low</w>''', '''er</w>''', '''lowest</w>''', '''newer</w>''', '''wider''', '''<unk>''', '''<|startoftext|>''', '''<|endoftext|>''']
									# fmt: on
									__a :						Dict			       =  dict(zip(_UpperCAmelCase	,			range(len(_UpperCAmelCase     )     )     )     )
									__a :						Any			       =  ['''#version: 0.2''', '''l o''', '''lo w</w>''', '''e r</w>''', '''''']
									__a :						str			       =  {'''unk_token''': '''<unk>'''}

									__a :						Optional[Any]			       =  os.path.join(self.tmpdirname	,			VOCAB_FILES_NAMES['''vocab_file''']     )
									__a :						Dict			       =  os.path.join(self.tmpdirname	,			VOCAB_FILES_NAMES['''merges_file''']     )
									with open(self.vocab_file	,			'''w'''	,			encoding='''utf-8'''     ) as fp:
													fp.write(json.dumps(_UpperCAmelCase     ) + '''\n'''     )
									with open(self.merges_file	,			'''w'''	,			encoding='''utf-8'''     ) as fp:
													fp.write('''\n'''.join(_UpperCAmelCase     )     )

									__a :						Any			       =  {
									    '''do_resize''': True,
									    '''size''': 20,
									    '''do_center_crop''': True,
									    '''crop_size''': 18,
									    '''do_normalize''': True,
									    '''image_mean''': [0.4_8_1_4_5_4_6_6, 0.4_5_7_8_2_7_5, 0.4_0_8_2_1_0_7_3],
									    '''image_std''': [0.2_6_8_6_2_9_5_4, 0.2_6_1_3_0_2_5_8, 0.2_7_5_7_7_7_1_1],
									}
									__a :						Union[str, Any]			       =  os.path.join(self.tmpdirname	,			_UpperCAmelCase     )
									with open(self.image_processor_file	,			'''w'''	,			encoding='''utf-8'''     ) as fp:
													json.dump(_UpperCAmelCase	,			_UpperCAmelCase     )







					def  _lowerCamelCase		(						self	,			**_UpperCAmelCase     ):
									return CLIPTokenizer.from_pretrained(self.tmpdirname	,			**_UpperCAmelCase     )







					def  _lowerCamelCase		(						self	,			**_UpperCAmelCase     ):
									return CLIPTokenizerFast.from_pretrained(self.tmpdirname	,			**_UpperCAmelCase     )







					def  _lowerCamelCase		(						self	,			**_UpperCAmelCase     ):
									return CLIPImageProcessor.from_pretrained(self.tmpdirname	,			**_UpperCAmelCase     )







					def  _lowerCamelCase		(						self     ):
									shutil.rmtree(self.tmpdirname     )







					def  _lowerCamelCase		(						self     ):
									__a :						Tuple			       =  [np.random.randint(255	,			size=(3, 30, 400)	,			dtype=np.uinta     )]

									__a :						Union[str, Any]			       =  [Image.fromarray(np.moveaxis(_UpperCAmelCase	,			0	,			-1     )     ) for x in image_inputs]

									return image_inputs







					def  _lowerCamelCase		(						self     ):
									__a :						str			       =  self.get_tokenizer()
									__a :						Any			       =  self.get_rust_tokenizer()
									__a :						Union[str, Any]			       =  self.get_image_processor()

									__a :						Any			       =  CLIPProcessor(tokenizer=_UpperCAmelCase	,			image_processor=_UpperCAmelCase     )
									processor_slow.save_pretrained(self.tmpdirname     )
									__a :						List[str]			       =  CLIPProcessor.from_pretrained(self.tmpdirname	,			use_fast=_UpperCAmelCase     )

									__a :						int			       =  CLIPProcessor(tokenizer=_UpperCAmelCase	,			image_processor=_UpperCAmelCase     )
									processor_fast.save_pretrained(self.tmpdirname     )
									__a :						Dict			       =  CLIPProcessor.from_pretrained(self.tmpdirname     )

									self.assertEqual(processor_slow.tokenizer.get_vocab()	,			tokenizer_slow.get_vocab()     )
									self.assertEqual(processor_fast.tokenizer.get_vocab()	,			tokenizer_fast.get_vocab()     )
									self.assertEqual(tokenizer_slow.get_vocab()	,			tokenizer_fast.get_vocab()     )
									self.assertIsInstance(processor_slow.tokenizer	,			_UpperCAmelCase     )
									self.assertIsInstance(processor_fast.tokenizer	,			_UpperCAmelCase     )

									self.assertEqual(processor_slow.image_processor.to_json_string()	,			image_processor.to_json_string()     )
									self.assertEqual(processor_fast.image_processor.to_json_string()	,			image_processor.to_json_string()     )
									self.assertIsInstance(processor_slow.image_processor	,			_UpperCAmelCase     )
									self.assertIsInstance(processor_fast.image_processor	,			_UpperCAmelCase     )







					def  _lowerCamelCase		(						self     ):
									__a :						Optional[int]			       =  CLIPProcessor(tokenizer=self.get_tokenizer()	,			image_processor=self.get_image_processor()     )
									processor.save_pretrained(self.tmpdirname     )

									__a :						int			       =  self.get_tokenizer(bos_token='''(BOS)'''	,			eos_token='''(EOS)'''     )
									__a :						Optional[Any]			       =  self.get_image_processor(do_normalize=_UpperCAmelCase	,			padding_value=1.0     )

									__a :						Dict			       =  CLIPProcessor.from_pretrained(
									    self.tmpdirname	,			bos_token='''(BOS)'''	,			eos_token='''(EOS)'''	,			do_normalize=_UpperCAmelCase	,			padding_value=1.0     )

									self.assertEqual(processor.tokenizer.get_vocab()	,			tokenizer_add_kwargs.get_vocab()     )
									self.assertIsInstance(processor.tokenizer	,			_UpperCAmelCase     )

									self.assertEqual(processor.image_processor.to_json_string()	,			image_processor_add_kwargs.to_json_string()     )
									self.assertIsInstance(processor.image_processor	,			_UpperCAmelCase     )







					def  _lowerCamelCase		(						self     ):
									__a :						Any			       =  self.get_image_processor()
									__a :						Tuple			       =  self.get_tokenizer()

									__a :						Optional[Any]			       =  CLIPProcessor(tokenizer=_UpperCAmelCase	,			image_processor=_UpperCAmelCase     )

									__a :						Optional[int]			       =  self.prepare_image_inputs()

									__a :						int			       =  image_processor(_UpperCAmelCase	,			return_tensors='''np'''     )
									__a :						Dict			       =  processor(images=_UpperCAmelCase	,			return_tensors='''np'''     )

									for key in input_image_proc.keys():
													self.assertAlmostEqual(input_image_proc[key].sum()	,			input_processor[key].sum()	,			delta=1e-2     )







					def  _lowerCamelCase		(						self     ):
									__a :						Union[str, Any]			       =  self.get_image_processor()
									__a :						List[str]			       =  self.get_tokenizer()

									__a :						Optional[int]			       =  CLIPProcessor(tokenizer=_UpperCAmelCase	,			image_processor=_UpperCAmelCase     )

									__a :						Optional[Any]			       =  '''lower newer'''

									__a :						List[Any]			       =  processor(text=_UpperCAmelCase     )

									__a :						Optional[Any]			       =  tokenizer(_UpperCAmelCase     )

									for key in encoded_tok.keys():
													self.assertListEqual(encoded_tok[key]	,			encoded_processor[key]     )







					def  _lowerCamelCase		(						self     ):
									__a :						Optional[int]			       =  self.get_image_processor()
									__a :						Tuple			       =  self.get_tokenizer()

									__a :						Dict			       =  CLIPProcessor(tokenizer=_UpperCAmelCase	,			image_processor=_UpperCAmelCase     )

									__a :						int			       =  '''lower newer'''
									__a :						List[Any]			       =  self.prepare_image_inputs()

									__a :						Union[str, Any]			       =  processor(text=_UpperCAmelCase	,			images=_UpperCAmelCase     )

									self.assertListEqual(list(inputs.keys()     )	,			['''input_ids''', '''attention_mask''', '''pixel_values''']     )

									# test if it raises when no input is passed
									with pytest.raises(_UpperCAmelCase     ):
													processor()







					def  _lowerCamelCase		(						self     ):
									__a :						Union[str, Any]			       =  self.get_image_processor()
									__a :						str			       =  self.get_tokenizer()

									__a :						Optional[int]			       =  CLIPProcessor(tokenizer=_UpperCAmelCase	,			image_processor=_UpperCAmelCase     )

									__a :						Optional[Any]			       =  [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

									__a :						int			       =  processor.batch_decode(_UpperCAmelCase     )
									__a :						Dict			       =  tokenizer.batch_decode(_UpperCAmelCase     )

									self.assertListEqual(_UpperCAmelCase	,			_UpperCAmelCase     )







					def  _lowerCamelCase		(						self     ):
									__a :						Tuple			       =  self.get_image_processor()
									__a :						Optional[int]			       =  self.get_tokenizer()

									__a :						Any			       =  CLIPProcessor(tokenizer=_UpperCAmelCase	,			image_processor=_UpperCAmelCase     )

									__a :						Union[str, Any]			       =  '''lower newer'''
									__a :						Tuple			       =  self.prepare_image_inputs()

									__a :						Dict			       =  processor(text=_UpperCAmelCase	,			images=_UpperCAmelCase     )

									self.assertListEqual(list(inputs.keys()     )	,			processor.model_input_names     )






"""simple docstring"""






import html

from ...feature_extraction_utils import BatchFeature, FeatureExtractionMixin
from ...utils import is_bsa_available, logging, requires_backends


if is_bsa_available():
		import bsa
		from bsa import BeautifulSoup


A 			=  logging.get_logger(__name__)




class 				__lowercase			(			_UpperCamelCase							):







					'''simple docstring'''




					def __init__(						self	,			**_UpperCAmelCase     ):
									requires_backends(self	,			['''bs4''']     )
									super().__init__(**_UpperCAmelCase     )







					def  _lowerCamelCase		(						self	,			_UpperCAmelCase     ):
									__a :						Tuple			       =  []
									__a :						Dict			       =  []
									__a :						int			       =  element if element.name else element.parent
									for parent in child.parents:  # type: bs4.element.Tag
													__a :						Dict			       =  parent.find_all(child.name	,			recursive=_UpperCAmelCase     )
													xpath_tags.append(child.name     )
													xpath_subscripts.append(
													    0 if 1 == len(_UpperCAmelCase     ) else next(i for i, s in enumerate(_UpperCAmelCase	,			1     ) if s is child     )     )
													__a :						Any			       =  parent
									xpath_tags.reverse()
									xpath_subscripts.reverse()
									return xpath_tags, xpath_subscripts







					def  _lowerCamelCase		(						self	,			_UpperCAmelCase     ):
									__a :						Dict			       =  BeautifulSoup(_UpperCAmelCase	,			'''html.parser'''     )

									__a :						Any			       =  []
									__a :						Dict			       =  []
									__a :						Optional[int]			       =  []

									for element in html_code.descendants:
													if type(_UpperCAmelCase     ) == bsa.element.NavigableString:
																	if type(element.parent     ) != bsa.element.Tag:
																					continue

																	__a :						Dict			       =  html.unescape(_UpperCAmelCase     ).strip()
																	if not text_in_this_tag:
																					continue

																	all_doc_strings.append(_UpperCAmelCase     )

																	__a       ,						__a :						List[Any]			       =  self.xpath_soup(_UpperCAmelCase     )
																	stringaxtag_seq.append(_UpperCAmelCase     )
																	stringaxsubs_seq.append(_UpperCAmelCase     )

									if len(_UpperCAmelCase     ) != len(_UpperCAmelCase     ):
													raise ValueError('''Number of doc strings and xtags does not correspond'''     )
									if len(_UpperCAmelCase     ) != len(_UpperCAmelCase     ):
													raise ValueError('''Number of doc strings and xsubs does not correspond'''     )

									return all_doc_strings, stringaxtag_seq, stringaxsubs_seq







					def  _lowerCamelCase		(						self	,			_UpperCAmelCase	,			_UpperCAmelCase     ):
									__a :						List[str]			       =  ''''''
									for tagname, subs in zip(_UpperCAmelCase	,			_UpperCAmelCase     ):
													xpath += f"""/{tagname}"""
													if subs != 0:
																	xpath += f"""[{subs}]"""
									return xpath







					def __call__(						self	,			_UpperCAmelCase     ):
									__a :						Optional[Any]			       =  False

									# Check that strings has a valid type
									if isinstance(_UpperCAmelCase	,			_UpperCAmelCase     ):
													__a :						str			       =  True
									elif isinstance(_UpperCAmelCase	,			(list, tuple)     ):
													if len(_UpperCAmelCase     ) == 0 or isinstance(html_strings[0]	,			_UpperCAmelCase     ):
																	__a :						Optional[Any]			       =  True

									if not valid_strings:
													raise ValueError(
													    '''HTML strings must of type `str`, `List[str]` (batch of examples), '''
													    f"""but is of type {type(_UpperCAmelCase     )}."""     )

									__a :						List[str]			       =  bool(isinstance(_UpperCAmelCase	,			(list, tuple)     ) and (isinstance(html_strings[0]	,			_UpperCAmelCase     ))     )

									if not is_batched:
													__a :						List[Any]			       =  [html_strings]

									# Get nodes + xpaths
									__a :						Tuple			       =  []
									__a :						int			       =  []
									for html_string in html_strings:
													__a       ,						__a       ,						__a :						Any			       =  self.get_three_from_single(_UpperCAmelCase     )
													nodes.append(_UpperCAmelCase     )
													__a :						List[Any]			       =  []
													for node, tag_list, sub_list in zip(_UpperCAmelCase	,			_UpperCAmelCase	,			_UpperCAmelCase     ):
																	__a :						Optional[int]			       =  self.construct_xpath(_UpperCAmelCase	,			_UpperCAmelCase     )
																	xpath_strings.append(_UpperCAmelCase     )
													xpaths.append(_UpperCAmelCase     )

									# return as Dict
									__a :						int			       =  {'''nodes''': nodes, '''xpaths''': xpaths}
									__a :						Any			       =  BatchFeature(data=_UpperCAmelCase	,			tensor_type=_UpperCAmelCase     )

									return encoded_inputs