






import html

from ...feature_extraction_utils import BatchFeature, FeatureExtractionMixin
from ...utils import is_bsa_available, logging, requires_backends


if is_bsa_available():
							import bsa
							from bsa import BeautifulSoup


SCREAMING_SNAKE_CASE__				     =			logging.get_logger(__name__)






class 			_UpperCamelCase(       __lowerCamelCase		):

		def __init__(					self		:    Union[str, Any]    ,       **SCREAMING_SNAKE_CASE__		:    List[Any]    ):



					'''simple docstring'''



					requires_backends(self    ,       ['bs4']    )
					super().__init__(**SCREAMING_SNAKE_CASE__    )

		def    __lowerCAmelCase  (					self		:    str    ,       SCREAMING_SNAKE_CASE__		:    Optional[int]    ):



					'''simple docstring'''



					__a						:  Any									=     []
					__a						:  Tuple									=     []
					__a						:  int									=     element if element.name else element.parent
					for parent in child.parents:  # type: bs4.element.Tag
								__a						:  Tuple									=     parent.find_all(child.name    ,       recursive=SCREAMING_SNAKE_CASE__    )
								xpath_tags.append(child.name    )
								xpath_subscripts.append(
								    0 if 1 == len(SCREAMING_SNAKE_CASE__    ) else next(i for i, s in enumerate(SCREAMING_SNAKE_CASE__    ,       1    ) if s is child    )    )
								__a						:  Union[str, Any]									=     parent
					xpath_tags.reverse()
					xpath_subscripts.reverse()
					return xpath_tags, xpath_subscripts

		def    __lowerCAmelCase  (					self		:    Optional[int]    ,       SCREAMING_SNAKE_CASE__		:    Any    ):



					'''simple docstring'''



					__a						:  Union[str, Any]									=     BeautifulSoup(SCREAMING_SNAKE_CASE__    ,       'html.parser'    )

					__a						:  Any									=     []
					__a						:  int									=     []
					__a						:  Union[str, Any]									=     []

					for element in html_code.descendants:
								if type(SCREAMING_SNAKE_CASE__    ) == bsa.element.NavigableString:
											if type(element.parent    ) != bsa.element.Tag:
														continue

											__a						:  str									=     html.unescape(SCREAMING_SNAKE_CASE__    ).strip()
											if not text_in_this_tag:
														continue

											all_doc_strings.append(SCREAMING_SNAKE_CASE__    )

											__a						,					__a						:  Optional[Any]									=     self.xpath_soup(SCREAMING_SNAKE_CASE__    )
											stringaxtag_seq.append(SCREAMING_SNAKE_CASE__    )
											stringaxsubs_seq.append(SCREAMING_SNAKE_CASE__    )

					if len(SCREAMING_SNAKE_CASE__    ) != len(SCREAMING_SNAKE_CASE__    ):
								raise ValueError('Number of doc strings and xtags does not correspond'    )
					if len(SCREAMING_SNAKE_CASE__    ) != len(SCREAMING_SNAKE_CASE__    ):
								raise ValueError('Number of doc strings and xsubs does not correspond'    )

					return all_doc_strings, stringaxtag_seq, stringaxsubs_seq

		def    __lowerCAmelCase  (					self		:    Optional[Any]    ,       SCREAMING_SNAKE_CASE__		:    Optional[Any]    ,       SCREAMING_SNAKE_CASE__		:    Optional[Any]    ):



					'''simple docstring'''



					__a						:  int									=     ''
					for tagname, subs in zip(SCREAMING_SNAKE_CASE__    ,       SCREAMING_SNAKE_CASE__    ):
								xpath += f'''/{tagname}'''
								if subs != 0:
											xpath += f'''[{subs}]'''
					return xpath



		def __call__(					self		:    str    ,       SCREAMING_SNAKE_CASE__		:    Optional[Any]    ):



					'''simple docstring'''



					__a						:  Union[str, Any]									=     False

					# Check that strings has a valid type
					if isinstance(SCREAMING_SNAKE_CASE__    ,       SCREAMING_SNAKE_CASE__    ):
								__a						:  Union[str, Any]									=     True
					elif isinstance(SCREAMING_SNAKE_CASE__    ,       (list, tuple)    ):
								if len(SCREAMING_SNAKE_CASE__    ) == 0 or isinstance(html_strings[0]    ,       SCREAMING_SNAKE_CASE__    ):
											__a						:  str									=     True

					if not valid_strings:
								raise ValueError(
								    'HTML strings must of type `str`, `List[str]` (batch of examples), '
								    f'''but is of type {type(SCREAMING_SNAKE_CASE__    )}.'''    )

					__a						:  Dict									=     bool(isinstance(SCREAMING_SNAKE_CASE__    ,       (list, tuple)    ) and (isinstance(html_strings[0]    ,       SCREAMING_SNAKE_CASE__    ))    )

					if not is_batched:
								__a						:  Any									=     [html_strings]

					# Get nodes + xpaths
					__a						:  str									=     []
					__a						:  List[str]									=     []
					for html_string in html_strings:
								__a						,					__a						,					__a						:  Optional[Any]									=     self.get_three_from_single(SCREAMING_SNAKE_CASE__    )
								nodes.append(SCREAMING_SNAKE_CASE__    )
								__a						:  Tuple									=     []
								for node, tag_list, sub_list in zip(SCREAMING_SNAKE_CASE__    ,       SCREAMING_SNAKE_CASE__    ,       SCREAMING_SNAKE_CASE__    ):
											__a						:  Tuple									=     self.construct_xpath(SCREAMING_SNAKE_CASE__    ,       SCREAMING_SNAKE_CASE__    )
											xpath_strings.append(SCREAMING_SNAKE_CASE__    )
								xpaths.append(SCREAMING_SNAKE_CASE__    )

					# return as Dict
					__a						:  str									=     {'nodes': nodes, 'xpaths': xpaths}
					__a						:  Tuple									=     BatchFeature(data=SCREAMING_SNAKE_CASE__    ,       tensor_type=SCREAMING_SNAKE_CASE__    )

					return encoded_inputs











'''simple docstring'''
from __future__ import annotations

import unittest

from transformers import FunnelConfig, is_tf_available
from transformers.testing_utils import require_tf

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
							import tensorflow as tf

							from transformers import (
							    TFFunnelBaseModel,
							    TFFunnelForMaskedLM,
							    TFFunnelForMultipleChoice,
							    TFFunnelForPreTraining,
							    TFFunnelForQuestionAnswering,
							    TFFunnelForSequenceClassification,
							    TFFunnelForTokenClassification,
							    TFFunnelModel,
							)
class   __UpperCAmelCase				:



					'''simple docstring'''





					def __init__(	self     :						str      ,       _lowercase     :						List[str]      ,       _lowercase     :						Dict=13      ,       _lowercase     :						List[str]=7      ,       _lowercase     :						Union[str, Any]=True      ,       _lowercase     :						int=True      ,       _lowercase     :						int=True      ,       _lowercase     :						Optional[Any]=True      ,       _lowercase     :						List[Any]=99      ,       _lowercase     :						Optional[Any]=[1, 1, 2]      ,       _lowercase     :						Optional[Any]=1      ,       _lowercase     :						List[str]=32      ,       _lowercase     :						Dict=4      ,       _lowercase     :						List[str]=8      ,       _lowercase     :						List[str]=37      ,       _lowercase     :						int="gelu_new"      ,       _lowercase     :						Optional[int]=0.1      ,       _lowercase     :						List[str]=0.1      ,       _lowercase     :						Union[str, Any]=0.0      ,       _lowercase     :						str=512      ,       _lowercase     :						Optional[Any]=3      ,       _lowercase     :						str=0.02      ,       _lowercase     :						Union[str, Any]=3      ,       _lowercase     :						Optional[Any]=4      ,       _lowercase     :						Tuple=None      ,       _lowercase     :						List[Any]=False      ,       )    ->							List[str]:
								A_									= parent
								A_									= batch_size
								A_									= seq_length
								A_									= is_training
								A_									= use_input_mask
								A_									= use_token_type_ids
								A_									= use_labels
								A_									= vocab_size
								A_									= block_sizes
								A_									= num_decoder_layers
								A_									= d_model
								A_									= n_head
								A_									= d_head
								A_									= d_inner
								A_									= hidden_act
								A_									= hidden_dropout
								A_									= attention_dropout
								A_									= activation_dropout
								A_									= max_position_embeddings
								A_									= type_vocab_size
								A_									= 2
								A_									= num_labels
								A_									= num_choices
								A_									= scope
								A_									= initializer_std

								# Used in the tests to check the size of the first attention layer
								A_									= n_head
								# Used in the tests to check the size of the first hidden state
								A_									= self.d_model
								# Used in the tests to check the number of output hidden states/attentions
								A_									= sum(self.block_sizes) + (0 if base else self.num_decoder_layers)
								# FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
								# the last hidden state of the first block (which is the first hidden state of the decoder).
								if not base:
											A_									= self.num_hidden_layers + 2



					def     __snake_case						(	self     :						str)    ->							List[str]:
								A_									= ids_tensor([self.batch_size, self.seq_length]      ,       self.vocab_size)

								A_									= None
								if self.use_input_mask:
											A_									= random_attention_mask([self.batch_size, self.seq_length])

								A_									= None
								if self.use_token_type_ids:
											A_									= ids_tensor([self.batch_size, self.seq_length]      ,       self.type_vocab_size)

								A_									= None
								A_									= None
								A_									= None
								if self.use_labels:
											A_									= ids_tensor([self.batch_size]      ,       self.type_sequence_label_size)
											A_									= ids_tensor([self.batch_size, self.seq_length]      ,       self.num_labels)
											A_									= ids_tensor([self.batch_size]      ,       self.num_choices)

								A_									= FunnelConfig(
								    vocab_size=self.vocab_size      ,       block_sizes=self.block_sizes      ,       num_decoder_layers=self.num_decoder_layers      ,       d_model=self.d_model      ,       n_head=self.n_head      ,       d_head=self.d_head      ,       d_inner=self.d_inner      ,       hidden_act=self.hidden_act      ,       hidden_dropout=self.hidden_dropout      ,       attention_dropout=self.attention_dropout      ,       activation_dropout=self.activation_dropout      ,       max_position_embeddings=self.max_position_embeddings      ,       type_vocab_size=self.type_vocab_size      ,       initializer_std=self.initializer_std      ,       )

								return (
								    config,
								    input_ids,
								    token_type_ids,
								    input_mask,
								    sequence_labels,
								    token_labels,
								    choice_labels,
								)



					def     __snake_case						(	self     :						Tuple      ,       _lowercase     :						Tuple      ,       _lowercase     :						str      ,       _lowercase     :						int      ,       _lowercase     :						Dict      ,       _lowercase     :						Optional[Any]      ,       _lowercase     :						str      ,       _lowercase     :						str      ,       )    ->							Dict:
								A_									= TFFunnelModel(config=_lowercase)
								A_									= {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
								A_									= model(_lowercase)

								A_									= [input_ids, input_mask]
								A_									= model(_lowercase)

								A_									= model(_lowercase)
								self.parent.assertEqual(result.last_hidden_state.shape      ,       (self.batch_size, self.seq_length, self.d_model))

								A_									= False
								A_									= TFFunnelModel(config=_lowercase)
								A_									= model(_lowercase)
								self.parent.assertEqual(result.last_hidden_state.shape      ,       (self.batch_size, self.seq_length, self.d_model))

								A_									= False
								A_									= TFFunnelModel(config=_lowercase)
								A_									= model(_lowercase)
								self.parent.assertEqual(result.last_hidden_state.shape      ,       (self.batch_size, self.seq_length, self.d_model))



					def     __snake_case						(	self     :						Union[str, Any]      ,       _lowercase     :						int      ,       _lowercase     :						Tuple      ,       _lowercase     :						int      ,       _lowercase     :						int      ,       _lowercase     :						Tuple      ,       _lowercase     :						List[str]      ,       _lowercase     :						Optional[int]      ,       )    ->							Dict:
								A_									= TFFunnelBaseModel(config=_lowercase)
								A_									= {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
								A_									= model(_lowercase)

								A_									= [input_ids, input_mask]
								A_									= model(_lowercase)

								A_									= model(_lowercase)
								self.parent.assertEqual(result.last_hidden_state.shape      ,       (self.batch_size, 2, self.d_model))

								A_									= False
								A_									= TFFunnelBaseModel(config=_lowercase)
								A_									= model(_lowercase)
								self.parent.assertEqual(result.last_hidden_state.shape      ,       (self.batch_size, 3, self.d_model))

								A_									= False
								A_									= TFFunnelBaseModel(config=_lowercase)
								A_									= model(_lowercase)
								self.parent.assertEqual(result.last_hidden_state.shape      ,       (self.batch_size, 2, self.d_model))



					def     __snake_case						(	self     :						int      ,       _lowercase     :						List[str]      ,       _lowercase     :						int      ,       _lowercase     :						List[str]      ,       _lowercase     :						str      ,       _lowercase     :						Any      ,       _lowercase     :						Optional[Any]      ,       _lowercase     :						Optional[Any]      ,       )    ->							List[Any]:
								A_									= TFFunnelForPreTraining(config=_lowercase)
								A_									= {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
								A_									= model(_lowercase)
								self.parent.assertEqual(result.logits.shape      ,       (self.batch_size, self.seq_length))



					def     __snake_case						(	self     :						List[str]      ,       _lowercase     :						str      ,       _lowercase     :						List[Any]      ,       _lowercase     :						List[str]      ,       _lowercase     :						List[Any]      ,       _lowercase     :						Union[str, Any]      ,       _lowercase     :						Optional[Any]      ,       _lowercase     :						Tuple      ,       )    ->							Union[str, Any]:
								A_									= TFFunnelForMaskedLM(config=_lowercase)
								A_									= {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
								A_									= model(_lowercase)
								self.parent.assertEqual(result.logits.shape      ,       (self.batch_size, self.seq_length, self.vocab_size))



					def     __snake_case						(	self     :						Union[str, Any]      ,       _lowercase     :						Dict      ,       _lowercase     :						List[Any]      ,       _lowercase     :						Any      ,       _lowercase     :						List[str]      ,       _lowercase     :						int      ,       _lowercase     :						Optional[int]      ,       _lowercase     :						str      ,       )    ->							Dict:
								A_									= self.num_labels
								A_									= TFFunnelForSequenceClassification(config=_lowercase)
								A_									= {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
								A_									= model(_lowercase)
								self.parent.assertEqual(result.logits.shape      ,       (self.batch_size, self.num_labels))



					def     __snake_case						(	self     :						Optional[int]      ,       _lowercase     :						int      ,       _lowercase     :						Optional[Any]      ,       _lowercase     :						Tuple      ,       _lowercase     :						Union[str, Any]      ,       _lowercase     :						Union[str, Any]      ,       _lowercase     :						Union[str, Any]      ,       _lowercase     :						str      ,       )    ->							List[str]:
								A_									= self.num_choices
								A_									= TFFunnelForMultipleChoice(config=_lowercase)
								A_									= tf.tile(tf.expand_dims(_lowercase      ,       1)      ,       (1, self.num_choices, 1))
								A_									= tf.tile(tf.expand_dims(_lowercase      ,       1)      ,       (1, self.num_choices, 1))
								A_									= tf.tile(tf.expand_dims(_lowercase      ,       1)      ,       (1, self.num_choices, 1))
								A_									= {
								    'input_ids': multiple_choice_inputs_ids,
								    'attention_mask': multiple_choice_input_mask,
								    'token_type_ids': multiple_choice_token_type_ids,
								}
								A_									= model(_lowercase)
								self.parent.assertEqual(result.logits.shape      ,       (self.batch_size, self.num_choices))



					def     __snake_case						(	self     :						Any      ,       _lowercase     :						Union[str, Any]      ,       _lowercase     :						Dict      ,       _lowercase     :						List[Any]      ,       _lowercase     :						List[str]      ,       _lowercase     :						int      ,       _lowercase     :						int      ,       _lowercase     :						Dict      ,       )    ->							List[str]:
								A_									= self.num_labels
								A_									= TFFunnelForTokenClassification(config=_lowercase)
								A_									= {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
								A_									= model(_lowercase)
								self.parent.assertEqual(result.logits.shape      ,       (self.batch_size, self.seq_length, self.num_labels))



					def     __snake_case						(	self     :						Optional[int]      ,       _lowercase     :						str      ,       _lowercase     :						Union[str, Any]      ,       _lowercase     :						List[str]      ,       _lowercase     :						List[Any]      ,       _lowercase     :						List[str]      ,       _lowercase     :						List[Any]      ,       _lowercase     :						Union[str, Any]      ,       )    ->							Optional[int]:
								A_									= TFFunnelForQuestionAnswering(config=_lowercase)
								A_									= {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}
								A_									= model(_lowercase)
								self.parent.assertEqual(result.start_logits.shape      ,       (self.batch_size, self.seq_length))
								self.parent.assertEqual(result.end_logits.shape      ,       (self.batch_size, self.seq_length))



					def     __snake_case						(	self     :						str)    ->							Union[str, Any]:
								A_									= self.prepare_config_and_inputs()
								(
								    (
								    A_
								)		, (
								    A_
								)		, (
								    A_
								)		, (
								    A_
								)		, (
								    A_
								)		, (
								    A_
								)		, (
								    A_
								)		, 
								)									= config_and_inputs
								A_									= {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
								return config, inputs_dict





@require_tf
class   __UpperCAmelCase				(     lowerCAmelCase			,lowerCAmelCase			,unittest.TestCase				):



					'''simple docstring'''





					_UpperCamelCase							=		(
					    (
					        TFFunnelModel,
					        TFFunnelForMaskedLM,
					        TFFunnelForPreTraining,
					        TFFunnelForQuestionAnswering,
					        TFFunnelForTokenClassification,
					    )
					    if is_tf_available()
					    else ()
					)
					_UpperCamelCase							=		(
					    {
					        """feature-extraction""": (TFFunnelBaseModel, TFFunnelModel),
					        """fill-mask""": TFFunnelForMaskedLM,
					        """question-answering""": TFFunnelForQuestionAnswering,
					        """text-classification""": TFFunnelForSequenceClassification,
					        """token-classification""": TFFunnelForTokenClassification,
					        """zero-shot""": TFFunnelForSequenceClassification,
					    }
					    if is_tf_available()
					    else {}
					)
					_UpperCamelCase							=		False
					_UpperCamelCase							=		False



					def     __snake_case						(	self     :						Optional[int])    ->							Optional[Any]:
								A_									= TFFunnelModelTester(self)
								A_									= ConfigTester(self      ,       config_class=_lowercase)



					def     __snake_case						(	self     :						Any)    ->							Optional[Any]:
								self.config_tester.run_common_tests()



					def     __snake_case						(	self     :						Union[str, Any])    ->							Union[str, Any]:
								A_									= self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_model(*_lowercase)



					def     __snake_case						(	self     :						List[Any])    ->							str:
								A_									= self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_pretraining(*_lowercase)



					def     __snake_case						(	self     :						Any)    ->							Optional[int]:
								A_									= self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_masked_lm(*_lowercase)



					def     __snake_case						(	self     :						List[str])    ->							Tuple:
								A_									= self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_token_classification(*_lowercase)



					def     __snake_case						(	self     :						int)    ->							Dict:
								A_									= self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_question_answering(*_lowercase)





@require_tf
class   __UpperCAmelCase				(     lowerCAmelCase			,unittest.TestCase				):



					'''simple docstring'''





					_UpperCamelCase							=		(
					    (TFFunnelBaseModel, TFFunnelForMultipleChoice, TFFunnelForSequenceClassification) if is_tf_available() else ()
					)
					_UpperCamelCase							=		False
					_UpperCamelCase							=		False



					def     __snake_case						(	self     :						str)    ->							Dict:
								A_									= TFFunnelModelTester(self      ,       base=_lowercase)
								A_									= ConfigTester(self      ,       config_class=_lowercase)



					def     __snake_case						(	self     :						Optional[Any])    ->							Tuple:
								self.config_tester.run_common_tests()



					def     __snake_case						(	self     :						List[str])    ->							List[str]:
								A_									= self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_base_model(*_lowercase)



					def     __snake_case						(	self     :						Dict)    ->							Dict:
								A_									= self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_sequence_classification(*_lowercase)



					def     __snake_case						(	self     :						Union[str, Any])    ->							int:
								A_									= self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_multiple_choice(*_lowercase)




