



import logging
import os
from dataclasses import dataclass
from enum import Enum
from typing import List, Optional, Union

from filelock import FileLock

from transformers import PreTrainedTokenizer, is_tf_available, is_torch_available


lowercase_										=  logging.getLogger(__name__)
@dataclass
class 					_snake_case      :
				UpperCamelCase__					:       Dict  		=4_2
				UpperCamelCase__					:       Dict  		=4_2
				UpperCamelCase__					:       Tuple  		=4_2
@dataclass
class 					_snake_case      :
				UpperCamelCase__					:       Optional[int]  		=4_2
				UpperCamelCase__					:       int  		=4_2
				UpperCamelCase__					:       List[Any]  		=None
				UpperCamelCase__					:       Optional[Any]  		=None
class 					_snake_case      (       _A):
				UpperCamelCase__					:       Tuple  		="""train"""
				UpperCamelCase__					:       Optional[Any]  		="""dev"""
				UpperCamelCase__					:       Optional[Any]  		="""test"""
class 					_snake_case      :




				@staticmethod
				def 				A__   (    __lowercase       :    List[Any],       __lowercase       :    Union[Split, str]    ):
							raise NotImplementedError




				@staticmethod
				def 				A__   (    __lowercase       :    str    ):
							raise NotImplementedError




				@staticmethod
				def 				A__   (    __lowercase       :    List[InputExample],       __lowercase       :    List[str],       __lowercase       :    int,       __lowercase       :    PreTrainedTokenizer,       __lowercase       :    Optional[Any]=False,       __lowercase       :    Tuple="[CLS]",       __lowercase       :    List[str]=1,       __lowercase       :    str="[SEP]",       __lowercase       :    int=False,       __lowercase       :    Dict=False,       __lowercase       :    List[str]=0,       __lowercase       :    str=0,       __lowercase       :    Optional[Any]=-100,       __lowercase       :    Union[str, Any]=0,       __lowercase       :    Optional[int]=True,       ):
							lowercase__       	=					{label: i for i, label in enumerate(UpperCamelCase__    )}

							lowercase__       	=					[]
							for ex_index, example in enumerate(UpperCamelCase__    ):
										if ex_index % 1_0000 == 0:
													logger.info("Writing example %d of %d",       UpperCamelCase__,       len(UpperCamelCase__    )    )

										lowercase__       	=					[]
										lowercase__       	=					[]
										for word, label in zip(example.words,       example.labels    ):
													lowercase__       	=					tokenizer.tokenize(UpperCamelCase__    )

													# bert-base-multilingual-cased sometimes output "nothing ([]) when calling tokenize with just a space.
													if len(UpperCamelCase__    ) > 0:
																tokens.extend(UpperCamelCase__    )
																# Use the real label id for the first token of the word, and padding ids for the remaining tokens
																label_ids.extend([label_map[label]] + [pad_token_label_id] * (len(UpperCamelCase__    ) - 1)    )

																  # Account for [CLS] and [SEP] with "- 2" and with "- 3" for RoBERTa.
										lowercase__       	=					tokenizer.num_special_tokens_to_add()
										if len(UpperCamelCase__    ) > max_seq_length - special_tokens_count:
													lowercase__       	=					tokens[: (max_seq_length - special_tokens_count)]
													lowercase__       	=					label_ids[: (max_seq_length - special_tokens_count)]

										# The convention in BERT is:
										# (a) For sequence pairs:
										#  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
										#  type_ids:   0   0  0    0    0     0       0   0   1  1  1  1   1   1
										# (b) For single sequences:
										#  tokens:   [CLS] the dog is hairy . [SEP]
										#  type_ids:   0   0   0   0  0     0   0
										#
										# Where "type_ids" are used to indicate whether this is the first
										# sequence or the second sequence. The embedding vectors for `type=0` and
										# `type=1` were learned during pre-training and are added to the wordpiece
										# embedding vector (and position vector). This is not *strictly* necessary
										# since the [SEP] token unambiguously separates the sequences, but it makes
										# it easier for the model to learn the concept of sequences.
										#
										# For classification tasks, the first vector (corresponding to [CLS]) is
										# used as the "sentence vector". Note that this only makes sense because
										# the entire model is fine-tuned.
										tokens += [sep_token]
										label_ids += [pad_token_label_id]
										if sep_token_extra:
													# roberta uses an extra separator b/w pairs of sentences
													tokens += [sep_token]
													label_ids += [pad_token_label_id]
										lowercase__       	=					[sequence_a_segment_id] * len(UpperCamelCase__    )

										if cls_token_at_end:
													tokens += [cls_token]
													label_ids += [pad_token_label_id]
													segment_ids += [cls_token_segment_id]
										else:
													lowercase__       	=					[cls_token] + tokens
													lowercase__       	=					[pad_token_label_id] + label_ids
													lowercase__       	=					[cls_token_segment_id] + segment_ids

										lowercase__       	=					tokenizer.convert_tokens_to_ids(UpperCamelCase__    )

										# The mask has 1 for real tokens and 0 for padding tokens. Only real
										# tokens are attended to.
										lowercase__       	=					[1 if mask_padding_with_zero else 0] * len(UpperCamelCase__    )

										# Zero-pad up to the sequence length.
										lowercase__       	=					max_seq_length - len(UpperCamelCase__    )
										if pad_on_left:
													lowercase__       	=					([pad_token] * padding_length) + input_ids
													lowercase__       	=					([0 if mask_padding_with_zero else 1] * padding_length) + input_mask
													lowercase__       	=					([pad_token_segment_id] * padding_length) + segment_ids
													lowercase__       	=					([pad_token_label_id] * padding_length) + label_ids
										else:
													input_ids += [pad_token] * padding_length
													input_mask += [0 if mask_padding_with_zero else 1] * padding_length
													segment_ids += [pad_token_segment_id] * padding_length
													label_ids += [pad_token_label_id] * padding_length

										assert len(UpperCamelCase__    ) == max_seq_length
										assert len(UpperCamelCase__    ) == max_seq_length
										assert len(UpperCamelCase__    ) == max_seq_length
										assert len(UpperCamelCase__    ) == max_seq_length

										if ex_index < 5:
													logger.info("*** Example ***"    )
													logger.info("guid: %s",       example.guid    )
													logger.info("tokens: %s",       " ".join([str(UpperCamelCase__    ) for x in tokens]    )    )
													logger.info("input_ids: %s",       " ".join([str(UpperCamelCase__    ) for x in input_ids]    )    )
													logger.info("input_mask: %s",       " ".join([str(UpperCamelCase__    ) for x in input_mask]    )    )
													logger.info("segment_ids: %s",       " ".join([str(UpperCamelCase__    ) for x in segment_ids]    )    )
													logger.info("label_ids: %s",       " ".join([str(UpperCamelCase__    ) for x in label_ids]    )    )

										if "token_type_ids" not in tokenizer.model_input_names:
													lowercase__       	=					None

										features.append(
										    InputFeatures(
										        input_ids=UpperCamelCase__,       attention_mask=UpperCamelCase__,       token_type_ids=UpperCamelCase__,       label_ids=UpperCamelCase__    )    )
							return features


if is_torch_available():
							import torch
							from torch import nn
							from torch.utils.data import Dataset
							class 					_snake_case      (       _A):
											UpperCamelCase__					:       List[Any]  		=4_2
											UpperCamelCase__					:       int  		=nn.CrossEntropyLoss().ignore_index




											def __init__(    self       :    Optional[Any],       __lowercase       :    TokenClassificationTask,       __lowercase       :    str,       __lowercase       :    PreTrainedTokenizer,       __lowercase       :    List[str],       __lowercase       :    str,       __lowercase       :    Optional[int] = None,       __lowercase       :    Optional[Any]=False,       __lowercase       :    Split = Split.train,       ):
														lowercase__       	=					os.path.join(
														    UpperCamelCase__,       "cached_{}_{}_{}".format(mode.value,       tokenizer.__class__.__name__,       str(UpperCamelCase__    )    ),       )

														# Make sure only the first process in distributed training processes the dataset,
														# and the others will use the cache.
														lowercase__       	=					cached_features_file + ".lock"
														with FileLock(UpperCamelCase__    ):
																	if os.path.exists(UpperCamelCase__    ) and not overwrite_cache:
																				logger.info(F'''Loading features from cached file {cached_features_file}'''    )
																				lowercase__       	=					torch.load(UpperCamelCase__    )
																	else:
																				logger.info(F'''Creating features from dataset file at {data_dir}'''    )
																				lowercase__       	=					token_classification_task.read_examples_from_file(UpperCamelCase__,       UpperCamelCase__    )
																				# TODO clean up all this to leverage built-in features of tokenizers
																				lowercase__       	=					token_classification_task.convert_examples_to_features(
																				    UpperCamelCase__,       UpperCamelCase__,       UpperCamelCase__,       UpperCamelCase__,       cls_token_at_end=bool(model_type in ["xlnet"]    ),       cls_token=tokenizer.cls_token,       cls_token_segment_id=2 if model_type in ["xlnet"] else 0,       sep_token=tokenizer.sep_token,       sep_token_extra=UpperCamelCase__,       pad_on_left=bool(tokenizer.padding_side == "left"    ),       pad_token=tokenizer.pad_token_id,       pad_token_segment_id=tokenizer.pad_token_type_id,       pad_token_label_id=self.pad_token_label_id,       )
																				logger.info(F'''Saving features into cached file {cached_features_file}'''    )
																				torch.save(self.features,       UpperCamelCase__    )




											def __len__(    self       :    List[str]    ):
														return len(self.features    )




											def __getitem__(    self       :    Tuple,       __lowercase       :    Union[str, Any]    ):
														return self.features[i]


if is_tf_available():
							import tensorflow as tf
							class 					_snake_case      :
											UpperCamelCase__					:       List[Any]  		=4_2
											UpperCamelCase__					:       Optional[int]  		=-1_0_0




											def __init__(    self       :    List[str],       __lowercase       :    TokenClassificationTask,       __lowercase       :    str,       __lowercase       :    PreTrainedTokenizer,       __lowercase       :    List[str],       __lowercase       :    str,       __lowercase       :    Optional[int] = None,       __lowercase       :    List[Any]=False,       __lowercase       :    Split = Split.train,       ):
														lowercase__       	=					token_classification_task.read_examples_from_file(UpperCamelCase__,       UpperCamelCase__    )
														# TODO clean up all this to leverage built-in features of tokenizers
														lowercase__       	=					token_classification_task.convert_examples_to_features(
														    UpperCamelCase__,       UpperCamelCase__,       UpperCamelCase__,       UpperCamelCase__,       cls_token_at_end=bool(model_type in ["xlnet"]    ),       cls_token=tokenizer.cls_token,       cls_token_segment_id=2 if model_type in ["xlnet"] else 0,       sep_token=tokenizer.sep_token,       sep_token_extra=UpperCamelCase__,       pad_on_left=bool(tokenizer.padding_side == "left"    ),       pad_token=tokenizer.pad_token_id,       pad_token_segment_id=tokenizer.pad_token_type_id,       pad_token_label_id=self.pad_token_label_id,       )

														def gen():
																	for ex in self.features:
																				if ex.token_type_ids is None:
																							yield (
																							    {"input_ids": ex.input_ids, "attention_mask": ex.attention_mask},
																							    ex.label_ids,
																							)
																				else:
																							yield (
																							    {
																							        "input_ids": ex.input_ids,
																							        "attention_mask": ex.attention_mask,
																							        "token_type_ids": ex.token_type_ids,
																							    },
																							    ex.label_ids,
																							)

														if "token_type_ids" not in tokenizer.model_input_names:
																	lowercase__       	=					tf.data.Dataset.from_generator(
																	    UpperCamelCase__,       ({"input_ids": tf.intaa, "attention_mask": tf.intaa}, tf.intaa),       (
																	        {"input_ids": tf.TensorShape([None]    ), "attention_mask": tf.TensorShape([None]    )},
																	        tf.TensorShape([None]    ),
																	    ),       )
														else:
																	lowercase__       	=					tf.data.Dataset.from_generator(
																	    UpperCamelCase__,       ({"input_ids": tf.intaa, "attention_mask": tf.intaa, "token_type_ids": tf.intaa}, tf.intaa),       (
																	        {
																	            "input_ids": tf.TensorShape([None]    ),
																	            "attention_mask": tf.TensorShape([None]    ),
																	            "token_type_ids": tf.TensorShape([None]    ),
																	        },
																	        tf.TensorShape([None]    ),
																	    ),       )




											def 				A__   (    self       :    Union[str, Any]    ):
														lowercase__       	=					self.dataset.apply(tf.data.experimental.assert_cardinality(len(self.features    )    )    )

														return self.dataset




											def __len__(    self       :    Tuple    ):
														return len(self.features    )




											def __getitem__(    self       :    Optional[Any],       __lowercase       :    Union[str, Any]    ):
														return self.features[i]






import html

from ...feature_extraction_utils import BatchFeature, FeatureExtractionMixin
from ...utils import is_bsa_available, logging, requires_backends


if is_bsa_available():
    import bsa
    from bsa import BeautifulSoup


__lowerCAmelCase     :	int       			=  logging.get_logger(__name__)




class 			UpperCAmelCase_     (					_A	):


  '''simple docstring'''







  def __init__(					self     :					Union[str, Any]     ,				**UpperCamelCase__     :					Union[str, Any]       )     ->      Union[str, Any]:







    """simple docstring"""






    requires_backends(self     ,				["""bs4"""]       )
    super().__init__(**UpperCamelCase__       )
  def 				_lowercase				(					self     :					Dict     ,				UpperCamelCase__     :					Union[str, Any]       )     ->      Optional[Any]:







    """simple docstring"""






    __magic_name__								=			[]
    __magic_name__								=			[]
    __magic_name__								=			element if element.name else element.parent
    for parent in child.parents:  # type: bs4.element.Tag
      __magic_name__								=			parent.find_all(child.name     ,				recursive=UpperCamelCase__       )
      xpath_tags.append(child.name       )
      xpath_subscripts.append(
          0 if 1 == len(UpperCamelCase__       ) else next(i for i, s in enumerate(UpperCamelCase__     ,				1       ) if s is child       )       )
      __magic_name__								=			parent
    xpath_tags.reverse()
    xpath_subscripts.reverse()
    return xpath_tags, xpath_subscripts
  def 				_lowercase				(					self     :					Union[str, Any]     ,				UpperCamelCase__     :					List[Any]       )     ->      str:







    """simple docstring"""






    __magic_name__								=			BeautifulSoup(UpperCamelCase__     ,				"""html.parser"""       )

    __magic_name__								=			[]
    __magic_name__								=			[]
    __magic_name__								=			[]

    for element in html_code.descendants:
      if type(UpperCamelCase__       ) == bsa.element.NavigableString:
        if type(element.parent       ) != bsa.element.Tag:
          continue

        __magic_name__								=			html.unescape(UpperCamelCase__       ).strip()
        if not text_in_this_tag:
          continue

        all_doc_strings.append(UpperCamelCase__       )

        __magic_name__	,       __magic_name__								=			self.xpath_soup(UpperCamelCase__       )
        stringaxtag_seq.append(UpperCamelCase__       )
        stringaxsubs_seq.append(UpperCamelCase__       )

    if len(UpperCamelCase__       ) != len(UpperCamelCase__       ):
      raise ValueError("""Number of doc strings and xtags does not correspond"""       )
    if len(UpperCamelCase__       ) != len(UpperCamelCase__       ):
      raise ValueError("""Number of doc strings and xsubs does not correspond"""       )

    return all_doc_strings, stringaxtag_seq, stringaxsubs_seq
  def 				_lowercase				(					self     :					List[Any]     ,				UpperCamelCase__     :					Dict     ,				UpperCamelCase__     :					List[Any]       )     ->      Optional[Any]:







    """simple docstring"""






    __magic_name__								=			""""""
    for tagname, subs in zip(UpperCamelCase__     ,				UpperCamelCase__       ):
      xpath += F'''/{tagname}'''
      if subs != 0:
        xpath += F'''[{subs}]'''
    return xpath





  def __call__(					self     :					str     ,				UpperCamelCase__     :					Tuple       )     ->      BatchFeature:







    """simple docstring"""






    __magic_name__								=			False

    # Check that strings has a valid type
    if isinstance(UpperCamelCase__     ,				UpperCamelCase__       ):
      __magic_name__								=			True
    elif isinstance(UpperCamelCase__     ,				(list, tuple)       ):
      if len(UpperCamelCase__       ) == 0 or isinstance(html_strings[0]     ,				UpperCamelCase__       ):
        __magic_name__								=			True

    if not valid_strings:
      raise ValueError(
          """HTML strings must of type `str`, `List[str]` (batch of examples), """
          F'''but is of type {type(UpperCamelCase__       )}.'''       )

    __magic_name__								=			bool(isinstance(UpperCamelCase__     ,				(list, tuple)       ) and (isinstance(html_strings[0]     ,				UpperCamelCase__       ))       )

    if not is_batched:
      __magic_name__								=			[html_strings]

    # Get nodes + xpaths
    __magic_name__								=			[]
    __magic_name__								=			[]
    for html_string in html_strings:
      __magic_name__	,       __magic_name__	,       __magic_name__								=			self.get_three_from_single(UpperCamelCase__       )
      nodes.append(UpperCamelCase__       )
      __magic_name__								=			[]
      for node, tag_list, sub_list in zip(UpperCamelCase__     ,				UpperCamelCase__     ,				UpperCamelCase__       ):
        __magic_name__								=			self.construct_xpath(UpperCamelCase__     ,				UpperCamelCase__       )
        xpath_strings.append(UpperCamelCase__       )
      xpaths.append(UpperCamelCase__       )

    # return as Dict
    __magic_name__								=			{"""nodes""": nodes, """xpaths""": xpaths}
    __magic_name__								=			BatchFeature(data=UpperCamelCase__     ,				tensor_type=UpperCamelCase__       )

    return encoded_inputs

