



'''simple docstring'''

import unicodedata
from dataclasses import dataclass
from typing import Optional, Union

import numpy as np

from transformers.data.data_collator import DataCollatorMixin
from transformers.file_utils import PaddingStrategy
from transformers.tokenization_utils_base import PreTrainedTokenizerBase



def 	__lowerCamelCase	( lowerCAmelCase_		,     lowerCAmelCase_		,     lowerCAmelCase_		,     lowerCAmelCase_    )     -> str:
		if isinstance(lowerCAmelCase_		,     lowerCAmelCase_    ):
				_a	: List[str]								=				np.full((len(lowerCAmelCase_    ), sequence_length, 2)		,     lowerCAmelCase_    )
		else:
				_a	: str								=				np.full((len(lowerCAmelCase_    ), sequence_length)		,     lowerCAmelCase_    )

		for i, tensor in enumerate(lowerCAmelCase_    ):
				if padding_side == "right":
						if isinstance(lowerCAmelCase_		,     lowerCAmelCase_    ):
								_a	: str								=				tensor[:sequence_length]
						else:
								_a	: Optional[int]								=				tensor[:sequence_length]
				else:
						if isinstance(lowerCAmelCase_		,     lowerCAmelCase_    ):
								_a	: List[str]								=				tensor[:sequence_length]
						else:
								_a	: Optional[Any]								=				tensor[:sequence_length]

		return out_tensor.tolist()



def 	__lowerCamelCase	( lowerCAmelCase_    )     -> Dict:
		_a	: Union[str, Any]								=				ord(lowerCAmelCase_    )
		if (cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126):
				return True
		_a	: List[Any]								=				unicodedata.category(lowerCAmelCase_    )
		if cat.startswith('P'    ):
				return True
		return False

@dataclass
class       __magic_name__		(       _UpperCamelCase  ):
							lowerCAmelCase			:       PreTrainedTokenizerBase
							lowerCAmelCase			:       Union[bool, str, PaddingStrategy]  							=   True
							lowerCAmelCase			:       Optional[int]  							=   None
							lowerCAmelCase			:       Optional[int]  							=   None
							lowerCAmelCase			:       int  							=   -1_0_0
							lowerCAmelCase			:       str  							=   "pt"







							def 				__lowercase     (				self   :			Union[str, Any]      ,_UpperCAmelCase   :			List[Any]						):
									import torch

									_a	: Optional[int]								=				'label' if 'label' in features[0].keys() else 'labels'
									_a	: Any								=				[feature[label_name] for feature in features] if label_name in features[0].keys() else None
									_a	: int								=				self.tokenizer.pad(
									    _UpperCAmelCase      ,padding=self.padding      ,max_length=self.max_length      ,pad_to_multiple_of=self.pad_to_multiple_of      ,return_tensors='pt' if labels is None else None      ,)

									if labels is None:
											return batch

									_a	: Any								=				torch.tensor(batch['entity_ids']						).shape[1]
									_a	: Dict								=				self.tokenizer.padding_side
									if padding_side == "right":
											_a	: Tuple								=				[
											    list(_UpperCAmelCase						) + [self.label_pad_token_id] * (sequence_length - len(_UpperCAmelCase						)) for label in labels
											]
									else:
											_a	: List[str]								=				[
											    [self.label_pad_token_id] * (sequence_length - len(_UpperCAmelCase						)) + list(_UpperCAmelCase						) for label in labels
											]

									_a	: Union[str, Any]								=				[feature['ner_tags'] for feature in features]
									_a	: List[str]								=				padding_tensor(_UpperCAmelCase      ,-1      ,_UpperCAmelCase      ,_UpperCAmelCase						)
									_a	: str								=				[feature['original_entity_spans'] for feature in features]
									_a	: str								=				padding_tensor(_UpperCAmelCase      ,(-1, -1)      ,_UpperCAmelCase      ,_UpperCAmelCase						)
									_a	: Dict								=				{k: torch.tensor(_UpperCAmelCase      ,dtype=torch.intaa						) for k, v in batch.items()}

									return batch









'''simple docstring'''

from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tensorflow_text_available, is_torch_available


__lowerCAmelCase		     =  {
    '''configuration_ernie''': ['''ERNIE_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''ErnieConfig''', '''ErnieOnnxConfig'''],
}

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__lowerCAmelCase		     =  [
							    '''ERNIE_PRETRAINED_MODEL_ARCHIVE_LIST''',
							    '''ErnieForCausalLM''',
							    '''ErnieForMaskedLM''',
							    '''ErnieForMultipleChoice''',
							    '''ErnieForNextSentencePrediction''',
							    '''ErnieForPreTraining''',
							    '''ErnieForQuestionAnswering''',
							    '''ErnieForSequenceClassification''',
							    '''ErnieForTokenClassification''',
							    '''ErnieModel''',
							    '''ErniePreTrainedModel''',
							]

if TYPE_CHECKING:
							from .configuration_ernie import ERNIE_PRETRAINED_CONFIG_ARCHIVE_MAP, ErnieConfig, ErnieOnnxConfig

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_ernie import (
														    ERNIE_PRETRAINED_MODEL_ARCHIVE_LIST,
														    ErnieForCausalLM,
														    ErnieForMaskedLM,
														    ErnieForMultipleChoice,
														    ErnieForNextSentencePrediction,
														    ErnieForPreTraining,
														    ErnieForQuestionAnswering,
														    ErnieForSequenceClassification,
														    ErnieForTokenClassification,
														    ErnieModel,
														    ErniePreTrainedModel,
														)

else:
							import sys

							__lowerCAmelCase		     =  _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)




