



"""simple docstring"""
import csv
import dataclasses
import json
from dataclasses import dataclass
from typing import List, Optional, Union

from ...utils import is_tf_available, is_torch_available, logging


__SCREAMING_SNAKE_CASE     :  Tuple										=					logging.get_logger(__name__)


@dataclass
class 							lowerCamelCase_:





					'''simple docstring'''



					lowercase__      :				str
					lowercase__      :				str
					lowercase__      :				Optional[str]        =       None
					lowercase__      :				Optional[str]        =       None

					def 	snake_case__					(			self      ):
									return json.dumps(dataclasses.asdict(self      ) ,				indent=2      ) + "\n"


@dataclass(frozen=A__			)
class 							lowerCamelCase_:





					'''simple docstring'''



					lowercase__      :				List[int]
					lowercase__      :				Optional[List[int]]        =       None
					lowercase__      :				Optional[List[int]]        =       None
					lowercase__      :				Optional[Union[int, float]]        =       None

					def 	snake_case__					(			self      ):
									return json.dumps(dataclasses.asdict(self      )      ) + "\n"


class 							lowerCamelCase_:





					'''simple docstring'''



					def 	snake_case__					(			self ,				lowerCamelCase__      ):
									raise NotImplementedError()

					def 	snake_case__					(			self ,				lowerCamelCase__      ):
									raise NotImplementedError()

					def 	snake_case__					(			self ,				lowerCamelCase__      ):
									raise NotImplementedError()

					def 	snake_case__					(			self ,				lowerCamelCase__      ):
									raise NotImplementedError()

					def 	snake_case__					(			self      ):
									raise NotImplementedError()

					def 	snake_case__					(			self ,				lowerCamelCase__      ):
									if len(self.get_labels()      ) > 1:
													_lowerCamelCase					     = self.get_labels()[int(example.label      )]
									return example





					@classmethod
					def 	snake_case__					(			cls ,				lowerCamelCase__ ,				lowerCamelCase__=None      ):
									with open(lowerCamelCase__ ,				'''r''' ,				encoding='''utf-8-sig'''      ) as f:
													return list(csv.reader(lowerCamelCase__ ,				delimiter='''\t''' ,				quotechar=lowerCamelCase__      )      )


class 							lowerCamelCase_(      A__			):





					'''simple docstring'''



					def __init__(			self ,				lowerCamelCase__=None ,				lowerCamelCase__=None ,				lowerCamelCase__="classification" ,				lowerCamelCase__=False      ):
									_lowerCamelCase					     = [] if labels is None else labels
									_lowerCamelCase					     = [] if examples is None else examples
									_lowerCamelCase					     = mode
									_lowerCamelCase					     = verbose

					def __len__(			self      ):
									return len(self.examples      )

					def __getitem__(			self ,				lowerCamelCase__      ):
									if isinstance(lowerCamelCase__ ,				lowerCamelCase__      ):
													return SingleSentenceClassificationProcessor(labels=self.labels ,				examples=self.examples[idx]      )
									return self.examples[idx]

					@classmethod
					def 	snake_case__					(			cls ,				lowerCamelCase__ ,				lowerCamelCase__="" ,				lowerCamelCase__=0 ,				lowerCamelCase__=1 ,				lowerCamelCase__=None ,				lowerCamelCase__=False ,				**lowerCamelCase__      ):
									_lowerCamelCase					     = cls(**lowerCamelCase__      )
									processor.add_examples_from_csv(
									    lowerCamelCase__ ,				split_name=lowerCamelCase__ ,				column_label=lowerCamelCase__ ,				column_text=lowerCamelCase__ ,				column_id=lowerCamelCase__ ,				skip_first_row=lowerCamelCase__ ,				overwrite_labels=lowerCamelCase__ ,				overwrite_examples=lowerCamelCase__ ,				)
									return processor

					@classmethod
					def 	snake_case__					(			cls ,				lowerCamelCase__ ,				lowerCamelCase__=None ,				**lowerCamelCase__      ):
									_lowerCamelCase					     = cls(**lowerCamelCase__      )
									processor.add_examples(lowerCamelCase__ ,				labels=lowerCamelCase__      )
									return processor

					def 	snake_case__					(			self ,				lowerCamelCase__ ,				lowerCamelCase__="" ,				lowerCamelCase__=0 ,				lowerCamelCase__=1 ,				lowerCamelCase__=None ,				lowerCamelCase__=False ,				lowerCamelCase__=False ,				lowerCamelCase__=False ,				):
									_lowerCamelCase					     = self._read_tsv(lowerCamelCase__      )
									if skip_first_row:
													_lowerCamelCase					     = lines[1:]
									_lowerCamelCase					     = []
									_lowerCamelCase					     = []
									_lowerCamelCase					     = []
									for i, line in enumerate(lowerCamelCase__      ):
													texts.append(line[column_text]      )
													labels.append(line[column_label]      )
													if column_id is not None:
																	ids.append(line[column_id]      )
													else:
																	_lowerCamelCase					     = F"""{split_name}-{i}""" if split_name else str(lowerCamelCase__      )
																	ids.append(lowerCamelCase__      )

									return self.add_examples(
									    lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__ ,				overwrite_labels=lowerCamelCase__ ,				overwrite_examples=lowerCamelCase__      )

					def 	snake_case__					(			self ,				lowerCamelCase__ ,				lowerCamelCase__=None ,				lowerCamelCase__=None ,				lowerCamelCase__=False ,				lowerCamelCase__=False      ):
									if labels is not None and len(lowerCamelCase__      ) != len(lowerCamelCase__      ):
													raise ValueError(
													    F"""Text and labels have mismatched lengths {len(lowerCamelCase__      )} and {len(lowerCamelCase__      )}"""      )
									if ids is not None and len(lowerCamelCase__      ) != len(lowerCamelCase__      ):
													raise ValueError(F"""Text and ids have mismatched lengths {len(lowerCamelCase__      )} and {len(lowerCamelCase__      )}"""      )
									if ids is None:
													_lowerCamelCase					     = [None] * len(lowerCamelCase__      )
									if labels is None:
													_lowerCamelCase					     = [None] * len(lowerCamelCase__      )
									_lowerCamelCase					     = []
									_lowerCamelCase					     = set()
									for text_or_text_and_label, label, guid in zip(lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__      ):
													if isinstance(lowerCamelCase__ ,				(tuple, list)      ) and label is None:
																	_lowerCamelCase     ,   _lowerCamelCase					     = text_or_text_and_label
													else:
																	_lowerCamelCase					     = text_or_text_and_label
													added_labels.add(lowerCamelCase__      )
													examples.append(InputExample(guid=lowerCamelCase__ ,				text_a=lowerCamelCase__ ,				text_b=lowerCamelCase__ ,				label=lowerCamelCase__      )      )

									# Update examples
									if overwrite_examples:
													_lowerCamelCase					     = examples
									else:
													self.examples.extend(lowerCamelCase__      )

									# Update labels
									if overwrite_labels:
													_lowerCamelCase					     = list(lowerCamelCase__      )
									else:
													_lowerCamelCase					     = list(set(self.labels      ).union(lowerCamelCase__      )      )

									return self.examples





					def 	snake_case__					(			self ,				lowerCamelCase__ ,				lowerCamelCase__=None ,				lowerCamelCase__=False ,				lowerCamelCase__=0 ,				lowerCamelCase__=True ,				lowerCamelCase__=None ,				):
									if max_length is None:
													_lowerCamelCase					     = tokenizer.max_len

									_lowerCamelCase					     = {label: i for i, label in enumerate(self.labels      )}

									_lowerCamelCase					     = []
									for ex_index, example in enumerate(self.examples      ):
													if ex_index % 1_0_0_0_0 == 0:
																	logger.info(F"""Tokenizing example {ex_index}"""      )

													_lowerCamelCase					     = tokenizer.encode(
													    example.text_a ,				add_special_tokens=lowerCamelCase__ ,				max_length=min(lowerCamelCase__ ,				tokenizer.max_len      ) ,				)
													all_input_ids.append(lowerCamelCase__      )

									_lowerCamelCase					     = max(len(lowerCamelCase__      ) for input_ids in all_input_ids      )

									_lowerCamelCase					     = []
									for ex_index, (input_ids, example) in enumerate(zip(lowerCamelCase__ ,				self.examples      )      ):
													if ex_index % 1_0_0_0_0 == 0:
																	logger.info(F"""Writing example {ex_index}/{len(self.examples      )}"""      )
													# The mask has 1 for real tokens and 0 for padding tokens. Only real
													# tokens are attended to.
													_lowerCamelCase					     = [1 if mask_padding_with_zero else 0] * len(lowerCamelCase__      )

													# Zero-pad up to the sequence length.
													_lowerCamelCase					     = batch_length - len(lowerCamelCase__      )
													if pad_on_left:
																	_lowerCamelCase					     = ([pad_token] * padding_length) + input_ids
																	_lowerCamelCase					     = ([0 if mask_padding_with_zero else 1] * padding_length) + attention_mask
													else:
																	_lowerCamelCase					     = input_ids + ([pad_token] * padding_length)
																	_lowerCamelCase					     = attention_mask + ([0 if mask_padding_with_zero else 1] * padding_length)

													if len(lowerCamelCase__      ) != batch_length:
																	raise ValueError(F"""Error with input length {len(lowerCamelCase__      )} vs {batch_length}"""      )
													if len(lowerCamelCase__      ) != batch_length:
																	raise ValueError(F"""Error with input length {len(lowerCamelCase__      )} vs {batch_length}"""      )

													if self.mode == "classification":
																	_lowerCamelCase					     = label_map[example.label]
													elif self.mode == "regression":
																	_lowerCamelCase					     = float(example.label      )
													else:
																	raise ValueError(self.mode      )

													if ex_index < 5 and self.verbose:
																	logger.info('''*** Example ***'''      )
																	logger.info(F"""guid: {example.guid}"""      )
																	logger.info(F"""input_ids: {" ".join([str(lowerCamelCase__      ) for x in input_ids]      )}"""      )
																	logger.info(F"""attention_mask: {" ".join([str(lowerCamelCase__      ) for x in attention_mask]      )}"""      )
																	logger.info(F"""label: {example.label} (id = {label})"""      )

													features.append(InputFeatures(input_ids=lowerCamelCase__ ,				attention_mask=lowerCamelCase__ ,				label=lowerCamelCase__      )      )

									if return_tensors is None:
													return features
									elif return_tensors == "tf":
													if not is_tf_available():
																	raise RuntimeError('''return_tensors set to \'tf\' but TensorFlow 2.0 can\'t be imported'''      )
													import tensorflow as tf

													def gen():
																	for ex in features:
																					yield ({"input_ids": ex.input_ids, "attention_mask": ex.attention_mask}, ex.label)

													_lowerCamelCase					     = tf.data.Dataset.from_generator(
													    lowerCamelCase__ ,				({'''input_ids''': tf.intaa, '''attention_mask''': tf.intaa}, tf.intaa) ,				({'''input_ids''': tf.TensorShape([None]      ), '''attention_mask''': tf.TensorShape([None]      )}, tf.TensorShape([]      )) ,				)
													return dataset
									elif return_tensors == "pt":
													if not is_torch_available():
																	raise RuntimeError('''return_tensors set to \'pt\' but PyTorch can\'t be imported'''      )
													import torch
													from torch.utils.data import TensorDataset

													_lowerCamelCase					     = torch.tensor([f.input_ids for f in features] ,				dtype=torch.long      )
													_lowerCamelCase					     = torch.tensor([f.attention_mask for f in features] ,				dtype=torch.long      )
													if self.mode == "classification":
																	_lowerCamelCase					     = torch.tensor([f.label for f in features] ,				dtype=torch.long      )
													elif self.mode == "regression":
																	_lowerCamelCase					     = torch.tensor([f.label for f in features] ,				dtype=torch.float      )

													_lowerCamelCase					     = TensorDataset(lowerCamelCase__ ,				lowerCamelCase__ ,				lowerCamelCase__      )
													return dataset
									else:
													raise ValueError('''return_tensors should be one of \'tf\' or \'pt\''''      )







"""simple docstring"""





from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import TensorType, is_torch_available, logging


_lowerCamelCase       =	logging.get_logger(__name__)

_lowerCamelCase       =	{
    '''facebook/m2m100_418M''': '''https://huggingface.co/facebook/m2m100_418M/resolve/main/config.json''',
    # See all M2M100 models at https://huggingface.co/models?filter=m2m_100
}





class 			snake_case		(   __UpperCAmelCase   ):
      lowerCamelCase__						= '''m2m_100'''
      lowerCamelCase__						= ['''past_key_values''']
      lowerCamelCase__						= {'''num_attention_heads''': '''encoder_attention_heads''', '''hidden_size''': '''d_model'''}

      def __init__(			self   :int						,							_lowerCamelCase   :Union[str, Any]=1_2_8_1_1_2						,							_lowerCamelCase   :str=1_0_2_4						,							_lowerCamelCase   :Any=1_2						,							_lowerCamelCase   :Dict=4_0_9_6						,							_lowerCamelCase   :List[str]=1_6						,							_lowerCamelCase   :Union[str, Any]=1_2						,							_lowerCamelCase   :Any=4_0_9_6						,							_lowerCamelCase   :str=1_6						,							_lowerCamelCase   :Union[str, Any]=0.0_5						,							_lowerCamelCase   :Any=0.0_5						,							_lowerCamelCase   :int=True						,							_lowerCamelCase   :Dict=True						,							_lowerCamelCase   :List[str]="relu"						,							_lowerCamelCase   :List[Any]=1_0_2_4						,							_lowerCamelCase   :Dict=0.1						,							_lowerCamelCase   :List[Any]=0.1						,							_lowerCamelCase   :Any=0.0						,							_lowerCamelCase   :Tuple=0.0_2						,							_lowerCamelCase   :Dict=2						,							_lowerCamelCase   :List[str]=True						,							_lowerCamelCase   :Any=1						,							_lowerCamelCase   :Dict=0						,							_lowerCamelCase   :List[Any]=2						,							**_lowerCamelCase   :Optional[Any]						,							):
       __SCREAMING_SNAKE_CASE :			Any												=       vocab_size
       __SCREAMING_SNAKE_CASE :			Any												=       max_position_embeddings
       __SCREAMING_SNAKE_CASE :			int												=       d_model
       __SCREAMING_SNAKE_CASE :			List[Any]												=       encoder_ffn_dim
       __SCREAMING_SNAKE_CASE :			Dict												=       encoder_layers
       __SCREAMING_SNAKE_CASE :			int												=       encoder_attention_heads
       __SCREAMING_SNAKE_CASE :			Dict												=       decoder_ffn_dim
       __SCREAMING_SNAKE_CASE :			Union[str, Any]												=       decoder_layers
       __SCREAMING_SNAKE_CASE :			Optional[Any]												=       decoder_attention_heads
       __SCREAMING_SNAKE_CASE :			Optional[Any]												=       dropout
       __SCREAMING_SNAKE_CASE :			Union[str, Any]												=       attention_dropout
       __SCREAMING_SNAKE_CASE :			List[str]												=       activation_dropout
       __SCREAMING_SNAKE_CASE :			List[Any]												=       activation_function
       __SCREAMING_SNAKE_CASE :			Any												=       init_std
       __SCREAMING_SNAKE_CASE :			List[Any]												=       encoder_layerdrop
       __SCREAMING_SNAKE_CASE :			Union[str, Any]												=       decoder_layerdrop
       __SCREAMING_SNAKE_CASE :			Union[str, Any]												=       use_cache
       __SCREAMING_SNAKE_CASE :			List[str]												=       encoder_layers
       __SCREAMING_SNAKE_CASE :			str												=       scale_embedding  # scale factor will be sqrt(d_model) if True

       super().__init__(
           pad_token_id=_lowerCamelCase						,							bos_token_id=_lowerCamelCase						,							eos_token_id=_lowerCamelCase						,							is_encoder_decoder=_lowerCamelCase						,							decoder_start_token_id=_lowerCamelCase						,							**_lowerCamelCase						,							)







class 			snake_case		(   __UpperCAmelCase   ):

      @property
      def 						SCREAMING_SNAKE_CASE_			(			self   :int						):
       __SCREAMING_SNAKE_CASE :			str												=       OrderedDict(
           [
               ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
               ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
           ]						)

       if self.use_past:
        __SCREAMING_SNAKE_CASE :			Dict												=       {0: '''batch'''}
        __SCREAMING_SNAKE_CASE :			Any												=       {0: '''batch''', 1: '''past_decoder_sequence + sequence'''}
       else:
        __SCREAMING_SNAKE_CASE :			int												=       {0: '''batch''', 1: '''decoder_sequence'''}
        __SCREAMING_SNAKE_CASE :			List[Any]												=       {0: '''batch''', 1: '''decoder_sequence'''}

       if self.use_past:
        self.fill_with_past_key_values_(_lowerCamelCase						,							direction='''inputs'''						)
       return common_inputs

      def 						SCREAMING_SNAKE_CASE_			(			self   :Tuple						,							_lowerCamelCase   :PreTrainedTokenizer						,							_lowerCamelCase   :int = -1						,							_lowerCamelCase   :int = -1						,							_lowerCamelCase   :bool = False						,							_lowerCamelCase   :Optional[TensorType] = None						,							):
       # Copied from OnnxConfig.generate_dummy_inputs
       # Did not use super(OnnxConfigWithPast, self).generate_dummy_inputs for code clarity.
       # If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX
       __SCREAMING_SNAKE_CASE :			List[Any]												=       compute_effective_axis_dimension(
           _lowerCamelCase						,							fixed_dimension=OnnxConfig.default_fixed_batch						,							num_token_to_add=0						)

       # If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
       __SCREAMING_SNAKE_CASE :			List[str]												=       tokenizer.num_special_tokens_to_add(_lowerCamelCase						)
       __SCREAMING_SNAKE_CASE :			int												=       compute_effective_axis_dimension(
           _lowerCamelCase						,							fixed_dimension=OnnxConfig.default_fixed_sequence						,							num_token_to_add=_lowerCamelCase						)

       # Generate dummy inputs according to compute batch and sequence
       __SCREAMING_SNAKE_CASE :			int												=       [''' '''.join([tokenizer.unk_token]						) * seq_length] * batch_size
       __SCREAMING_SNAKE_CASE :			Union[str, Any]												=       dict(tokenizer(_lowerCamelCase						,							return_tensors=_lowerCamelCase						)						)
       return common_inputs

      def 						SCREAMING_SNAKE_CASE_			(			self   :str						,							_lowerCamelCase   :PreTrainedTokenizer						,							_lowerCamelCase   :int = -1						,							_lowerCamelCase   :int = -1						,							_lowerCamelCase   :bool = False						,							_lowerCamelCase   :Optional[TensorType] = None						,							):
       __SCREAMING_SNAKE_CASE :			Dict												=       self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
           _lowerCamelCase						,							_lowerCamelCase						,							_lowerCamelCase						,							_lowerCamelCase						,							_lowerCamelCase						)

       # Generate decoder inputs
       __SCREAMING_SNAKE_CASE :			Dict												=       seq_length if not self.use_past else 1
       __SCREAMING_SNAKE_CASE :			Any												=       self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
           _lowerCamelCase						,							_lowerCamelCase						,							_lowerCamelCase						,							_lowerCamelCase						,							_lowerCamelCase						)
       __SCREAMING_SNAKE_CASE :			Optional[int]												=       {f'''decoder_{name}''': tensor for name, tensor in decoder_inputs.items()}
       __SCREAMING_SNAKE_CASE :			Optional[Any]												=       dict(**_lowerCamelCase						,							**_lowerCamelCase						)

       if self.use_past:
        if not is_torch_available():
         raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''						)
        else:
         import torch
        __SCREAMING_SNAKE_CASE							,      __SCREAMING_SNAKE_CASE :			int												=       common_inputs['''input_ids'''].shape
        __SCREAMING_SNAKE_CASE :			Tuple												=       common_inputs['''decoder_input_ids'''].shape[1]
        __SCREAMING_SNAKE_CASE							,      __SCREAMING_SNAKE_CASE :			Optional[Any]												=       self.num_attention_heads
        __SCREAMING_SNAKE_CASE :			List[str]												=       (
            batch,
            num_encoder_attention_heads,
            encoder_seq_length,
            self._config.hidden_size // num_encoder_attention_heads,
        )
        __SCREAMING_SNAKE_CASE :			Union[str, Any]												=       decoder_seq_length + 3
        __SCREAMING_SNAKE_CASE :			List[str]												=       (
            batch,
            num_decoder_attention_heads,
            decoder_past_length,
            self._config.hidden_size // num_decoder_attention_heads,
        )

        __SCREAMING_SNAKE_CASE :			Any												=       torch.cat(
            [common_inputs['''decoder_attention_mask'''], torch.ones(_lowerCamelCase						,							_lowerCamelCase						)]						,							dim=1						)

        __SCREAMING_SNAKE_CASE :			Union[str, Any]												=       []
        # If the number of encoder and decoder layers are present in the model configuration, both are considered
        __SCREAMING_SNAKE_CASE							,      __SCREAMING_SNAKE_CASE :			Union[str, Any]												=       self.num_layers
        __SCREAMING_SNAKE_CASE :			Any												=       min(_lowerCamelCase						,							_lowerCamelCase						)
        __SCREAMING_SNAKE_CASE :			Tuple												=       max(_lowerCamelCase						,							_lowerCamelCase						) - min_num_layers
        __SCREAMING_SNAKE_CASE :			Optional[int]												=       '''encoder''' if num_encoder_layers > num_decoder_layers else '''decoder'''

        for _ in range(_lowerCamelCase						):
         common_inputs["past_key_values"].append(
             (
                 torch.zeros(_lowerCamelCase						),
                 torch.zeros(_lowerCamelCase						),
                 torch.zeros(_lowerCamelCase						),
                 torch.zeros(_lowerCamelCase						),
             )						)
        # TODO: test this.
        __SCREAMING_SNAKE_CASE :			Any												=       encoder_shape if remaining_side_name == '''encoder''' else decoder_shape
        for _ in range(_lowerCamelCase						,							_lowerCamelCase						):
         common_inputs["past_key_values"].append((torch.zeros(_lowerCamelCase						), torch.zeros(_lowerCamelCase						))						)
       return common_inputs



      lowerCamelCase__						= _generate_dummy_inputs_for_default_and_seqaseq_lm

