






import logging
import os
from dataclasses import dataclass
from enum import Enum
from typing import List, Optional, Union

from filelock import FileLock

from transformers import PreTrainedTokenizer, is_tf_available, is_torch_available


a_							=    logging.getLogger(__name__)


@dataclass
class      lowercase__     :
	a_         =42
	a_         =42
	a_         =42





@dataclass
class      lowercase__     :
	a_         =42
	a_         =42
	a_         =None
	a_         =None





class      lowercase__     (			A_	):
	a_         ='''train'''
	a_         ='''dev'''
	a_         ='''test'''





class      lowercase__     :

	@staticmethod
	def 		UpperCAmelCase  (							__UpperCAmelCase		,    __UpperCAmelCase       )->					Dict:







							'''simple docstring'''





							raise NotImplementedError

	@staticmethod
	def 		UpperCAmelCase  (							__UpperCAmelCase       )->					Tuple:







							'''simple docstring'''





							raise NotImplementedError


	@staticmethod
	def 		UpperCAmelCase  (							__UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase=False		,    __UpperCAmelCase="[CLS]"		,    __UpperCAmelCase=1		,    __UpperCAmelCase="[SEP]"		,    __UpperCAmelCase=False		,    __UpperCAmelCase=False		,    __UpperCAmelCase=0		,    __UpperCAmelCase=0		,    __UpperCAmelCase=-100		,    __UpperCAmelCase=0		,    __UpperCAmelCase=True		,    )->					str:







							'''simple docstring'''





							lowerCAmelCase__       =						{label: i for i, label in enumerate(__UpperCAmelCase       )}

							lowerCAmelCase__       =						[]
							for ex_index, example in enumerate(__UpperCAmelCase       ):
													if ex_index % 10000 == 0:
																			logger.info("Writing example %d of %d"		,    __UpperCAmelCase		,    len(__UpperCAmelCase       )       )

													lowerCAmelCase__       =						[]
													lowerCAmelCase__       =						[]
													for word, label in zip(example.words		,    example.labels       ):
																			lowerCAmelCase__       =						tokenizer.tokenize(__UpperCAmelCase       )

																			# bert-base-multilingual-cased sometimes output "nothing ([]) when calling tokenize with just a space.
																			if len(__UpperCAmelCase       ) > 0:
																									tokens.extend(__UpperCAmelCase       )
																									# Use the real label id for the first token of the word, and padding ids for the remaining tokens
																									label_ids.extend([label_map[label]] + [pad_token_label_id] * (len(__UpperCAmelCase       ) - 1)       )

            # Account for [CLS] and [SEP] with "- 2" and with "- 3" for RoBERTa.
													lowerCAmelCase__       =						tokenizer.num_special_tokens_to_add()
													if len(__UpperCAmelCase       ) > max_seq_length - special_tokens_count:
																			lowerCAmelCase__       =						tokens[: (max_seq_length - special_tokens_count)]
																			lowerCAmelCase__       =						label_ids[: (max_seq_length - special_tokens_count)]

													# The convention in BERT is:
													# (a) For sequence pairs:
													#  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
													#  type_ids:   0   0  0    0    0     0       0   0   1  1  1  1   1   1
													# (b) For single sequences:
													#  tokens:   [CLS] the dog is hairy . [SEP]
													#  type_ids:   0   0   0   0  0     0   0
													#
													# Where "type_ids" are used to indicate whether this is the first
													# sequence or the second sequence. The embedding vectors for `type=0` and
													# `type=1` were learned during pre-training and are added to the wordpiece
													# embedding vector (and position vector). This is not *strictly* necessary
													# since the [SEP] token unambiguously separates the sequences, but it makes
													# it easier for the model to learn the concept of sequences.
													#
													# For classification tasks, the first vector (corresponding to [CLS]) is
													# used as the "sentence vector". Note that this only makes sense because
													# the entire model is fine-tuned.
													tokens += [sep_token]
													label_ids += [pad_token_label_id]
													if sep_token_extra:
																			# roberta uses an extra separator b/w pairs of sentences
																			tokens += [sep_token]
																			label_ids += [pad_token_label_id]
													lowerCAmelCase__       =						[sequence_a_segment_id] * len(__UpperCAmelCase       )

													if cls_token_at_end:
																			tokens += [cls_token]
																			label_ids += [pad_token_label_id]
																			segment_ids += [cls_token_segment_id]
													else:
																			lowerCAmelCase__       =						[cls_token] + tokens
																			lowerCAmelCase__       =						[pad_token_label_id] + label_ids
																			lowerCAmelCase__       =						[cls_token_segment_id] + segment_ids

													lowerCAmelCase__       =						tokenizer.convert_tokens_to_ids(__UpperCAmelCase       )

													# The mask has 1 for real tokens and 0 for padding tokens. Only real
													# tokens are attended to.
													lowerCAmelCase__       =						[1 if mask_padding_with_zero else 0] * len(__UpperCAmelCase       )

													# Zero-pad up to the sequence length.
													lowerCAmelCase__       =						max_seq_length - len(__UpperCAmelCase       )
													if pad_on_left:
																			lowerCAmelCase__       =						([pad_token] * padding_length) + input_ids
																			lowerCAmelCase__       =						([0 if mask_padding_with_zero else 1] * padding_length) + input_mask
																			lowerCAmelCase__       =						([pad_token_segment_id] * padding_length) + segment_ids
																			lowerCAmelCase__       =						([pad_token_label_id] * padding_length) + label_ids
													else:
																			input_ids += [pad_token] * padding_length
																			input_mask += [0 if mask_padding_with_zero else 1] * padding_length
																			segment_ids += [pad_token_segment_id] * padding_length
																			label_ids += [pad_token_label_id] * padding_length

													assert len(__UpperCAmelCase       ) == max_seq_length
													assert len(__UpperCAmelCase       ) == max_seq_length
													assert len(__UpperCAmelCase       ) == max_seq_length
													assert len(__UpperCAmelCase       ) == max_seq_length

													if ex_index < 5:
																			logger.info("*** Example ***"       )
																			logger.info("guid: %s"		,    example.guid       )
																			logger.info("tokens: %s"		,    " ".join([str(__UpperCAmelCase       ) for x in tokens]       )       )
																			logger.info("input_ids: %s"		,    " ".join([str(__UpperCAmelCase       ) for x in input_ids]       )       )
																			logger.info("input_mask: %s"		,    " ".join([str(__UpperCAmelCase       ) for x in input_mask]       )       )
																			logger.info("segment_ids: %s"		,    " ".join([str(__UpperCAmelCase       ) for x in segment_ids]       )       )
																			logger.info("label_ids: %s"		,    " ".join([str(__UpperCAmelCase       ) for x in label_ids]       )       )

													if "token_type_ids" not in tokenizer.model_input_names:
																			lowerCAmelCase__       =						None

													features.append(
													    InputFeatures(
													        input_ids=__UpperCAmelCase		,    attention_mask=__UpperCAmelCase		,    token_type_ids=__UpperCAmelCase		,    label_ids=__UpperCAmelCase       )       )
							return features


if is_torch_available():
					import torch
					from torch import nn
					from torch.utils.data import Dataset


					class      lowercase__     (			A_	):
						a_         =42
						a_         =nn.CrossEntropyLoss().ignore_index

						def __init__(							self		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase = None		,    __UpperCAmelCase=False		,    __UpperCAmelCase = Split.train		,    )->					Optional[int]:







												'''simple docstring'''





												lowerCAmelCase__       =						os.path.join(
												    __UpperCAmelCase		,    "cached_{}_{}_{}".format(mode.value		,    tokenizer.__class__.__name__		,    str(__UpperCAmelCase       )       )		,    )

												# Make sure only the first process in distributed training processes the dataset,
												# and the others will use the cache.
												lowerCAmelCase__       =						cached_features_file + """.lock"""
												with FileLock(__UpperCAmelCase       ):
																		if os.path.exists(__UpperCAmelCase       ) and not overwrite_cache:
																								logger.info(F"Loading features from cached file {cached_features_file}"       )
																								lowerCAmelCase__       =						torch.load(__UpperCAmelCase       )
																		else:
																								logger.info(F"Creating features from dataset file at {data_dir}"       )
																								lowerCAmelCase__       =						token_classification_task.read_examples_from_file(__UpperCAmelCase		,    __UpperCAmelCase       )
																								# TODO clean up all this to leverage built-in features of tokenizers
																								lowerCAmelCase__       =						token_classification_task.convert_examples_to_features(
																								    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    cls_token_at_end=bool(model_type in ["xlnet"]       )		,    cls_token=tokenizer.cls_token		,    cls_token_segment_id=2 if model_type in ["xlnet"] else 0		,    sep_token=tokenizer.sep_token		,    sep_token_extra=__UpperCAmelCase		,    pad_on_left=bool(tokenizer.padding_side == "left"       )		,    pad_token=tokenizer.pad_token_id		,    pad_token_segment_id=tokenizer.pad_token_type_id		,    pad_token_label_id=self.pad_token_label_id		,    )
																								logger.info(F"Saving features into cached file {cached_features_file}"       )
																								torch.save(self.features		,    __UpperCAmelCase       )

						def __len__(							self       )->					Tuple:







												'''simple docstring'''





												return len(self.features       )


						def __getitem__(							self		,    __UpperCAmelCase       )->					Dict:







												'''simple docstring'''





												return self.features[i]


if is_tf_available():
					import tensorflow as tf


					class      lowercase__     :
						a_         =42
						a_         =-100

						def __init__(							self		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase = None		,    __UpperCAmelCase=False		,    __UpperCAmelCase = Split.train		,    )->					Any:







												'''simple docstring'''





												lowerCAmelCase__       =						token_classification_task.read_examples_from_file(__UpperCAmelCase		,    __UpperCAmelCase       )
												# TODO clean up all this to leverage built-in features of tokenizers
												lowerCAmelCase__       =						token_classification_task.convert_examples_to_features(
												    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    cls_token_at_end=bool(model_type in ["xlnet"]       )		,    cls_token=tokenizer.cls_token		,    cls_token_segment_id=2 if model_type in ["xlnet"] else 0		,    sep_token=tokenizer.sep_token		,    sep_token_extra=__UpperCAmelCase		,    pad_on_left=bool(tokenizer.padding_side == "left"       )		,    pad_token=tokenizer.pad_token_id		,    pad_token_segment_id=tokenizer.pad_token_type_id		,    pad_token_label_id=self.pad_token_label_id		,    )

												def gen():
																		for ex in self.features:
																								if ex.token_type_ids is None:
																														yield (
																														    {"input_ids": ex.input_ids, "attention_mask": ex.attention_mask},
																														    ex.label_ids,
																														)
																								else:
																														yield (
																														    {
																														        "input_ids": ex.input_ids,
																														        "attention_mask": ex.attention_mask,
																														        "token_type_ids": ex.token_type_ids,
																														    },
																														    ex.label_ids,
																														)

												if "token_type_ids" not in tokenizer.model_input_names:
																		lowerCAmelCase__       =						tf.data.Dataset.from_generator(
																		    __UpperCAmelCase		,    ({"input_ids": tf.intaa, "attention_mask": tf.intaa}, tf.intaa)		,    (
																		        {"input_ids": tf.TensorShape([None]       ), "attention_mask": tf.TensorShape([None]       )},
																		        tf.TensorShape([None]       ),
																		    )		,    )
												else:
																		lowerCAmelCase__       =						tf.data.Dataset.from_generator(
																		    __UpperCAmelCase		,    ({"input_ids": tf.intaa, "attention_mask": tf.intaa, "token_type_ids": tf.intaa}, tf.intaa)		,    (
																		        {
																		            "input_ids": tf.TensorShape([None]       ),
																		            "attention_mask": tf.TensorShape([None]       ),
																		            "token_type_ids": tf.TensorShape([None]       ),
																		        },
																		        tf.TensorShape([None]       ),
																		    )		,    )

						def 		UpperCAmelCase  (							self       )->					str:







												'''simple docstring'''





												lowerCAmelCase__       =						self.dataset.apply(tf.data.experimental.assert_cardinality(len(self.features       )       )       )

												return self.dataset

						def __len__(							self       )->					Optional[Any]:







												'''simple docstring'''





												return len(self.features       )


						def __getitem__(							self		,    __UpperCAmelCase       )->					Tuple:







												'''simple docstring'''





												return self.features[i]














from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


a_							=    logging.get_logger(__name__)


a_							=    {
    '''Salesforce/codegen-350M-nl''': '''https://huggingface.co/Salesforce/codegen-350M-nl/resolve/main/config.json''',
    '''Salesforce/codegen-350M-multi''': '''https://huggingface.co/Salesforce/codegen-350M-multi/resolve/main/config.json''',
    '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/config.json''',
    '''Salesforce/codegen-2B-nl''': '''https://huggingface.co/Salesforce/codegen-2B-nl/resolve/main/config.json''',
    '''Salesforce/codegen-2B-multi''': '''https://huggingface.co/Salesforce/codegen-2B-multi/resolve/main/config.json''',
    '''Salesforce/codegen-2B-mono''': '''https://huggingface.co/Salesforce/codegen-2B-mono/resolve/main/config.json''',
    '''Salesforce/codegen-6B-nl''': '''https://huggingface.co/Salesforce/codegen-6B-nl/resolve/main/config.json''',
    '''Salesforce/codegen-6B-multi''': '''https://huggingface.co/Salesforce/codegen-6B-multi/resolve/main/config.json''',
    '''Salesforce/codegen-6B-mono''': '''https://huggingface.co/Salesforce/codegen-6B-mono/resolve/main/config.json''',
    '''Salesforce/codegen-16B-nl''': '''https://huggingface.co/Salesforce/codegen-16B-nl/resolve/main/config.json''',
    '''Salesforce/codegen-16B-multi''': '''https://huggingface.co/Salesforce/codegen-16B-multi/resolve/main/config.json''',
    '''Salesforce/codegen-16B-mono''': '''https://huggingface.co/Salesforce/codegen-16B-mono/resolve/main/config.json''',
}


class      lowercase__     (			_UpperCAmelCase	):
	a_         ="""codegen"""
	a_         ={
	    """max_position_embeddings""": """n_positions""",
	    """hidden_size""": """n_embd""",
	    """num_attention_heads""": """n_head""",
	    """num_hidden_layers""": """n_layer""",
	}

	def __init__(							self		,    __UpperCAmelCase=50400		,    __UpperCAmelCase=2048		,    __UpperCAmelCase=2048		,    __UpperCAmelCase=4096		,    __UpperCAmelCase=28		,    __UpperCAmelCase=16		,    __UpperCAmelCase=64		,    __UpperCAmelCase=None		,    __UpperCAmelCase="gelu_new"		,    __UpperCAmelCase=0.0		,    __UpperCAmelCase=0.0		,    __UpperCAmelCase=0.0		,    __UpperCAmelCase=1E-5		,    __UpperCAmelCase=0.02		,    __UpperCAmelCase=True		,    __UpperCAmelCase=50256		,    __UpperCAmelCase=50256		,    __UpperCAmelCase=False		,    **__UpperCAmelCase		,    )->					Optional[int]:







							'''simple docstring'''





							lowerCAmelCase__       =						vocab_size
							lowerCAmelCase__       =						n_ctx
							lowerCAmelCase__       =						n_positions
							lowerCAmelCase__       =						n_embd
							lowerCAmelCase__       =						n_layer
							lowerCAmelCase__       =						n_head
							lowerCAmelCase__       =						n_inner
							lowerCAmelCase__       =						rotary_dim
							lowerCAmelCase__       =						activation_function
							lowerCAmelCase__       =						resid_pdrop
							lowerCAmelCase__       =						embd_pdrop
							lowerCAmelCase__       =						attn_pdrop
							lowerCAmelCase__       =						layer_norm_epsilon
							lowerCAmelCase__       =						initializer_range
							lowerCAmelCase__       =						use_cache

							lowerCAmelCase__       =						bos_token_id
							lowerCAmelCase__       =						eos_token_id

							super().__init__(
							    bos_token_id=__UpperCAmelCase		,    eos_token_id=__UpperCAmelCase		,    tie_word_embeddings=__UpperCAmelCase		,    **__UpperCAmelCase       )





class      lowercase__     (			_UpperCAmelCase	):

	def __init__(							self		,    __UpperCAmelCase		,    __UpperCAmelCase = "default"		,    __UpperCAmelCase = None		,    __UpperCAmelCase = False		,    )->					Any:







							'''simple docstring'''





							super().__init__(__UpperCAmelCase		,    task=__UpperCAmelCase		,    patching_specs=__UpperCAmelCase		,    use_past=__UpperCAmelCase       )
							if not getattr(self._config		,    "pad_token_id"		,    __UpperCAmelCase       ):
													# TODO: how to do that better?
													lowerCAmelCase__       =						0

	@property
	def 		UpperCAmelCase  (							self       )->					Mapping[str, Mapping[int, str]]:







							'''simple docstring'''





							lowerCAmelCase__       =						OrderedDict({"input_ids": {0: "batch", 1: "sequence"}}       )
							if self.use_past:
													self.fill_with_past_key_values_(__UpperCAmelCase		,    direction="inputs"       )
													lowerCAmelCase__       =						{0: "batch", 1: "past_sequence + sequence"}
							else:
													lowerCAmelCase__       =						{0: "batch", 1: "sequence"}

							return common_inputs

	@property
	def 		UpperCAmelCase  (							self       )->					int:







							'''simple docstring'''





							return self._config.n_layer

	@property
	def 		UpperCAmelCase  (							self       )->					int:







							'''simple docstring'''





							return self._config.n_head

	def 		UpperCAmelCase  (							self		,    __UpperCAmelCase		,    __UpperCAmelCase = -1		,    __UpperCAmelCase = -1		,    __UpperCAmelCase = False		,    __UpperCAmelCase = None		,    )->					Mapping[str, Any]:







							'''simple docstring'''





							lowerCAmelCase__       =						super(__UpperCAmelCase		,    self       ).generate_dummy_inputs(
							    __UpperCAmelCase		,    batch_size=__UpperCAmelCase		,    seq_length=__UpperCAmelCase		,    is_pair=__UpperCAmelCase		,    framework=__UpperCAmelCase       )

							# We need to order the input in the way they appears in the forward()
							lowerCAmelCase__       =						OrderedDict({"input_ids": common_inputs["input_ids"]}       )

							# Need to add the past_keys
							if self.use_past:
													if not is_torch_available():
																			raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed."       )
													else:
																			import torch

																			lowerCAmelCase__      ,							lowerCAmelCase__       =						common_inputs["input_ids"].shape
																			# Not using the same length for past_key_values
																			lowerCAmelCase__       =						seqlen + 2
																			lowerCAmelCase__       =						(
																			    batch,
																			    self.num_attention_heads,
																			    past_key_values_length,
																			    self._config.hidden_size // self.num_attention_heads,
																			)
																			lowerCAmelCase__       =						[
																			    (torch.zeros(__UpperCAmelCase       ), torch.zeros(__UpperCAmelCase       )) for _ in range(self.num_layers       )
																			]

							lowerCAmelCase__       =						common_inputs["attention_mask"]
							if self.use_past:
													lowerCAmelCase__       =						ordered_inputs["attention_mask"].dtype
													lowerCAmelCase__       =						torch.cat(
													    [ordered_inputs["attention_mask"], torch.ones(__UpperCAmelCase		,    __UpperCAmelCase		,    dtype=__UpperCAmelCase       )]		,    dim=1       )

							return ordered_inputs


	@property
	def 		UpperCAmelCase  (							self       )->					int:







							'''simple docstring'''





							return 13






