





"""simple docstring"""





from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


SCREAMING_SNAKE_CASE__   =  logging.get_logger(__name__)


SCREAMING_SNAKE_CASE__   =  {
    "Salesforce/codegen-350M-nl": "https://huggingface.co/Salesforce/codegen-350M-nl/resolve/main/config.json",
    "Salesforce/codegen-350M-multi": "https://huggingface.co/Salesforce/codegen-350M-multi/resolve/main/config.json",
    "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/config.json",
    "Salesforce/codegen-2B-nl": "https://huggingface.co/Salesforce/codegen-2B-nl/resolve/main/config.json",
    "Salesforce/codegen-2B-multi": "https://huggingface.co/Salesforce/codegen-2B-multi/resolve/main/config.json",
    "Salesforce/codegen-2B-mono": "https://huggingface.co/Salesforce/codegen-2B-mono/resolve/main/config.json",
    "Salesforce/codegen-6B-nl": "https://huggingface.co/Salesforce/codegen-6B-nl/resolve/main/config.json",
    "Salesforce/codegen-6B-multi": "https://huggingface.co/Salesforce/codegen-6B-multi/resolve/main/config.json",
    "Salesforce/codegen-6B-mono": "https://huggingface.co/Salesforce/codegen-6B-mono/resolve/main/config.json",
    "Salesforce/codegen-16B-nl": "https://huggingface.co/Salesforce/codegen-16B-nl/resolve/main/config.json",
    "Salesforce/codegen-16B-multi": "https://huggingface.co/Salesforce/codegen-16B-multi/resolve/main/config.json",
    "Salesforce/codegen-16B-mono": "https://huggingface.co/Salesforce/codegen-16B-mono/resolve/main/config.json",
}

class 						lowercase   (   _UpperCAmelCase   ):
      _SCREAMING_SNAKE_CASE							      = 'codegen'
      _SCREAMING_SNAKE_CASE							      = {
          'max_position_embeddings': 'n_positions',
          'hidden_size': 'n_embd',
          'num_attention_heads': 'n_head',
          'num_hidden_layers': 'n_layer',
      }





      def __init__(			self ,		lowercase=50_400 ,		lowercase=2_048 ,		lowercase=2_048 ,		lowercase=4_096 ,		lowercase=28 ,		lowercase=16 ,		lowercase=64 ,		lowercase=None ,		lowercase="gelu_new" ,		lowercase=0.0 ,		lowercase=0.0 ,		lowercase=0.0 ,		lowercase=1e-5 ,		lowercase=0.02 ,		lowercase=True ,		lowercase=50_256 ,		lowercase=50_256 ,		lowercase=False ,		**lowercase ,		)						->	str:
            lowerCAmelCase    		=			vocab_size
            lowerCAmelCase    		=			n_ctx
            lowerCAmelCase    		=			n_positions
            lowerCAmelCase    		=			n_embd
            lowerCAmelCase    		=			n_layer
            lowerCAmelCase    		=			n_head
            lowerCAmelCase    		=			n_inner
            lowerCAmelCase    		=			rotary_dim
            lowerCAmelCase    		=			activation_function
            lowerCAmelCase    		=			resid_pdrop
            lowerCAmelCase    		=			embd_pdrop
            lowerCAmelCase    		=			attn_pdrop
            lowerCAmelCase    		=			layer_norm_epsilon
            lowerCAmelCase    		=			initializer_range
            lowerCAmelCase    		=			use_cache

            lowerCAmelCase    		=			bos_token_id
            lowerCAmelCase    		=			eos_token_id

            super().__init__(
                bos_token_id=lowercase ,		eos_token_id=lowercase ,		tie_word_embeddings=lowercase ,		**lowercase					)







class 						lowercase   (   _UpperCAmelCase   ):





      def __init__(			self ,		lowercase ,		lowercase = "default" ,		lowercase = None ,		lowercase = False ,		)						->	Optional[Any]:
            super().__init__(lowercase ,		task=lowercase ,		patching_specs=lowercase ,		use_past=lowercase					)
            if not getattr(self._config ,		"""pad_token_id""" ,		lowercase					):
                  # TODO: how to do that better?
                  lowerCAmelCase    		=			0





      @property
      def        _snake_case     (			self					)						->	Mapping[str, Mapping[int, str]]:
            lowerCAmelCase    		=			OrderedDict({"""input_ids""": {0: """batch""", 1: """sequence"""}}					)
            if self.use_past:
                  self.fill_with_past_key_values_(lowercase ,		direction="""inputs"""					)
                  lowerCAmelCase    		=			{0: """batch""", 1: """past_sequence + sequence"""}
            else:
                  lowerCAmelCase    		=			{0: """batch""", 1: """sequence"""}

            return common_inputs





      @property
      def        _snake_case     (			self					)						->	int:
            return self._config.n_layer





      @property
      def        _snake_case     (			self					)						->	int:
            return self._config.n_head





      def        _snake_case     (			self ,		lowercase ,		lowercase = -1 ,		lowercase = -1 ,		lowercase = False ,		lowercase = None ,		)						->	Mapping[str, Any]:
            lowerCAmelCase    		=			super(lowercase ,		self					).generate_dummy_inputs(
                lowercase ,		batch_size=lowercase ,		seq_length=lowercase ,		is_pair=lowercase ,		framework=lowercase					)

            # We need to order the input in the way they appears in the forward()
            lowerCAmelCase    		=			OrderedDict({"""input_ids""": common_inputs["""input_ids"""]}					)

            # Need to add the past_keys
            if self.use_past:
                  if not is_torch_available():
                        raise ValueError("""Cannot generate dummy past_keys inputs without PyTorch installed."""					)
                  else:
                        import torch

                        lowerCAmelCase		,						lowerCAmelCase    		=			common_inputs["""input_ids"""].shape
                        # Not using the same length for past_key_values
                        lowerCAmelCase    		=			seqlen + 2
                        lowerCAmelCase    		=			(
                            batch,
                            self.num_attention_heads,
                            past_key_values_length,
                            self._config.hidden_size // self.num_attention_heads,
                        )
                        lowerCAmelCase    		=			[
                            (torch.zeros(lowercase					), torch.zeros(lowercase					)) for _ in range(self.num_layers					)
                        ]

            lowerCAmelCase    		=			common_inputs["""attention_mask"""]
            if self.use_past:
                  lowerCAmelCase    		=			ordered_inputs["""attention_mask"""].dtype
                  lowerCAmelCase    		=			torch.cat(
                      [ordered_inputs["""attention_mask"""], torch.ones(lowercase ,		lowercase ,		dtype=lowercase					)] ,		dim=1					)

            return ordered_inputs





      @property
      def        _snake_case     (			self					)						->	int:
            return 13













"""simple docstring"""





import logging
import os
from dataclasses import dataclass
from enum import Enum
from typing import List, Optional, Union

from filelock import FileLock

from transformers import PreTrainedTokenizer, is_tf_available, is_torch_available


SCREAMING_SNAKE_CASE__   =  logging.getLogger(__name__)

@dataclass
class 						lowercase   :
      _SCREAMING_SNAKE_CASE							      = 42
      _SCREAMING_SNAKE_CASE							      = 42
      _SCREAMING_SNAKE_CASE							      = 42







@dataclass
class 						lowercase   :
      _SCREAMING_SNAKE_CASE							      = 42
      _SCREAMING_SNAKE_CASE							      = 42
      _SCREAMING_SNAKE_CASE							      = None
      _SCREAMING_SNAKE_CASE							      = None







class 						lowercase   (   _UpperCAmelCase   ):
      _SCREAMING_SNAKE_CASE							      = 'train'
      _SCREAMING_SNAKE_CASE							      = 'dev'
      _SCREAMING_SNAKE_CASE							      = 'test'







class 						lowercase   :





      @staticmethod
      def        _snake_case     (			lowercase ,		lowercase					)						->	List[InputExample]:
            raise NotImplementedError





      @staticmethod
      def        _snake_case     (			lowercase					)						->	List[str]:
            raise NotImplementedError





      @staticmethod
      def        _snake_case     (			lowercase ,		lowercase ,		lowercase ,		lowercase ,		lowercase=False ,		lowercase="[CLS]" ,		lowercase=1 ,		lowercase="[SEP]" ,		lowercase=False ,		lowercase=False ,		lowercase=0 ,		lowercase=0 ,		lowercase=-100 ,		lowercase=0 ,		lowercase=True ,		)						->	List[InputFeatures]:
            lowerCAmelCase    		=			{label: i for i, label in enumerate(lowercase					)}

            lowerCAmelCase    		=			[]
            for ex_index, example in enumerate(lowercase					):
                  if ex_index % 10_000 == 0:
                        logger.info("""Writing example %d of %d""" ,		lowercase ,		len(lowercase					)					)

                  lowerCAmelCase    		=			[]
                  lowerCAmelCase    		=			[]
                  for word, label in zip(example.words ,		example.labels					):
                        lowerCAmelCase    		=			tokenizer.tokenize(lowercase					)

                        # bert-base-multilingual-cased sometimes output "nothing ([]) when calling tokenize with just a space.
                        if len(lowercase					) > 0:
                              tokens.extend(lowercase					)
                              # Use the real label id for the first token of the word, and padding ids for the remaining tokens
                              label_ids.extend([label_map[label]] + [pad_token_label_id] * (len(lowercase					) - 1)					)

            # Account for [CLS] and [SEP] with "- 2" and with "- 3" for RoBERTa.
                  lowerCAmelCase    		=			tokenizer.num_special_tokens_to_add()
                  if len(lowercase					) > max_seq_length - special_tokens_count:
                        lowerCAmelCase    		=			tokens[: (max_seq_length - special_tokens_count)]
                        lowerCAmelCase    		=			label_ids[: (max_seq_length - special_tokens_count)]

                  # The convention in BERT is:
                  # (a) For sequence pairs:
                  #  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
                  #  type_ids:   0   0  0    0    0     0       0   0   1  1  1  1   1   1
                  # (b) For single sequences:
                  #  tokens:   [CLS] the dog is hairy . [SEP]
                  #  type_ids:   0   0   0   0  0     0   0
                  #
                  # Where "type_ids" are used to indicate whether this is the first
                  # sequence or the second sequence. The embedding vectors for `type=0` and
                  # `type=1` were learned during pre-training and are added to the wordpiece
                  # embedding vector (and position vector). This is not *strictly* necessary
                  # since the [SEP] token unambiguously separates the sequences, but it makes
                  # it easier for the model to learn the concept of sequences.
                  #
                  # For classification tasks, the first vector (corresponding to [CLS]) is
                  # used as the "sentence vector". Note that this only makes sense because
                  # the entire model is fine-tuned.
                  tokens += [sep_token]
                  label_ids += [pad_token_label_id]
                  if sep_token_extra:
                        # roberta uses an extra separator b/w pairs of sentences
                        tokens += [sep_token]
                        label_ids += [pad_token_label_id]
                  lowerCAmelCase    		=			[sequence_a_segment_id] * len(lowercase					)

                  if cls_token_at_end:
                        tokens += [cls_token]
                        label_ids += [pad_token_label_id]
                        segment_ids += [cls_token_segment_id]
                  else:
                        lowerCAmelCase    		=			[cls_token] + tokens
                        lowerCAmelCase    		=			[pad_token_label_id] + label_ids
                        lowerCAmelCase    		=			[cls_token_segment_id] + segment_ids

                  lowerCAmelCase    		=			tokenizer.convert_tokens_to_ids(lowercase					)

                  # The mask has 1 for real tokens and 0 for padding tokens. Only real
                  # tokens are attended to.
                  lowerCAmelCase    		=			[1 if mask_padding_with_zero else 0] * len(lowercase					)

                  # Zero-pad up to the sequence length.
                  lowerCAmelCase    		=			max_seq_length - len(lowercase					)
                  if pad_on_left:
                        lowerCAmelCase    		=			([pad_token] * padding_length) + input_ids
                        lowerCAmelCase    		=			([0 if mask_padding_with_zero else 1] * padding_length) + input_mask
                        lowerCAmelCase    		=			([pad_token_segment_id] * padding_length) + segment_ids
                        lowerCAmelCase    		=			([pad_token_label_id] * padding_length) + label_ids
                  else:
                        input_ids += [pad_token] * padding_length
                        input_mask += [0 if mask_padding_with_zero else 1] * padding_length
                        segment_ids += [pad_token_segment_id] * padding_length
                        label_ids += [pad_token_label_id] * padding_length

                  assert len(lowercase					) == max_seq_length
                  assert len(lowercase					) == max_seq_length
                  assert len(lowercase					) == max_seq_length
                  assert len(lowercase					) == max_seq_length

                  if ex_index < 5:
                        logger.info("""*** Example ***"""					)
                        logger.info("""guid: %s""" ,		example.guid					)
                        logger.info("""tokens: %s""" ,		""" """.join([str(lowercase					) for x in tokens]					)					)
                        logger.info("""input_ids: %s""" ,		""" """.join([str(lowercase					) for x in input_ids]					)					)
                        logger.info("""input_mask: %s""" ,		""" """.join([str(lowercase					) for x in input_mask]					)					)
                        logger.info("""segment_ids: %s""" ,		""" """.join([str(lowercase					) for x in segment_ids]					)					)
                        logger.info("""label_ids: %s""" ,		""" """.join([str(lowercase					) for x in label_ids]					)					)

                  if "token_type_ids" not in tokenizer.model_input_names:
                        lowerCAmelCase    		=			None

                  features.append(
                      InputFeatures(
                          input_ids=lowercase ,		attention_mask=lowercase ,		token_type_ids=lowercase ,		label_ids=lowercase					)					)
            return features


if is_torch_available():
       import torch
       from torch import nn
       from torch.utils.data import Dataset

       class 						lowercase   (   _UpperCAmelCase   ):
             _SCREAMING_SNAKE_CASE							      = 42
             _SCREAMING_SNAKE_CASE							      = nn.CrossEntropyLoss().ignore_index





             def __init__(			self ,		lowercase ,		lowercase ,		lowercase ,		lowercase ,		lowercase ,		lowercase = None ,		lowercase=False ,		lowercase = Split.train ,		)						->	List[str]:
                   # Load data features from cache or dataset file
                   lowerCAmelCase    		=			os.path.join(
                       lowercase ,		"""cached_{}_{}_{}""".format(mode.value ,		tokenizer.__class__.__name__ ,		str(lowercase					)					) ,		)

                   # Make sure only the first process in distributed training processes the dataset,
                   # and the others will use the cache.
                   lowerCAmelCase    		=			cached_features_file + """.lock"""
                   with FileLock(lowercase					):
                         if os.path.exists(lowercase					) and not overwrite_cache:
                               logger.info(f'Loading features from cached file {cached_features_file}'					)
                               lowerCAmelCase    		=			torch.load(lowercase					)
                         else:
                               logger.info(f'Creating features from dataset file at {data_dir}'					)
                               lowerCAmelCase    		=			token_classification_task.read_examples_from_file(lowercase ,		lowercase					)
                               # TODO clean up all this to leverage built-in features of tokenizers
                               lowerCAmelCase    		=			token_classification_task.convert_examples_to_features(
                                   lowercase ,		lowercase ,		lowercase ,		lowercase ,		cls_token_at_end=bool(model_type in ["""xlnet"""]					) ,		cls_token=tokenizer.cls_token ,		cls_token_segment_id=2 if model_type in ["""xlnet"""] else 0 ,		sep_token=tokenizer.sep_token ,		sep_token_extra=lowercase ,		pad_on_left=bool(tokenizer.padding_side == """left"""					) ,		pad_token=tokenizer.pad_token_id ,		pad_token_segment_id=tokenizer.pad_token_type_id ,		pad_token_label_id=self.pad_token_label_id ,		)
                               logger.info(f'Saving features into cached file {cached_features_file}'					)
                               torch.save(self.features ,		lowercase					)





             def __len__(			self					)						->	int:
                   return len(self.features					)





             def __getitem__(			self ,		lowercase					)						->	InputFeatures:
                   return self.features[i]


if is_tf_available():
       import tensorflow as tf

       class 						lowercase   :
             _SCREAMING_SNAKE_CASE							      = 42
             _SCREAMING_SNAKE_CASE							      = -100





             def __init__(			self ,		lowercase ,		lowercase ,		lowercase ,		lowercase ,		lowercase ,		lowercase = None ,		lowercase=False ,		lowercase = Split.train ,		)						->	Any:
                   lowerCAmelCase    		=			token_classification_task.read_examples_from_file(lowercase ,		lowercase					)
                   # TODO clean up all this to leverage built-in features of tokenizers
                   lowerCAmelCase    		=			token_classification_task.convert_examples_to_features(
                       lowercase ,		lowercase ,		lowercase ,		lowercase ,		cls_token_at_end=bool(model_type in ["""xlnet"""]					) ,		cls_token=tokenizer.cls_token ,		cls_token_segment_id=2 if model_type in ["""xlnet"""] else 0 ,		sep_token=tokenizer.sep_token ,		sep_token_extra=lowercase ,		pad_on_left=bool(tokenizer.padding_side == """left"""					) ,		pad_token=tokenizer.pad_token_id ,		pad_token_segment_id=tokenizer.pad_token_type_id ,		pad_token_label_id=self.pad_token_label_id ,		)

                   def gen():
                         for ex in self.features:
                               if ex.token_type_ids is None:
                                     yield (
                                         {"input_ids": ex.input_ids, "attention_mask": ex.attention_mask},
                                         ex.label_ids,
                                     )
                               else:
                                     yield (
                                         {
                                             "input_ids": ex.input_ids,
                                             "attention_mask": ex.attention_mask,
                                             "token_type_ids": ex.token_type_ids,
                                         },
                                         ex.label_ids,
                                     )

                   if "token_type_ids" not in tokenizer.model_input_names:
                         lowerCAmelCase    		=			tf.data.Dataset.from_generator(
                             lowercase ,		({"""input_ids""": tf.intaa, """attention_mask""": tf.intaa}, tf.intaa) ,		(
                                 {"""input_ids""": tf.TensorShape([None]					), """attention_mask""": tf.TensorShape([None]					)},
                                 tf.TensorShape([None]					),
                             ) ,		)
                   else:
                         lowerCAmelCase    		=			tf.data.Dataset.from_generator(
                             lowercase ,		({"""input_ids""": tf.intaa, """attention_mask""": tf.intaa, """token_type_ids""": tf.intaa}, tf.intaa) ,		(
                                 {
                                     """input_ids""": tf.TensorShape([None]					),
                                     """attention_mask""": tf.TensorShape([None]					),
                                     """token_type_ids""": tf.TensorShape([None]					),
                                 },
                                 tf.TensorShape([None]					),
                             ) ,		)





             def        _snake_case     (			self					)						->	List[str]:
                   lowerCAmelCase    		=			self.dataset.apply(tf.data.experimental.assert_cardinality(len(self.features					)					)					)

                   return self.dataset





             def __len__(			self					)						->	Optional[int]:
                   return len(self.features					)





             def __getitem__(			self ,		lowercase					)						->	InputFeatures:
                   return self.features[i]






