Upload config and modelling files

config.json

@@ -1,271 +1,25 @@
 {
-  "_name_or_path": "distributed/optimized-gpt2-2b",
+  "_name_or_path": "distributed/optimized-gpt2-2b-vtestnet-v1",
   "activation_function": "gelu_new",
   "all_reduce_scores": {
     "0": "SUCCESS",
     "1": "SUCCESS",
-    "10": "SUCCESS",
-    "100": "SUCCESS",
-    "101": "SUCCESS",
-    "102": "SUCCESS",
-    "103": "SUCCESS",
-    "104": "SUCCESS",
-    "105": "SUCCESS",
-    "106": "SUCCESS",
-    "107": "SUCCESS",
-    "108": "SUCCESS",
-    "109": "SUCCESS",
-    "11": "SUCCESS",
-    "110": "SUCCESS",
-    "111": "SUCCESS",
-    "112": "SUCCESS",
-    "113": "SUCCESS",
-    "114": "SUCCESS",
-    "115": "SUCCESS",
-    "116": "SUCCESS",
-    "117": "SUCCESS",
-    "118": "SUCCESS",
-    "119": "SUCCESS",
-    "12": "SUCCESS",
-    "120": "SUCCESS",
-    "121": "SUCCESS",
-    "122": "SUCCESS",
-    "123": "SUCCESS",
-    "124": "SUCCESS",
-    "125": "SUCCESS",
-    "126": "SUCCESS",
-    "127": "SUCCESS",
-    "128": "SUCCESS",
-    "129": "SUCCESS",
-    "13": "SUCCESS",
-    "130": "SUCCESS",
-    "131": "SUCCESS",
-    "132": "SUCCESS",
-    "133": "SUCCESS",
-    "134": "SUCCESS",
-    "135": "SUCCESS",
-    "136": "SUCCESS",
-    "137": "SUCCESS",
-    "138": "SUCCESS",
-    "139": "SUCCESS",
-    "14": "SUCCESS",
-    "140": "SUCCESS",
-    "141": "SUCCESS",
-    "142": "SUCCESS",
-    "143": "SUCCESS",
-    "144": "SUCCESS",
-    "145": "SUCCESS",
-    "146": "SUCCESS",
-    "147": "SUCCESS",
-    "148": "SUCCESS",
-    "149": "SUCCESS",
-    "15": "SUCCESS",
-    "150": "SUCCESS",
-    "151": "SUCCESS",
-    "152": "SUCCESS",
-    "153": "SUCCESS",
-    "154": "SUCCESS",
-    "155": "SUCCESS",
-    "156": "SUCCESS",
-    "157": "SUCCESS",
-    "158": "SUCCESS",
-    "159": "SUCCESS",
-    "16": "SUCCESS",
-    "160": "SUCCESS",
-    "161": "SUCCESS",
-    "162": "SUCCESS",
-    "163": "SUCCESS",
-    "164": "SUCCESS",
-    "165": "SUCCESS",
-    "166": "SUCCESS",
-    "167": "SUCCESS",
-    "168": "SUCCESS",
-    "169": "SUCCESS",
-    "17": "SUCCESS",
-    "170": "SUCCESS",
-    "171": "SUCCESS",
-    "172": "SUCCESS",
-    "173": "SUCCESS",
-    "174": "SUCCESS",
-    "175": "SUCCESS",
-    "176": "SUCCESS",
-    "177": "SUCCESS",
-    "178": "SUCCESS",
-    "179": "SUCCESS",
-    "18": "SUCCESS",
-    "180": "SUCCESS",
-    "181": "SUCCESS",
-    "182": "SUCCESS",
-    "183": "SUCCESS",
-    "184": "SUCCESS",
-    "185": "SUCCESS",
-    "186": "SUCCESS",
-    "187": "SUCCESS",
-    "188": "SUCCESS",
-    "189": "SUCCESS",
-    "19": "SUCCESS",
-    "190": "SUCCESS",
-    "191": "SUCCESS",
-    "192": "SUCCESS",
-    "193": "SUCCESS",
-    "194": "SUCCESS",
-    "195": "SUCCESS",
-    "196": "SUCCESS",
-    "197": "SUCCESS",
-    "198": "SUCCESS",
-    "199": "SUCCESS",
     "2": "SUCCESS",
-    "20": "SUCCESS",
-    "200": "SUCCESS",
-    "201": "SUCCESS",
-    "202": "SUCCESS",
-    "203": "SUCCESS",
-    "204": "SUCCESS",
-    "205": "SUCCESS",
-    "206": "SUCCESS",
-    "207": "SUCCESS",
-    "208": "SUCCESS",
-    "209": "SUCCESS",
-    "21": "SUCCESS",
-    "210": "SUCCESS",
-    "211": "SUCCESS",
-    "212": "SUCCESS",
-    "213": "SUCCESS",
-    "214": "SUCCESS",
-    "215": "SUCCESS",
-    "216": "SUCCESS",
-    "217": "SUCCESS",
-    "218": "SUCCESS",
-    "219": "SUCCESS",
-    "22": "SUCCESS",
-    "220": "SUCCESS",
-    "221": "SUCCESS",
-    "222": "SUCCESS",
-    "223": "SUCCESS",
-    "224": "SUCCESS",
-    "225": "SUCCESS",
-    "226": "SUCCESS",
-    "227": "SUCCESS",
-    "228": "SUCCESS",
-    "229": "SUCCESS",
-    "23": "SUCCESS",
-    "230": "SUCCESS",
-    "231": "SUCCESS",
-    "232": "SUCCESS",
-    "233": "SUCCESS",
-    "234": "SUCCESS",
-    "235": "SUCCESS",
-    "236": "SUCCESS",
-    "237": "SUCCESS",
-    "238": "SUCCESS",
-    "239": "SUCCESS",
-    "24": "SUCCESS",
-    "240": "SUCCESS",
-    "241": "SUCCESS",
-    "242": "SUCCESS",
-    "243": "SUCCESS",
-    "244": "SUCCESS",
-    "245": "SUCCESS",
-    "246": "SUCCESS",
-    "247": "SUCCESS",
-    "248": "SUCCESS",
-    "249": "SUCCESS",
-    "25": "SUCCESS",
-    "250": "SUCCESS",
-    "251": "SUCCESS",
-    "252": "SUCCESS",
-    "253": "SUCCESS",
-    "254": "SUCCESS",
-    "255": "SUCCESS",
-    "26": "SUCCESS",
-    "27": "SUCCESS",
-    "28": "SUCCESS",
-    "29": "SUCCESS",
-    "3": "SUCCESS",
-    "30": "SUCCESS",
-    "31": "SUCCESS",
-    "32": "SUCCESS",
-    "33": "SUCCESS",
-    "34": "SUCCESS",
-    "35": "SUCCESS",
-    "36": "SUCCESS",
-    "37": "SUCCESS",
-    "38": "SUCCESS",
-    "39": "SUCCESS",
-    "4": "SUCCESS",
-    "40": "SUCCESS",
-    "41": "SUCCESS",
-    "42": "SUCCESS",
-    "43": "SUCCESS",
-    "44": "SUCCESS",
-    "45": "SUCCESS",
-    "46": "SUCCESS",
-    "47": "SUCCESS",
-    "48": "SUCCESS",
-    "49": "SUCCESS",
-    "5": "SUCCESS",
-    "50": "SUCCESS",
-    "51": "SUCCESS",
-    "52": "SUCCESS",
-    "53": "SUCCESS",
-    "54": "SUCCESS",
-    "55": "SUCCESS",
-    "56": "SUCCESS",
-    "57": "SUCCESS",
-    "58": "SUCCESS",
-    "59": "SUCCESS",
-    "6": "SUCCESS",
-    "60": "SUCCESS",
-    "61": "SUCCESS",
-    "62": "SUCCESS",
-    "63": "SUCCESS",
-    "64": "SUCCESS",
-    "65": "SUCCESS",
-    "66": "SUCCESS",
-    "67": "SUCCESS",
-    "68": "SUCCESS",
-    "69": "SUCCESS",
-    "7": "SUCCESS",
-    "70": "SUCCESS",
-    "71": "SUCCESS",
-    "72": "SUCCESS",
-    "73": "SUCCESS",
-    "74": "SUCCESS",
-    "75": "SUCCESS",
-    "76": "SUCCESS",
-    "77": "SUCCESS",
-    "78": "SUCCESS",
-    "79": "SUCCESS",
-    "8": "SUCCESS",
-    "80": "SUCCESS",
-    "81": "SUCCESS",
-    "82": "SUCCESS",
-    "83": "SUCCESS",
-    "84": "SUCCESS",
-    "85": "SUCCESS",
-    "86": "SUCCESS",
-    "87": "SUCCESS",
-    "88": "SUCCESS",
-    "89": "SUCCESS",
-    "9": "SUCCESS",
-    "90": "SUCCESS",
-    "91": "SUCCESS",
-    "92": "SUCCESS",
-    "93": "SUCCESS",
-    "94": "SUCCESS",
-    "95": "SUCCESS",
-    "96": "SUCCESS",
-    "97": "SUCCESS",
-    "98": "SUCCESS",
-    "99": "SUCCESS"
+    "3": "NON_PARTICIPATING",
+    "4": "NON_PARTICIPATING",
+    "5": "NON_PARTICIPATING",
+    "6": "NON_PARTICIPATING",
+    "7": "NON_PARTICIPATING",
+    "8": "NON_PARTICIPATING",
+    "9": "NON_PARTICIPATING"
   },
   "architectures": [
     "GPTOptim"
   ],
   "attn_pdrop": 0.1,
   "auto_map": {
-    "AutoConfig": "distributed/optimized-gpt2-500m--configuration_gpt_optimized.GPTOptimConfig",
-    "AutoModelForCausalLM": "distributed/optimized-gpt2-500m--modeling_gpt_optimized.GPTOptim"
+    "AutoConfig": "configuration_gpt_optimized.GPTOptimConfig",
+    "AutoModelForCausalLM": "modeling_gpt_optimized.GPTOptim"
   },
   "block_size": 1024,
   "bos_token_id": 50256,

configuration_gpt_optimized.py

@@ -0,0 +1,22 @@
+from transformers import PretrainedConfig, GPT2Config
+from typing import List
+
+
+class GPTOptimConfig(GPT2Config):
+    model_type = "gpt_optimized"
+
+    def __init__(
+        self,
+        block_size: int = 1024, # max sequence length
+        vocab_size: int = 50257, # number of tokens: 50,000 BPE merges + 256 bytes tokens + 1 <|endoftext|> token
+        n_layer: int = 16, # number of layers
+        n_head: int = 16, # number of heads
+        n_embd: int = 1024, # embedding dimension
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.block_size = block_size
+        self.vocab_size = vocab_size
+        self.n_layer = n_layer
+        self.n_head = n_head
+        self.n_embd = n_embd

modeling_gpt_optimized.py

@@ -0,0 +1,189 @@
+import torch
+import torch.nn as nn
+import bitsandbytes
+from torch.nn import CrossEntropyLoss, functional as F
+from transformers import PreTrainedModel, GPT2PreTrainedModel
+from .configuration_gpt_optimized import GPTOptimConfig
+from transformers.modeling_outputs import CausalLMOutputWithCrossAttentions, BaseModelOutputWithPastAndCrossAttentions
+from transformers.utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward
+from transformers.modeling_attn_mask_utils import _prepare_4d_attention_mask_for_sdpa, _prepare_4d_causal_attention_mask_for_sdpa
+from typing import Optional, Tuple, Union
+
+_CHECKPOINT_FOR_DOC = "openai-community/gpt2"
+_CONFIG_FOR_DOC = "GPT2Config"
+
+GPT2_INPUTS_DOCSTRING = r"""
+    Args:
+        input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`):
+            `input_ids_length` = `sequence_length` if `past_key_values` is `None` else
+            `past_key_values[0][0].shape[-2]` (`sequence_length` of input past key value states). Indices of input
+            sequence tokens in the vocabulary.
+            If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as
+            `input_ids`.
+            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+            [`PreTrainedTokenizer.__call__`] for details.
+            [What are input IDs?](../glossary#input-ids)
+        past_key_values (`Tuple[Tuple[torch.Tensor]]` of length `config.n_layers`):
+            Contains precomputed hidden-states (key and values in the attention blocks) as computed by the model (see
+            `past_key_values` output below). Can be used to speed up sequential decoding. The `input_ids` which have
+            their past given to this model should not be passed as `input_ids` as they have already been computed.
+        attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
+            - 1 for tokens that are **not masked**,
+            - 0 for tokens that are **masked**.
+            If `past_key_values` is used, `attention_mask` needs to contain the masking strategy that was used for
+            `past_key_values`. In other words, the `attention_mask` always has to have the length:
+            `len(past_key_values) + len(input_ids)`
+            [What are attention masks?](../glossary#attention-mask)
+        token_type_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`, *optional*):
+            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
+            1]`:
+            - 0 corresponds to a *sentence A* token,
+            - 1 corresponds to a *sentence B* token.
+            [What are token type IDs?](../glossary#token-type-ids)
+        position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+            config.max_position_embeddings - 1]`.
+            [What are position IDs?](../glossary#position-ids)
+        head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
+            Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:
+            - 1 indicates the head is **not masked**,
+            - 0 indicates the head is **masked**.
+        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
+            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+            model's internal embedding lookup matrix.
+            If `past_key_values` is used, optionally only the last `inputs_embeds` have to be input (see
+            `past_key_values`).
+        use_cache (`bool`, *optional*):
+            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+            `past_key_values`).
+        output_attentions (`bool`, *optional*):
+            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
+            tensors for more detail.
+        output_hidden_states (`bool`, *optional*):
+            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
+            more detail.
+        return_dict (`bool`, *optional*):
+            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+"""
+
+class CausalSelfAttention(nn.Module):
+
+    def __init__(self, config):
+        super().__init__()
+        assert config.n_embd % config.n_head == 0
+        # key, query, value projections for all heads, but in a batch
+        self.c_attn = nn.Linear(config.n_embd, 3 * config.n_embd)
+        # output projection
+        self.c_proj = nn.Linear(config.n_embd, config.n_embd)
+        self.c_proj.NANOGPT_SCALE_INIT = 1
+        # regularization
+        self.n_head = config.n_head
+        self.n_embd = config.n_embd
+
+    def forward(self, x):
+        B, T, C = x.size() # batch size, sequence length, embedding dimensionality (n_embd)
+        # calculate query, key, values for all heads in batch and move head forward to be the batch dim
+        # nh is "number of heads", hs is "head size", and C (number of channels) = nh * hs
+        # e.g. in GPT-2 (124M), n_head=12, hs=64, so nh*hs=C=768 channels in the Transformer
+        qkv = self.c_attn(x)
+        q, k, v = qkv.split(self.n_embd, dim=2)
+        k = k.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs)
+        q = q.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs)
+        v = v.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs)
+        y = F.scaled_dot_product_attention(q, k, v, is_causal=True) # flash attention
+        y = y.transpose(1, 2).contiguous().view(B, T, C) # re-assemble all head outputs side by side
+        # output projection
+        y = self.c_proj(y)
+        return y
+
+class MLP(nn.Module):
+
+    def __init__(self, config):
+        super().__init__()
+        self.c_fc    = nn.Linear(config.n_embd, 4 * config.n_embd)
+        self.gelu    = nn.GELU(approximate='tanh')
+        self.c_proj  = nn.Linear(4 * config.n_embd, config.n_embd)
+        self.c_proj.NANOGPT_SCALE_INIT = 1
+
+    def forward(self, x):
+        x = self.c_fc(x)
+        x = self.gelu(x)
+        x = self.c_proj(x)
+        return x
+
+class Block(nn.Module):
+
+    def __init__(self, config):
+        super().__init__()
+        self.ln_1 = nn.LayerNorm(config.n_embd)
+        self.attn = CausalSelfAttention(config)
+        self.ln_2 = nn.LayerNorm(config.n_embd)
+        self.mlp = MLP(config)
+
+    def forward(self, x):
+        x = x + self.attn(self.ln_1(x))
+        x = x + self.mlp(self.ln_2(x))
+        return x
+
+class GPT(nn.Module):
+
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+
+        self.transformer = nn.ModuleDict(dict(
+            wte = bitsandbytes.nn.StableEmbedding(config.vocab_size, config.n_embd),
+            wpe = bitsandbytes.nn.StableEmbedding(config.block_size, config.n_embd),
+            h = nn.ModuleList([Block(config) for _ in range(config.n_layer)]),
+            ln_f = nn.LayerNorm(config.n_embd),
+        ))
+        self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False)
+
+        # weight sharing scheme
+        self.transformer.wte.weight = self.lm_head.weight
+
+        # init params
+        self.apply(self._init_weights)
+
+    def _init_weights(self, module):
+        if isinstance(module, nn.Linear):
+            std = 0.02
+            if hasattr(module, 'NANOGPT_SCALE_INIT'):
+                std *= (2 * self.config.n_layer) ** -0.5
+            torch.nn.init.normal_(module.weight, mean=0.0, std=std)
+            if module.bias is not None:
+                torch.nn.init.zeros_(module.bias)
+        elif isinstance(module, nn.Embedding):
+            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
+
+class GPTOptim(GPT2PreTrainedModel):
+    config_class = GPTOptimConfig
+    
+    def __init__(self, config):
+        super().__init__(config)
+        self.model = GPT(
+            config
+        )
+        self.config = config
+    
+    def forward(self, input_ids, labels=None):
+        # input_ids is of shape (B, T)
+        B, T = input_ids.size()
+        assert T <= self.config.block_size, f"Cannot forward sequence of length {T}, block size is only {self.config.block_size}"
+        # forward the token and posisition embeddings
+        pos = torch.arange(0, T, dtype=torch.long, device=input_ids.device) # shape (T)
+        pos_emb = self.model.transformer.wpe(pos) # position embeddings of shape (T, n_embd)
+        tok_emb = self.model.transformer.wte(input_ids) # token embeddings of shape (B, T, n_embd)
+        x = tok_emb + pos_emb
+        # forward the blocks of the transformer
+        for block in self.model.transformer.h:
+            x = block(x)
+        # forward the final layernorm and the classifier
+        x = self.model.transformer.ln_f(x)
+        logits = self.model.lm_head(x) # (B, T, vocab_size)
+        loss = None
+        if labels is not None:
+            loss = F.cross_entropy(logits.view(-1, logits.size(-1)), labels.view(-1), ignore_index=self.config.eos_token_id)
+        return logits, loss