Upload sCT

config.json

@@ -18,7 +18,7 @@
   "layer_norm_eps": 1e-05,
   "mask_token_id": 5,
   "max_positions": 20480,
-  "model_type": "sCellTransformer",
+  "model_type": "sCT",
   "num_cells": 50,
   "num_downsamples": 8,
   "num_hidden_layers_head": 1,

config.py

@@ -1,4 +1,4 @@
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 from typing import Tuple
 
 from transformers import PretrainedConfig
@@ -6,7 +6,7 @@ from transformers import PretrainedConfig
 
 @dataclass
 class sCTConfig(PretrainedConfig):  # noqa: N801
-    model_type = "sCellTransformer"
+    model_type = "sCT"
 
     def __init__(self, **kwargs):  # type: ignore
         super().__init__()

sct.py

@@ -672,9 +672,7 @@ class sCT(PreTrainedModel):  # noqa: N801
         for _idx, conv_block in enumerate(self.conv_tower):
             x, res = conv_block(x)
             residuals.append(res)
-        outs["residuals"] = residuals
         residuals = residuals[::-1]
-        conv_block_out = x
         x = x.permute(0, 2, 1)
 
         for layer_idx, transformer in enumerate(self.transformer_layers):
@@ -686,16 +684,11 @@ class sCT(PreTrainedModel):  # noqa: N801
                 for map_number in self._attention_maps_per_layer_to_save[layer_idx + 1]:
                     dkey = f"attention_map_layer_{layer_idx + 1}_number_{map_number}"
                     outs[dkey] = output["attention_weights"][:, map_number + 1]
-        transformer_output = x
         x = x.permute(0, 2, 1)
         for deconv_block, res in zip(self.deconv_tower, residuals):
             x = deconv_block(x, res)
-        deconv_block_out = x
         x = x.permute(0, 2, 1)
         logits = self.lm_head(x)
         outs["logits"] = logits
-        outs["transformer_output"] = transformer_output
-        outs["conv_out"] = conv_block_out
-        outs["deconv_out"] = deconv_block_out
 
         return outs