Upload model

config.json

@@ -0,0 +1,35 @@
+{
+  "architectures": [
+    "HierarchicalBertModel"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "auto_map": {
+    "AutoModel": "hierarchical_bert_architecture.HierarchicalBertModel"
+  },
+  "classifier_dropout": null,
+  "directionality": "bidi",
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "num_main_segment": 0,
+  "num_sub_segment": 0,
+  "pad_token_id": 0,
+  "pooler_fc_size": 768,
+  "pooler_num_attention_heads": 12,
+  "pooler_num_fc_layers": 3,
+  "pooler_size_per_head": 128,
+  "pooler_type": "first_token_transform",
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.22.2",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 105879
+}

hierarchical_bert_architecture.py

@@ -0,0 +1,25 @@
+from torch import nn
+import torch
+from transformers import BertModel, PreTrainedModel
+
+from typing import Tuple
+
+
+class HierarchicalBertModel(PreTrainedModel):
+
+    def __init__(self, config, num_main_segment=None, num_sub_segment=None):
+        super(HierarchicalBertModel, self).__init__(config=config)
+        self.num_main_segment = num_main_segment if num_main_segment else config.num_main_segment
+        self.num_sub_segment = num_sub_segment if num_sub_segment else config.num_sub_segment
+        self.bert = BertModel.from_pretrained("bert-base-multilingual-uncased")
+        self.dropout = nn.Dropout(0.1)
+        self.hidden_2 = nn.Linear(768, 768)
+        self.fc_main = nn.Linear(768, self.num_main_segment)
+        self.fc_sub = nn.Linear(768, self.num_sub_segment)
+
+    def forward(self, input_ids: torch.tensor, attention_masks: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
+        outputs = self.bert(input_ids=input_ids,
+                            attention_mask=attention_masks)
+        last_hidden_state_cls = outputs[0][:, 0, :]
+        out = self.hidden_2(last_hidden_state_cls)
+        return self.fc_main(last_hidden_state_cls), self.fc_sub(last_hidden_state_cls)

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba54163add74c9f167845c3e6e366bd441ad30e9236e1beeeba668fe552041b9
+size 671857125