Upload 10 files

README.md

@@ -0,0 +1,68 @@
+---
+language:
+- en
+license: apache-2.0
+datasets:
+- wikimedia/wikipedia
+- bookcorpus
+- nomic-ai/nomic-bert-2048-pretraining-data
+inference: false
+---
+
+# nomic-bert-2048: A 2048 Sequence Length Pretrained BERT
+
+`nomic-bert-2048` is a BERT model pretrained on `wikipedia` and `bookcorpus` with a max sequence length of 2048. 
+
+We make several modifications to our BERT training procedure similar to [MosaicBERT](https://www.databricks.com/blog/mosaicbert).
+Namely, we add:
+- Use [Rotary Position Embeddings](https://arxiv.org/pdf/2104.09864.pdf) to allow for context length extrapolation.
+- Use SwiGLU activations as it has [been shown](https://arxiv.org/abs/2002.05202) to [improve model performance](https://www.databricks.com/blog/mosaicbert)
+- Set dropout to 0
+
+We evaluate the quality of nomic-bert-2048 on the standard [GLUE](https://gluebenchmark.com/) benchmark. We find
+it performs comparably to other BERT models but with the advantage of a significantly longer context length.
+
+| Model       | Bsz | Steps | Seq   | Avg      | Cola     | SST2     | MRPC | STSB | QQP  | MNLI | QNLI | RTE  |
+|-------------|-----|-------|-------|----------|----------|----------|------|------|------|------|------|------|
+| NomicBERT   | 4k  | 100k  | 2048  | 0.84     | 0.50     | 0.93     | 0.88 | 0.90 | 0.92 | 0.86 | 0.92 | 0.82 |
+| RobertaBase | 8k  | 500k  | 512   | 0.86     | 0.64     | 0.95     | 0.90 | 0.91 | 0.92 | 0.88 | 0.93 | 0.79 |
+| JinaBERTBase| 4k  | 100k  | 512   | 0.83     | 0.51     | 0.95     | 0.88 | 0.90 | 0.81 | 0.86 | 0.92 | 0.79 |
+| MosaicBERT  | 4k  | 178k  | 128   | 0.85     | 0.59     | 0.94     | 0.89 | 0.90 | 0.92 | 0.86 | 0.91 | 0.83 |
+
+## Pretraining Data
+
+We use [BookCorpus](https://huggingface.co/datasets/bookcorpus) and a 2023 dump of [wikipedia](https://huggingface.co/datasets/wikimedia/wikipedia). 
+We pack and tokenize the sequences to 2048 tokens. If a document is shorter than 2048 tokens, we append another document until it fits 2048 tokens.
+If a document is greater than 2048 tokens, we split it across multiple documents. We release the dataset [here](https://huggingface.co/datasets/nomic-ai/nomic-bert-2048-pretraining-data/)
+
+
+# Usage
+
+```python
+from transformers import AutoModelForMaskedLM, AutoConfig, AutoTokenizer, pipeline
+
+tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased') # `nomic-bert-2048` uses the standard BERT tokenizer
+
+config = AutoConfig.from_pretrained('nomic-ai/nomic-bert-2048', trust_remote_code=True) # the config needs to be passed in
+model = AutoModelForMaskedLM.from_pretrained('nomic-ai/nomic-bert-2048',config=config, trust_remote_code=True)
+
+# To use this model directly for masked language modeling
+classifier = pipeline('fill-mask', model=model, tokenizer=tokenizer,device="cpu")
+
+print(classifier("I [MASK] to the store yesterday."))
+```
+To finetune the model for a Sequence Classification task, you can use the following snippet
+
+```python
+from transformers import AutoConfig, AutoModelForSequenceClassification
+model_path = "nomic-ai/nomic-bert-2048"
+config = AutoConfig.from_pretrained(model_path, trust_remote_code=True)
+# strict needs to be false here since we're initializing some new params
+model = AutoModelForSequenceClassification.from_pretrained(model_path, config=config, trust_remote_code=True, strict=False)
+```
+
+# Join the Nomic Community
+
+- Nomic: [https://nomic.ai](https://nomic.ai)
+- Discord: [https://discord.gg/myY5YDR8z8](https://discord.gg/myY5YDR8z8)
+- Twitter: [https://twitter.com/nomic_ai](https://twitter.com/nomic_ai)

config.json

@@ -0,0 +1,59 @@
+{
+  "activation_function": "swiglu",
+  "architectures": [
+    "NomicBertForPreTraining"
+  ],
+  "attn_pdrop": 0.0,
+  "auto_map": {
+    "AutoConfig": "configuration_hf_nomic_bert.NomicBertConfig",
+    "AutoModel": "modeling_hf_nomic_bert.NomicBertModel",
+    "AutoModelForMaskedLM": "modeling_hf_nomic_bert.NomicBertForPreTraining",
+    "AutoModelForSequenceClassification": "modeling_hf_nomic_bert.NomicBertForSequenceClassification",
+    "AutoModelForMultipleChoice": "modeling_hf_nomic_bert.NomicBertForMultipleChoice",
+    "AutoModelForQuestionAnswering": "modeling_hf_nomic_bert.NomicBertForQuestionAnswering",
+    "AutoModelForTokenClassification": "modeling_hf_nomic_bert.NomicBertForTokenClassification"
+  },
+  "bos_token_id": null,
+  "causal": false,
+  "dense_seq_output": true,
+  "embd_pdrop": 0.1,
+  "eos_token_id": null,
+  "fused_bias_fc": true,
+  "fused_dropout_add_ln": true,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-12,
+  "mlp_fc1_bias": false,
+  "mlp_fc2_bias": false,
+  "model_type": "nomic_bert",
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": 3072,
+  "n_layer": 12,
+  "n_positions": 2048,
+  "pad_vocab_size_multiple": 64,
+  "parallel_block": false,
+  "parallel_block_tied_norm": false,
+  "prenorm": false,
+  "qkv_proj_bias": false,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "rotary_emb_base": 1000,
+  "rotary_emb_fraction": 1.0,
+  "rotary_emb_interleaved": false,
+  "rotary_emb_scale_base": null,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.34.0",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "use_flash_attn": true,
+  "use_rms_norm": false,
+  "use_xentropy": true,
+  "vocab_size": 30528
+}

configuration_hf_nomic_bert.py

@@ -0,0 +1,56 @@
+from transformers import GPT2Config
+
+
+class NomicBertConfig(GPT2Config):
+    model_type = "nomic_bert"
+
+    def __init__(
+        self,
+        prenorm=False,
+        parallel_block=False,
+        parallel_block_tied_norm=False,
+        rotary_emb_fraction=0.0,
+        fused_dropout_add_ln=False,
+        fused_bias_fc=False,
+        use_flash_attn=False,
+        use_xentropy=False,
+        qkv_proj_bias=True,
+        rotary_emb_base=10_000,
+        rotary_emb_scale_base=None,
+        rotary_emb_interleaved=False,
+        mlp_fc1_bias=True,
+        mlp_fc2_bias=True,
+        use_rms_norm=False,
+        causal=False,
+        type_vocab_size=2,
+        dense_seq_output=True,
+        pad_vocab_size_multiple=1,
+        tie_word_embeddings=True,
+        rotary_scaling_factor=None,
+        max_trained_positions=2048,
+        **kwargs,
+    ):
+        self.prenorm = prenorm
+        self.parallel_block = parallel_block
+        self.parallel_block_tied_norm = parallel_block_tied_norm
+        self.rotary_emb_fraction = rotary_emb_fraction
+        self.tie_word_embeddings = tie_word_embeddings
+        self.fused_dropout_add_ln = fused_dropout_add_ln
+        self.fused_bias_fc = fused_bias_fc
+        self.use_flash_attn = use_flash_attn
+        self.use_xentropy = use_xentropy
+        self.qkv_proj_bias = qkv_proj_bias
+        self.rotary_emb_base = rotary_emb_base
+        self.rotary_emb_scale_base = rotary_emb_scale_base
+        self.rotary_emb_interleaved = rotary_emb_interleaved
+        self.mlp_fc1_bias = mlp_fc1_bias
+        self.mlp_fc2_bias = mlp_fc2_bias
+        self.use_rms_norm = use_rms_norm
+        self.causal = causal
+        self.type_vocab_size = type_vocab_size
+        self.dense_seq_output = dense_seq_output
+        self.pad_vocab_size_multiple = pad_vocab_size_multiple
+        self.rotary_scaling_factor = rotary_scaling_factor
+        self.max_trained_positions = max_trained_positions
+
+        super().__init__(**kwargs)

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9acacdb7c24a8eac0fc7c11b00c4a2e7986a49aacbd40c9b30ae12b2763ead19
+size 549304584

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0b1c7db843aeae0c744716acf3c7b7da60d142e7ca31edd11853f5b163c8776
+size 549328982

special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

tokenizer_config.json

@@ -0,0 +1,55 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "101": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "102": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "103": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_lower_case": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 8192,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}