Llamacpp quants

.gitattributes

@@ -33,3 +33,16 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+Hercules-3.1-Mistral-7B-Q2_K.gguf filter=lfs diff=lfs merge=lfs -text
+Hercules-3.1-Mistral-7B-Q3_K_L.gguf filter=lfs diff=lfs merge=lfs -text
+Hercules-3.1-Mistral-7B-Q3_K_M.gguf filter=lfs diff=lfs merge=lfs -text
+Hercules-3.1-Mistral-7B-Q3_K_S.gguf filter=lfs diff=lfs merge=lfs -text
+Hercules-3.1-Mistral-7B-Q4_0.gguf filter=lfs diff=lfs merge=lfs -text
+Hercules-3.1-Mistral-7B-Q4_K_M.gguf filter=lfs diff=lfs merge=lfs -text
+Hercules-3.1-Mistral-7B-Q4_K_S.gguf filter=lfs diff=lfs merge=lfs -text
+Hercules-3.1-Mistral-7B-Q5_0.gguf filter=lfs diff=lfs merge=lfs -text
+Hercules-3.1-Mistral-7B-Q5_K_M.gguf filter=lfs diff=lfs merge=lfs -text
+Hercules-3.1-Mistral-7B-Q5_K_S.gguf filter=lfs diff=lfs merge=lfs -text
+Hercules-3.1-Mistral-7B-Q6_K.gguf filter=lfs diff=lfs merge=lfs -text
+Hercules-3.1-Mistral-7B-Q8_0.gguf filter=lfs diff=lfs merge=lfs -text
+Hercules-3.1-Mistral-7B-fp16.gguf filter=lfs diff=lfs merge=lfs -text

Hercules-3.1-Mistral-7B-Q2_K.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8597174984cb9e2f7f914c9ee767cbc6c248703222ba8fa4bb1a9b23a8ccac4a
+size 2719241984

Hercules-3.1-Mistral-7B-Q3_K_L.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:512f541d3886a7d3a661ef3bd044ac7b74c168c28ab22ee5a4b44d4f68f17aa4
+size 3822024448

Hercules-3.1-Mistral-7B-Q3_K_M.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9df8d1cab8ec62358be6314d6ab881e10de3cc807d688d127b6e747574e831df
+size 3518985984

Hercules-3.1-Mistral-7B-Q3_K_S.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6bf20da6761353fb4514f24cf051cbe5c20d3c860f7a31eaef1e0ad9835a88d8
+size 3164567296

Hercules-3.1-Mistral-7B-Q4_0.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:745629bf898b12b963cbccfc2c7c8e4d849171782e077e88115a1d8758560c87
+size 4108916480

Hercules-3.1-Mistral-7B-Q4_K_M.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e6645d32437f9765ee350e808ec0acb58499a76cb2454d7de6fbc38f34968c9
+size 4368439040

Hercules-3.1-Mistral-7B-Q4_K_S.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83ba3ee3a3a2f06d6503f4d1d8ac0e3e5b9e936d68f7f958a9cde1d0f2c27f1e
+size 4140373760

Hercules-3.1-Mistral-7B-Q5_0.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a4a13c86c5389881b55116a57181e815749b2408f9eafe3d15f8948f21c2fe2
+size 4997715712

Hercules-3.1-Mistral-7B-Q5_K_M.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6df84d2cf1d8c8a783163d87ab6199e0669946de40d04b195d2a5ebf1798245
+size 5131409152

Hercules-3.1-Mistral-7B-Q5_K_S.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf665a4af9f745b2999318653691ab310652bdcb9162198417bcfddbfd20f0b2
+size 4997715712

Hercules-3.1-Mistral-7B-Q6_K.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cdf618abf854cb0e702f9f6c62c8033a89d0a0457033471da3b9ab4ce677216f
+size 5942064896

Hercules-3.1-Mistral-7B-Q8_0.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:829b60d1fadde45c7389b07f096cc8419a9e413595781938341fb4ce96ab4561
+size 7695857408

Hercules-3.1-Mistral-7B-fp16.gguf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4529fbd18f86c37abf59e5749a7e3617f7e2809dd68f34210969bf506bc05e2d
+size 14484731584

README.md

@@ -0,0 +1,142 @@
+---
+license: apache-2.0
+library_name: transformers
+tags:
+- chemistry
+- biology
+- code
+- medical
+- not-for-all-audiences
+datasets:
+- Locutusque/Hercules-v3.0
+model-index:
+- name: Hercules-3.1-Mistral-7B
+  results:
+  - task:
+      type: text-generation
+      name: Text Generation
+    dataset:
+      name: AI2 Reasoning Challenge (25-Shot)
+      type: ai2_arc
+      config: ARC-Challenge
+      split: test
+      args:
+        num_few_shot: 25
+    metrics:
+    - type: acc_norm
+      value: 61.18
+      name: normalized accuracy
+    source:
+      url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=Locutusque/Hercules-3.1-Mistral-7B
+      name: Open LLM Leaderboard
+  - task:
+      type: text-generation
+      name: Text Generation
+    dataset:
+      name: HellaSwag (10-Shot)
+      type: hellaswag
+      split: validation
+      args:
+        num_few_shot: 10
+    metrics:
+    - type: acc_norm
+      value: 83.55
+      name: normalized accuracy
+    source:
+      url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=Locutusque/Hercules-3.1-Mistral-7B
+      name: Open LLM Leaderboard
+  - task:
+      type: text-generation
+      name: Text Generation
+    dataset:
+      name: MMLU (5-Shot)
+      type: cais/mmlu
+      config: all
+      split: test
+      args:
+        num_few_shot: 5
+    metrics:
+    - type: acc
+      value: 63.65
+      name: accuracy
+    source:
+      url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=Locutusque/Hercules-3.1-Mistral-7B
+      name: Open LLM Leaderboard
+  - task:
+      type: text-generation
+      name: Text Generation
+    dataset:
+      name: TruthfulQA (0-shot)
+      type: truthful_qa
+      config: multiple_choice
+      split: validation
+      args:
+        num_few_shot: 0
+    metrics:
+    - type: mc2
+      value: 42.83
+    source:
+      url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=Locutusque/Hercules-3.1-Mistral-7B
+      name: Open LLM Leaderboard
+  - task:
+      type: text-generation
+      name: Text Generation
+    dataset:
+      name: Winogrande (5-shot)
+      type: winogrande
+      config: winogrande_xl
+      split: validation
+      args:
+        num_few_shot: 5
+    metrics:
+    - type: acc
+      value: 79.01
+      name: accuracy
+    source:
+      url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=Locutusque/Hercules-3.1-Mistral-7B
+      name: Open LLM Leaderboard
+  - task:
+      type: text-generation
+      name: Text Generation
+    dataset:
+      name: GSM8k (5-shot)
+      type: gsm8k
+      config: main
+      split: test
+      args:
+        num_few_shot: 5
+    metrics:
+    - type: acc
+      value: 42.3
+      name: accuracy
+    source:
+      url: https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard?query=Locutusque/Hercules-3.1-Mistral-7B
+      name: Open LLM Leaderboard
+quantized_by: bartowski
+pipeline_tag: text-generation
+---
+
+## Llamacpp Quantizations of Hercules-3.1-Mistral-7B
+
+Using <a href="https://github.com/ggerganov/llama.cpp/commit/fa974646e1a2024fc7dc9e6f27cf1f2f5d4a3763">llama.cpp commit fa97464</a> for quantization.
+
+Original model: https://huggingface.co/Locutusque/Hercules-3.1-Mistral-7B
+
+Download a file (not the whole branch) from below:
+
+| Filename | Quant type | File Size | Description |
+| -------- | ---------- | --------- | ----------- |
+| [Hercules-3.1-Mistral-7B-Q8_0.gguf](https://huggingface.co/bartowski/Hercules-3.1-Mistral-7B-GGUF/blob/main/Hercules-3.1-Mistral-7B-Q8_0.gguf) | Q8_0 | 7.69GB | Extremely high quality, generally unneeded but max available quant. |
+| [Hercules-3.1-Mistral-7B-Q6_K.gguf](https://huggingface.co/bartowski/Hercules-3.1-Mistral-7B-GGUF/blob/main/Hercules-3.1-Mistral-7B-Q6_K.gguf) | Q6_K | 5.94GB | Very high quality, near perfect, *recommended*. |
+| [Hercules-3.1-Mistral-7B-Q5_K_M.gguf](https://huggingface.co/bartowski/Hercules-3.1-Mistral-7B-GGUF/blob/main/Hercules-3.1-Mistral-7B-Q5_K_M.gguf) | Q5_K_M | 5.13GB | High quality, very usable. |
+| [Hercules-3.1-Mistral-7B-Q5_K_S.gguf](https://huggingface.co/bartowski/Hercules-3.1-Mistral-7B-GGUF/blob/main/Hercules-3.1-Mistral-7B-Q5_K_S.gguf) | Q5_K_S | 4.99GB | High quality, very usable. |
+| [Hercules-3.1-Mistral-7B-Q5_0.gguf](https://huggingface.co/bartowski/Hercules-3.1-Mistral-7B-GGUF/blob/main/Hercules-3.1-Mistral-7B-Q5_0.gguf) | Q5_0 | 4.99GB | High quality, older format, generally not recommended. |
+| [Hercules-3.1-Mistral-7B-Q4_K_M.gguf](https://huggingface.co/bartowski/Hercules-3.1-Mistral-7B-GGUF/blob/main/Hercules-3.1-Mistral-7B-Q4_K_M.gguf) | Q4_K_M | 4.36GB | Good quality, similar to 4.25 bpw. |
+| [Hercules-3.1-Mistral-7B-Q4_K_S.gguf](https://huggingface.co/bartowski/Hercules-3.1-Mistral-7B-GGUF/blob/main/Hercules-3.1-Mistral-7B-Q4_K_S.gguf) | Q4_K_S | 4.14GB | Slightly lower quality with small space savings. |
+| [Hercules-3.1-Mistral-7B-Q4_0.gguf](https://huggingface.co/bartowski/Hercules-3.1-Mistral-7B-GGUF/blob/main/Hercules-3.1-Mistral-7B-Q4_0.gguf) | Q4_0 | 4.10GB | Decent quality, older format, generally not recommended. |
+| [Hercules-3.1-Mistral-7B-Q3_K_L.gguf](https://huggingface.co/bartowski/Hercules-3.1-Mistral-7B-GGUF/blob/main/Hercules-3.1-Mistral-7B-Q3_K_L.gguf) | Q3_K_L | 3.82GB | Lower quality but usable, good for low RAM availability. |
+| [Hercules-3.1-Mistral-7B-Q3_K_M.gguf](https://huggingface.co/bartowski/Hercules-3.1-Mistral-7B-GGUF/blob/main/Hercules-3.1-Mistral-7B-Q3_K_M.gguf) | Q3_K_M | 3.51GB | Even lower quality. |
+| [Hercules-3.1-Mistral-7B-Q3_K_S.gguf](https://huggingface.co/bartowski/Hercules-3.1-Mistral-7B-GGUF/blob/main/Hercules-3.1-Mistral-7B-Q3_K_S.gguf) | Q3_K_S | 3.16GB | Low quality, not recommended. |
+| [Hercules-3.1-Mistral-7B-Q2_K.gguf](https://huggingface.co/bartowski/Hercules-3.1-Mistral-7B-GGUF/blob/main/Hercules-3.1-Mistral-7B-Q2_K.gguf) | Q2_K | 2.71GB | Extremely low quality, *not* recommended.
+
+Want to support my work? Visit my ko-fi page here: https://ko-fi.com/bartowski