Add new SentenceTransformer model
Browse files- 2_Dense/config.json +6 -0
- 2_Dense/model.safetensors +3 -0
- 3_Dense/config.json +6 -0
- 3_Dense/model.safetensors +3 -0
- README.md +5 -72
- modules.json +12 -0
2_Dense/config.json
ADDED
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"in_features": 768,
|
3 |
+
"out_features": 768,
|
4 |
+
"bias": true,
|
5 |
+
"activation_function": "torch.nn.modules.activation.ReLU"
|
6 |
+
}
|
2_Dense/model.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:d5e0d234a36d4cd513a1f95bf413a6eea8972f8c835c5270a3e57d4eabf1b5ed
|
3 |
+
size 1181344
|
3_Dense/config.json
ADDED
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"in_features": 768,
|
3 |
+
"out_features": 768,
|
4 |
+
"bias": true,
|
5 |
+
"activation_function": "torch.nn.modules.linear.Identity"
|
6 |
+
}
|
3_Dense/model.safetensors
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:38a849eb316f199a702045d159bf2fd1eec62ad05f2bc051329e42e986c6731a
|
3 |
+
size 1181344
|
README.md
CHANGED
@@ -63,49 +63,6 @@ datasets:
|
|
63 |
- redis/langcache-sentencepairs-v2
|
64 |
pipeline_tag: sentence-similarity
|
65 |
library_name: sentence-transformers
|
66 |
-
metrics:
|
67 |
-
- cosine_accuracy@1
|
68 |
-
- cosine_precision@1
|
69 |
-
- cosine_recall@1
|
70 |
-
- cosine_ndcg@10
|
71 |
-
- cosine_mrr@1
|
72 |
-
- cosine_map@100
|
73 |
-
- cosine_auc_precision_cache_hit_ratio
|
74 |
-
- cosine_auc_similarity_distribution
|
75 |
-
model-index:
|
76 |
-
- name: Redis fine-tuned BiEncoder model for semantic caching on LangCache
|
77 |
-
results:
|
78 |
-
- task:
|
79 |
-
type: custom-information-retrieval
|
80 |
-
name: Custom Information Retrieval
|
81 |
-
dataset:
|
82 |
-
name: test
|
83 |
-
type: test
|
84 |
-
metrics:
|
85 |
-
- type: cosine_accuracy@1
|
86 |
-
value: 0.5953768980477223
|
87 |
-
name: Cosine Accuracy@1
|
88 |
-
- type: cosine_precision@1
|
89 |
-
value: 0.5953768980477223
|
90 |
-
name: Cosine Precision@1
|
91 |
-
- type: cosine_recall@1
|
92 |
-
value: 0.5778879609728815
|
93 |
-
name: Cosine Recall@1
|
94 |
-
- type: cosine_ndcg@10
|
95 |
-
value: 0.7775436499957671
|
96 |
-
name: Cosine Ndcg@10
|
97 |
-
- type: cosine_mrr@1
|
98 |
-
value: 0.5953768980477223
|
99 |
-
name: Cosine Mrr@1
|
100 |
-
- type: cosine_map@100
|
101 |
-
value: 0.7274666565910912
|
102 |
-
name: Cosine Map@100
|
103 |
-
- type: cosine_auc_precision_cache_hit_ratio
|
104 |
-
value: 0.36387321267916206
|
105 |
-
name: Cosine Auc Precision Cache Hit Ratio
|
106 |
-
- type: cosine_auc_similarity_distribution
|
107 |
-
value: 0.15403918371209657
|
108 |
-
name: Cosine Auc Similarity Distribution
|
109 |
---
|
110 |
|
111 |
# Redis fine-tuned BiEncoder model for semantic caching on LangCache
|
@@ -137,6 +94,8 @@ This is a [sentence-transformers](https://www.SBERT.net) model finetuned from [A
|
|
137 |
SentenceTransformer(
|
138 |
(0): Transformer({'max_seq_length': 100, 'do_lower_case': False, 'architecture': 'ModernBertModel'})
|
139 |
(1): Pooling({'word_embedding_dimension': 768, 'pooling_mode_cls_token': True, 'pooling_mode_mean_tokens': False, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False, 'pooling_mode_weightedmean_tokens': False, 'pooling_mode_lasttoken': False, 'include_prompt': True})
|
|
|
|
|
140 |
)
|
141 |
```
|
142 |
|
@@ -169,9 +128,9 @@ print(embeddings.shape)
|
|
169 |
# Get the similarity scores for the embeddings
|
170 |
similarities = model.similarity(embeddings, embeddings)
|
171 |
print(similarities)
|
172 |
-
# tensor([[1.
|
173 |
-
# [1.
|
174 |
-
# [0.
|
175 |
```
|
176 |
|
177 |
<!--
|
@@ -198,26 +157,6 @@ You can finetune this model on your own dataset.
|
|
198 |
*List how the model may foreseeably be misused and address what users ought not to do with the model.*
|
199 |
-->
|
200 |
|
201 |
-
## Evaluation
|
202 |
-
|
203 |
-
### Metrics
|
204 |
-
|
205 |
-
#### Custom Information Retrieval
|
206 |
-
|
207 |
-
* Dataset: `test`
|
208 |
-
* Evaluated with <code>ir_evaluator.CustomInformationRetrievalEvaluator</code>
|
209 |
-
|
210 |
-
| Metric | Value |
|
211 |
-
|:-------------------------------------|:-----------|
|
212 |
-
| cosine_accuracy@1 | 0.5954 |
|
213 |
-
| cosine_precision@1 | 0.5954 |
|
214 |
-
| cosine_recall@1 | 0.5779 |
|
215 |
-
| **cosine_ndcg@10** | **0.7775** |
|
216 |
-
| cosine_mrr@1 | 0.5954 |
|
217 |
-
| cosine_map@100 | 0.7275 |
|
218 |
-
| cosine_auc_precision_cache_hit_ratio | 0.3639 |
|
219 |
-
| cosine_auc_similarity_distribution | 0.154 |
|
220 |
-
|
221 |
<!--
|
222 |
## Bias, Risks and Limitations
|
223 |
|
@@ -433,12 +372,6 @@ You can finetune this model on your own dataset.
|
|
433 |
|
434 |
</details>
|
435 |
|
436 |
-
### Training Logs
|
437 |
-
| Epoch | Step | test_cosine_ndcg@10 |
|
438 |
-
|:-----:|:----:|:-------------------:|
|
439 |
-
| -1 | -1 | 0.7775 |
|
440 |
-
|
441 |
-
|
442 |
### Framework Versions
|
443 |
- Python: 3.12.3
|
444 |
- Sentence Transformers: 5.1.0
|
|
|
63 |
- redis/langcache-sentencepairs-v2
|
64 |
pipeline_tag: sentence-similarity
|
65 |
library_name: sentence-transformers
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
66 |
---
|
67 |
|
68 |
# Redis fine-tuned BiEncoder model for semantic caching on LangCache
|
|
|
94 |
SentenceTransformer(
|
95 |
(0): Transformer({'max_seq_length': 100, 'do_lower_case': False, 'architecture': 'ModernBertModel'})
|
96 |
(1): Pooling({'word_embedding_dimension': 768, 'pooling_mode_cls_token': True, 'pooling_mode_mean_tokens': False, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False, 'pooling_mode_weightedmean_tokens': False, 'pooling_mode_lasttoken': False, 'include_prompt': True})
|
97 |
+
(mlp_hidden): Dense({'in_features': 768, 'out_features': 768, 'bias': True, 'activation_function': 'torch.nn.modules.activation.ReLU'})
|
98 |
+
(mlp_out): Dense({'in_features': 768, 'out_features': 768, 'bias': True, 'activation_function': 'torch.nn.modules.linear.Identity'})
|
99 |
)
|
100 |
```
|
101 |
|
|
|
128 |
# Get the similarity scores for the embeddings
|
129 |
similarities = model.similarity(embeddings, embeddings)
|
130 |
print(similarities)
|
131 |
+
# tensor([[1.0000, 1.0000, 0.7693],
|
132 |
+
# [1.0000, 1.0000, 0.7693],
|
133 |
+
# [0.7693, 0.7693, 1.0000]])
|
134 |
```
|
135 |
|
136 |
<!--
|
|
|
157 |
*List how the model may foreseeably be misused and address what users ought not to do with the model.*
|
158 |
-->
|
159 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
160 |
<!--
|
161 |
## Bias, Risks and Limitations
|
162 |
|
|
|
372 |
|
373 |
</details>
|
374 |
|
|
|
|
|
|
|
|
|
|
|
|
|
375 |
### Framework Versions
|
376 |
- Python: 3.12.3
|
377 |
- Sentence Transformers: 5.1.0
|
modules.json
CHANGED
@@ -10,5 +10,17 @@
|
|
10 |
"name": "1",
|
11 |
"path": "1_Pooling",
|
12 |
"type": "sentence_transformers.models.Pooling"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
13 |
}
|
14 |
]
|
|
|
10 |
"name": "1",
|
11 |
"path": "1_Pooling",
|
12 |
"type": "sentence_transformers.models.Pooling"
|
13 |
+
},
|
14 |
+
{
|
15 |
+
"idx": 2,
|
16 |
+
"name": "mlp_hidden",
|
17 |
+
"path": "2_Dense",
|
18 |
+
"type": "sentence_transformers.models.Dense"
|
19 |
+
},
|
20 |
+
{
|
21 |
+
"idx": 3,
|
22 |
+
"name": "mlp_out",
|
23 |
+
"path": "3_Dense",
|
24 |
+
"type": "sentence_transformers.models.Dense"
|
25 |
}
|
26 |
]
|