|
|
--- |
|
|
language: |
|
|
- en |
|
|
license: apache-2.0 |
|
|
tags: |
|
|
- sentence-transformers |
|
|
- sparse-encoder |
|
|
- sparse |
|
|
- splade |
|
|
- generated_from_trainer |
|
|
- dataset_size:99000 |
|
|
- loss:SpladeLoss |
|
|
- loss:SparseMultipleNegativesRankingLoss |
|
|
- loss:FlopsLoss |
|
|
base_model: distilbert/distilbert-base-uncased |
|
|
widget: |
|
|
- text: 'The term emergent literacy signals a belief that, in a literate society, |
|
|
young children even one and two year olds, are in the process of becoming literate”. |
|
|
... Gray (1956:21) notes: Functional literacy is used for the training of adults |
|
|
to ''meet independently the reading and writing demands placed on them''.' |
|
|
- text: Rey is seemingly confirmed as being The Chosen One per a quote by a Lucasfilm |
|
|
production designer who worked on The Rise of Skywalker. |
|
|
- text: are union gun safes fireproof? |
|
|
- text: Fruit is an essential part of a healthy diet — and may aid weight loss. Most |
|
|
fruits are low in calories while high in nutrients and fiber, which can boost |
|
|
your fullness. Keep in mind that it's best to eat fruits whole rather than juiced. |
|
|
What's more, simply eating fruit is not the key to weight loss. |
|
|
- text: Treatment of suspected bacterial infection is with antibiotics, such as amoxicillin/clavulanate |
|
|
or doxycycline, given for 5 to 7 days for acute sinusitis and for up to 6 weeks |
|
|
for chronic sinusitis. |
|
|
datasets: |
|
|
- sentence-transformers/gooaq |
|
|
pipeline_tag: feature-extraction |
|
|
library_name: sentence-transformers |
|
|
metrics: |
|
|
- dot_accuracy@1 |
|
|
- dot_accuracy@3 |
|
|
- dot_accuracy@5 |
|
|
- dot_accuracy@10 |
|
|
- dot_precision@1 |
|
|
- dot_precision@3 |
|
|
- dot_precision@5 |
|
|
- dot_precision@10 |
|
|
- dot_recall@1 |
|
|
- dot_recall@3 |
|
|
- dot_recall@5 |
|
|
- dot_recall@10 |
|
|
- dot_ndcg@10 |
|
|
- dot_mrr@10 |
|
|
- dot_map@100 |
|
|
- query_active_dims |
|
|
- query_sparsity_ratio |
|
|
- corpus_active_dims |
|
|
- corpus_sparsity_ratio |
|
|
co2_eq_emissions: |
|
|
emissions: 1.0881870582723092 |
|
|
energy_consumed: 0.019418388234485075 |
|
|
source: codecarbon |
|
|
training_type: fine-tuning |
|
|
on_cloud: false |
|
|
cpu_model: AMD Ryzen 9 6900HX with Radeon Graphics |
|
|
ram_total_size: 30.6114501953125 |
|
|
hours_used: 0.174 |
|
|
hardware_used: 1 x NVIDIA GeForce RTX 3070 Ti Laptop GPU |
|
|
model-index: |
|
|
- name: splade-distilbert-base-uncased trained on GooAQ |
|
|
results: |
|
|
- task: |
|
|
type: sparse-information-retrieval |
|
|
name: Sparse Information Retrieval |
|
|
dataset: |
|
|
name: NanoMSMARCO |
|
|
type: NanoMSMARCO |
|
|
metrics: |
|
|
- type: dot_accuracy@1 |
|
|
value: 0.22 |
|
|
name: Dot Accuracy@1 |
|
|
- type: dot_accuracy@3 |
|
|
value: 0.46 |
|
|
name: Dot Accuracy@3 |
|
|
- type: dot_accuracy@5 |
|
|
value: 0.54 |
|
|
name: Dot Accuracy@5 |
|
|
- type: dot_accuracy@10 |
|
|
value: 0.7 |
|
|
name: Dot Accuracy@10 |
|
|
- type: dot_precision@1 |
|
|
value: 0.22 |
|
|
name: Dot Precision@1 |
|
|
- type: dot_precision@3 |
|
|
value: 0.15333333333333332 |
|
|
name: Dot Precision@3 |
|
|
- type: dot_precision@5 |
|
|
value: 0.10800000000000001 |
|
|
name: Dot Precision@5 |
|
|
- type: dot_precision@10 |
|
|
value: 0.07 |
|
|
name: Dot Precision@10 |
|
|
- type: dot_recall@1 |
|
|
value: 0.22 |
|
|
name: Dot Recall@1 |
|
|
- type: dot_recall@3 |
|
|
value: 0.46 |
|
|
name: Dot Recall@3 |
|
|
- type: dot_recall@5 |
|
|
value: 0.54 |
|
|
name: Dot Recall@5 |
|
|
- type: dot_recall@10 |
|
|
value: 0.7 |
|
|
name: Dot Recall@10 |
|
|
- type: dot_ndcg@10 |
|
|
value: 0.44470504856183124 |
|
|
name: Dot Ndcg@10 |
|
|
- type: dot_mrr@10 |
|
|
value: 0.3652460317460317 |
|
|
name: Dot Mrr@10 |
|
|
- type: dot_map@100 |
|
|
value: 0.37928248813494486 |
|
|
name: Dot Map@100 |
|
|
- type: query_active_dims |
|
|
value: 125.86000061035156 |
|
|
name: Query Active Dims |
|
|
- type: query_sparsity_ratio |
|
|
value: 0.9958764169906837 |
|
|
name: Query Sparsity Ratio |
|
|
- type: corpus_active_dims |
|
|
value: 296.2349853515625 |
|
|
name: Corpus Active Dims |
|
|
- type: corpus_sparsity_ratio |
|
|
value: 0.9902943783057611 |
|
|
name: Corpus Sparsity Ratio |
|
|
- type: dot_accuracy@1 |
|
|
value: 0.24 |
|
|
name: Dot Accuracy@1 |
|
|
- type: dot_accuracy@3 |
|
|
value: 0.5 |
|
|
name: Dot Accuracy@3 |
|
|
- type: dot_accuracy@5 |
|
|
value: 0.58 |
|
|
name: Dot Accuracy@5 |
|
|
- type: dot_accuracy@10 |
|
|
value: 0.72 |
|
|
name: Dot Accuracy@10 |
|
|
- type: dot_precision@1 |
|
|
value: 0.24 |
|
|
name: Dot Precision@1 |
|
|
- type: dot_precision@3 |
|
|
value: 0.16666666666666663 |
|
|
name: Dot Precision@3 |
|
|
- type: dot_precision@5 |
|
|
value: 0.11600000000000002 |
|
|
name: Dot Precision@5 |
|
|
- type: dot_precision@10 |
|
|
value: 0.07200000000000001 |
|
|
name: Dot Precision@10 |
|
|
- type: dot_recall@1 |
|
|
value: 0.24 |
|
|
name: Dot Recall@1 |
|
|
- type: dot_recall@3 |
|
|
value: 0.5 |
|
|
name: Dot Recall@3 |
|
|
- type: dot_recall@5 |
|
|
value: 0.58 |
|
|
name: Dot Recall@5 |
|
|
- type: dot_recall@10 |
|
|
value: 0.72 |
|
|
name: Dot Recall@10 |
|
|
- type: dot_ndcg@10 |
|
|
value: 0.47847271089832977 |
|
|
name: Dot Ndcg@10 |
|
|
- type: dot_mrr@10 |
|
|
value: 0.40169047619047615 |
|
|
name: Dot Mrr@10 |
|
|
- type: dot_map@100 |
|
|
value: 0.4140044816294816 |
|
|
name: Dot Map@100 |
|
|
- type: query_active_dims |
|
|
value: 109.69999694824219 |
|
|
name: Query Active Dims |
|
|
- type: query_sparsity_ratio |
|
|
value: 0.9964058712748758 |
|
|
name: Query Sparsity Ratio |
|
|
- type: corpus_active_dims |
|
|
value: 265.6180725097656 |
|
|
name: Corpus Active Dims |
|
|
- type: corpus_sparsity_ratio |
|
|
value: 0.9912974879591847 |
|
|
name: Corpus Sparsity Ratio |
|
|
- task: |
|
|
type: sparse-information-retrieval |
|
|
name: Sparse Information Retrieval |
|
|
dataset: |
|
|
name: NanoNFCorpus |
|
|
type: NanoNFCorpus |
|
|
metrics: |
|
|
- type: dot_accuracy@1 |
|
|
value: 0.36 |
|
|
name: Dot Accuracy@1 |
|
|
- type: dot_accuracy@3 |
|
|
value: 0.5 |
|
|
name: Dot Accuracy@3 |
|
|
- type: dot_accuracy@5 |
|
|
value: 0.56 |
|
|
name: Dot Accuracy@5 |
|
|
- type: dot_accuracy@10 |
|
|
value: 0.58 |
|
|
name: Dot Accuracy@10 |
|
|
- type: dot_precision@1 |
|
|
value: 0.36 |
|
|
name: Dot Precision@1 |
|
|
- type: dot_precision@3 |
|
|
value: 0.2866666666666666 |
|
|
name: Dot Precision@3 |
|
|
- type: dot_precision@5 |
|
|
value: 0.276 |
|
|
name: Dot Precision@5 |
|
|
- type: dot_precision@10 |
|
|
value: 0.20400000000000001 |
|
|
name: Dot Precision@10 |
|
|
- type: dot_recall@1 |
|
|
value: 0.020432228546915038 |
|
|
name: Dot Recall@1 |
|
|
- type: dot_recall@3 |
|
|
value: 0.05966030415500706 |
|
|
name: Dot Recall@3 |
|
|
- type: dot_recall@5 |
|
|
value: 0.08546529551494754 |
|
|
name: Dot Recall@5 |
|
|
- type: dot_recall@10 |
|
|
value: 0.10325648585391117 |
|
|
name: Dot Recall@10 |
|
|
- type: dot_ndcg@10 |
|
|
value: 0.2586742055175529 |
|
|
name: Dot Ndcg@10 |
|
|
- type: dot_mrr@10 |
|
|
value: 0.444 |
|
|
name: Dot Mrr@10 |
|
|
- type: dot_map@100 |
|
|
value: 0.10277044614671307 |
|
|
name: Dot Map@100 |
|
|
- type: query_active_dims |
|
|
value: 160.10000610351562 |
|
|
name: Query Active Dims |
|
|
- type: query_sparsity_ratio |
|
|
value: 0.9947546030370383 |
|
|
name: Query Sparsity Ratio |
|
|
- type: corpus_active_dims |
|
|
value: 409.76904296875 |
|
|
name: Corpus Active Dims |
|
|
- type: corpus_sparsity_ratio |
|
|
value: 0.986574633281936 |
|
|
name: Corpus Sparsity Ratio |
|
|
- type: dot_accuracy@1 |
|
|
value: 0.38 |
|
|
name: Dot Accuracy@1 |
|
|
- type: dot_accuracy@3 |
|
|
value: 0.5 |
|
|
name: Dot Accuracy@3 |
|
|
- type: dot_accuracy@5 |
|
|
value: 0.52 |
|
|
name: Dot Accuracy@5 |
|
|
- type: dot_accuracy@10 |
|
|
value: 0.66 |
|
|
name: Dot Accuracy@10 |
|
|
- type: dot_precision@1 |
|
|
value: 0.38 |
|
|
name: Dot Precision@1 |
|
|
- type: dot_precision@3 |
|
|
value: 0.29333333333333333 |
|
|
name: Dot Precision@3 |
|
|
- type: dot_precision@5 |
|
|
value: 0.268 |
|
|
name: Dot Precision@5 |
|
|
- type: dot_precision@10 |
|
|
value: 0.22799999999999998 |
|
|
name: Dot Precision@10 |
|
|
- type: dot_recall@1 |
|
|
value: 0.03979891140267026 |
|
|
name: Dot Recall@1 |
|
|
- type: dot_recall@3 |
|
|
value: 0.05843303142773433 |
|
|
name: Dot Recall@3 |
|
|
- type: dot_recall@5 |
|
|
value: 0.07656018207627424 |
|
|
name: Dot Recall@5 |
|
|
- type: dot_recall@10 |
|
|
value: 0.10998150964383814 |
|
|
name: Dot Recall@10 |
|
|
- type: dot_ndcg@10 |
|
|
value: 0.28162049888840096 |
|
|
name: Dot Ndcg@10 |
|
|
- type: dot_mrr@10 |
|
|
value: 0.4571904761904762 |
|
|
name: Dot Mrr@10 |
|
|
- type: dot_map@100 |
|
|
value: 0.11433559983443616 |
|
|
name: Dot Map@100 |
|
|
- type: query_active_dims |
|
|
value: 140.05999755859375 |
|
|
name: Query Active Dims |
|
|
- type: query_sparsity_ratio |
|
|
value: 0.9954111789018218 |
|
|
name: Query Sparsity Ratio |
|
|
- type: corpus_active_dims |
|
|
value: 371.9038391113281 |
|
|
name: Corpus Active Dims |
|
|
- type: corpus_sparsity_ratio |
|
|
value: 0.9878152205258068 |
|
|
name: Corpus Sparsity Ratio |
|
|
- task: |
|
|
type: sparse-information-retrieval |
|
|
name: Sparse Information Retrieval |
|
|
dataset: |
|
|
name: NanoNQ |
|
|
type: NanoNQ |
|
|
metrics: |
|
|
- type: dot_accuracy@1 |
|
|
value: 0.32 |
|
|
name: Dot Accuracy@1 |
|
|
- type: dot_accuracy@3 |
|
|
value: 0.54 |
|
|
name: Dot Accuracy@3 |
|
|
- type: dot_accuracy@5 |
|
|
value: 0.64 |
|
|
name: Dot Accuracy@5 |
|
|
- type: dot_accuracy@10 |
|
|
value: 0.74 |
|
|
name: Dot Accuracy@10 |
|
|
- type: dot_precision@1 |
|
|
value: 0.32 |
|
|
name: Dot Precision@1 |
|
|
- type: dot_precision@3 |
|
|
value: 0.18 |
|
|
name: Dot Precision@3 |
|
|
- type: dot_precision@5 |
|
|
value: 0.136 |
|
|
name: Dot Precision@5 |
|
|
- type: dot_precision@10 |
|
|
value: 0.08 |
|
|
name: Dot Precision@10 |
|
|
- type: dot_recall@1 |
|
|
value: 0.3 |
|
|
name: Dot Recall@1 |
|
|
- type: dot_recall@3 |
|
|
value: 0.5 |
|
|
name: Dot Recall@3 |
|
|
- type: dot_recall@5 |
|
|
value: 0.62 |
|
|
name: Dot Recall@5 |
|
|
- type: dot_recall@10 |
|
|
value: 0.7 |
|
|
name: Dot Recall@10 |
|
|
- type: dot_ndcg@10 |
|
|
value: 0.5013957867971872 |
|
|
name: Dot Ndcg@10 |
|
|
- type: dot_mrr@10 |
|
|
value: 0.4491904761904762 |
|
|
name: Dot Mrr@10 |
|
|
- type: dot_map@100 |
|
|
value: 0.44262111936629595 |
|
|
name: Dot Map@100 |
|
|
- type: query_active_dims |
|
|
value: 128.39999389648438 |
|
|
name: Query Active Dims |
|
|
- type: query_sparsity_ratio |
|
|
value: 0.9957931985487031 |
|
|
name: Query Sparsity Ratio |
|
|
- type: corpus_active_dims |
|
|
value: 359.4007873535156 |
|
|
name: Corpus Active Dims |
|
|
- type: corpus_sparsity_ratio |
|
|
value: 0.9882248611705158 |
|
|
name: Corpus Sparsity Ratio |
|
|
- type: dot_accuracy@1 |
|
|
value: 0.36 |
|
|
name: Dot Accuracy@1 |
|
|
- type: dot_accuracy@3 |
|
|
value: 0.58 |
|
|
name: Dot Accuracy@3 |
|
|
- type: dot_accuracy@5 |
|
|
value: 0.66 |
|
|
name: Dot Accuracy@5 |
|
|
- type: dot_accuracy@10 |
|
|
value: 0.72 |
|
|
name: Dot Accuracy@10 |
|
|
- type: dot_precision@1 |
|
|
value: 0.36 |
|
|
name: Dot Precision@1 |
|
|
- type: dot_precision@3 |
|
|
value: 0.19333333333333333 |
|
|
name: Dot Precision@3 |
|
|
- type: dot_precision@5 |
|
|
value: 0.136 |
|
|
name: Dot Precision@5 |
|
|
- type: dot_precision@10 |
|
|
value: 0.078 |
|
|
name: Dot Precision@10 |
|
|
- type: dot_recall@1 |
|
|
value: 0.34 |
|
|
name: Dot Recall@1 |
|
|
- type: dot_recall@3 |
|
|
value: 0.54 |
|
|
name: Dot Recall@3 |
|
|
- type: dot_recall@5 |
|
|
value: 0.63 |
|
|
name: Dot Recall@5 |
|
|
- type: dot_recall@10 |
|
|
value: 0.69 |
|
|
name: Dot Recall@10 |
|
|
- type: dot_ndcg@10 |
|
|
value: 0.5189963924532662 |
|
|
name: Dot Ndcg@10 |
|
|
- type: dot_mrr@10 |
|
|
value: 0.4757777777777777 |
|
|
name: Dot Mrr@10 |
|
|
- type: dot_map@100 |
|
|
value: 0.46913515575703424 |
|
|
name: Dot Map@100 |
|
|
- type: query_active_dims |
|
|
value: 115.30000305175781 |
|
|
name: Query Active Dims |
|
|
- type: query_sparsity_ratio |
|
|
value: 0.9962223968595846 |
|
|
name: Query Sparsity Ratio |
|
|
- type: corpus_active_dims |
|
|
value: 336.913818359375 |
|
|
name: Corpus Active Dims |
|
|
- type: corpus_sparsity_ratio |
|
|
value: 0.9889616074189316 |
|
|
name: Corpus Sparsity Ratio |
|
|
- task: |
|
|
type: sparse-nano-beir |
|
|
name: Sparse Nano BEIR |
|
|
dataset: |
|
|
name: NanoBEIR mean |
|
|
type: NanoBEIR_mean |
|
|
metrics: |
|
|
- type: dot_accuracy@1 |
|
|
value: 0.3 |
|
|
name: Dot Accuracy@1 |
|
|
- type: dot_accuracy@3 |
|
|
value: 0.5 |
|
|
name: Dot Accuracy@3 |
|
|
- type: dot_accuracy@5 |
|
|
value: 0.5800000000000001 |
|
|
name: Dot Accuracy@5 |
|
|
- type: dot_accuracy@10 |
|
|
value: 0.6733333333333332 |
|
|
name: Dot Accuracy@10 |
|
|
- type: dot_precision@1 |
|
|
value: 0.3 |
|
|
name: Dot Precision@1 |
|
|
- type: dot_precision@3 |
|
|
value: 0.20666666666666664 |
|
|
name: Dot Precision@3 |
|
|
- type: dot_precision@5 |
|
|
value: 0.17333333333333334 |
|
|
name: Dot Precision@5 |
|
|
- type: dot_precision@10 |
|
|
value: 0.11800000000000001 |
|
|
name: Dot Precision@10 |
|
|
- type: dot_recall@1 |
|
|
value: 0.180144076182305 |
|
|
name: Dot Recall@1 |
|
|
- type: dot_recall@3 |
|
|
value: 0.339886768051669 |
|
|
name: Dot Recall@3 |
|
|
- type: dot_recall@5 |
|
|
value: 0.4151550985049825 |
|
|
name: Dot Recall@5 |
|
|
- type: dot_recall@10 |
|
|
value: 0.501085495284637 |
|
|
name: Dot Recall@10 |
|
|
- type: dot_ndcg@10 |
|
|
value: 0.40159168029219044 |
|
|
name: Dot Ndcg@10 |
|
|
- type: dot_mrr@10 |
|
|
value: 0.41947883597883595 |
|
|
name: Dot Mrr@10 |
|
|
- type: dot_map@100 |
|
|
value: 0.30822468454931795 |
|
|
name: Dot Map@100 |
|
|
- type: query_active_dims |
|
|
value: 138.12000020345053 |
|
|
name: Query Active Dims |
|
|
- type: query_sparsity_ratio |
|
|
value: 0.9954747395254749 |
|
|
name: Query Sparsity Ratio |
|
|
- type: corpus_active_dims |
|
|
value: 346.36973212643693 |
|
|
name: Corpus Active Dims |
|
|
- type: corpus_sparsity_ratio |
|
|
value: 0.9886518009263339 |
|
|
name: Corpus Sparsity Ratio |
|
|
- type: dot_accuracy@1 |
|
|
value: 0.4301726844583988 |
|
|
name: Dot Accuracy@1 |
|
|
- type: dot_accuracy@3 |
|
|
value: 0.6182417582417583 |
|
|
name: Dot Accuracy@3 |
|
|
- type: dot_accuracy@5 |
|
|
value: 0.6783359497645213 |
|
|
name: Dot Accuracy@5 |
|
|
- type: dot_accuracy@10 |
|
|
value: 0.7722135007849293 |
|
|
name: Dot Accuracy@10 |
|
|
- type: dot_precision@1 |
|
|
value: 0.4301726844583988 |
|
|
name: Dot Precision@1 |
|
|
- type: dot_precision@3 |
|
|
value: 0.274160125588697 |
|
|
name: Dot Precision@3 |
|
|
- type: dot_precision@5 |
|
|
value: 0.21524646781789644 |
|
|
name: Dot Precision@5 |
|
|
- type: dot_precision@10 |
|
|
value: 0.1563861852433281 |
|
|
name: Dot Precision@10 |
|
|
- type: dot_recall@1 |
|
|
value: 0.24332694326060123 |
|
|
name: Dot Recall@1 |
|
|
- type: dot_recall@3 |
|
|
value: 0.38912806185875454 |
|
|
name: Dot Recall@3 |
|
|
- type: dot_recall@5 |
|
|
value: 0.4466126446755131 |
|
|
name: Dot Recall@5 |
|
|
- type: dot_recall@10 |
|
|
value: 0.5378480354517308 |
|
|
name: Dot Recall@10 |
|
|
- type: dot_ndcg@10 |
|
|
value: 0.48091561944614786 |
|
|
name: Dot Ndcg@10 |
|
|
- type: dot_mrr@10 |
|
|
value: 0.5383367720714658 |
|
|
name: Dot Mrr@10 |
|
|
- type: dot_map@100 |
|
|
value: 0.40550699373209664 |
|
|
name: Dot Map@100 |
|
|
- type: query_active_dims |
|
|
value: 161.59013707612073 |
|
|
name: Query Active Dims |
|
|
- type: query_sparsity_ratio |
|
|
value: 0.9947057814993735 |
|
|
name: Query Sparsity Ratio |
|
|
- type: corpus_active_dims |
|
|
value: 302.84806046588795 |
|
|
name: Corpus Active Dims |
|
|
- type: corpus_sparsity_ratio |
|
|
value: 0.99007771245443 |
|
|
name: Corpus Sparsity Ratio |
|
|
- task: |
|
|
type: sparse-information-retrieval |
|
|
name: Sparse Information Retrieval |
|
|
dataset: |
|
|
name: NanoClimateFEVER |
|
|
type: NanoClimateFEVER |
|
|
metrics: |
|
|
- type: dot_accuracy@1 |
|
|
value: 0.26 |
|
|
name: Dot Accuracy@1 |
|
|
- type: dot_accuracy@3 |
|
|
value: 0.4 |
|
|
name: Dot Accuracy@3 |
|
|
- type: dot_accuracy@5 |
|
|
value: 0.42 |
|
|
name: Dot Accuracy@5 |
|
|
- type: dot_accuracy@10 |
|
|
value: 0.64 |
|
|
name: Dot Accuracy@10 |
|
|
- type: dot_precision@1 |
|
|
value: 0.26 |
|
|
name: Dot Precision@1 |
|
|
- type: dot_precision@3 |
|
|
value: 0.14 |
|
|
name: Dot Precision@3 |
|
|
- type: dot_precision@5 |
|
|
value: 0.09200000000000001 |
|
|
name: Dot Precision@5 |
|
|
- type: dot_precision@10 |
|
|
value: 0.08 |
|
|
name: Dot Precision@10 |
|
|
- type: dot_recall@1 |
|
|
value: 0.13 |
|
|
name: Dot Recall@1 |
|
|
- type: dot_recall@3 |
|
|
value: 0.18 |
|
|
name: Dot Recall@3 |
|
|
- type: dot_recall@5 |
|
|
value: 0.19 |
|
|
name: Dot Recall@5 |
|
|
- type: dot_recall@10 |
|
|
value: 0.30733333333333335 |
|
|
name: Dot Recall@10 |
|
|
- type: dot_ndcg@10 |
|
|
value: 0.2528315611912319 |
|
|
name: Dot Ndcg@10 |
|
|
- type: dot_mrr@10 |
|
|
value: 0.3483253968253968 |
|
|
name: Dot Mrr@10 |
|
|
- type: dot_map@100 |
|
|
value: 0.195000428587257 |
|
|
name: Dot Map@100 |
|
|
- type: query_active_dims |
|
|
value: 215.39999389648438 |
|
|
name: Query Active Dims |
|
|
- type: query_sparsity_ratio |
|
|
value: 0.9929427955606944 |
|
|
name: Query Sparsity Ratio |
|
|
- type: corpus_active_dims |
|
|
value: 334.818359375 |
|
|
name: Corpus Active Dims |
|
|
- type: corpus_sparsity_ratio |
|
|
value: 0.9890302614712339 |
|
|
name: Corpus Sparsity Ratio |
|
|
- task: |
|
|
type: sparse-information-retrieval |
|
|
name: Sparse Information Retrieval |
|
|
dataset: |
|
|
name: NanoDBPedia |
|
|
type: NanoDBPedia |
|
|
metrics: |
|
|
- type: dot_accuracy@1 |
|
|
value: 0.54 |
|
|
name: Dot Accuracy@1 |
|
|
- type: dot_accuracy@3 |
|
|
value: 0.68 |
|
|
name: Dot Accuracy@3 |
|
|
- type: dot_accuracy@5 |
|
|
value: 0.76 |
|
|
name: Dot Accuracy@5 |
|
|
- type: dot_accuracy@10 |
|
|
value: 0.9 |
|
|
name: Dot Accuracy@10 |
|
|
- type: dot_precision@1 |
|
|
value: 0.54 |
|
|
name: Dot Precision@1 |
|
|
- type: dot_precision@3 |
|
|
value: 0.43333333333333335 |
|
|
name: Dot Precision@3 |
|
|
- type: dot_precision@5 |
|
|
value: 0.4 |
|
|
name: Dot Precision@5 |
|
|
- type: dot_precision@10 |
|
|
value: 0.35999999999999993 |
|
|
name: Dot Precision@10 |
|
|
- type: dot_recall@1 |
|
|
value: 0.04725330037285543 |
|
|
name: Dot Recall@1 |
|
|
- type: dot_recall@3 |
|
|
value: 0.09136010229983793 |
|
|
name: Dot Recall@3 |
|
|
- type: dot_recall@5 |
|
|
value: 0.12256470056683391 |
|
|
name: Dot Recall@5 |
|
|
- type: dot_recall@10 |
|
|
value: 0.24664786941021674 |
|
|
name: Dot Recall@10 |
|
|
- type: dot_ndcg@10 |
|
|
value: 0.43054704834652313 |
|
|
name: Dot Ndcg@10 |
|
|
- type: dot_mrr@10 |
|
|
value: 0.6440714285714284 |
|
|
name: Dot Mrr@10 |
|
|
- type: dot_map@100 |
|
|
value: 0.3239717199251123 |
|
|
name: Dot Map@100 |
|
|
- type: query_active_dims |
|
|
value: 147.72000122070312 |
|
|
name: Query Active Dims |
|
|
- type: query_sparsity_ratio |
|
|
value: 0.9951602122658835 |
|
|
name: Query Sparsity Ratio |
|
|
- type: corpus_active_dims |
|
|
value: 295.1452331542969 |
|
|
name: Corpus Active Dims |
|
|
- type: corpus_sparsity_ratio |
|
|
value: 0.9903300821324194 |
|
|
name: Corpus Sparsity Ratio |
|
|
- task: |
|
|
type: sparse-information-retrieval |
|
|
name: Sparse Information Retrieval |
|
|
dataset: |
|
|
name: NanoFEVER |
|
|
type: NanoFEVER |
|
|
metrics: |
|
|
- type: dot_accuracy@1 |
|
|
value: 0.56 |
|
|
name: Dot Accuracy@1 |
|
|
- type: dot_accuracy@3 |
|
|
value: 0.78 |
|
|
name: Dot Accuracy@3 |
|
|
- type: dot_accuracy@5 |
|
|
value: 0.86 |
|
|
name: Dot Accuracy@5 |
|
|
- type: dot_accuracy@10 |
|
|
value: 0.92 |
|
|
name: Dot Accuracy@10 |
|
|
- type: dot_precision@1 |
|
|
value: 0.56 |
|
|
name: Dot Precision@1 |
|
|
- type: dot_precision@3 |
|
|
value: 0.26 |
|
|
name: Dot Precision@3 |
|
|
- type: dot_precision@5 |
|
|
value: 0.172 |
|
|
name: Dot Precision@5 |
|
|
- type: dot_precision@10 |
|
|
value: 0.096 |
|
|
name: Dot Precision@10 |
|
|
- type: dot_recall@1 |
|
|
value: 0.5466666666666666 |
|
|
name: Dot Recall@1 |
|
|
- type: dot_recall@3 |
|
|
value: 0.7466666666666666 |
|
|
name: Dot Recall@3 |
|
|
- type: dot_recall@5 |
|
|
value: 0.8066666666666668 |
|
|
name: Dot Recall@5 |
|
|
- type: dot_recall@10 |
|
|
value: 0.8766666666666667 |
|
|
name: Dot Recall@10 |
|
|
- type: dot_ndcg@10 |
|
|
value: 0.7202530021492869 |
|
|
name: Dot Ndcg@10 |
|
|
- type: dot_mrr@10 |
|
|
value: 0.6843809523809523 |
|
|
name: Dot Mrr@10 |
|
|
- type: dot_map@100 |
|
|
value: 0.6647642136958143 |
|
|
name: Dot Map@100 |
|
|
- type: query_active_dims |
|
|
value: 201.5399932861328 |
|
|
name: Query Active Dims |
|
|
- type: query_sparsity_ratio |
|
|
value: 0.9933968942636088 |
|
|
name: Query Sparsity Ratio |
|
|
- type: corpus_active_dims |
|
|
value: 374.9945983886719 |
|
|
name: Corpus Active Dims |
|
|
- type: corpus_sparsity_ratio |
|
|
value: 0.9877139571984578 |
|
|
name: Corpus Sparsity Ratio |
|
|
- task: |
|
|
type: sparse-information-retrieval |
|
|
name: Sparse Information Retrieval |
|
|
dataset: |
|
|
name: NanoFiQA2018 |
|
|
type: NanoFiQA2018 |
|
|
metrics: |
|
|
- type: dot_accuracy@1 |
|
|
value: 0.36 |
|
|
name: Dot Accuracy@1 |
|
|
- type: dot_accuracy@3 |
|
|
value: 0.52 |
|
|
name: Dot Accuracy@3 |
|
|
- type: dot_accuracy@5 |
|
|
value: 0.54 |
|
|
name: Dot Accuracy@5 |
|
|
- type: dot_accuracy@10 |
|
|
value: 0.62 |
|
|
name: Dot Accuracy@10 |
|
|
- type: dot_precision@1 |
|
|
value: 0.36 |
|
|
name: Dot Precision@1 |
|
|
- type: dot_precision@3 |
|
|
value: 0.24666666666666667 |
|
|
name: Dot Precision@3 |
|
|
- type: dot_precision@5 |
|
|
value: 0.168 |
|
|
name: Dot Precision@5 |
|
|
- type: dot_precision@10 |
|
|
value: 0.106 |
|
|
name: Dot Precision@10 |
|
|
- type: dot_recall@1 |
|
|
value: 0.18857936507936507 |
|
|
name: Dot Recall@1 |
|
|
- type: dot_recall@3 |
|
|
value: 0.3216825396825396 |
|
|
name: Dot Recall@3 |
|
|
- type: dot_recall@5 |
|
|
value: 0.3532380952380953 |
|
|
name: Dot Recall@5 |
|
|
- type: dot_recall@10 |
|
|
value: 0.4552380952380953 |
|
|
name: Dot Recall@10 |
|
|
- type: dot_ndcg@10 |
|
|
value: 0.3784249151812378 |
|
|
name: Dot Ndcg@10 |
|
|
- type: dot_mrr@10 |
|
|
value: 0.44319047619047613 |
|
|
name: Dot Mrr@10 |
|
|
- type: dot_map@100 |
|
|
value: 0.31981273776184116 |
|
|
name: Dot Map@100 |
|
|
- type: query_active_dims |
|
|
value: 87.62000274658203 |
|
|
name: Query Active Dims |
|
|
- type: query_sparsity_ratio |
|
|
value: 0.9971292837053083 |
|
|
name: Query Sparsity Ratio |
|
|
- type: corpus_active_dims |
|
|
value: 275.46795654296875 |
|
|
name: Corpus Active Dims |
|
|
- type: corpus_sparsity_ratio |
|
|
value: 0.9909747737191872 |
|
|
name: Corpus Sparsity Ratio |
|
|
- task: |
|
|
type: sparse-information-retrieval |
|
|
name: Sparse Information Retrieval |
|
|
dataset: |
|
|
name: NanoHotpotQA |
|
|
type: NanoHotpotQA |
|
|
metrics: |
|
|
- type: dot_accuracy@1 |
|
|
value: 0.66 |
|
|
name: Dot Accuracy@1 |
|
|
- type: dot_accuracy@3 |
|
|
value: 0.86 |
|
|
name: Dot Accuracy@3 |
|
|
- type: dot_accuracy@5 |
|
|
value: 0.92 |
|
|
name: Dot Accuracy@5 |
|
|
- type: dot_accuracy@10 |
|
|
value: 0.92 |
|
|
name: Dot Accuracy@10 |
|
|
- type: dot_precision@1 |
|
|
value: 0.66 |
|
|
name: Dot Precision@1 |
|
|
- type: dot_precision@3 |
|
|
value: 0.4333333333333333 |
|
|
name: Dot Precision@3 |
|
|
- type: dot_precision@5 |
|
|
value: 0.2879999999999999 |
|
|
name: Dot Precision@5 |
|
|
- type: dot_precision@10 |
|
|
value: 0.15599999999999997 |
|
|
name: Dot Precision@10 |
|
|
- type: dot_recall@1 |
|
|
value: 0.33 |
|
|
name: Dot Recall@1 |
|
|
- type: dot_recall@3 |
|
|
value: 0.65 |
|
|
name: Dot Recall@3 |
|
|
- type: dot_recall@5 |
|
|
value: 0.72 |
|
|
name: Dot Recall@5 |
|
|
- type: dot_recall@10 |
|
|
value: 0.78 |
|
|
name: Dot Recall@10 |
|
|
- type: dot_ndcg@10 |
|
|
value: 0.6985941766475363 |
|
|
name: Dot Ndcg@10 |
|
|
- type: dot_mrr@10 |
|
|
value: 0.7596666666666667 |
|
|
name: Dot Mrr@10 |
|
|
- type: dot_map@100 |
|
|
value: 0.632578269203448 |
|
|
name: Dot Map@100 |
|
|
- type: query_active_dims |
|
|
value: 131.75999450683594 |
|
|
name: Query Active Dims |
|
|
- type: query_sparsity_ratio |
|
|
value: 0.9956831139995139 |
|
|
name: Query Sparsity Ratio |
|
|
- type: corpus_active_dims |
|
|
value: 330.9889831542969 |
|
|
name: Corpus Active Dims |
|
|
- type: corpus_sparsity_ratio |
|
|
value: 0.9891557242921729 |
|
|
name: Corpus Sparsity Ratio |
|
|
- task: |
|
|
type: sparse-information-retrieval |
|
|
name: Sparse Information Retrieval |
|
|
dataset: |
|
|
name: NanoQuoraRetrieval |
|
|
type: NanoQuoraRetrieval |
|
|
metrics: |
|
|
- type: dot_accuracy@1 |
|
|
value: 0.58 |
|
|
name: Dot Accuracy@1 |
|
|
- type: dot_accuracy@3 |
|
|
value: 0.76 |
|
|
name: Dot Accuracy@3 |
|
|
- type: dot_accuracy@5 |
|
|
value: 0.86 |
|
|
name: Dot Accuracy@5 |
|
|
- type: dot_accuracy@10 |
|
|
value: 0.94 |
|
|
name: Dot Accuracy@10 |
|
|
- type: dot_precision@1 |
|
|
value: 0.58 |
|
|
name: Dot Precision@1 |
|
|
- type: dot_precision@3 |
|
|
value: 0.26 |
|
|
name: Dot Precision@3 |
|
|
- type: dot_precision@5 |
|
|
value: 0.184 |
|
|
name: Dot Precision@5 |
|
|
- type: dot_precision@10 |
|
|
value: 0.11199999999999999 |
|
|
name: Dot Precision@10 |
|
|
- type: dot_recall@1 |
|
|
value: 0.57 |
|
|
name: Dot Recall@1 |
|
|
- type: dot_recall@3 |
|
|
value: 0.7233333333333334 |
|
|
name: Dot Recall@3 |
|
|
- type: dot_recall@5 |
|
|
value: 0.8233333333333333 |
|
|
name: Dot Recall@5 |
|
|
- type: dot_recall@10 |
|
|
value: 0.8953333333333333 |
|
|
name: Dot Recall@10 |
|
|
- type: dot_ndcg@10 |
|
|
value: 0.7379320795882585 |
|
|
name: Dot Ndcg@10 |
|
|
- type: dot_mrr@10 |
|
|
value: 0.6864126984126984 |
|
|
name: Dot Mrr@10 |
|
|
- type: dot_map@100 |
|
|
value: 0.6882004324782192 |
|
|
name: Dot Map@100 |
|
|
- type: query_active_dims |
|
|
value: 56.70000076293945 |
|
|
name: Query Active Dims |
|
|
- type: query_sparsity_ratio |
|
|
value: 0.9981423235448876 |
|
|
name: Query Sparsity Ratio |
|
|
- type: corpus_active_dims |
|
|
value: 63.429447174072266 |
|
|
name: Corpus Active Dims |
|
|
- type: corpus_sparsity_ratio |
|
|
value: 0.9979218449913483 |
|
|
name: Corpus Sparsity Ratio |
|
|
- task: |
|
|
type: sparse-information-retrieval |
|
|
name: Sparse Information Retrieval |
|
|
dataset: |
|
|
name: NanoSCIDOCS |
|
|
type: NanoSCIDOCS |
|
|
metrics: |
|
|
- type: dot_accuracy@1 |
|
|
value: 0.4 |
|
|
name: Dot Accuracy@1 |
|
|
- type: dot_accuracy@3 |
|
|
value: 0.58 |
|
|
name: Dot Accuracy@3 |
|
|
- type: dot_accuracy@5 |
|
|
value: 0.66 |
|
|
name: Dot Accuracy@5 |
|
|
- type: dot_accuracy@10 |
|
|
value: 0.74 |
|
|
name: Dot Accuracy@10 |
|
|
- type: dot_precision@1 |
|
|
value: 0.4 |
|
|
name: Dot Precision@1 |
|
|
- type: dot_precision@3 |
|
|
value: 0.2533333333333333 |
|
|
name: Dot Precision@3 |
|
|
- type: dot_precision@5 |
|
|
value: 0.228 |
|
|
name: Dot Precision@5 |
|
|
- type: dot_precision@10 |
|
|
value: 0.154 |
|
|
name: Dot Precision@10 |
|
|
- type: dot_recall@1 |
|
|
value: 0.08466666666666667 |
|
|
name: Dot Recall@1 |
|
|
- type: dot_recall@3 |
|
|
value: 0.15866666666666668 |
|
|
name: Dot Recall@3 |
|
|
- type: dot_recall@5 |
|
|
value: 0.23566666666666666 |
|
|
name: Dot Recall@5 |
|
|
- type: dot_recall@10 |
|
|
value: 0.31666666666666665 |
|
|
name: Dot Recall@10 |
|
|
- type: dot_ndcg@10 |
|
|
value: 0.307302076202993 |
|
|
name: Dot Ndcg@10 |
|
|
- type: dot_mrr@10 |
|
|
value: 0.5031111111111111 |
|
|
name: Dot Mrr@10 |
|
|
- type: dot_map@100 |
|
|
value: 0.2314013330851555 |
|
|
name: Dot Map@100 |
|
|
- type: query_active_dims |
|
|
value: 219.97999572753906 |
|
|
name: Query Active Dims |
|
|
- type: query_sparsity_ratio |
|
|
value: 0.9927927398031735 |
|
|
name: Query Sparsity Ratio |
|
|
- type: corpus_active_dims |
|
|
value: 370.2647399902344 |
|
|
name: Corpus Active Dims |
|
|
- type: corpus_sparsity_ratio |
|
|
value: 0.98786892274457 |
|
|
name: Corpus Sparsity Ratio |
|
|
- task: |
|
|
type: sparse-information-retrieval |
|
|
name: Sparse Information Retrieval |
|
|
dataset: |
|
|
name: NanoArguAna |
|
|
type: NanoArguAna |
|
|
metrics: |
|
|
- type: dot_accuracy@1 |
|
|
value: 0.1 |
|
|
name: Dot Accuracy@1 |
|
|
- type: dot_accuracy@3 |
|
|
value: 0.38 |
|
|
name: Dot Accuracy@3 |
|
|
- type: dot_accuracy@5 |
|
|
value: 0.46 |
|
|
name: Dot Accuracy@5 |
|
|
- type: dot_accuracy@10 |
|
|
value: 0.54 |
|
|
name: Dot Accuracy@10 |
|
|
- type: dot_precision@1 |
|
|
value: 0.1 |
|
|
name: Dot Precision@1 |
|
|
- type: dot_precision@3 |
|
|
value: 0.12666666666666665 |
|
|
name: Dot Precision@3 |
|
|
- type: dot_precision@5 |
|
|
value: 0.09200000000000001 |
|
|
name: Dot Precision@5 |
|
|
- type: dot_precision@10 |
|
|
value: 0.05400000000000001 |
|
|
name: Dot Precision@10 |
|
|
- type: dot_recall@1 |
|
|
value: 0.1 |
|
|
name: Dot Recall@1 |
|
|
- type: dot_recall@3 |
|
|
value: 0.38 |
|
|
name: Dot Recall@3 |
|
|
- type: dot_recall@5 |
|
|
value: 0.46 |
|
|
name: Dot Recall@5 |
|
|
- type: dot_recall@10 |
|
|
value: 0.54 |
|
|
name: Dot Recall@10 |
|
|
- type: dot_ndcg@10 |
|
|
value: 0.314067080699688 |
|
|
name: Dot Ndcg@10 |
|
|
- type: dot_mrr@10 |
|
|
value: 0.24191269841269844 |
|
|
name: Dot Mrr@10 |
|
|
- type: dot_map@100 |
|
|
value: 0.2544871127158089 |
|
|
name: Dot Map@100 |
|
|
- type: query_active_dims |
|
|
value: 392.3999938964844 |
|
|
name: Query Active Dims |
|
|
- type: query_sparsity_ratio |
|
|
value: 0.98714369982647 |
|
|
name: Query Sparsity Ratio |
|
|
- type: corpus_active_dims |
|
|
value: 371.9895324707031 |
|
|
name: Corpus Active Dims |
|
|
- type: corpus_sparsity_ratio |
|
|
value: 0.9878124129326157 |
|
|
name: Corpus Sparsity Ratio |
|
|
- task: |
|
|
type: sparse-information-retrieval |
|
|
name: Sparse Information Retrieval |
|
|
dataset: |
|
|
name: NanoSciFact |
|
|
type: NanoSciFact |
|
|
metrics: |
|
|
- type: dot_accuracy@1 |
|
|
value: 0.54 |
|
|
name: Dot Accuracy@1 |
|
|
- type: dot_accuracy@3 |
|
|
value: 0.64 |
|
|
name: Dot Accuracy@3 |
|
|
- type: dot_accuracy@5 |
|
|
value: 0.66 |
|
|
name: Dot Accuracy@5 |
|
|
- type: dot_accuracy@10 |
|
|
value: 0.78 |
|
|
name: Dot Accuracy@10 |
|
|
- type: dot_precision@1 |
|
|
value: 0.54 |
|
|
name: Dot Precision@1 |
|
|
- type: dot_precision@3 |
|
|
value: 0.22 |
|
|
name: Dot Precision@3 |
|
|
- type: dot_precision@5 |
|
|
value: 0.14400000000000002 |
|
|
name: Dot Precision@5 |
|
|
- type: dot_precision@10 |
|
|
value: 0.08599999999999998 |
|
|
name: Dot Precision@10 |
|
|
- type: dot_recall@1 |
|
|
value: 0.505 |
|
|
name: Dot Recall@1 |
|
|
- type: dot_recall@3 |
|
|
value: 0.6 |
|
|
name: Dot Recall@3 |
|
|
- type: dot_recall@5 |
|
|
value: 0.635 |
|
|
name: Dot Recall@5 |
|
|
- type: dot_recall@10 |
|
|
value: 0.76 |
|
|
name: Dot Recall@10 |
|
|
- type: dot_ndcg@10 |
|
|
value: 0.6330847757650383 |
|
|
name: Dot Ndcg@10 |
|
|
- type: dot_mrr@10 |
|
|
value: 0.6099365079365079 |
|
|
name: Dot Mrr@10 |
|
|
- type: dot_map@100 |
|
|
value: 0.5921039809068559 |
|
|
name: Dot Map@100 |
|
|
- type: query_active_dims |
|
|
value: 239.02000427246094 |
|
|
name: Query Active Dims |
|
|
- type: query_sparsity_ratio |
|
|
value: 0.9921689271911257 |
|
|
name: Query Sparsity Ratio |
|
|
- type: corpus_active_dims |
|
|
value: 362.61492919921875 |
|
|
name: Corpus Active Dims |
|
|
- type: corpus_sparsity_ratio |
|
|
value: 0.9881195554288966 |
|
|
name: Corpus Sparsity Ratio |
|
|
- task: |
|
|
type: sparse-information-retrieval |
|
|
name: Sparse Information Retrieval |
|
|
dataset: |
|
|
name: NanoTouche2020 |
|
|
type: NanoTouche2020 |
|
|
metrics: |
|
|
- type: dot_accuracy@1 |
|
|
value: 0.6122448979591837 |
|
|
name: Dot Accuracy@1 |
|
|
- type: dot_accuracy@3 |
|
|
value: 0.8571428571428571 |
|
|
name: Dot Accuracy@3 |
|
|
- type: dot_accuracy@5 |
|
|
value: 0.9183673469387755 |
|
|
name: Dot Accuracy@5 |
|
|
- type: dot_accuracy@10 |
|
|
value: 0.9387755102040817 |
|
|
name: Dot Accuracy@10 |
|
|
- type: dot_precision@1 |
|
|
value: 0.6122448979591837 |
|
|
name: Dot Precision@1 |
|
|
- type: dot_precision@3 |
|
|
value: 0.5374149659863945 |
|
|
name: Dot Precision@3 |
|
|
- type: dot_precision@5 |
|
|
value: 0.5102040816326532 |
|
|
name: Dot Precision@5 |
|
|
- type: dot_precision@10 |
|
|
value: 0.4510204081632653 |
|
|
name: Dot Precision@10 |
|
|
- type: dot_recall@1 |
|
|
value: 0.04128535219959204 |
|
|
name: Dot Recall@1 |
|
|
- type: dot_recall@3 |
|
|
value: 0.10852246408702973 |
|
|
name: Dot Recall@3 |
|
|
- type: dot_recall@5 |
|
|
value: 0.17293473623380118 |
|
|
name: Dot Recall@5 |
|
|
- type: dot_recall@10 |
|
|
value: 0.29415698658034994 |
|
|
name: Dot Recall@10 |
|
|
- type: dot_ndcg@10 |
|
|
value: 0.4997767347881314 |
|
|
name: Dot Ndcg@10 |
|
|
- type: dot_mrr@10 |
|
|
value: 0.7427113702623908 |
|
|
name: Dot Mrr@10 |
|
|
- type: dot_map@100 |
|
|
value: 0.37179545293679184 |
|
|
name: Dot Map@100 |
|
|
- type: query_active_dims |
|
|
value: 41.06122589111328 |
|
|
name: Query Active Dims |
|
|
- type: query_sparsity_ratio |
|
|
value: 0.9986547006784905 |
|
|
name: Query Sparsity Ratio |
|
|
- type: corpus_active_dims |
|
|
value: 307.7058410644531 |
|
|
name: Corpus Active Dims |
|
|
- type: corpus_sparsity_ratio |
|
|
value: 0.9899185557609445 |
|
|
name: Corpus Sparsity Ratio |
|
|
--- |
|
|
|
|
|
# splade-distilbert-base-uncased trained on GooAQ |
|
|
|
|
|
This is a [SPLADE Sparse Encoder](https://www.sbert.net/docs/sparse_encoder/usage/usage.html) model finetuned from [distilbert/distilbert-base-uncased](https://huggingface.co/distilbert/distilbert-base-uncased) on the [gooaq](https://huggingface.co/datasets/sentence-transformers/gooaq) dataset using the [sentence-transformers](https://www.SBERT.net) library. It maps sentences & paragraphs to a 30522-dimensional sparse vector space and can be used for semantic search and sparse retrieval. |
|
|
## Model Details |
|
|
|
|
|
### Model Description |
|
|
- **Model Type:** SPLADE Sparse Encoder |
|
|
- **Base model:** [distilbert/distilbert-base-uncased](https://huggingface.co/distilbert/distilbert-base-uncased) <!-- at revision 12040accade4e8a0f71eabdb258fecc2e7e948be --> |
|
|
- **Maximum Sequence Length:** 256 tokens |
|
|
- **Output Dimensionality:** 30522 dimensions |
|
|
- **Similarity Function:** Dot Product |
|
|
- **Training Dataset:** |
|
|
- [gooaq](https://huggingface.co/datasets/sentence-transformers/gooaq) |
|
|
- **Language:** en |
|
|
- **License:** apache-2.0 |
|
|
|
|
|
### Model Sources |
|
|
|
|
|
- **Documentation:** [Sentence Transformers Documentation](https://sbert.net) |
|
|
- **Documentation:** [Sparse Encoder Documentation](https://www.sbert.net/docs/sparse_encoder/usage/usage.html) |
|
|
- **Repository:** [Sentence Transformers on GitHub](https://github.com/UKPLab/sentence-transformers) |
|
|
- **Hugging Face:** [Sparse Encoders on Hugging Face](https://huggingface.co/models?library=sentence-transformers&other=sparse-encoder) |
|
|
|
|
|
### Full Model Architecture |
|
|
|
|
|
``` |
|
|
SparseEncoder( |
|
|
(0): MLMTransformer({'max_seq_length': 256, 'do_lower_case': False}) with MLMTransformer model: DistilBertForMaskedLM |
|
|
(1): SpladePooling({'pooling_strategy': 'max', 'activation_function': 'relu', 'word_embedding_dimension': 30522}) |
|
|
) |
|
|
``` |
|
|
|
|
|
## Usage |
|
|
|
|
|
### Direct Usage (Sentence Transformers) |
|
|
|
|
|
First install the Sentence Transformers library: |
|
|
|
|
|
```bash |
|
|
pip install -U sentence-transformers |
|
|
``` |
|
|
|
|
|
Then you can load this model and run inference. |
|
|
```python |
|
|
from sentence_transformers import SparseEncoder |
|
|
|
|
|
# Download from the 🤗 Hub |
|
|
model = SparseEncoder("arthurbresnu/splade-distilbert-base-uncased-gooaq") |
|
|
# Run inference |
|
|
sentences = [ |
|
|
'how many days for doxycycline to work on sinus infection?', |
|
|
'Treatment of suspected bacterial infection is with antibiotics, such as amoxicillin/clavulanate or doxycycline, given for 5 to 7 days for acute sinusitis and for up to 6 weeks for chronic sinusitis.', |
|
|
'Most engagements typically have a cocktail dress code, calling for dresses at, or slightly above, knee-length and high heels. If your party states a different dress code, however, such as semi-formal or dressy-casual, you may need to dress up or down accordingly.', |
|
|
] |
|
|
embeddings = model.encode(sentences) |
|
|
print(embeddings.shape) |
|
|
# (3, 30522) |
|
|
|
|
|
# Get the similarity scores for the embeddings |
|
|
similarities = model.similarity(embeddings, embeddings) |
|
|
print(similarities.shape) |
|
|
# [3, 3] |
|
|
``` |
|
|
|
|
|
<!-- |
|
|
### Direct Usage (Transformers) |
|
|
|
|
|
<details><summary>Click to see the direct usage in Transformers</summary> |
|
|
|
|
|
</details> |
|
|
--> |
|
|
|
|
|
<!-- |
|
|
### Downstream Usage (Sentence Transformers) |
|
|
|
|
|
You can finetune this model on your own dataset. |
|
|
|
|
|
<details><summary>Click to expand</summary> |
|
|
|
|
|
</details> |
|
|
--> |
|
|
|
|
|
<!-- |
|
|
### Out-of-Scope Use |
|
|
|
|
|
*List how the model may foreseeably be misused and address what users ought not to do with the model.* |
|
|
--> |
|
|
|
|
|
## Evaluation |
|
|
|
|
|
### Metrics |
|
|
|
|
|
#### Sparse Information Retrieval |
|
|
|
|
|
* Datasets: `NanoMSMARCO`, `NanoNFCorpus`, `NanoNQ`, `NanoClimateFEVER`, `NanoDBPedia`, `NanoFEVER`, `NanoFiQA2018`, `NanoHotpotQA`, `NanoMSMARCO`, `NanoNFCorpus`, `NanoNQ`, `NanoQuoraRetrieval`, `NanoSCIDOCS`, `NanoArguAna`, `NanoSciFact` and `NanoTouche2020` |
|
|
* Evaluated with [<code>SparseInformationRetrievalEvaluator</code>](https://sbert.net/docs/package_reference/sparse_encoder/evaluation.html#sentence_transformers.sparse_encoder.evaluation.SparseInformationRetrievalEvaluator) |
|
|
|
|
|
| Metric | NanoMSMARCO | NanoNFCorpus | NanoNQ | NanoClimateFEVER | NanoDBPedia | NanoFEVER | NanoFiQA2018 | NanoHotpotQA | NanoQuoraRetrieval | NanoSCIDOCS | NanoArguAna | NanoSciFact | NanoTouche2020 | |
|
|
|:----------------------|:------------|:-------------|:----------|:-----------------|:------------|:-----------|:-------------|:-------------|:-------------------|:------------|:------------|:------------|:---------------| |
|
|
| dot_accuracy@1 | 0.24 | 0.38 | 0.36 | 0.26 | 0.54 | 0.56 | 0.36 | 0.66 | 0.58 | 0.4 | 0.1 | 0.54 | 0.6122 | |
|
|
| dot_accuracy@3 | 0.5 | 0.5 | 0.58 | 0.4 | 0.68 | 0.78 | 0.52 | 0.86 | 0.76 | 0.58 | 0.38 | 0.64 | 0.8571 | |
|
|
| dot_accuracy@5 | 0.58 | 0.52 | 0.66 | 0.42 | 0.76 | 0.86 | 0.54 | 0.92 | 0.86 | 0.66 | 0.46 | 0.66 | 0.9184 | |
|
|
| dot_accuracy@10 | 0.72 | 0.66 | 0.72 | 0.64 | 0.9 | 0.92 | 0.62 | 0.92 | 0.94 | 0.74 | 0.54 | 0.78 | 0.9388 | |
|
|
| dot_precision@1 | 0.24 | 0.38 | 0.36 | 0.26 | 0.54 | 0.56 | 0.36 | 0.66 | 0.58 | 0.4 | 0.1 | 0.54 | 0.6122 | |
|
|
| dot_precision@3 | 0.1667 | 0.2933 | 0.1933 | 0.14 | 0.4333 | 0.26 | 0.2467 | 0.4333 | 0.26 | 0.2533 | 0.1267 | 0.22 | 0.5374 | |
|
|
| dot_precision@5 | 0.116 | 0.268 | 0.136 | 0.092 | 0.4 | 0.172 | 0.168 | 0.288 | 0.184 | 0.228 | 0.092 | 0.144 | 0.5102 | |
|
|
| dot_precision@10 | 0.072 | 0.228 | 0.078 | 0.08 | 0.36 | 0.096 | 0.106 | 0.156 | 0.112 | 0.154 | 0.054 | 0.086 | 0.451 | |
|
|
| dot_recall@1 | 0.24 | 0.0398 | 0.34 | 0.13 | 0.0473 | 0.5467 | 0.1886 | 0.33 | 0.57 | 0.0847 | 0.1 | 0.505 | 0.0413 | |
|
|
| dot_recall@3 | 0.5 | 0.0584 | 0.54 | 0.18 | 0.0914 | 0.7467 | 0.3217 | 0.65 | 0.7233 | 0.1587 | 0.38 | 0.6 | 0.1085 | |
|
|
| dot_recall@5 | 0.58 | 0.0766 | 0.63 | 0.19 | 0.1226 | 0.8067 | 0.3532 | 0.72 | 0.8233 | 0.2357 | 0.46 | 0.635 | 0.1729 | |
|
|
| dot_recall@10 | 0.72 | 0.11 | 0.69 | 0.3073 | 0.2466 | 0.8767 | 0.4552 | 0.78 | 0.8953 | 0.3167 | 0.54 | 0.76 | 0.2942 | |
|
|
| **dot_ndcg@10** | **0.4785** | **0.2816** | **0.519** | **0.2528** | **0.4305** | **0.7203** | **0.3784** | **0.6986** | **0.7379** | **0.3073** | **0.3141** | **0.6331** | **0.4998** | |
|
|
| dot_mrr@10 | 0.4017 | 0.4572 | 0.4758 | 0.3483 | 0.6441 | 0.6844 | 0.4432 | 0.7597 | 0.6864 | 0.5031 | 0.2419 | 0.6099 | 0.7427 | |
|
|
| dot_map@100 | 0.414 | 0.1143 | 0.4691 | 0.195 | 0.324 | 0.6648 | 0.3198 | 0.6326 | 0.6882 | 0.2314 | 0.2545 | 0.5921 | 0.3718 | |
|
|
| query_active_dims | 109.7 | 140.06 | 115.3 | 215.4 | 147.72 | 201.54 | 87.62 | 131.76 | 56.7 | 219.98 | 392.4 | 239.02 | 41.0612 | |
|
|
| query_sparsity_ratio | 0.9964 | 0.9954 | 0.9962 | 0.9929 | 0.9952 | 0.9934 | 0.9971 | 0.9957 | 0.9981 | 0.9928 | 0.9871 | 0.9922 | 0.9987 | |
|
|
| corpus_active_dims | 265.6181 | 371.9038 | 336.9138 | 334.8184 | 295.1452 | 374.9946 | 275.468 | 330.989 | 63.4294 | 370.2647 | 371.9895 | 362.6149 | 307.7058 | |
|
|
| corpus_sparsity_ratio | 0.9913 | 0.9878 | 0.989 | 0.989 | 0.9903 | 0.9877 | 0.991 | 0.9892 | 0.9979 | 0.9879 | 0.9878 | 0.9881 | 0.9899 | |
|
|
|
|
|
#### Sparse Nano BEIR |
|
|
|
|
|
* Dataset: `NanoBEIR_mean` |
|
|
* Evaluated with [<code>SparseNanoBEIREvaluator</code>](https://sbert.net/docs/package_reference/sparse_encoder/evaluation.html#sentence_transformers.sparse_encoder.evaluation.SparseNanoBEIREvaluator) with these parameters: |
|
|
```json |
|
|
{ |
|
|
"dataset_names": [ |
|
|
"msmarco", |
|
|
"nfcorpus", |
|
|
"nq" |
|
|
] |
|
|
} |
|
|
``` |
|
|
|
|
|
| Metric | Value | |
|
|
|:----------------------|:-----------| |
|
|
| dot_accuracy@1 | 0.3 | |
|
|
| dot_accuracy@3 | 0.5 | |
|
|
| dot_accuracy@5 | 0.58 | |
|
|
| dot_accuracy@10 | 0.6733 | |
|
|
| dot_precision@1 | 0.3 | |
|
|
| dot_precision@3 | 0.2067 | |
|
|
| dot_precision@5 | 0.1733 | |
|
|
| dot_precision@10 | 0.118 | |
|
|
| dot_recall@1 | 0.1801 | |
|
|
| dot_recall@3 | 0.3399 | |
|
|
| dot_recall@5 | 0.4152 | |
|
|
| dot_recall@10 | 0.5011 | |
|
|
| **dot_ndcg@10** | **0.4016** | |
|
|
| dot_mrr@10 | 0.4195 | |
|
|
| dot_map@100 | 0.3082 | |
|
|
| query_active_dims | 138.12 | |
|
|
| query_sparsity_ratio | 0.9955 | |
|
|
| corpus_active_dims | 346.3697 | |
|
|
| corpus_sparsity_ratio | 0.9887 | |
|
|
|
|
|
#### Sparse Nano BEIR |
|
|
|
|
|
* Dataset: `NanoBEIR_mean` |
|
|
* Evaluated with [<code>SparseNanoBEIREvaluator</code>](https://sbert.net/docs/package_reference/sparse_encoder/evaluation.html#sentence_transformers.sparse_encoder.evaluation.SparseNanoBEIREvaluator) with these parameters: |
|
|
```json |
|
|
{ |
|
|
"dataset_names": [ |
|
|
"climatefever", |
|
|
"dbpedia", |
|
|
"fever", |
|
|
"fiqa2018", |
|
|
"hotpotqa", |
|
|
"msmarco", |
|
|
"nfcorpus", |
|
|
"nq", |
|
|
"quoraretrieval", |
|
|
"scidocs", |
|
|
"arguana", |
|
|
"scifact", |
|
|
"touche2020" |
|
|
] |
|
|
} |
|
|
``` |
|
|
|
|
|
| Metric | Value | |
|
|
|:----------------------|:-----------| |
|
|
| dot_accuracy@1 | 0.4302 | |
|
|
| dot_accuracy@3 | 0.6182 | |
|
|
| dot_accuracy@5 | 0.6783 | |
|
|
| dot_accuracy@10 | 0.7722 | |
|
|
| dot_precision@1 | 0.4302 | |
|
|
| dot_precision@3 | 0.2742 | |
|
|
| dot_precision@5 | 0.2152 | |
|
|
| dot_precision@10 | 0.1564 | |
|
|
| dot_recall@1 | 0.2433 | |
|
|
| dot_recall@3 | 0.3891 | |
|
|
| dot_recall@5 | 0.4466 | |
|
|
| dot_recall@10 | 0.5378 | |
|
|
| **dot_ndcg@10** | **0.4809** | |
|
|
| dot_mrr@10 | 0.5383 | |
|
|
| dot_map@100 | 0.4055 | |
|
|
| query_active_dims | 161.5901 | |
|
|
| query_sparsity_ratio | 0.9947 | |
|
|
| corpus_active_dims | 302.8481 | |
|
|
| corpus_sparsity_ratio | 0.9901 | |
|
|
|
|
|
<!-- |
|
|
## Bias, Risks and Limitations |
|
|
|
|
|
*What are the known or foreseeable issues stemming from this model? You could also flag here known failure cases or weaknesses of the model.* |
|
|
--> |
|
|
|
|
|
<!-- |
|
|
### Recommendations |
|
|
|
|
|
*What are recommendations with respect to the foreseeable issues? For example, filtering explicit content.* |
|
|
--> |
|
|
|
|
|
## Training Details |
|
|
|
|
|
### Training Dataset |
|
|
|
|
|
#### gooaq |
|
|
|
|
|
* Dataset: [gooaq](https://huggingface.co/datasets/sentence-transformers/gooaq) at [b089f72](https://huggingface.co/datasets/sentence-transformers/gooaq/tree/b089f728748a068b7bc5234e5bcf5b25e3c8279c) |
|
|
* Size: 99,000 training samples |
|
|
* Columns: <code>question</code> and <code>answer</code> |
|
|
* Approximate statistics based on the first 1000 samples: |
|
|
| | question | answer | |
|
|
|:--------|:----------------------------------------------------------------------------------|:------------------------------------------------------------------------------------| |
|
|
| type | string | string | |
|
|
| details | <ul><li>min: 8 tokens</li><li>mean: 11.79 tokens</li><li>max: 24 tokens</li></ul> | <ul><li>min: 14 tokens</li><li>mean: 60.02 tokens</li><li>max: 153 tokens</li></ul> | |
|
|
* Samples: |
|
|
| question | answer | |
|
|
|:-----------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| |
|
|
| <code>what are the 5 characteristics of a star?</code> | <code>Key Concept: Characteristics used to classify stars include color, temperature, size, composition, and brightness.</code> | |
|
|
| <code>are copic markers alcohol ink?</code> | <code>Copic Ink is alcohol-based and flammable. Keep away from direct sunlight and extreme temperatures.</code> | |
|
|
| <code>what is the difference between appellate term and appellate division?</code> | <code>Appellate terms An appellate term is an intermediate appellate court that hears appeals from the inferior courts within their designated counties or judicial districts, and are intended to ease the workload on the Appellate Division and provide a less expensive forum closer to the people.</code> | |
|
|
* Loss: [<code>SpladeLoss</code>](https://sbert.net/docs/package_reference/sparse_encoder/losses.html#spladeloss) with these parameters: |
|
|
```json |
|
|
{ |
|
|
"loss": "SparseMultipleNegativesRankingLoss(scale=1.0, similarity_fct='dot_score')", |
|
|
"lambda_corpus": 3e-05, |
|
|
"lambda_query": 5e-05 |
|
|
} |
|
|
``` |
|
|
|
|
|
### Evaluation Dataset |
|
|
|
|
|
#### gooaq |
|
|
|
|
|
* Dataset: [gooaq](https://huggingface.co/datasets/sentence-transformers/gooaq) at [b089f72](https://huggingface.co/datasets/sentence-transformers/gooaq/tree/b089f728748a068b7bc5234e5bcf5b25e3c8279c) |
|
|
* Size: 1,000 evaluation samples |
|
|
* Columns: <code>question</code> and <code>answer</code> |
|
|
* Approximate statistics based on the first 1000 samples: |
|
|
| | question | answer | |
|
|
|:--------|:----------------------------------------------------------------------------------|:------------------------------------------------------------------------------------| |
|
|
| type | string | string | |
|
|
| details | <ul><li>min: 8 tokens</li><li>mean: 11.93 tokens</li><li>max: 25 tokens</li></ul> | <ul><li>min: 14 tokens</li><li>mean: 60.84 tokens</li><li>max: 127 tokens</li></ul> | |
|
|
* Samples: |
|
|
| question | answer | |
|
|
|:-----------------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| |
|
|
| <code>should you take ibuprofen with high blood pressure?</code> | <code>In general, people with high blood pressure should use acetaminophen or possibly aspirin for over-the-counter pain relief. Unless your health care provider has said it's OK, you should not use ibuprofen, ketoprofen, or naproxen sodium. If aspirin or acetaminophen doesn't help with your pain, call your doctor.</code> | |
|
|
| <code>how old do you have to be to work in sc?</code> | <code>The general minimum age of employment for South Carolina youth is 14, although the state allows younger children who are performers to work in show business. If their families are agricultural workers, children younger than age 14 may also participate in farm labor.</code> | |
|
|
| <code>how to write a topic proposal for a research paper?</code> | <code>['Write down the main topic of your paper. ... ', 'Write two or three short sentences under the main topic that explain why you chose that topic. ... ', 'Write a thesis sentence that states the angle and purpose of your research paper. ... ', 'List the items you will cover in the body of the paper that support your thesis statement.']</code> | |
|
|
* Loss: [<code>SpladeLoss</code>](https://sbert.net/docs/package_reference/sparse_encoder/losses.html#spladeloss) with these parameters: |
|
|
```json |
|
|
{ |
|
|
"loss": "SparseMultipleNegativesRankingLoss(scale=1.0, similarity_fct='dot_score')", |
|
|
"lambda_corpus": 3e-05, |
|
|
"lambda_query": 5e-05 |
|
|
} |
|
|
``` |
|
|
|
|
|
### Training Hyperparameters |
|
|
#### Non-Default Hyperparameters |
|
|
|
|
|
- `eval_strategy`: steps |
|
|
- `per_device_train_batch_size`: 32 |
|
|
- `per_device_eval_batch_size`: 32 |
|
|
- `learning_rate`: 2e-05 |
|
|
- `num_train_epochs`: 1 |
|
|
- `bf16`: True |
|
|
- `load_best_model_at_end`: True |
|
|
- `batch_sampler`: no_duplicates |
|
|
|
|
|
#### All Hyperparameters |
|
|
<details><summary>Click to expand</summary> |
|
|
|
|
|
- `overwrite_output_dir`: False |
|
|
- `do_predict`: False |
|
|
- `eval_strategy`: steps |
|
|
- `prediction_loss_only`: True |
|
|
- `per_device_train_batch_size`: 32 |
|
|
- `per_device_eval_batch_size`: 32 |
|
|
- `per_gpu_train_batch_size`: None |
|
|
- `per_gpu_eval_batch_size`: None |
|
|
- `gradient_accumulation_steps`: 1 |
|
|
- `eval_accumulation_steps`: None |
|
|
- `torch_empty_cache_steps`: None |
|
|
- `learning_rate`: 2e-05 |
|
|
- `weight_decay`: 0.0 |
|
|
- `adam_beta1`: 0.9 |
|
|
- `adam_beta2`: 0.999 |
|
|
- `adam_epsilon`: 1e-08 |
|
|
- `max_grad_norm`: 1.0 |
|
|
- `num_train_epochs`: 1 |
|
|
- `max_steps`: -1 |
|
|
- `lr_scheduler_type`: linear |
|
|
- `lr_scheduler_kwargs`: {} |
|
|
- `warmup_ratio`: 0.0 |
|
|
- `warmup_steps`: 0 |
|
|
- `log_level`: passive |
|
|
- `log_level_replica`: warning |
|
|
- `log_on_each_node`: True |
|
|
- `logging_nan_inf_filter`: True |
|
|
- `save_safetensors`: True |
|
|
- `save_on_each_node`: False |
|
|
- `save_only_model`: False |
|
|
- `restore_callback_states_from_checkpoint`: False |
|
|
- `no_cuda`: False |
|
|
- `use_cpu`: False |
|
|
- `use_mps_device`: False |
|
|
- `seed`: 42 |
|
|
- `data_seed`: None |
|
|
- `jit_mode_eval`: False |
|
|
- `use_ipex`: False |
|
|
- `bf16`: True |
|
|
- `fp16`: False |
|
|
- `fp16_opt_level`: O1 |
|
|
- `half_precision_backend`: auto |
|
|
- `bf16_full_eval`: False |
|
|
- `fp16_full_eval`: False |
|
|
- `tf32`: None |
|
|
- `local_rank`: 0 |
|
|
- `ddp_backend`: None |
|
|
- `tpu_num_cores`: None |
|
|
- `tpu_metrics_debug`: False |
|
|
- `debug`: [] |
|
|
- `dataloader_drop_last`: False |
|
|
- `dataloader_num_workers`: 0 |
|
|
- `dataloader_prefetch_factor`: None |
|
|
- `past_index`: -1 |
|
|
- `disable_tqdm`: False |
|
|
- `remove_unused_columns`: True |
|
|
- `label_names`: None |
|
|
- `load_best_model_at_end`: True |
|
|
- `ignore_data_skip`: False |
|
|
- `fsdp`: [] |
|
|
- `fsdp_min_num_params`: 0 |
|
|
- `fsdp_config`: {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False} |
|
|
- `tp_size`: 0 |
|
|
- `fsdp_transformer_layer_cls_to_wrap`: None |
|
|
- `accelerator_config`: {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None} |
|
|
- `deepspeed`: None |
|
|
- `label_smoothing_factor`: 0.0 |
|
|
- `optim`: adamw_torch |
|
|
- `optim_args`: None |
|
|
- `adafactor`: False |
|
|
- `group_by_length`: False |
|
|
- `length_column_name`: length |
|
|
- `ddp_find_unused_parameters`: None |
|
|
- `ddp_bucket_cap_mb`: None |
|
|
- `ddp_broadcast_buffers`: False |
|
|
- `dataloader_pin_memory`: True |
|
|
- `dataloader_persistent_workers`: False |
|
|
- `skip_memory_metrics`: True |
|
|
- `use_legacy_prediction_loop`: False |
|
|
- `push_to_hub`: False |
|
|
- `resume_from_checkpoint`: None |
|
|
- `hub_model_id`: None |
|
|
- `hub_strategy`: every_save |
|
|
- `hub_private_repo`: None |
|
|
- `hub_always_push`: False |
|
|
- `gradient_checkpointing`: False |
|
|
- `gradient_checkpointing_kwargs`: None |
|
|
- `include_inputs_for_metrics`: False |
|
|
- `include_for_metrics`: [] |
|
|
- `eval_do_concat_batches`: True |
|
|
- `fp16_backend`: auto |
|
|
- `push_to_hub_model_id`: None |
|
|
- `push_to_hub_organization`: None |
|
|
- `mp_parameters`: |
|
|
- `auto_find_batch_size`: False |
|
|
- `full_determinism`: False |
|
|
- `torchdynamo`: None |
|
|
- `ray_scope`: last |
|
|
- `ddp_timeout`: 1800 |
|
|
- `torch_compile`: False |
|
|
- `torch_compile_backend`: None |
|
|
- `torch_compile_mode`: None |
|
|
- `dispatch_batches`: None |
|
|
- `split_batches`: None |
|
|
- `include_tokens_per_second`: False |
|
|
- `include_num_input_tokens_seen`: False |
|
|
- `neftune_noise_alpha`: None |
|
|
- `optim_target_modules`: None |
|
|
- `batch_eval_metrics`: False |
|
|
- `eval_on_start`: False |
|
|
- `use_liger_kernel`: False |
|
|
- `eval_use_gather_object`: False |
|
|
- `average_tokens_across_devices`: False |
|
|
- `prompts`: None |
|
|
- `batch_sampler`: no_duplicates |
|
|
- `multi_dataset_batch_sampler`: proportional |
|
|
|
|
|
</details> |
|
|
|
|
|
### Training Logs |
|
|
| Epoch | Step | Training Loss | Validation Loss | NanoMSMARCO_dot_ndcg@10 | NanoNFCorpus_dot_ndcg@10 | NanoNQ_dot_ndcg@10 | NanoBEIR_mean_dot_ndcg@10 | NanoClimateFEVER_dot_ndcg@10 | NanoDBPedia_dot_ndcg@10 | NanoFEVER_dot_ndcg@10 | NanoFiQA2018_dot_ndcg@10 | NanoHotpotQA_dot_ndcg@10 | NanoQuoraRetrieval_dot_ndcg@10 | NanoSCIDOCS_dot_ndcg@10 | NanoArguAna_dot_ndcg@10 | NanoSciFact_dot_ndcg@10 | NanoTouche2020_dot_ndcg@10 | |
|
|
|:----------:|:--------:|:-------------:|:---------------:|:-----------------------:|:------------------------:|:------------------:|:-------------------------:|:----------------------------:|:-----------------------:|:---------------------:|:------------------------:|:------------------------:|:------------------------------:|:-----------------------:|:-----------------------:|:-----------------------:|:--------------------------:| |
|
|
| 0.0323 | 100 | 15.2006 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
|
|
| 0.0646 | 200 | 0.2384 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
|
|
| 0.0970 | 300 | 0.1932 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
|
|
| 0.1293 | 400 | 0.1428 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
|
|
| 0.1616 | 500 | 0.144 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
|
|
| 0.1939 | 600 | 0.1345 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
|
|
| 0.1972 | 610 | - | 0.1199 | 0.4364 | 0.2195 | 0.4998 | 0.3853 | - | - | - | - | - | - | - | - | - | - | |
|
|
| 0.2262 | 700 | 0.1406 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
|
|
| 0.2586 | 800 | 0.1012 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
|
|
| 0.2909 | 900 | 0.112 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
|
|
| 0.3232 | 1000 | 0.0736 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
|
|
| 0.3555 | 1100 | 0.0943 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
|
|
| 0.3878 | 1200 | 0.0901 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
|
|
| 0.3943 | 1220 | - | 0.1126 | 0.4706 | 0.2490 | 0.5154 | 0.4117 | - | - | - | - | - | - | - | - | - | - | |
|
|
| 0.4202 | 1300 | 0.0988 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
|
|
| 0.4525 | 1400 | 0.0953 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
|
|
| 0.4848 | 1500 | 0.1145 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
|
|
| 0.5171 | 1600 | 0.0928 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
|
|
| 0.5495 | 1700 | 0.0963 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
|
|
| 0.5818 | 1800 | 0.0724 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
|
|
| 0.5915 | 1830 | - | 0.0736 | 0.4576 | 0.2457 | 0.5015 | 0.4016 | - | - | - | - | - | - | - | - | - | - | |
|
|
| 0.6141 | 1900 | 0.0753 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
|
|
| 0.6464 | 2000 | 0.0657 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
|
|
| 0.6787 | 2100 | 0.0741 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
|
|
| 0.7111 | 2200 | 0.0671 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
|
|
| 0.7434 | 2300 | 0.1013 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
|
|
| 0.7757 | 2400 | 0.0795 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
|
|
| **0.7886** | **2440** | **-** | **0.0719** | **0.4785** | **0.2816** | **0.519** | **0.4264** | **-** | **-** | **-** | **-** | **-** | **-** | **-** | **-** | **-** | **-** | |
|
|
| 0.8080 | 2500 | 0.0666 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
|
|
| 0.8403 | 2600 | 0.0589 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
|
|
| 0.8727 | 2700 | 0.0569 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
|
|
| 0.9050 | 2800 | 0.0754 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
|
|
| 0.9373 | 2900 | 0.0724 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
|
|
| 0.9696 | 3000 | 0.0658 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | |
|
|
| 0.9858 | 3050 | - | 0.0661 | 0.4447 | 0.2587 | 0.5014 | 0.4016 | - | - | - | - | - | - | - | - | - | - | |
|
|
| -1 | -1 | - | - | 0.4785 | 0.2816 | 0.5190 | 0.4809 | 0.2528 | 0.4305 | 0.7203 | 0.3784 | 0.6986 | 0.7379 | 0.3073 | 0.3141 | 0.6331 | 0.4998 | |
|
|
|
|
|
* The bold row denotes the saved checkpoint. |
|
|
|
|
|
### Environmental Impact |
|
|
Carbon emissions were measured using [CodeCarbon](https://github.com/mlco2/codecarbon). |
|
|
- **Energy Consumed**: 0.019 kWh |
|
|
- **Carbon Emitted**: 0.001 kg of CO2 |
|
|
- **Hours Used**: 0.174 hours |
|
|
|
|
|
### Training Hardware |
|
|
- **On Cloud**: No |
|
|
- **GPU Model**: 1 x NVIDIA GeForce RTX 3070 Ti Laptop GPU |
|
|
- **CPU Model**: AMD Ryzen 9 6900HX with Radeon Graphics |
|
|
- **RAM Size**: 30.61 GB |
|
|
|
|
|
### Framework Versions |
|
|
- Python: 3.12.9 |
|
|
- Sentence Transformers: 4.2.0.dev0 |
|
|
- Transformers: 4.50.3 |
|
|
- PyTorch: 2.6.0+cu124 |
|
|
- Accelerate: 1.6.0 |
|
|
- Datasets: 3.5.0 |
|
|
- Tokenizers: 0.21.1 |
|
|
|
|
|
## Citation |
|
|
|
|
|
### BibTeX |
|
|
|
|
|
#### Sentence Transformers |
|
|
```bibtex |
|
|
@inproceedings{reimers-2019-sentence-bert, |
|
|
title = "Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks", |
|
|
author = "Reimers, Nils and Gurevych, Iryna", |
|
|
booktitle = "Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing", |
|
|
month = "11", |
|
|
year = "2019", |
|
|
publisher = "Association for Computational Linguistics", |
|
|
url = "https://arxiv.org/abs/1908.10084", |
|
|
} |
|
|
``` |
|
|
|
|
|
#### SpladeLoss |
|
|
```bibtex |
|
|
@misc{formal2022distillationhardnegativesampling, |
|
|
title={From Distillation to Hard Negative Sampling: Making Sparse Neural IR Models More Effective}, |
|
|
author={Thibault Formal and Carlos Lassance and Benjamin Piwowarski and Stéphane Clinchant}, |
|
|
year={2022}, |
|
|
eprint={2205.04733}, |
|
|
archivePrefix={arXiv}, |
|
|
primaryClass={cs.IR}, |
|
|
url={https://arxiv.org/abs/2205.04733}, |
|
|
} |
|
|
``` |
|
|
|
|
|
#### SparseMultipleNegativesRankingLoss |
|
|
```bibtex |
|
|
@misc{henderson2017efficient, |
|
|
title={Efficient Natural Language Response Suggestion for Smart Reply}, |
|
|
author={Matthew Henderson and Rami Al-Rfou and Brian Strope and Yun-hsuan Sung and Laszlo Lukacs and Ruiqi Guo and Sanjiv Kumar and Balint Miklos and Ray Kurzweil}, |
|
|
year={2017}, |
|
|
eprint={1705.00652}, |
|
|
archivePrefix={arXiv}, |
|
|
primaryClass={cs.CL} |
|
|
} |
|
|
``` |
|
|
|
|
|
#### FlopsLoss |
|
|
```bibtex |
|
|
@article{paria2020minimizing, |
|
|
title={Minimizing flops to learn efficient sparse representations}, |
|
|
author={Paria, Biswajit and Yeh, Chih-Kuan and Yen, Ian EH and Xu, Ning and Ravikumar, Pradeep and P{'o}czos, Barnab{'a}s}, |
|
|
journal={arXiv preprint arXiv:2004.05665}, |
|
|
year={2020} |
|
|
} |
|
|
``` |
|
|
|
|
|
<!-- |
|
|
## Glossary |
|
|
|
|
|
*Clearly define terms in order to be accessible across audiences.* |
|
|
--> |
|
|
|
|
|
<!-- |
|
|
## Model Card Authors |
|
|
|
|
|
*Lists the people who create the model card, providing recognition and accountability for the detailed work that goes into its construction.* |
|
|
--> |
|
|
|
|
|
<!-- |
|
|
## Model Card Contact |
|
|
|
|
|
*Provides a way for people who have updates to the Model Card, suggestions, or questions, to contact the Model Card authors.* |
|
|
--> |
