metadata

language:
  - en
license: apache-2.0
tags:
  - sentence-transformers
  - sparse-encoder
  - sparse
  - splade
  - generated_from_trainer
  - dataset_size:99000
  - loss:SpladeLoss
  - loss:SparseMultipleNegativesRankingLoss
  - loss:FlopsLoss
base_model: distilbert/distilbert-base-uncased
widget:
  - text: How do I know if a girl likes me at school?
  - text: What are some five star hotel in Jaipur?
  - text: Is it normal to fantasize your wife having sex with another man?
  - text: >-
      What is the Sahara, and how do the average temperatures there compare to
      the ones in the Simpson Desert?
  - text: >-
      What are Hillary Clinton's most recognized accomplishments while Secretary
      of State?
datasets:
  - sentence-transformers/quora-duplicates
pipeline_tag: feature-extraction
library_name: sentence-transformers
metrics:
  - cosine_accuracy
  - cosine_accuracy_threshold
  - cosine_f1
  - cosine_f1_threshold
  - cosine_precision
  - cosine_recall
  - cosine_ap
  - cosine_mcc
  - dot_accuracy
  - dot_accuracy_threshold
  - dot_f1
  - dot_f1_threshold
  - dot_precision
  - dot_recall
  - dot_ap
  - dot_mcc
  - euclidean_accuracy
  - euclidean_accuracy_threshold
  - euclidean_f1
  - euclidean_f1_threshold
  - euclidean_precision
  - euclidean_recall
  - euclidean_ap
  - euclidean_mcc
  - manhattan_accuracy
  - manhattan_accuracy_threshold
  - manhattan_f1
  - manhattan_f1_threshold
  - manhattan_precision
  - manhattan_recall
  - manhattan_ap
  - manhattan_mcc
  - max_accuracy
  - max_accuracy_threshold
  - max_f1
  - max_f1_threshold
  - max_precision
  - max_recall
  - max_ap
  - max_mcc
  - active_dims
  - sparsity_ratio
  - dot_accuracy@1
  - dot_accuracy@3
  - dot_accuracy@5
  - dot_accuracy@10
  - dot_precision@1
  - dot_precision@3
  - dot_precision@5
  - dot_precision@10
  - dot_recall@1
  - dot_recall@3
  - dot_recall@5
  - dot_recall@10
  - dot_ndcg@10
  - dot_mrr@10
  - dot_map@100
  - query_active_dims
  - query_sparsity_ratio
  - corpus_active_dims
  - corpus_sparsity_ratio
co2_eq_emissions:
  emissions: 1.4164940270091377
  energy_consumed: 0.02527693261851813
  source: codecarbon
  training_type: fine-tuning
  on_cloud: false
  cpu_model: AMD Ryzen 9 6900HX with Radeon Graphics
  ram_total_size: 30.6114501953125
  hours_used: 0.222
  hardware_used: 1 x NVIDIA GeForce RTX 3070 Ti Laptop GPU
model-index:
  - name: splade-distilbert-base-uncased trained on Quora Duplicates Questions
    results:
      - task:
          type: sparse-binary-classification
          name: Sparse Binary Classification
        dataset:
          name: quora duplicates dev
          type: quora_duplicates_dev
        metrics:
          - type: cosine_accuracy
            value: 0.758
            name: Cosine Accuracy
          - type: cosine_accuracy_threshold
            value: 0.8166326284408569
            name: Cosine Accuracy Threshold
          - type: cosine_f1
            value: 0.6792899408284023
            name: Cosine F1
          - type: cosine_f1_threshold
            value: 0.5695896148681641
            name: Cosine F1 Threshold
          - type: cosine_precision
            value: 0.5487571701720841
            name: Cosine Precision
          - type: cosine_recall
            value: 0.8913043478260869
            name: Cosine Recall
          - type: cosine_ap
            value: 0.6887627674706448
            name: Cosine Ap
          - type: cosine_mcc
            value: 0.508171027288805
            name: Cosine Mcc
          - type: dot_accuracy
            value: 0.765
            name: Dot Accuracy
          - type: dot_accuracy_threshold
            value: 51.6699104309082
            name: Dot Accuracy Threshold
          - type: dot_f1
            value: 0.6762028608582575
            name: Dot F1
          - type: dot_f1_threshold
            value: 46.524925231933594
            name: Dot F1 Threshold
          - type: dot_precision
            value: 0.5816554809843401
            name: Dot Precision
          - type: dot_recall
            value: 0.8074534161490683
            name: Dot Recall
          - type: dot_ap
            value: 0.6335823489360819
            name: Dot Ap
          - type: dot_mcc
            value: 0.4996270089694481
            name: Dot Mcc
          - type: euclidean_accuracy
            value: 0.677
            name: Euclidean Accuracy
          - type: euclidean_accuracy_threshold
            value: -14.272356986999512
            name: Euclidean Accuracy Threshold
          - type: euclidean_f1
            value: 0.48599545798637395
            name: Euclidean F1
          - type: euclidean_f1_threshold
            value: -0.6444530487060547
            name: Euclidean F1 Threshold
          - type: euclidean_precision
            value: 0.3213213213213213
            name: Euclidean Precision
          - type: euclidean_recall
            value: 0.9968944099378882
            name: Euclidean Recall
          - type: euclidean_ap
            value: 0.2032823056922341
            name: Euclidean Ap
          - type: euclidean_mcc
            value: -0.04590966956831287
            name: Euclidean Mcc
          - type: manhattan_accuracy
            value: 0.677
            name: Manhattan Accuracy
          - type: manhattan_accuracy_threshold
            value: -161.77682495117188
            name: Manhattan Accuracy Threshold
          - type: manhattan_f1
            value: 0.48599545798637395
            name: Manhattan F1
          - type: manhattan_f1_threshold
            value: -3.0494537353515625
            name: Manhattan F1 Threshold
          - type: manhattan_precision
            value: 0.3213213213213213
            name: Manhattan Precision
          - type: manhattan_recall
            value: 0.9968944099378882
            name: Manhattan Recall
          - type: manhattan_ap
            value: 0.20444314945561334
            name: Manhattan Ap
          - type: manhattan_mcc
            value: -0.04590966956831287
            name: Manhattan Mcc
          - type: max_accuracy
            value: 0.765
            name: Max Accuracy
          - type: max_accuracy_threshold
            value: 51.6699104309082
            name: Max Accuracy Threshold
          - type: max_f1
            value: 0.6792899408284023
            name: Max F1
          - type: max_f1_threshold
            value: 46.524925231933594
            name: Max F1 Threshold
          - type: max_precision
            value: 0.5816554809843401
            name: Max Precision
          - type: max_recall
            value: 0.9968944099378882
            name: Max Recall
          - type: max_ap
            value: 0.6887627674706448
            name: Max Ap
          - type: max_mcc
            value: 0.508171027288805
            name: Max Mcc
          - type: active_dims
            value: 78.32280731201172
            name: Active Dims
          - type: sparsity_ratio
            value: 0.9974338900690646
            name: Sparsity Ratio
      - task:
          type: sparse-information-retrieval
          name: Sparse Information Retrieval
        dataset:
          name: NanoMSMARCO
          type: NanoMSMARCO
        metrics:
          - type: dot_accuracy@1
            value: 0.22
            name: Dot Accuracy@1
          - type: dot_accuracy@3
            value: 0.42
            name: Dot Accuracy@3
          - type: dot_accuracy@5
            value: 0.52
            name: Dot Accuracy@5
          - type: dot_accuracy@10
            value: 0.76
            name: Dot Accuracy@10
          - type: dot_precision@1
            value: 0.22
            name: Dot Precision@1
          - type: dot_precision@3
            value: 0.13999999999999999
            name: Dot Precision@3
          - type: dot_precision@5
            value: 0.10400000000000001
            name: Dot Precision@5
          - type: dot_precision@10
            value: 0.07600000000000001
            name: Dot Precision@10
          - type: dot_recall@1
            value: 0.22
            name: Dot Recall@1
          - type: dot_recall@3
            value: 0.42
            name: Dot Recall@3
          - type: dot_recall@5
            value: 0.52
            name: Dot Recall@5
          - type: dot_recall@10
            value: 0.76
            name: Dot Recall@10
          - type: dot_ndcg@10
            value: 0.45321847177875746
            name: Dot Ndcg@10
          - type: dot_mrr@10
            value: 0.3601269841269841
            name: Dot Mrr@10
          - type: dot_map@100
            value: 0.37334906504034243
            name: Dot Map@100
          - type: query_active_dims
            value: 74.76000213623047
            name: Query Active Dims
          - type: query_sparsity_ratio
            value: 0.9975506191554868
            name: Query Sparsity Ratio
          - type: corpus_active_dims
            value: 103.06523895263672
            name: Corpus Active Dims
          - type: corpus_sparsity_ratio
            value: 0.9966232475279261
            name: Corpus Sparsity Ratio
          - type: dot_accuracy@1
            value: 0.22
            name: Dot Accuracy@1
          - type: dot_accuracy@3
            value: 0.42
            name: Dot Accuracy@3
          - type: dot_accuracy@5
            value: 0.52
            name: Dot Accuracy@5
          - type: dot_accuracy@10
            value: 0.76
            name: Dot Accuracy@10
          - type: dot_precision@1
            value: 0.22
            name: Dot Precision@1
          - type: dot_precision@3
            value: 0.13999999999999999
            name: Dot Precision@3
          - type: dot_precision@5
            value: 0.10400000000000001
            name: Dot Precision@5
          - type: dot_precision@10
            value: 0.07600000000000001
            name: Dot Precision@10
          - type: dot_recall@1
            value: 0.22
            name: Dot Recall@1
          - type: dot_recall@3
            value: 0.42
            name: Dot Recall@3
          - type: dot_recall@5
            value: 0.52
            name: Dot Recall@5
          - type: dot_recall@10
            value: 0.76
            name: Dot Recall@10
          - type: dot_ndcg@10
            value: 0.45321847177875746
            name: Dot Ndcg@10
          - type: dot_mrr@10
            value: 0.3601269841269841
            name: Dot Mrr@10
          - type: dot_map@100
            value: 0.37334906504034243
            name: Dot Map@100
          - type: query_active_dims
            value: 74.76000213623047
            name: Query Active Dims
          - type: query_sparsity_ratio
            value: 0.9975506191554868
            name: Query Sparsity Ratio
          - type: corpus_active_dims
            value: 103.06523895263672
            name: Corpus Active Dims
          - type: corpus_sparsity_ratio
            value: 0.9966232475279261
            name: Corpus Sparsity Ratio
      - task:
          type: sparse-information-retrieval
          name: Sparse Information Retrieval
        dataset:
          name: NanoNQ
          type: NanoNQ
        metrics:
          - type: dot_accuracy@1
            value: 0.38
            name: Dot Accuracy@1
          - type: dot_accuracy@3
            value: 0.54
            name: Dot Accuracy@3
          - type: dot_accuracy@5
            value: 0.62
            name: Dot Accuracy@5
          - type: dot_accuracy@10
            value: 0.62
            name: Dot Accuracy@10
          - type: dot_precision@1
            value: 0.38
            name: Dot Precision@1
          - type: dot_precision@3
            value: 0.18
            name: Dot Precision@3
          - type: dot_precision@5
            value: 0.12400000000000003
            name: Dot Precision@5
          - type: dot_precision@10
            value: 0.06400000000000002
            name: Dot Precision@10
          - type: dot_recall@1
            value: 0.36
            name: Dot Recall@1
          - type: dot_recall@3
            value: 0.52
            name: Dot Recall@3
          - type: dot_recall@5
            value: 0.6
            name: Dot Recall@5
          - type: dot_recall@10
            value: 0.61
            name: Dot Recall@10
          - type: dot_ndcg@10
            value: 0.4828377104499333
            name: Dot Ndcg@10
          - type: dot_mrr@10
            value: 0.4536666666666666
            name: Dot Mrr@10
          - type: dot_map@100
            value: 0.445384784044708
            name: Dot Map@100
          - type: query_active_dims
            value: 74.73999786376953
            name: Query Active Dims
          - type: query_sparsity_ratio
            value: 0.9975512745605213
            name: Query Sparsity Ratio
          - type: corpus_active_dims
            value: 141.31478881835938
            name: Corpus Active Dims
          - type: corpus_sparsity_ratio
            value: 0.9953700678586476
            name: Corpus Sparsity Ratio
          - type: dot_accuracy@1
            value: 0.38
            name: Dot Accuracy@1
          - type: dot_accuracy@3
            value: 0.54
            name: Dot Accuracy@3
          - type: dot_accuracy@5
            value: 0.62
            name: Dot Accuracy@5
          - type: dot_accuracy@10
            value: 0.62
            name: Dot Accuracy@10
          - type: dot_precision@1
            value: 0.38
            name: Dot Precision@1
          - type: dot_precision@3
            value: 0.18
            name: Dot Precision@3
          - type: dot_precision@5
            value: 0.12400000000000003
            name: Dot Precision@5
          - type: dot_precision@10
            value: 0.06400000000000002
            name: Dot Precision@10
          - type: dot_recall@1
            value: 0.36
            name: Dot Recall@1
          - type: dot_recall@3
            value: 0.52
            name: Dot Recall@3
          - type: dot_recall@5
            value: 0.6
            name: Dot Recall@5
          - type: dot_recall@10
            value: 0.61
            name: Dot Recall@10
          - type: dot_ndcg@10
            value: 0.4828377104499333
            name: Dot Ndcg@10
          - type: dot_mrr@10
            value: 0.4536666666666666
            name: Dot Mrr@10
          - type: dot_map@100
            value: 0.445384784044708
            name: Dot Map@100
          - type: query_active_dims
            value: 74.73999786376953
            name: Query Active Dims
          - type: query_sparsity_ratio
            value: 0.9975512745605213
            name: Query Sparsity Ratio
          - type: corpus_active_dims
            value: 141.31478881835938
            name: Corpus Active Dims
          - type: corpus_sparsity_ratio
            value: 0.9953700678586476
            name: Corpus Sparsity Ratio
      - task:
          type: sparse-information-retrieval
          name: Sparse Information Retrieval
        dataset:
          name: NanoNFCorpus
          type: NanoNFCorpus
        metrics:
          - type: dot_accuracy@1
            value: 0.34
            name: Dot Accuracy@1
          - type: dot_accuracy@3
            value: 0.5
            name: Dot Accuracy@3
          - type: dot_accuracy@5
            value: 0.54
            name: Dot Accuracy@5
          - type: dot_accuracy@10
            value: 0.58
            name: Dot Accuracy@10
          - type: dot_precision@1
            value: 0.34
            name: Dot Precision@1
          - type: dot_precision@3
            value: 0.30666666666666664
            name: Dot Precision@3
          - type: dot_precision@5
            value: 0.26
            name: Dot Precision@5
          - type: dot_precision@10
            value: 0.198
            name: Dot Precision@10
          - type: dot_recall@1
            value: 0.011597172822497613
            name: Dot Recall@1
          - type: dot_recall@3
            value: 0.06058581579610722
            name: Dot Recall@3
          - type: dot_recall@5
            value: 0.08260772201759854
            name: Dot Recall@5
          - type: dot_recall@10
            value: 0.09800124609193644
            name: Dot Recall@10
          - type: dot_ndcg@10
            value: 0.2466972614666078
            name: Dot Ndcg@10
          - type: dot_mrr@10
            value: 0.42200000000000004
            name: Dot Mrr@10
          - type: dot_map@100
            value: 0.09401937795309984
            name: Dot Map@100
          - type: query_active_dims
            value: 79.69999694824219
            name: Query Active Dims
          - type: query_sparsity_ratio
            value: 0.9973887688569477
            name: Query Sparsity Ratio
          - type: corpus_active_dims
            value: 202.17269897460938
            name: Corpus Active Dims
          - type: corpus_sparsity_ratio
            value: 0.9933761647672298
            name: Corpus Sparsity Ratio
          - type: dot_accuracy@1
            value: 0.34
            name: Dot Accuracy@1
          - type: dot_accuracy@3
            value: 0.5
            name: Dot Accuracy@3
          - type: dot_accuracy@5
            value: 0.54
            name: Dot Accuracy@5
          - type: dot_accuracy@10
            value: 0.58
            name: Dot Accuracy@10
          - type: dot_precision@1
            value: 0.34
            name: Dot Precision@1
          - type: dot_precision@3
            value: 0.30666666666666664
            name: Dot Precision@3
          - type: dot_precision@5
            value: 0.26
            name: Dot Precision@5
          - type: dot_precision@10
            value: 0.198
            name: Dot Precision@10
          - type: dot_recall@1
            value: 0.011597172822497613
            name: Dot Recall@1
          - type: dot_recall@3
            value: 0.06058581579610722
            name: Dot Recall@3
          - type: dot_recall@5
            value: 0.08260772201759854
            name: Dot Recall@5
          - type: dot_recall@10
            value: 0.09800124609193644
            name: Dot Recall@10
          - type: dot_ndcg@10
            value: 0.2466972614666078
            name: Dot Ndcg@10
          - type: dot_mrr@10
            value: 0.42200000000000004
            name: Dot Mrr@10
          - type: dot_map@100
            value: 0.09401937795309984
            name: Dot Map@100
          - type: query_active_dims
            value: 79.69999694824219
            name: Query Active Dims
          - type: query_sparsity_ratio
            value: 0.9973887688569477
            name: Query Sparsity Ratio
          - type: corpus_active_dims
            value: 202.17269897460938
            name: Corpus Active Dims
          - type: corpus_sparsity_ratio
            value: 0.9933761647672298
            name: Corpus Sparsity Ratio
      - task:
          type: sparse-information-retrieval
          name: Sparse Information Retrieval
        dataset:
          name: NanoQuoraRetrieval
          type: NanoQuoraRetrieval
        metrics:
          - type: dot_accuracy@1
            value: 0.94
            name: Dot Accuracy@1
          - type: dot_accuracy@3
            value: 0.98
            name: Dot Accuracy@3
          - type: dot_accuracy@5
            value: 0.98
            name: Dot Accuracy@5
          - type: dot_accuracy@10
            value: 0.98
            name: Dot Accuracy@10
          - type: dot_precision@1
            value: 0.94
            name: Dot Precision@1
          - type: dot_precision@3
            value: 0.3933333333333333
            name: Dot Precision@3
          - type: dot_precision@5
            value: 0.24799999999999997
            name: Dot Precision@5
          - type: dot_precision@10
            value: 0.13199999999999998
            name: Dot Precision@10
          - type: dot_recall@1
            value: 0.8173333333333332
            name: Dot Recall@1
          - type: dot_recall@3
            value: 0.9279999999999999
            name: Dot Recall@3
          - type: dot_recall@5
            value: 0.946
            name: Dot Recall@5
          - type: dot_recall@10
            value: 0.97
            name: Dot Recall@10
          - type: dot_ndcg@10
            value: 0.9467235239993945
            name: Dot Ndcg@10
          - type: dot_mrr@10
            value: 0.96
            name: Dot Mrr@10
          - type: dot_map@100
            value: 0.9290737327188939
            name: Dot Map@100
          - type: query_active_dims
            value: 76.58000183105469
            name: Query Active Dims
          - type: query_sparsity_ratio
            value: 0.9974909900455063
            name: Query Sparsity Ratio
          - type: corpus_active_dims
            value: 77.59056854248047
            name: Corpus Active Dims
          - type: corpus_sparsity_ratio
            value: 0.9974578805929336
            name: Corpus Sparsity Ratio
          - type: dot_accuracy@1
            value: 0.94
            name: Dot Accuracy@1
          - type: dot_accuracy@3
            value: 0.98
            name: Dot Accuracy@3
          - type: dot_accuracy@5
            value: 0.98
            name: Dot Accuracy@5
          - type: dot_accuracy@10
            value: 0.98
            name: Dot Accuracy@10
          - type: dot_precision@1
            value: 0.94
            name: Dot Precision@1
          - type: dot_precision@3
            value: 0.3933333333333333
            name: Dot Precision@3
          - type: dot_precision@5
            value: 0.24799999999999997
            name: Dot Precision@5
          - type: dot_precision@10
            value: 0.13199999999999998
            name: Dot Precision@10
          - type: dot_recall@1
            value: 0.8173333333333332
            name: Dot Recall@1
          - type: dot_recall@3
            value: 0.9279999999999999
            name: Dot Recall@3
          - type: dot_recall@5
            value: 0.946
            name: Dot Recall@5
          - type: dot_recall@10
            value: 0.97
            name: Dot Recall@10
          - type: dot_ndcg@10
            value: 0.9467235239993945
            name: Dot Ndcg@10
          - type: dot_mrr@10
            value: 0.96
            name: Dot Mrr@10
          - type: dot_map@100
            value: 0.9290737327188939
            name: Dot Map@100
          - type: query_active_dims
            value: 76.58000183105469
            name: Query Active Dims
          - type: query_sparsity_ratio
            value: 0.9974909900455063
            name: Query Sparsity Ratio
          - type: corpus_active_dims
            value: 77.59056854248047
            name: Corpus Active Dims
          - type: corpus_sparsity_ratio
            value: 0.9974578805929336
            name: Corpus Sparsity Ratio
      - task:
          type: sparse-nano-beir
          name: Sparse Nano BEIR
        dataset:
          name: NanoBEIR mean
          type: NanoBEIR_mean
        metrics:
          - type: dot_accuracy@1
            value: 0.47
            name: Dot Accuracy@1
          - type: dot_accuracy@3
            value: 0.61
            name: Dot Accuracy@3
          - type: dot_accuracy@5
            value: 0.665
            name: Dot Accuracy@5
          - type: dot_accuracy@10
            value: 0.735
            name: Dot Accuracy@10
          - type: dot_precision@1
            value: 0.47
            name: Dot Precision@1
          - type: dot_precision@3
            value: 0.255
            name: Dot Precision@3
          - type: dot_precision@5
            value: 0.184
            name: Dot Precision@5
          - type: dot_precision@10
            value: 0.1175
            name: Dot Precision@10
          - type: dot_recall@1
            value: 0.3522326265389577
            name: Dot Recall@1
          - type: dot_recall@3
            value: 0.4821464539490268
            name: Dot Recall@3
          - type: dot_recall@5
            value: 0.5371519305043997
            name: Dot Recall@5
          - type: dot_recall@10
            value: 0.6095003115229841
            name: Dot Recall@10
          - type: dot_ndcg@10
            value: 0.5323692419236733
            name: Dot Ndcg@10
          - type: dot_mrr@10
            value: 0.5489484126984127
            name: Dot Mrr@10
          - type: dot_map@100
            value: 0.46045673993926106
            name: Dot Map@100
          - type: query_active_dims
            value: 76.44499969482422
            name: Query Active Dims
          - type: query_sparsity_ratio
            value: 0.9974954131546155
            name: Query Sparsity Ratio
          - type: corpus_active_dims
            value: 122.79780664247188
            name: Corpus Active Dims
          - type: corpus_sparsity_ratio
            value: 0.9959767444255792
            name: Corpus Sparsity Ratio
          - type: dot_accuracy@1
            value: 0.4359811616954475
            name: Dot Accuracy@1
          - type: dot_accuracy@3
            value: 0.6088540031397174
            name: Dot Accuracy@3
          - type: dot_accuracy@5
            value: 0.6659026687598116
            name: Dot Accuracy@5
          - type: dot_accuracy@10
            value: 0.7383987441130299
            name: Dot Accuracy@10
          - type: dot_precision@1
            value: 0.4359811616954475
            name: Dot Precision@1
          - type: dot_precision@3
            value: 0.2725170068027211
            name: Dot Precision@3
          - type: dot_precision@5
            value: 0.2089481946624804
            name: Dot Precision@5
          - type: dot_precision@10
            value: 0.14605965463108322
            name: Dot Precision@10
          - type: dot_recall@1
            value: 0.2532746332292894
            name: Dot Recall@1
          - type: dot_recall@3
            value: 0.3813452238818861
            name: Dot Recall@3
          - type: dot_recall@5
            value: 0.4363867898661836
            name: Dot Recall@5
          - type: dot_recall@10
            value: 0.5099503000039356
            name: Dot Recall@10
          - type: dot_ndcg@10
            value: 0.4684519639817077
            name: Dot Ndcg@10
          - type: dot_mrr@10
            value: 0.5328029827315542
            name: Dot Mrr@10
          - type: dot_map@100
            value: 0.39738635557561647
            name: Dot Map@100
          - type: query_active_dims
            value: 90.39137197532713
            name: Query Active Dims
          - type: query_sparsity_ratio
            value: 0.9970384846348428
            name: Query Sparsity Ratio
          - type: corpus_active_dims
            value: 152.36685474307478
            name: Corpus Active Dims
          - type: corpus_sparsity_ratio
            value: 0.9950079662295042
            name: Corpus Sparsity Ratio
      - task:
          type: sparse-information-retrieval
          name: Sparse Information Retrieval
        dataset:
          name: NanoClimateFEVER
          type: NanoClimateFEVER
        metrics:
          - type: dot_accuracy@1
            value: 0.18
            name: Dot Accuracy@1
          - type: dot_accuracy@3
            value: 0.32
            name: Dot Accuracy@3
          - type: dot_accuracy@5
            value: 0.4
            name: Dot Accuracy@5
          - type: dot_accuracy@10
            value: 0.48
            name: Dot Accuracy@10
          - type: dot_precision@1
            value: 0.18
            name: Dot Precision@1
          - type: dot_precision@3
            value: 0.10666666666666666
            name: Dot Precision@3
          - type: dot_precision@5
            value: 0.08400000000000002
            name: Dot Precision@5
          - type: dot_precision@10
            value: 0.054000000000000006
            name: Dot Precision@10
          - type: dot_recall@1
            value: 0.085
            name: Dot Recall@1
          - type: dot_recall@3
            value: 0.14666666666666667
            name: Dot Recall@3
          - type: dot_recall@5
            value: 0.17833333333333332
            name: Dot Recall@5
          - type: dot_recall@10
            value: 0.215
            name: Dot Recall@10
          - type: dot_ndcg@10
            value: 0.1845115403570178
            name: Dot Ndcg@10
          - type: dot_mrr@10
            value: 0.2674126984126984
            name: Dot Mrr@10
          - type: dot_map@100
            value: 0.1475834110231865
            name: Dot Map@100
          - type: query_active_dims
            value: 89.86000061035156
            name: Query Active Dims
          - type: query_sparsity_ratio
            value: 0.9970558940891701
            name: Query Sparsity Ratio
          - type: corpus_active_dims
            value: 221.75527954101562
            name: Corpus Active Dims
          - type: corpus_sparsity_ratio
            value: 0.992734575730915
            name: Corpus Sparsity Ratio
      - task:
          type: sparse-information-retrieval
          name: Sparse Information Retrieval
        dataset:
          name: NanoDBPedia
          type: NanoDBPedia
        metrics:
          - type: dot_accuracy@1
            value: 0.6
            name: Dot Accuracy@1
          - type: dot_accuracy@3
            value: 0.84
            name: Dot Accuracy@3
          - type: dot_accuracy@5
            value: 0.84
            name: Dot Accuracy@5
          - type: dot_accuracy@10
            value: 0.92
            name: Dot Accuracy@10
          - type: dot_precision@1
            value: 0.6
            name: Dot Precision@1
          - type: dot_precision@3
            value: 0.5266666666666666
            name: Dot Precision@3
          - type: dot_precision@5
            value: 0.456
            name: Dot Precision@5
          - type: dot_precision@10
            value: 0.4220000000000001
            name: Dot Precision@10
          - type: dot_recall@1
            value: 0.04570544957623723
            name: Dot Recall@1
          - type: dot_recall@3
            value: 0.15367137863132574
            name: Dot Recall@3
          - type: dot_recall@5
            value: 0.1908008582920462
            name: Dot Recall@5
          - type: dot_recall@10
            value: 0.293554014064817
            name: Dot Recall@10
          - type: dot_ndcg@10
            value: 0.5070720730882787
            name: Dot Ndcg@10
          - type: dot_mrr@10
            value: 0.7147222222222225
            name: Dot Mrr@10
          - type: dot_map@100
            value: 0.3906658166774757
            name: Dot Map@100
          - type: query_active_dims
            value: 69.5199966430664
            name: Query Active Dims
          - type: query_sparsity_ratio
            value: 0.997722298779796
            name: Query Sparsity Ratio
          - type: corpus_active_dims
            value: 135.93350219726562
            name: Corpus Active Dims
          - type: corpus_sparsity_ratio
            value: 0.9955463763122578
            name: Corpus Sparsity Ratio
      - task:
          type: sparse-information-retrieval
          name: Sparse Information Retrieval
        dataset:
          name: NanoFEVER
          type: NanoFEVER
        metrics:
          - type: dot_accuracy@1
            value: 0.58
            name: Dot Accuracy@1
          - type: dot_accuracy@3
            value: 0.76
            name: Dot Accuracy@3
          - type: dot_accuracy@5
            value: 0.8
            name: Dot Accuracy@5
          - type: dot_accuracy@10
            value: 0.86
            name: Dot Accuracy@10
          - type: dot_precision@1
            value: 0.58
            name: Dot Precision@1
          - type: dot_precision@3
            value: 0.26666666666666666
            name: Dot Precision@3
          - type: dot_precision@5
            value: 0.16799999999999998
            name: Dot Precision@5
          - type: dot_precision@10
            value: 0.09
            name: Dot Precision@10
          - type: dot_recall@1
            value: 0.5466666666666666
            name: Dot Recall@1
          - type: dot_recall@3
            value: 0.7466666666666667
            name: Dot Recall@3
          - type: dot_recall@5
            value: 0.7866666666666667
            name: Dot Recall@5
          - type: dot_recall@10
            value: 0.8466666666666667
            name: Dot Recall@10
          - type: dot_ndcg@10
            value: 0.7069849294263234
            name: Dot Ndcg@10
          - type: dot_mrr@10
            value: 0.6765000000000001
            name: Dot Mrr@10
          - type: dot_map@100
            value: 0.6651380090497737
            name: Dot Map@100
          - type: query_active_dims
            value: 89.87999725341797
            name: Query Active Dims
          - type: query_sparsity_ratio
            value: 0.9970552389340994
            name: Query Sparsity Ratio
          - type: corpus_active_dims
            value: 221.215576171875
            name: Corpus Active Dims
          - type: corpus_sparsity_ratio
            value: 0.9927522581688004
            name: Corpus Sparsity Ratio
      - task:
          type: sparse-information-retrieval
          name: Sparse Information Retrieval
        dataset:
          name: NanoFiQA2018
          type: NanoFiQA2018
        metrics:
          - type: dot_accuracy@1
            value: 0.28
            name: Dot Accuracy@1
          - type: dot_accuracy@3
            value: 0.42
            name: Dot Accuracy@3
          - type: dot_accuracy@5
            value: 0.46
            name: Dot Accuracy@5
          - type: dot_accuracy@10
            value: 0.5
            name: Dot Accuracy@10
          - type: dot_precision@1
            value: 0.28
            name: Dot Precision@1
          - type: dot_precision@3
            value: 0.18
            name: Dot Precision@3
          - type: dot_precision@5
            value: 0.136
            name: Dot Precision@5
          - type: dot_precision@10
            value: 0.08399999999999999
            name: Dot Precision@10
          - type: dot_recall@1
            value: 0.14183333333333334
            name: Dot Recall@1
          - type: dot_recall@3
            value: 0.24288888888888888
            name: Dot Recall@3
          - type: dot_recall@5
            value: 0.27715873015873016
            name: Dot Recall@5
          - type: dot_recall@10
            value: 0.3288730158730159
            name: Dot Recall@10
          - type: dot_ndcg@10
            value: 0.28813286680239514
            name: Dot Ndcg@10
          - type: dot_mrr@10
            value: 0.3561904761904763
            name: Dot Mrr@10
          - type: dot_map@100
            value: 0.2415362537997973
            name: Dot Map@100
          - type: query_active_dims
            value: 82.86000061035156
            name: Query Active Dims
          - type: query_sparsity_ratio
            value: 0.9972852368583202
            name: Query Sparsity Ratio
          - type: corpus_active_dims
            value: 130.93699645996094
            name: Corpus Active Dims
          - type: corpus_sparsity_ratio
            value: 0.9957100780925245
            name: Corpus Sparsity Ratio
      - task:
          type: sparse-information-retrieval
          name: Sparse Information Retrieval
        dataset:
          name: NanoHotpotQA
          type: NanoHotpotQA
        metrics:
          - type: dot_accuracy@1
            value: 0.78
            name: Dot Accuracy@1
          - type: dot_accuracy@3
            value: 0.84
            name: Dot Accuracy@3
          - type: dot_accuracy@5
            value: 0.92
            name: Dot Accuracy@5
          - type: dot_accuracy@10
            value: 0.98
            name: Dot Accuracy@10
          - type: dot_precision@1
            value: 0.78
            name: Dot Precision@1
          - type: dot_precision@3
            value: 0.3733333333333333
            name: Dot Precision@3
          - type: dot_precision@5
            value: 0.28400000000000003
            name: Dot Precision@5
          - type: dot_precision@10
            value: 0.16
            name: Dot Precision@10
          - type: dot_recall@1
            value: 0.39
            name: Dot Recall@1
          - type: dot_recall@3
            value: 0.56
            name: Dot Recall@3
          - type: dot_recall@5
            value: 0.71
            name: Dot Recall@5
          - type: dot_recall@10
            value: 0.8
            name: Dot Recall@10
          - type: dot_ndcg@10
            value: 0.7143331285788386
            name: Dot Ndcg@10
          - type: dot_mrr@10
            value: 0.8361904761904762
            name: Dot Mrr@10
          - type: dot_map@100
            value: 0.6181181734895289
            name: Dot Map@100
          - type: query_active_dims
            value: 91.9800033569336
            name: Query Active Dims
          - type: query_sparsity_ratio
            value: 0.9969864359033833
            name: Query Sparsity Ratio
          - type: corpus_active_dims
            value: 152.01571655273438
            name: Corpus Active Dims
          - type: corpus_sparsity_ratio
            value: 0.9950194706587794
            name: Corpus Sparsity Ratio
      - task:
          type: sparse-information-retrieval
          name: Sparse Information Retrieval
        dataset:
          name: NanoSCIDOCS
          type: NanoSCIDOCS
        metrics:
          - type: dot_accuracy@1
            value: 0.36
            name: Dot Accuracy@1
          - type: dot_accuracy@3
            value: 0.58
            name: Dot Accuracy@3
          - type: dot_accuracy@5
            value: 0.68
            name: Dot Accuracy@5
          - type: dot_accuracy@10
            value: 0.76
            name: Dot Accuracy@10
          - type: dot_precision@1
            value: 0.36
            name: Dot Precision@1
          - type: dot_precision@3
            value: 0.2733333333333333
            name: Dot Precision@3
          - type: dot_precision@5
            value: 0.21199999999999997
            name: Dot Precision@5
          - type: dot_precision@10
            value: 0.15199999999999997
            name: Dot Precision@10
          - type: dot_recall@1
            value: 0.07566666666666666
            name: Dot Recall@1
          - type: dot_recall@3
            value: 0.16966666666666666
            name: Dot Recall@3
          - type: dot_recall@5
            value: 0.21766666666666665
            name: Dot Recall@5
          - type: dot_recall@10
            value: 0.31066666666666665
            name: Dot Recall@10
          - type: dot_ndcg@10
            value: 0.30291194083231554
            name: Dot Ndcg@10
          - type: dot_mrr@10
            value: 0.4943888888888889
            name: Dot Mrr@10
          - type: dot_map@100
            value: 0.21666464487074008
            name: Dot Map@100
          - type: query_active_dims
            value: 94.30000305175781
            name: Query Active Dims
          - type: query_sparsity_ratio
            value: 0.996910425167035
            name: Query Sparsity Ratio
          - type: corpus_active_dims
            value: 199.64630126953125
            name: Corpus Active Dims
          - type: corpus_sparsity_ratio
            value: 0.9934589377737524
            name: Corpus Sparsity Ratio
      - task:
          type: sparse-information-retrieval
          name: Sparse Information Retrieval
        dataset:
          name: NanoArguAna
          type: NanoArguAna
        metrics:
          - type: dot_accuracy@1
            value: 0.1
            name: Dot Accuracy@1
          - type: dot_accuracy@3
            value: 0.34
            name: Dot Accuracy@3
          - type: dot_accuracy@5
            value: 0.42
            name: Dot Accuracy@5
          - type: dot_accuracy@10
            value: 0.44
            name: Dot Accuracy@10
          - type: dot_precision@1
            value: 0.1
            name: Dot Precision@1
          - type: dot_precision@3
            value: 0.1133333333333333
            name: Dot Precision@3
          - type: dot_precision@5
            value: 0.084
            name: Dot Precision@5
          - type: dot_precision@10
            value: 0.044000000000000004
            name: Dot Precision@10
          - type: dot_recall@1
            value: 0.1
            name: Dot Recall@1
          - type: dot_recall@3
            value: 0.34
            name: Dot Recall@3
          - type: dot_recall@5
            value: 0.42
            name: Dot Recall@5
          - type: dot_recall@10
            value: 0.44
            name: Dot Recall@10
          - type: dot_ndcg@10
            value: 0.2781554838544819
            name: Dot Ndcg@10
          - type: dot_mrr@10
            value: 0.22466666666666665
            name: Dot Mrr@10
          - type: dot_map@100
            value: 0.2332757160696607
            name: Dot Map@100
          - type: query_active_dims
            value: 189.10000610351562
            name: Query Active Dims
          - type: query_sparsity_ratio
            value: 0.9938044687077021
            name: Query Sparsity Ratio
          - type: corpus_active_dims
            value: 164.03329467773438
            name: Corpus Active Dims
          - type: corpus_sparsity_ratio
            value: 0.9946257357093985
            name: Corpus Sparsity Ratio
      - task:
          type: sparse-information-retrieval
          name: Sparse Information Retrieval
        dataset:
          name: NanoSciFact
          type: NanoSciFact
        metrics:
          - type: dot_accuracy@1
            value: 0.52
            name: Dot Accuracy@1
          - type: dot_accuracy@3
            value: 0.62
            name: Dot Accuracy@3
          - type: dot_accuracy@5
            value: 0.64
            name: Dot Accuracy@5
          - type: dot_accuracy@10
            value: 0.76
            name: Dot Accuracy@10
          - type: dot_precision@1
            value: 0.52
            name: Dot Precision@1
          - type: dot_precision@3
            value: 0.21333333333333332
            name: Dot Precision@3
          - type: dot_precision@5
            value: 0.14
            name: Dot Precision@5
          - type: dot_precision@10
            value: 0.08399999999999999
            name: Dot Precision@10
          - type: dot_recall@1
            value: 0.475
            name: Dot Recall@1
          - type: dot_recall@3
            value: 0.58
            name: Dot Recall@3
          - type: dot_recall@5
            value: 0.615
            name: Dot Recall@5
          - type: dot_recall@10
            value: 0.74
            name: Dot Recall@10
          - type: dot_ndcg@10
            value: 0.6020710919940331
            name: Dot Ndcg@10
          - type: dot_mrr@10
            value: 0.5799047619047619
            name: Dot Mrr@10
          - type: dot_map@100
            value: 0.5551340236204781
            name: Dot Map@100
          - type: query_active_dims
            value: 82.45999908447266
            name: Query Active Dims
          - type: query_sparsity_ratio
            value: 0.9972983422094073
            name: Query Sparsity Ratio
          - type: corpus_active_dims
            value: 194.24940490722656
            name: Corpus Active Dims
          - type: corpus_sparsity_ratio
            value: 0.9936357576532591
            name: Corpus Sparsity Ratio
      - task:
          type: sparse-information-retrieval
          name: Sparse Information Retrieval
        dataset:
          name: NanoTouche2020
          type: NanoTouche2020
        metrics:
          - type: dot_accuracy@1
            value: 0.3877551020408163
            name: Dot Accuracy@1
          - type: dot_accuracy@3
            value: 0.7551020408163265
            name: Dot Accuracy@3
          - type: dot_accuracy@5
            value: 0.8367346938775511
            name: Dot Accuracy@5
          - type: dot_accuracy@10
            value: 0.9591836734693877
            name: Dot Accuracy@10
          - type: dot_precision@1
            value: 0.3877551020408163
            name: Dot Precision@1
          - type: dot_precision@3
            value: 0.4693877551020407
            name: Dot Precision@3
          - type: dot_precision@5
            value: 0.4163265306122449
            name: Dot Precision@5
          - type: dot_precision@10
            value: 0.33877551020408164
            name: Dot Precision@10
          - type: dot_recall@1
            value: 0.02376760958202688
            name: Dot Recall@1
          - type: dot_recall@3
            value: 0.08934182714819683
            name: Dot Recall@3
          - type: dot_recall@5
            value: 0.12879429112534482
            name: Dot Recall@5
          - type: dot_recall@10
            value: 0.21659229068805946
            name: Dot Recall@10
          - type: dot_ndcg@10
            value: 0.37622550913382224
            name: Dot Ndcg@10
          - type: dot_mrr@10
            value: 0.5806689342403627
            name: Dot Mrr@10
          - type: dot_map@100
            value: 0.2560796141253303
            name: Dot Map@100
          - type: query_active_dims
            value: 79.12245178222656
            name: Query Active Dims
          - type: query_sparsity_ratio
            value: 0.9974076911151881
            name: Query Sparsity Ratio
          - type: corpus_active_dims
            value: 135.00782775878906
            name: Corpus Active Dims
          - type: corpus_sparsity_ratio
            value: 0.9955767044178366
            name: Corpus Sparsity Ratio

splade-distilbert-base-uncased trained on Quora Duplicates Questions

This is a SPLADE Sparse Encoder model finetuned from distilbert/distilbert-base-uncased on the quora-duplicates dataset using the sentence-transformers library. It maps sentences & paragraphs to a 30522-dimensional sparse vector space and can be used for semantic search and sparse retrieval.

Model Details

Model Description

Model Type: SPLADE Sparse Encoder
Base model: distilbert/distilbert-base-uncased
Maximum Sequence Length: 256 tokens
Output Dimensionality: 30522 dimensions
Similarity Function: Dot Product
Training Dataset:
- quora-duplicates
Language: en
License: apache-2.0

Model Sources

Documentation: Sentence Transformers Documentation
Documentation: Sparse Encoder Documentation
Repository: Sentence Transformers on GitHub
Hugging Face: Sparse Encoders on Hugging Face

Full Model Architecture

SparseEncoder(
  (0): MLMTransformer({'max_seq_length': 256, 'do_lower_case': False}) with MLMTransformer model: DistilBertForMaskedLM 
  (1): SpladePooling({'pooling_strategy': 'max', 'activation_function': 'relu', 'word_embedding_dimension': 30522})
)

Usage

Direct Usage (Sentence Transformers)

First install the Sentence Transformers library:

pip install -U sentence-transformers

Then you can load this model and run inference.

from sentence_transformers import SparseEncoder

# Download from the 🤗 Hub
model = SparseEncoder("arthurbresnu/splade-distilbert-base-uncased-quora-duplicates")
# Run inference
sentences = [
    'What accomplishments did Hillary Clinton achieve during her time as Secretary of State?',
    "What are Hillary Clinton's most recognized accomplishments while Secretary of State?",
    'What are Hillary Clinton’s qualifications to be President?',
]
embeddings = model.encode(sentences)
print(embeddings.shape)
# (3, 30522)

# Get the similarity scores for the embeddings
similarities = model.similarity(embeddings, embeddings)
print(similarities.shape)
# [3, 3]

Evaluation

Metrics

Sparse Binary Classification

Dataset: quora_duplicates_dev
Evaluated with SparseBinaryClassificationEvaluator

Metric	Value
cosine_accuracy	0.758
cosine_accuracy_threshold	0.8166
cosine_f1	0.6793
cosine_f1_threshold	0.5696
cosine_precision	0.5488
cosine_recall	0.8913
cosine_ap	0.6888
cosine_mcc	0.5082
dot_accuracy	0.765
dot_accuracy_threshold	51.6699
dot_f1	0.6762
dot_f1_threshold	46.5249
dot_precision	0.5817
dot_recall	0.8075
dot_ap	0.6336
dot_mcc	0.4996
euclidean_accuracy	0.677
euclidean_accuracy_threshold	-14.2724
euclidean_f1	0.486
euclidean_f1_threshold	-0.6445
euclidean_precision	0.3213
euclidean_recall	0.9969
euclidean_ap	0.2033
euclidean_mcc	-0.0459
manhattan_accuracy	0.677
manhattan_accuracy_threshold	-161.7768
manhattan_f1	0.486
manhattan_f1_threshold	-3.0495
manhattan_precision	0.3213
manhattan_recall	0.9969
manhattan_ap	0.2044
manhattan_mcc	-0.0459
max_accuracy	0.765
max_accuracy_threshold	51.6699
max_f1	0.6793
max_f1_threshold	46.5249
max_precision	0.5817
max_recall	0.9969
max_ap	0.6888
max_mcc	0.5082
active_dims	78.3228
sparsity_ratio	0.9974

Sparse Information Retrieval

Datasets: NanoMSMARCO, NanoNQ, NanoNFCorpus, NanoQuoraRetrieval, NanoClimateFEVER, NanoDBPedia, NanoFEVER, NanoFiQA2018, NanoHotpotQA, NanoMSMARCO, NanoNFCorpus, NanoNQ, NanoQuoraRetrieval, NanoSCIDOCS, NanoArguAna, NanoSciFact and NanoTouche2020
Evaluated with SparseInformationRetrievalEvaluator

Metric	NanoMSMARCO	NanoNQ	NanoNFCorpus	NanoQuoraRetrieval	NanoClimateFEVER	NanoDBPedia	NanoFEVER	NanoFiQA2018	NanoHotpotQA	NanoSCIDOCS	NanoArguAna	NanoSciFact	NanoTouche2020
dot_accuracy@1	0.22	0.38	0.34	0.94	0.18	0.6	0.58	0.28	0.78	0.36	0.1	0.52	0.3878
dot_accuracy@3	0.42	0.54	0.5	0.98	0.32	0.84	0.76	0.42	0.84	0.58	0.34	0.62	0.7551
dot_accuracy@5	0.52	0.62	0.54	0.98	0.4	0.84	0.8	0.46	0.92	0.68	0.42	0.64	0.8367
dot_accuracy@10	0.76	0.62	0.58	0.98	0.48	0.92	0.86	0.5	0.98	0.76	0.44	0.76	0.9592
dot_precision@1	0.22	0.38	0.34	0.94	0.18	0.6	0.58	0.28	0.78	0.36	0.1	0.52	0.3878
dot_precision@3	0.14	0.18	0.3067	0.3933	0.1067	0.5267	0.2667	0.18	0.3733	0.2733	0.1133	0.2133	0.4694
dot_precision@5	0.104	0.124	0.26	0.248	0.084	0.456	0.168	0.136	0.284	0.212	0.084	0.14	0.4163
dot_precision@10	0.076	0.064	0.198	0.132	0.054	0.422	0.09	0.084	0.16	0.152	0.044	0.084	0.3388
dot_recall@1	0.22	0.36	0.0116	0.8173	0.085	0.0457	0.5467	0.1418	0.39	0.0757	0.1	0.475	0.0238
dot_recall@3	0.42	0.52	0.0606	0.928	0.1467	0.1537	0.7467	0.2429	0.56	0.1697	0.34	0.58	0.0893
dot_recall@5	0.52	0.6	0.0826	0.946	0.1783	0.1908	0.7867	0.2772	0.71	0.2177	0.42	0.615	0.1288
dot_recall@10	0.76	0.61	0.098	0.97	0.215	0.2936	0.8467	0.3289	0.8	0.3107	0.44	0.74	0.2166
dot_ndcg@10	0.4532	0.4828	0.2467	0.9467	0.1845	0.5071	0.707	0.2881	0.7143	0.3029	0.2782	0.6021	0.3762
dot_mrr@10	0.3601	0.4537	0.422	0.96	0.2674	0.7147	0.6765	0.3562	0.8362	0.4944	0.2247	0.5799	0.5807
dot_map@100	0.3733	0.4454	0.094	0.9291	0.1476	0.3907	0.6651	0.2415	0.6181	0.2167	0.2333	0.5551	0.2561
query_active_dims	74.76	74.74	79.7	76.58	89.86	69.52	89.88	82.86	91.98	94.3	189.1	82.46	79.1225
query_sparsity_ratio	0.9976	0.9976	0.9974	0.9975	0.9971	0.9977	0.9971	0.9973	0.997	0.9969	0.9938	0.9973	0.9974
corpus_active_dims	103.0652	141.3148	202.1727	77.5906	221.7553	135.9335	221.2156	130.937	152.0157	199.6463	164.0333	194.2494	135.0078
corpus_sparsity_ratio	0.9966	0.9954	0.9934	0.9975	0.9927	0.9955	0.9928	0.9957	0.995	0.9935	0.9946	0.9936	0.9956

Sparse Nano BEIR

Dataset: NanoBEIR_mean

Evaluated with SparseNanoBEIREvaluator with these parameters:

{
    "dataset_names": [
        "msmarco",
        "nq",
        "nfcorpus",
        "quoraretrieval"
    ]
}

Metric	Value
dot_accuracy@1	0.47
dot_accuracy@3	0.61
dot_accuracy@5	0.665
dot_accuracy@10	0.735
dot_precision@1	0.47
dot_precision@3	0.255
dot_precision@5	0.184
dot_precision@10	0.1175
dot_recall@1	0.3522
dot_recall@3	0.4821
dot_recall@5	0.5372
dot_recall@10	0.6095
dot_ndcg@10	0.5324
dot_mrr@10	0.5489
dot_map@100	0.4605
query_active_dims	76.445
query_sparsity_ratio	0.9975
corpus_active_dims	122.7978
corpus_sparsity_ratio	0.996

Sparse Nano BEIR

Dataset: NanoBEIR_mean

Evaluated with SparseNanoBEIREvaluator with these parameters:

{
    "dataset_names": [
        "climatefever",
        "dbpedia",
        "fever",
        "fiqa2018",
        "hotpotqa",
        "msmarco",
        "nfcorpus",
        "nq",
        "quoraretrieval",
        "scidocs",
        "arguana",
        "scifact",
        "touche2020"
    ]
}

Metric	Value
dot_accuracy@1	0.436
dot_accuracy@3	0.6089
dot_accuracy@5	0.6659
dot_accuracy@10	0.7384
dot_precision@1	0.436
dot_precision@3	0.2725
dot_precision@5	0.2089
dot_precision@10	0.1461
dot_recall@1	0.2533
dot_recall@3	0.3813
dot_recall@5	0.4364
dot_recall@10	0.51
dot_ndcg@10	0.4685
dot_mrr@10	0.5328
dot_map@100	0.3974
query_active_dims	90.3914
query_sparsity_ratio	0.997
corpus_active_dims	152.3669
corpus_sparsity_ratio	0.995

Training Details

Training Dataset

quora-duplicates

Dataset: quora-duplicates at 451a485
Size: 99,000 training samples
Columns: anchor, positive, and negative

Approximate statistics based on the first 1000 samples:

	anchor	positive	negative
type	string	string	string
details	min: 6 tokens mean: 14.1 tokens max: 39 tokens	min: 6 tokens mean: 13.83 tokens max: 41 tokens	min: 6 tokens mean: 15.21 tokens max: 75 tokens

Samples:

anchor	positive	negative
`What are the best GMAT coaching institutes in Delhi NCR?`	`Which are the best GMAT coaching institutes in Delhi/NCR?`	`What are the best GMAT coaching institutes in Delhi-Noida Area?`
`Is a third world war coming?`	`Is World War 3 more imminent than expected?`	`Since the UN is unable to control terrorism and groups like ISIS, al-Qaeda and countries that promote terrorism (even though it consumed those countries), can we assume that the world is heading towards World War III?`
`Should I build iOS or Android apps first?`	`Should people choose Android or iOS first to build their App?`	`How much more effort is it to build your app on both iOS and Android?`

Loss: SpladeLoss with these parameters:

{
    "loss": "SparseMultipleNegativesRankingLoss(scale=1.0, similarity_fct='dot_score')",
    "lambda_corpus": 3e-05,
    "lambda_query": 5e-05
}

Evaluation Dataset

quora-duplicates

Dataset: quora-duplicates at 451a485
Size: 1,000 evaluation samples
Columns: anchor, positive, and negative

Approximate statistics based on the first 1000 samples:

	anchor	positive	negative
type	string	string	string
details	min: 6 tokens mean: 14.05 tokens max: 40 tokens	min: 6 tokens mean: 14.14 tokens max: 44 tokens	min: 6 tokens mean: 14.56 tokens max: 60 tokens

Samples:

anchor	positive	negative
`What happens if we use petrol in diesel vehicles?`	`Why can't we use petrol in diesel?`	`Why are diesel engines noisier than petrol engines?`
`Why is Saltwater taffy candy imported in Switzerland?`	`Why is Saltwater taffy candy imported in Laos?`	`Is salt a consumer product?`
`Which is your favourite film in 2016?`	`What movie is the best movie of 2016?`	`What will the best movie of 2017 be?`

Loss: SpladeLoss with these parameters:

{
    "loss": "SparseMultipleNegativesRankingLoss(scale=1.0, similarity_fct='dot_score')",
    "lambda_corpus": 3e-05,
    "lambda_query": 5e-05
}

Training Hyperparameters

Non-Default Hyperparameters

eval_strategy: steps
per_device_train_batch_size: 12
per_device_eval_batch_size: 12
learning_rate: 2e-05
num_train_epochs: 1
bf16: True
load_best_model_at_end: True
batch_sampler: no_duplicates

All Hyperparameters

Click to expand

overwrite_output_dir: False
do_predict: False
eval_strategy: steps
prediction_loss_only: True
per_device_train_batch_size: 12
per_device_eval_batch_size: 12
per_gpu_train_batch_size: None
per_gpu_eval_batch_size: None
gradient_accumulation_steps: 1
eval_accumulation_steps: None
torch_empty_cache_steps: None
learning_rate: 2e-05
weight_decay: 0.0
adam_beta1: 0.9
adam_beta2: 0.999
adam_epsilon: 1e-08
max_grad_norm: 1.0
num_train_epochs: 1
max_steps: -1
lr_scheduler_type: linear
lr_scheduler_kwargs: {}
warmup_ratio: 0.0
warmup_steps: 0
log_level: passive
log_level_replica: warning
log_on_each_node: True
logging_nan_inf_filter: True
save_safetensors: True
save_on_each_node: False
save_only_model: False
restore_callback_states_from_checkpoint: False
no_cuda: False
use_cpu: False
use_mps_device: False
seed: 42
data_seed: None
jit_mode_eval: False
use_ipex: False
bf16: True
fp16: False
fp16_opt_level: O1
half_precision_backend: auto
bf16_full_eval: False
fp16_full_eval: False
tf32: None
local_rank: 0
ddp_backend: None
tpu_num_cores: None
tpu_metrics_debug: False
debug: []
dataloader_drop_last: False
dataloader_num_workers: 0
dataloader_prefetch_factor: None
past_index: -1
disable_tqdm: False
remove_unused_columns: True
label_names: None
load_best_model_at_end: True
ignore_data_skip: False
fsdp: []
fsdp_min_num_params: 0
fsdp_config: {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}
tp_size: 0
fsdp_transformer_layer_cls_to_wrap: None
accelerator_config: {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}
deepspeed: None
label_smoothing_factor: 0.0
optim: adamw_torch
optim_args: None
adafactor: False
group_by_length: False
length_column_name: length
ddp_find_unused_parameters: None
ddp_bucket_cap_mb: None
ddp_broadcast_buffers: False
dataloader_pin_memory: True
dataloader_persistent_workers: False
skip_memory_metrics: True
use_legacy_prediction_loop: False
push_to_hub: False
resume_from_checkpoint: None
hub_model_id: None
hub_strategy: every_save
hub_private_repo: None
hub_always_push: False
gradient_checkpointing: False
gradient_checkpointing_kwargs: None
include_inputs_for_metrics: False
include_for_metrics: []
eval_do_concat_batches: True
fp16_backend: auto
push_to_hub_model_id: None
push_to_hub_organization: None
mp_parameters:
auto_find_batch_size: False
full_determinism: False
torchdynamo: None
ray_scope: last
ddp_timeout: 1800
torch_compile: False
torch_compile_backend: None
torch_compile_mode: None
dispatch_batches: None
split_batches: None
include_tokens_per_second: False
include_num_input_tokens_seen: False
neftune_noise_alpha: None
optim_target_modules: None
batch_eval_metrics: False
eval_on_start: False
use_liger_kernel: False
eval_use_gather_object: False
average_tokens_across_devices: False
prompts: None
batch_sampler: no_duplicates
multi_dataset_batch_sampler: proportional

Training Logs

Epoch	Step	Training Loss	Validation Loss	quora_duplicates_dev_max_ap	NanoMSMARCO_dot_ndcg@10	NanoNQ_dot_ndcg@10	NanoNFCorpus_dot_ndcg@10	NanoQuoraRetrieval_dot_ndcg@10	NanoBEIR_mean_dot_ndcg@10	NanoClimateFEVER_dot_ndcg@10	NanoDBPedia_dot_ndcg@10	NanoFEVER_dot_ndcg@10	NanoFiQA2018_dot_ndcg@10	NanoHotpotQA_dot_ndcg@10	NanoSCIDOCS_dot_ndcg@10	NanoArguAna_dot_ndcg@10	NanoSciFact_dot_ndcg@10	NanoTouche2020_dot_ndcg@10
0.0242	200	8.3389	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-
0.0485	400	0.4397	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-
0.0727	600	0.3737	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-
0.0970	800	0.2666	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-
0.1212	1000	0.288	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-
0.1455	1200	0.1977	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-
0.1697	1400	0.2707	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-
0.1939	1600	0.1951	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-
0.2	1650	-	0.1669	0.6472	0.3052	0.2793	0.1711	0.9281	0.4209	-	-	-	-	-	-	-	-	-
0.2182	1800	0.2178	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-
0.2424	2000	0.2174	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-
0.2667	2200	0.1832	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-
0.2909	2400	0.1879	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-
0.3152	2600	0.1723	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-
0.3394	2800	0.1543	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-
0.3636	3000	0.1559	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-
0.3879	3200	0.1575	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-
0.4	3300	-	0.1149	0.6749	0.3894	0.4467	0.2360	0.9292	0.5003	-	-	-	-	-	-	-	-	-
0.4121	3400	0.1395	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-
0.4364	3600	0.1596	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-
0.4606	3800	0.1595	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-
0.4848	4000	0.1211	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-
0.5091	4200	0.1163	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-
0.5333	4400	0.1182	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-
0.5576	4600	0.1337	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-
0.5818	4800	0.1362	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-
0.6	4950	-	0.1001	0.6802	0.4093	0.4269	0.2341	0.9365	0.5017	-	-	-	-	-	-	-	-	-
0.6061	5000	0.1112	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-
0.6303	5200	0.1064	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-
0.6545	5400	0.119	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-
0.6788	5600	0.1077	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-
0.7030	5800	0.1398	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-
0.7273	6000	0.09	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-
0.7515	6200	0.0903	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-
0.7758	6400	0.1082	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-
0.8	6600	0.1122	0.0901	0.6941	0.4451	0.4757	0.2542	0.9411	0.5290	-	-	-	-	-	-	-	-	-
0.8242	6800	0.0708	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-
0.8485	7000	0.1291	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-
0.8727	7200	0.1165	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-
0.8970	7400	0.0735	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-
0.9212	7600	0.0775	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-
0.9455	7800	0.0945	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-
0.9697	8000	0.0912	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-
0.9939	8200	0.104	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-	-
1.0	8250	-	0.0686	0.6888	0.4532	0.4828	0.2467	0.9467	0.5324	-	-	-	-	-	-	-	-	-
-1	-1	-	-	-	0.4532	0.4828	0.2467	0.9467	0.4685	0.1845	0.5071	0.7070	0.2881	0.7143	0.3029	0.2782	0.6021	0.3762

The bold row denotes the saved checkpoint.

Environmental Impact

Carbon emissions were measured using CodeCarbon.

Energy Consumed: 0.025 kWh
Carbon Emitted: 0.001 kg of CO2
Hours Used: 0.222 hours

Training Hardware

On Cloud: No
GPU Model: 1 x NVIDIA GeForce RTX 3070 Ti Laptop GPU
CPU Model: AMD Ryzen 9 6900HX with Radeon Graphics
RAM Size: 30.61 GB

Framework Versions

Python: 3.12.9
Sentence Transformers: 4.2.0.dev0
Transformers: 4.50.3
PyTorch: 2.6.0+cu124
Accelerate: 1.6.0
Datasets: 3.5.0
Tokenizers: 0.21.1

Citation

BibTeX

Sentence Transformers

@inproceedings{reimers-2019-sentence-bert,
    title = "Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks",
    author = "Reimers, Nils and Gurevych, Iryna",
    booktitle = "Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing",
    month = "11",
    year = "2019",
    publisher = "Association for Computational Linguistics",
    url = "https://arxiv.org/abs/1908.10084",
}

SpladeLoss

@misc{formal2022distillationhardnegativesampling,
      title={From Distillation to Hard Negative Sampling: Making Sparse Neural IR Models More Effective},
      author={Thibault Formal and Carlos Lassance and Benjamin Piwowarski and Stéphane Clinchant},
      year={2022},
      eprint={2205.04733},
      archivePrefix={arXiv},
      primaryClass={cs.IR},
      url={https://arxiv.org/abs/2205.04733},
}

SparseMultipleNegativesRankingLoss

@misc{henderson2017efficient,
    title={Efficient Natural Language Response Suggestion for Smart Reply},
    author={Matthew Henderson and Rami Al-Rfou and Brian Strope and Yun-hsuan Sung and Laszlo Lukacs and Ruiqi Guo and Sanjiv Kumar and Balint Miklos and Ray Kurzweil},
    year={2017},
    eprint={1705.00652},
    archivePrefix={arXiv},
    primaryClass={cs.CL}
}

FlopsLoss

@article{paria2020minimizing,
    title={Minimizing flops to learn efficient sparse representations},
    author={Paria, Biswajit and Yeh, Chih-Kuan and Yen, Ian EH and Xu, Ning and Ravikumar, Pradeep and P{'o}czos, Barnab{'a}s},
    journal={arXiv preprint arXiv:2004.05665},
    year={2020}
    }