colbert-muvera-small / README.md

davidmezzetti

Update README.md

9ede8c4 verified 28 days ago

preview code

raw

history blame contribute delete

31.6 kB

metadata

tags:
  - ColBERT
  - PyLate
  - sentence-transformers
  - sentence-similarity
  - feature-extraction
  - generated_from_trainer
  - dataset_size:640000
  - loss:Distillation
base_model: bclavie/mini-base
datasets:
  - lightonai/ms-marco-en-bge-gemma-unnormalized
pipeline_tag: sentence-similarity
library_name: PyLate
license: apache-2.0
metrics:
  - MaxSim_accuracy@1
  - MaxSim_accuracy@3
  - MaxSim_accuracy@5
  - MaxSim_accuracy@10
  - MaxSim_precision@1
  - MaxSim_precision@3
  - MaxSim_precision@5
  - MaxSim_precision@10
  - MaxSim_recall@1
  - MaxSim_recall@3
  - MaxSim_recall@5
  - MaxSim_recall@10
  - MaxSim_ndcg@10
  - MaxSim_mrr@10
  - MaxSim_map@100
model-index:
  - name: ColBERT MUVERA Small
    results:
      - task:
          type: py-late-information-retrieval
          name: Py Late Information Retrieval
        dataset:
          name: NanoClimateFEVER
          type: NanoClimateFEVER
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.28
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.38
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 0.52
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 0.64
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.28
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.14666666666666667
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.12
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.08
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.13166666666666665
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.21
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.265
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.335
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.27666051264859415
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.3671349206349206
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.22158617300046946
            name: Maxsim Map@100
      - task:
          type: py-late-information-retrieval
          name: Py Late Information Retrieval
        dataset:
          name: NanoDBPedia
          type: NanoDBPedia
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.8
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.88
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 0.92
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 0.96
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.8
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.6333333333333332
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.556
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.48399999999999993
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.10583280294731091
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.1747980000610803
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.2211728749541224
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.3392671917074792
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.6189072509940752
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.8510238095238097
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.47586135688175013
            name: Maxsim Map@100
      - task:
          type: py-late-information-retrieval
          name: Py Late Information Retrieval
        dataset:
          name: NanoFEVER
          type: NanoFEVER
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.88
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.96
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 1
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 1
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.88
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.33333333333333326
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.21199999999999997
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.10799999999999997
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.8166666666666668
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.9133333333333333
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.9633333333333333
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.9733333333333333
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.9208334669406996
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.929
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.8912380952380953
            name: Maxsim Map@100
      - task:
          type: py-late-information-retrieval
          name: Py Late Information Retrieval
        dataset:
          name: NanoFiQA2018
          type: NanoFiQA2018
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.44
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.56
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 0.68
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 0.76
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.44
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.26666666666666666
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.22
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.13599999999999998
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.22257936507936507
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.36418253968253966
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.5042063492063492
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.5963968253968254
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.4781894440800092
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.5321666666666666
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.39543817074336585
            name: Maxsim Map@100
      - task:
          type: py-late-information-retrieval
          name: Py Late Information Retrieval
        dataset:
          name: NanoHotpotQA
          type: NanoHotpotQA
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.84
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.96
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 0.98
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 0.98
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.84
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.5066666666666666
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.324
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.17399999999999996
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.42
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.76
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.81
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.87
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.813477163259318
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.8973333333333333
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.7469519155158202
            name: Maxsim Map@100
      - task:
          type: py-late-information-retrieval
          name: Py Late Information Retrieval
        dataset:
          name: NanoMSMARCO
          type: NanoMSMARCO
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.48
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.64
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 0.68
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 0.8
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.48
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.21333333333333332
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.136
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.08
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.48
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.64
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.68
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.8
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.6277729303272284
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.574547619047619
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.585980942367483
            name: Maxsim Map@100
      - task:
          type: py-late-information-retrieval
          name: Py Late Information Retrieval
        dataset:
          name: NanoNFCorpus
          type: NanoNFCorpus
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.5
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.62
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 0.64
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 0.74
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.5
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.4
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.3440000000000001
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.292
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.06602691624937523
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.09818050757008642
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.11806464030634821
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.15514192209178235
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.37561452677051027
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.5691666666666667
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.18300178358234423
            name: Maxsim Map@100
      - task:
          type: py-late-information-retrieval
          name: Py Late Information Retrieval
        dataset:
          name: NanoNQ
          type: NanoNQ
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.56
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.76
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 0.78
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 0.82
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.56
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.2533333333333333
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.16399999999999998
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.088
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.54
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.71
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.75
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.79
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.6818400710905007
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.6597222222222223
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.6459882013890279
            name: Maxsim Map@100
      - task:
          type: py-late-information-retrieval
          name: Py Late Information Retrieval
        dataset:
          name: NanoQuoraRetrieval
          type: NanoQuoraRetrieval
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.86
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.98
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 0.98
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 1
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.86
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.40666666666666657
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.264
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.13799999999999998
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.764
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.9453333333333334
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.97
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.9966666666666666
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.9414269581610836
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.9228571428571428
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.9205543345543344
            name: Maxsim Map@100
      - task:
          type: py-late-information-retrieval
          name: Py Late Information Retrieval
        dataset:
          name: NanoSCIDOCS
          type: NanoSCIDOCS
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.5
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.74
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 0.76
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 0.82
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.5
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.3933333333333333
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.292
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.18599999999999997
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.10466666666666667
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.24366666666666664
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.29966666666666664
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.3826666666666666
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.39084995006976664
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.6197222222222222
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.3153590016638529
            name: Maxsim Map@100
      - task:
          type: py-late-information-retrieval
          name: Py Late Information Retrieval
        dataset:
          name: NanoArguAna
          type: NanoArguAna
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.28
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.56
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 0.64
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 0.84
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.28
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.18666666666666668
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.12800000000000003
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.08399999999999999
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.28
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.56
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.64
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.84
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.5432952971404568
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.450484126984127
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.4551681906230779
            name: Maxsim Map@100
      - task:
          type: py-late-information-retrieval
          name: Py Late Information Retrieval
        dataset:
          name: NanoSciFact
          type: NanoSciFact
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.7
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.82
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 0.84
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 0.88
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.7
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.29333333333333333
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.18799999999999997
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.09799999999999998
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.665
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.81
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.84
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.87
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.7883940477308562
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.7645238095238096
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.7622104923007755
            name: Maxsim Map@100
      - task:
          type: py-late-information-retrieval
          name: Py Late Information Retrieval
        dataset:
          name: NanoTouche2020
          type: NanoTouche2020
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.673469387755102
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.9183673469387755
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 0.9591836734693877
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 1
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.673469387755102
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.6326530612244898
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.6040816326530614
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.5020408163265305
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.04919462393895531
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.13143050077268048
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.20505385244507174
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.3259510245836729
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.5631037374817277
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.7906462585034014
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.41297955687388305
            name: Maxsim Map@100
      - task:
          type: nano-beir
          name: Nano BEIR
        dataset:
          name: NanoBEIR mean
          type: NanoBEIR_mean
        metrics:
          - type: MaxSim_accuracy@1
            value: 0.5994976452119309
            name: Maxsim Accuracy@1
          - type: MaxSim_accuracy@3
            value: 0.7521821036106752
            name: Maxsim Accuracy@3
          - type: MaxSim_accuracy@5
            value: 0.7983987441130298
            name: Maxsim Accuracy@5
          - type: MaxSim_accuracy@10
            value: 0.8646153846153847
            name: Maxsim Accuracy@10
          - type: MaxSim_precision@1
            value: 0.5994976452119309
            name: Maxsim Precision@1
          - type: MaxSim_precision@3
            value: 0.3589220303506017
            name: Maxsim Precision@3
          - type: MaxSim_precision@5
            value: 0.2732370486656201
            name: Maxsim Precision@5
          - type: MaxSim_precision@10
            value: 0.18846467817896384
            name: Maxsim Precision@10
          - type: MaxSim_recall@1
            value: 0.35735643909346215
            name: Maxsim Recall@1
          - type: MaxSim_recall@3
            value: 0.5046865293399786
            name: Maxsim Recall@3
          - type: MaxSim_recall@5
            value: 0.5589613628393763
            name: Maxsim Recall@5
          - type: MaxSim_recall@10
            value: 0.6364941254189559
            name: Maxsim Recall@10
          - type: MaxSim_ndcg@10
            value: 0.6169511812842173
            name: Maxsim Ndcg@10
          - type: MaxSim_mrr@10
            value: 0.6867945229373801
            name: Maxsim Mrr@10
          - type: MaxSim_map@100
            value: 0.5394090934410984
            name: Maxsim Map@100

ColBERT MUVERA Small

This is a PyLate model finetuned from bclavie/mini-base on the msmarco-en-bge-gemma-unnormalized dataset. It maps sentences & paragraphs to sequences of 128-dimensional dense vectors and can be used for semantic textual similarity using the MaxSim operator.

This model is trained with un-normalized scores, making it compatible with MUVERA fixed-dimensional encoding.

Usage (txtai)

This model can be used to build embeddings databases with txtai for semantic search and/or as a knowledge source for retrieval augmented generation (RAG).

Note: txtai 9.0+ is required for late interaction model support

import txtai

embeddings = txtai.Embeddings(
  path="neuml/colbert-muvera-small",
  content=True
)
embeddings.index(documents())

# Run a query
embeddings.search("query to run")

Late interaction models excel as reranker pipelines.

from txtai.pipeline import Reranker, Similarity

similarity = Similarity(path="neuml/colbert-muvera-small", lateencode=True)
ranker = Reranker(embeddings, similarity)
ranker("query to run")

Usage (PyLate)

Alternatively, the model can be loaded with PyLate.

from pylate import rank, models

queries = [
    "query A",
    "query B",
]

documents = [
    ["document A", "document B"],
    ["document 1", "document C", "document B"],
]

documents_ids = [
    [1, 2],
    [1, 3, 2],
]

model = models.ColBERT(
    model_name_or_path="neuml/colbert-muvera-small",
)

queries_embeddings = model.encode(
    queries,
    is_query=True,
)

documents_embeddings = model.encode(
    documents,
    is_query=False,
)

reranked_documents = rank.rerank(
    documents_ids=documents_ids,
    queries_embeddings=queries_embeddings,
    documents_embeddings=documents_embeddings,
)

Full Model Architecture

ColBERT(
  (0): Transformer({'max_seq_length': 299, 'do_lower_case': False}) with Transformer model: BertModel 
  (1): Dense({'in_features': 384, 'out_features': 128, 'bias': False, 'activation_function': 'torch.nn.modules.linear.Identity'})
)

Evaluation

BEIR Subset

The following table shows a subset of BEIR scored with the txtai benchmarks script.

Scores reported are ndcg@10 and grouped into the following three categories.

FULL multi-vector maxsim

Model	Parameters	ArguAna	NFCorpus	SciFact	Average
AnswerAI ColBERT Small v1	33M	0.4440	0.3649	0.7423	0.5171
ColBERT v2	110M	0.4595	0.3165	0.6456	0.4739
ColBERT MUVERA Micro	4M	0.3947	0.3235	0.6676	0.4619
ColBERT MUVERA Small	33M	0.4455	0.3502	0.7145	0.5034
GTE ModernColBERT v1	149M	0.4946	0.3717	0.7529	0.5397

MUVERA encoding + maxsim re-ranking of the top 100 results per MUVERA paper

Model	Parameters	ArguAna	NFCorpus	SciFact	Average
AnswerAI ColBERT Small v1	33M	0.0317	0.1135	0.0836	0.0763
ColBERT v2	110M	0.4562	0.3025	0.6278	0.4622
ColBERT MUVERA Micro	4M	0.3849	0.3095	0.6464	0.4469
ColBERT MUVERA Small	33M	0.4451	0.3537	0.7148	0.5045
GTE ModernColBERT v1	149M	0.0265	0.1052	0.0556	0.0624

MUVERA encoding only

Model	Parameters	ArguAna	NFCorpus	SciFact	Average
AnswerAI ColBERT Small v1	33M	0.0024	0.0201	0.0047	0.0091
ColBERT v2	110M	0.3463	0.2356	0.5002	0.3607
ColBERT MUVERA Micro	4M	0.2795	0.2348	0.4875	0.3339
ColBERT MUVERA Small	33M	0.3850	0.2928	0.6357	0.4378
GTE ModernColBERT v1	149M	0.0003	0.0203	0.0013	0.0073

Note: The scores reported don't match scores reported in the respective papers due to different default settings in the txtai benchmark scripts.

As noted earlier, models trained with min-max score normalization don't perform well with MUVERA encoding. See this GitHub Issue for more.

In reviewing the scores, this model is surprisingly and unreasonably competitive with the original ColBERT v2 model at only 3% of the size!

Nano BEIR

Dataset: NanoBEIR_mean
Evaluated with pylate.evaluation.nano_beir_evaluator.NanoBEIREvaluator

Metric	Value
MaxSim_accuracy@1	0.5995
MaxSim_accuracy@3	0.7522
MaxSim_accuracy@5	0.7984
MaxSim_accuracy@10	0.8646
MaxSim_precision@1	0.5995
MaxSim_precision@3	0.3589
MaxSim_precision@5	0.2732
MaxSim_precision@10	0.1885
MaxSim_recall@1	0.3574
MaxSim_recall@3	0.5047
MaxSim_recall@5	0.559
MaxSim_recall@10	0.6365
MaxSim_ndcg@10	0.617
MaxSim_mrr@10	0.6868
MaxSim_map@100	0.5394

Training Details

Training Hyperparameters

Non-Default Hyperparameters

eval_strategy: steps
gradient_accumulation_steps: 4
learning_rate: 3e-06
num_train_epochs: 1
warmup_ratio: 0.05
bf16: True

All Hyperparameters

Click to expand

overwrite_output_dir: False
do_predict: False
eval_strategy: steps
prediction_loss_only: True
per_device_train_batch_size: 8
per_device_eval_batch_size: 8
per_gpu_train_batch_size: None
per_gpu_eval_batch_size: None
gradient_accumulation_steps: 4
eval_accumulation_steps: None
torch_empty_cache_steps: None
learning_rate: 3e-06
weight_decay: 0.0
adam_beta1: 0.9
adam_beta2: 0.999
adam_epsilon: 1e-08
max_grad_norm: 1.0
num_train_epochs: 1
max_steps: -1
lr_scheduler_type: linear
lr_scheduler_kwargs: {}
warmup_ratio: 0.05
warmup_steps: 0
log_level: passive
log_level_replica: warning
log_on_each_node: True
logging_nan_inf_filter: True
save_safetensors: True
save_on_each_node: False
save_only_model: False
restore_callback_states_from_checkpoint: False
no_cuda: False
use_cpu: False
use_mps_device: False
seed: 42
data_seed: None
jit_mode_eval: False
use_ipex: False
bf16: True
fp16: False
fp16_opt_level: O1
half_precision_backend: auto
bf16_full_eval: False
fp16_full_eval: False
tf32: None
local_rank: 0
ddp_backend: None
tpu_num_cores: None
tpu_metrics_debug: False
debug: []
dataloader_drop_last: False
dataloader_num_workers: 0
dataloader_prefetch_factor: None
past_index: -1
disable_tqdm: False
remove_unused_columns: True
label_names: None
load_best_model_at_end: False
ignore_data_skip: False
fsdp: []
fsdp_min_num_params: 0
fsdp_config: {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}
fsdp_transformer_layer_cls_to_wrap: None
accelerator_config: {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}
deepspeed: None
label_smoothing_factor: 0.0
optim: adamw_torch
optim_args: None
adafactor: False
group_by_length: False
length_column_name: length
ddp_find_unused_parameters: None
ddp_bucket_cap_mb: None
ddp_broadcast_buffers: False
dataloader_pin_memory: True
dataloader_persistent_workers: False
skip_memory_metrics: True
use_legacy_prediction_loop: False
push_to_hub: False
resume_from_checkpoint: None
hub_model_id: None
hub_strategy: every_save
hub_private_repo: None
hub_always_push: False
gradient_checkpointing: False
gradient_checkpointing_kwargs: None
include_inputs_for_metrics: False
include_for_metrics: []
eval_do_concat_batches: True
fp16_backend: auto
push_to_hub_model_id: None
push_to_hub_organization: None
mp_parameters:
auto_find_batch_size: False
full_determinism: False
torchdynamo: None
ray_scope: last
ddp_timeout: 1800
torch_compile: False
torch_compile_backend: None
torch_compile_mode: None
include_tokens_per_second: False
include_num_input_tokens_seen: False
neftune_noise_alpha: None
optim_target_modules: None
batch_eval_metrics: False
eval_on_start: False
use_liger_kernel: False
eval_use_gather_object: False
average_tokens_across_devices: False
prompts: None
batch_sampler: batch_sampler
multi_dataset_batch_sampler: proportional

Framework Versions

Python: 3.10.18
Sentence Transformers: 4.0.2
PyLate: 1.3.0
Transformers: 4.52.3
PyTorch: 2.8.0+cu128
Accelerate: 1.10.1
Datasets: 4.0.0
Tokenizers: 0.21.4

Citation

BibTeX

Sentence Transformers

@inproceedings{reimers-2019-sentence-bert,
    title = "Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks",
    author = "Reimers, Nils and Gurevych, Iryna",
    booktitle = "Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing",
    month = "11",
    year = "2019",
    publisher = "Association for Computational Linguistics",
    url = "https://arxiv.org/abs/1908.10084"
}

PyLate

@misc{PyLate,
title={PyLate: Flexible Training and Retrieval for Late Interaction Models},
author={Chaffin, Antoine and Sourty, Raphaël},
url={https://github.com/lightonai/pylate},
year={2024}
}