llm/Lib/site-packages/faiss/contrib/factory_tools.py

# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

import faiss
import re


def get_code_size(d, indexkey):
    """ size of one vector in an index in dimension d
    constructed with factory string indexkey"""

    if indexkey == "Flat":
        return d * 4

    if indexkey.endswith(",RFlat"):
        return d * 4 + get_code_size(d, indexkey[:-len(",RFlat")])

    mo = re.match("IVF\\d+(_HNSW32)?,(.*)$", indexkey)
    if mo:
        return get_code_size(d, mo.group(2))

    mo = re.match("IVF\\d+\\(.*\\)?,(.*)$", indexkey)
    if mo:
        return get_code_size(d, mo.group(1))

    mo = re.match("IMI\\d+x2,(.*)$", indexkey)
    if mo:
        return get_code_size(d, mo.group(1))

    mo = re.match("(.*),Refine\\((.*)\\)$", indexkey)
    if mo:
        return get_code_size(d, mo.group(1)) + get_code_size(d, mo.group(2))

    mo = re.match('PQ(\\d+)x(\\d+)(fs|fsr)?$', indexkey)
    if mo:
        return (int(mo.group(1)) * int(mo.group(2)) + 7) // 8

    mo = re.match('PQ(\\d+)\\+(\\d+)$', indexkey)
    if mo:
        return (int(mo.group(1)) + int(mo.group(2)))

    mo = re.match('PQ(\\d+)$', indexkey)
    if mo:
        return int(mo.group(1))

    if indexkey == "HNSW32" or indexkey == "HNSW32,Flat":
        return d * 4 + 64 * 4 # roughly

    if indexkey == 'SQ8':
        return d
    elif indexkey == 'SQ4':
        return (d + 1) // 2
    elif indexkey == 'SQ6':
        return (d * 6 + 7) // 8
    elif indexkey == 'SQfp16':
        return d * 2

    mo = re.match('PCAR?(\\d+),(.*)$', indexkey)
    if mo:
        return get_code_size(int(mo.group(1)), mo.group(2))
    mo = re.match('OPQ\\d+_(\\d+),(.*)$', indexkey)
    if mo:
        return get_code_size(int(mo.group(1)), mo.group(2))
    mo = re.match('OPQ\\d+,(.*)$', indexkey)
    if mo:
        return get_code_size(d, mo.group(1))
    mo = re.match('RR(\\d+),(.*)$', indexkey)
    if mo:
        return get_code_size(int(mo.group(1)), mo.group(2))
    raise RuntimeError("cannot parse " + indexkey)


def get_hnsw_M(index):
    return index.hnsw.cum_nneighbor_per_level.at(1) // 2


def reverse_index_factory(index):
    """
    attempts to get the factory string the index was built with
    """
    index = faiss.downcast_index(index)
    if isinstance(index, faiss.IndexFlat):
        return "Flat"
    elif isinstance(index, faiss.IndexIVF):
        quantizer = faiss.downcast_index(index.quantizer)

        if isinstance(quantizer, faiss.IndexFlat):
            prefix = f"IVF{index.nlist}"
        elif isinstance(quantizer, faiss.MultiIndexQuantizer):
            prefix = f"IMI{quantizer.pq.M}x{quantizer.pq.nbits}"
        elif isinstance(quantizer, faiss.IndexHNSW):
            prefix = f"IVF{index.nlist}_HNSW{get_hnsw_M(quantizer)}"
        else:
            prefix = f"IVF{index.nlist}({reverse_index_factory(quantizer)})"

        if isinstance(index, faiss.IndexIVFFlat):
            return prefix + ",Flat"
        if isinstance(index, faiss.IndexIVFScalarQuantizer):
            return prefix + ",SQ8"
        if isinstance(index, faiss.IndexIVFPQ):
            return prefix + f",PQ{index.pq.M}x{index.pq.nbits}"
        if isinstance(index, faiss.IndexIVFPQFastScan):
            return prefix + f",PQ{index.pq.M}x{index.pq.nbits}fs"

    elif isinstance(index, faiss.IndexPreTransform):
        if index.chain.size() != 1:
            raise NotImplementedError()
        vt = faiss.downcast_VectorTransform(index.chain.at(0))
        if isinstance(vt, faiss.OPQMatrix):
            prefix = f"OPQ{vt.M}_{vt.d_out}"
        elif isinstance(vt, faiss.ITQTransform):
            prefix = f"ITQ{vt.itq.d_out}"
        elif isinstance(vt, faiss.PCAMatrix):
            assert vt.eigen_power == 0
            prefix = "PCA" + ("R" if vt.random_rotation else "") + str(vt.d_out)
        else:
            raise NotImplementedError()
        return f"{prefix},{reverse_index_factory(index.index)}"

    elif isinstance(index, faiss.IndexHNSW):
        return f"HNSW{get_hnsw_M(index)}"

    elif isinstance(index, faiss.IndexRefine):
        return f"{reverse_index_factory(index.base_index)},Refine({reverse_index_factory(index.refine_index)})"

    elif isinstance(index, faiss.IndexPQFastScan):
        return f"PQ{index.pq.M}x{index.pq.nbits}fs"

    elif isinstance(index, faiss.IndexPQ):
        return f"PQ{index.pq.M}x{index.pq.nbits}"

    elif isinstance(index, faiss.IndexLSH):
        return "LSH" + ("r" if index.rotate_data else "") + ("t" if index.train_thresholds else "")

    elif isinstance(index, faiss.IndexScalarQuantizer):
        sqtypes = {
            faiss.ScalarQuantizer.QT_8bit: "8",
            faiss.ScalarQuantizer.QT_4bit: "4",
            faiss.ScalarQuantizer.QT_6bit: "6",
            faiss.ScalarQuantizer.QT_fp16: "fp16",
        }
        return f"SQ{sqtypes[index.sq.qtype]}"

    raise NotImplementedError()