multi-layer repository/Static Index/VQ_quantizer.cpp
void VQ_quantizer::search_in_group(const float * query, const idx_t group_id, float * result_dists, idx_t * result_labels, size_t k){
        
        assert(group_id < nc_upper);
        //If no enough centroids, return error, need to check before use
        if (k > exact_nc_in_groups[group_id]){
            std::cout << "No enough centroids in group " << group_id << " in VQ layer" << std::endl;
            exit(0);
        }
        if (use_HNSW){
            //The distance result for search kNN is in reverse 
            size_t search_para = k > this->HNSW_quantizers[group_id]->efSearch ? k : this->HNSW_quantizers[group_id]->efSearch;

            auto result_queue = this->HNSW_quantizers[group_id]->searchBaseLayer(query, search_para);

            // The result of search result
            for (size_t j = 0; j < this->max_nc_per_group; j++){
                if (j < search_para){
                    result_dists[search_para - j - 1] = result_queue.top().first;
                    result_labels[search_para - j -1] = CentroidDistributionMap[group_id] + result_queue.top().second;
                    result_queue.pop();           
                }
                else{
                    result_dists[j] = MAX_DIST;
                    result_labels[j] = INVALID_ID;
                }
            }
        }
        else{
//#pragma omp parallel for
            for (size_t j = 0; j < this->max_nc_per_group; j++){
                if (j < exact_nc_in_groups[group_id]){
                    const float * target_centroid = this->L2_quantizers[group_id]->xb.data() + j * this->dimension;
                    result_dists[j] = faiss::fvec_L2sqr(target_centroid, query, dimension);
                    result_labels[j] = CentroidDistributionMap[group_id] + j;
                }
                else{
                    result_dists[j] = MAX_DIST;
                    result_labels[j] = INVALID_ID;
                }
            }
        }
    }
/**
     * 
     * With the shrink mode, we must ensure the k is smaller than the group size
     * 
     * For a query sequence queries, search in their group and return the k cloest centroids
     * 
     * Notice: 
     * nc_per_group should be larger than efSearch and k should be smaller than efSearch
     * Use parallel when n is large, in searching, ignore parallel
     * 
     * Input:
     * Queries: the query data points                                               size: n * dimension
     * group_ids: the group id for relative query: which group should we search in  size: n
     * 
     * Output:
     * result_labels in L2 search: the search result labels in L2 search mode       size: n * max_nc_per_group
     * result_labels in HNSW search: the search result labels in HNSW search mode   size: n * k (n * efSearch?)
     * result_dists in L2 search:                                                   size: n * max_nc_per_group
     * result_dists in HNSW search:                                                 size: n * k (n * efSearch?)
     * The output result for HNSW search and L2 search is different
     **/
With the shrink mode, we must ensure the k is smaller than the group size

For a query sequence queries, search in their group and return the k cloest centroids

nc_per_group should be larger than efSearch and k should be smaller than efSearch
Use parallel when n is large, in searching, ignore parallel

the query data points                                               size: n * dimension
group_ids: the group id for relative query: which group should we search in  size: n


With the shrink mode we must ensure the k is smaller than the group size For a query sequence queries search in their group and return the k cloest centroids nc_per_group should be larger than efSearch and k should be smaller than efSearch Use parallel when n is large in searching ignore parallel the query data points size : n * dimension group_ids : the group id for relative query : which group should we search in size : n
void VQ_quantizer::search_in_group(const float * query, const idx_t group_id, float * result_dists, idx_t * result_labels, size_t k){
        assert(group_id < nc_upper);
        if (k > exact_nc_in_groups[group_id]){
            std::cout << "No enough centroids in group " << group_id << " in VQ layer" << std::endl;
            exit(0);
        }
        if (use_HNSW){
            size_t search_para = k > this->HNSW_quantizers[group_id]->efSearch ? k : this->HNSW_quantizers[group_id]->efSearch;
            auto result_queue = this->HNSW_quantizers[group_id]->searchBaseLayer(query, search_para);
            for (size_t j = 0; j < this->max_nc_per_group; j++){
                if (j < search_para){
                    result_dists[search_para - j - 1] = result_queue.top().first;
                    result_labels[search_para - j -1] = CentroidDistributionMap[group_id] + result_queue.top().second;
                    result_queue.pop();           
                }
                else{
                    result_dists[j] = MAX_DIST;
                    result_labels[j] = INVALID_ID;
                }
            }
        }
        else{
            for (size_t j = 0; j < this->max_nc_per_group; j++){
                if (j < exact_nc_in_groups[group_id]){
                    const float * target_centroid = this->L2_quantizers[group_id]->xb.data() + j * this->dimension;
                    result_dists[j] = faiss::fvec_L2sqr(target_centroid, query, dimension);
                    result_labels[j] = CentroidDistributionMap[group_id] + j;
                }
                else{
                    result_dists[j] = MAX_DIST;
                    result_labels[j] = INVALID_ID;
                }
            }
        }
    }
void VQ_quantizer :: search_in_group ( const float * query , const idx_t group_id , float * result_dists , idx_t * result_labels , size_t k ) { assert ( group_id < nc_upper ) ; if ( k > exact_nc_in_groups [ group_id ] ) { std :: cout << " " << group_id << " " << std :: endl ; exit ( 0 ) ; } if ( use_HNSW ) { size_t search_para = k > this -> HNSW_quantizers [ group_id ] -> efSearch ? k : this -> HNSW_quantizers [ group_id ] -> efSearch ; auto result_queue = this -> HNSW_quantizers [ group_id ] -> searchBaseLayer ( query , search_para ) ; for ( size_t j = 0 ; j < this -> max_nc_per_group ; j ++ ) { if ( j < search_para ) { result_dists [ search_para - j - 1 ] = result_queue . top ( ) . first ; result_labels [ search_para - j - 1 ] = CentroidDistributionMap [ group_id ] + result_queue . top ( ) . second ; result_queue . pop ( ) ; } else { result_dists [ j ] = MAX_DIST ; result_labels [ j ] = INVALID_ID ; } } } else { for ( size_t j = 0 ; j < this -> max_nc_per_group ; j ++ ) { if ( j < exact_nc_in_groups [ group_id ] ) { const float * target_centroid = this -> L2_quantizers [ group_id ] -> xb . data ( ) + j * this -> dimension ; result_dists [ j ] = faiss :: fvec_L2sqr ( target_centroid , query , dimension ) ; result_labels [ j ] = CentroidDistributionMap [ group_id ] + j ; } else { result_dists [ j ] = MAX_DIST ; result_labels [ j ] = INVALID_ID ; } } } }
With the shrink mode, we must ensure the k is smaller than the group size
With the shrink mode we must ensure the k is smaller than the group size
//If no enough centroids, return error, need to check before use //The distance result for search kNN is in reverse  // The result of search result //#pragma omp parallel for