scripts/example.py
def main():
    """
    A brief demo script showing how to train various LOPQ models with brief
    discussion of trade offs.
    """

    # Get the oxford dataset
    data = load_oxford_data()

    # Compute PCA of oxford dataset. See README in data/oxford for details
    # about this dataset.
    P, mu = pca(data)

    # Mean center and rotate the data; includes dimension permutation.
    # It is worthwhile see how this affects recall performance. On this
    # dataset, which is already PCA'd from higher dimensional features,
    # this additional step to variance balance the dimensions typically
    # improves recall@1 by 3-5%. The benefit can be much greater depending
    # on the dataset.
    data = data - mu
    data = np.dot(data, P)

    # Create a train and test split. The test split will become
    # a set of queries for which we will compute the true nearest neighbors.
    train, test = train_test_split(data, test_size=0.2)

    # Compute distance-sorted neighbors in training set for each point in test set.
    # These will be our groundtruth for recall evaluation.
    nns = compute_all_neighbors(test, train)

    # Fit model
    m = LOPQModel(V=16, M=8)
    m.fit(train, n_init=1)

    # Note that we didn't specify a random seed for fitting the model, so different
    # runs will be different. You may also see a warning that some local projections
    # can't be estimated because too few points fall in a cluster. This is ok for the
    # purposes of this demo, but you might want to avoid this by increasing the amount
    # of training data or decreasing the number of clusters (the V hyperparameter).

    # With a model in hand, we can test it's recall. We populate a LOPQSearcher
    # instance with data and get recall stats. By default, we will retrieve 1000
    # ranked results for each query vector for recall evaluation.
    searcher = LOPQSearcher(m)
    searcher.add_data(train)
    recall, _ = get_recall(searcher, test, nns)
    print 'Recall (V=%d, M=%d, subquants=%d): %s' % (m.V, m.M, m.subquantizer_clusters, str(recall))

    # We can experiment with other hyperparameters without discarding all
    # parameters everytime. Here we train a new model that uses the same coarse
    # quantizers but a higher number of subquantizers, i.e. we increase M.
    m2 = LOPQModel(V=16, M=16, parameters=(m.Cs, None, None, None))
    m2.fit(train, n_init=1)

    # Let's evaluate again.
    searcher = LOPQSearcher(m2)
    searcher.add_data(train)
    recall, _ = get_recall(searcher, test, nns)
    print 'Recall (V=%d, M=%d, subquants=%d): %s' % (m2.V, m2.M, m2.subquantizer_clusters, str(recall))

    # The recall is probably higher. We got better recall with a finer quantization
    # at the expense of more data required for index items.

    # We can also hold both coarse quantizers and rotations fixed and see what
    # increasing the number of subquantizer clusters does to performance.
    m3 = LOPQModel(V=16, M=8, subquantizer_clusters=512, parameters=(m.Cs, m.Rs, m.mus, None))
    m3.fit(train, n_init=1)

    searcher = LOPQSearcher(m3)
    searcher.add_data(train)
    recall, _ = get_recall(searcher, test, nns)
    print 'Recall (V=%d, M=%d, subquants=%d): %s' % (m3.V, m3.M, m3.subquantizer_clusters, str(recall))

    # The recall is probably better than the first but worse than the second. We increased recall
    # only a little by increasing the number of model parameters (double the subquatizer centroids),
    # the index storage requirement (another bit for each fine code), and distance computation time
    # (double the subquantizer centroids).

    A brief demo script showing how to train various LOPQ models with brief
    discussion of trade offs.
    
A brief demo script showing how to train various LOPQ models with brief
discussion of trade offs.
A brief demo script showing how to train various LOPQ models with brief discussion of trade offs .
def main():
    data = load_oxford_data()
    P, mu = pca(data)
    data = data - mu
    data = np.dot(data, P)
    train, test = train_test_split(data, test_size=0.2)
    nns = compute_all_neighbors(test, train)
    m = LOPQModel(V=16, M=8)
    m.fit(train, n_init=1)
    searcher = LOPQSearcher(m)
    searcher.add_data(train)
    recall, _ = get_recall(searcher, test, nns)
    print 'Recall (V=%d, M=%d, subquants=%d): %s' % (m.V, m.M, m.subquantizer_clusters, str(recall))
    m2 = LOPQModel(V=16, M=16, parameters=(m.Cs, None, None, None))
    m2.fit(train, n_init=1)
    searcher = LOPQSearcher(m2)
    searcher.add_data(train)
    recall, _ = get_recall(searcher, test, nns)
    print 'Recall (V=%d, M=%d, subquants=%d): %s' % (m2.V, m2.M, m2.subquantizer_clusters, str(recall))
    m3 = LOPQModel(V=16, M=8, subquantizer_clusters=512, parameters=(m.Cs, m.Rs, m.mus, None))
    m3.fit(train, n_init=1)
    searcher = LOPQSearcher(m3)
    searcher.add_data(train)
    recall, _ = get_recall(searcher, test, nns)
    print 'Recall (V=%d, M=%d, subquants=%d): %s' % (m3.V, m3.M, m3.subquantizer_clusters, str(recall))
def main ( ) : data = load_oxford_data ( ) P , mu = pca ( data ) data = data - mu data = np . dot ( data , P ) train , test = train_test_split ( data , test_size = 0.2 ) nns = compute_all_neighbors ( test , train ) m = LOPQModel ( V = 16 , M = 8 ) m . fit ( train , n_init = 1 ) searcher = LOPQSearcher ( m ) searcher . add_data ( train ) recall , _ = get_recall ( searcher , test , nns ) print 'Recall (V=%d, M=%d, subquants=%d): %s' % ( m . V , m . M , m . subquantizer_clusters , str ( recall ) ) m2 = LOPQModel ( V = 16 , M = 16 , parameters = ( m . Cs , None , None , None ) ) m2 . fit ( train , n_init = 1 ) searcher = LOPQSearcher ( m2 ) searcher . add_data ( train ) recall , _ = get_recall ( searcher , test , nns ) print 'Recall (V=%d, M=%d, subquants=%d): %s' % ( m2 . V , m2 . M , m2 . subquantizer_clusters , str ( recall ) ) m3 = LOPQModel ( V = 16 , M = 8 , subquantizer_clusters = 512 , parameters = ( m . Cs , m . Rs , m . mus , None ) ) m3 . fit ( train , n_init = 1 ) searcher = LOPQSearcher ( m3 ) searcher . add_data ( train ) recall , _ = get_recall ( searcher , test , nns ) print 'Recall (V=%d, M=%d, subquants=%d): %s' % ( m3 . V , m3 . M , m3 . subquantizer_clusters , str ( recall ) )
A brief demo script showing how to train various LOPQ models with brief
discussion of trade offs.
A brief demo script showing how to train various LOPQ models with brief discussion of trade offs .
"""
    A brief demo script showing how to train various LOPQ models with brief
    discussion of trade offs.
    """ # Get the oxford dataset # Compute PCA of oxford dataset. See README in data/oxford for details # about this dataset. # Mean center and rotate the data; includes dimension permutation. # It is worthwhile see how this affects recall performance. On this # dataset, which is already PCA'd from higher dimensional features, # this additional step to variance balance the dimensions typically # improves recall@1 by 3-5%. The benefit can be much greater depending # on the dataset. # Create a train and test split. The test split will become # a set of queries for which we will compute the true nearest neighbors. # Compute distance-sorted neighbors in training set for each point in test set. # These will be our groundtruth for recall evaluation. # Fit model # Note that we didn't specify a random seed for fitting the model, so different # runs will be different. You may also see a warning that some local projections # can't be estimated because too few points fall in a cluster. This is ok for the # purposes of this demo, but you might want to avoid this by increasing the amount # of training data or decreasing the number of clusters (the V hyperparameter). # With a model in hand, we can test it's recall. We populate a LOPQSearcher # instance with data and get recall stats. By default, we will retrieve 1000 # ranked results for each query vector for recall evaluation. # We can experiment with other hyperparameters without discarding all # parameters everytime. Here we train a new model that uses the same coarse # quantizers but a higher number of subquantizers, i.e. we increase M. # Let's evaluate again. # The recall is probably higher. We got better recall with a finer quantization # at the expense of more data required for index items. # We can also hold both coarse quantizers and rotations fixed and see what # increasing the number of subquantizer clusters does to performance. # The recall is probably better than the first but worse than the second. We increased recall # only a little by increasing the number of model parameters (double the subquatizer centroids), # the index storage requirement (another bit for each fine code), and distance computation time # (double the subquantizer centroids).