BatchCollection.py
def compareBatchBlock(args):
    """
    Process a rectangular block of the all pairs score matrix
    between all of the songs.  Return score matrices for each
    individual type of feature, in addition to one for early
    similarity network fusion
    :param idxs: [start1, end1, start2, end2] range of rectangular
        block of songs to compare
    :param Kappa: Percent nearest neighbors to use both for
        binary cross-similarity and similarity network fusion
    :param CSMTypes: Dictionary of types of features and
        associated cross-similarity comparisons to do
    :param allFiles: List of all files that are being compared
        from which this block is drawn
    :param scratchDir: Path to directory for storing block results
    """
    (idxs, Kappa, CSMTypes, allFiles, scratchDir) = args
    DsFilename = "%s/D%i_%i_%i_%i.mat"%(scratchDir, idxs[0], idxs[1], idxs[2], idxs[3])
    if os.path.exists(DsFilename):
        return sio.loadmat(DsFilename)
    #Figure out block size thisM x thisN
    thisM = idxs[1] - idxs[0]
    thisN = idxs[3] - idxs[2]
    D = np.zeros((thisM, thisN))

    AllFeatures = {}
    tic = time.time()
    allidxs = [i + idxs[0] for i in range(thisM)]
    allidxs += [j + idxs[2] for j in range(thisN)]
    allidxs = np.unique(np.array(allidxs))
    #Preload features and Ws for SSM parts
    ticfeatures = time.time()
    count = 1
    for idx in allidxs:
        filename = getMatFilename(scratchDir, allFiles[idx])
        AllFeatures[idx] = sio.loadmat(filename)
        for key, val in AllFeatures[idx].items():
            if type(val) is np.ndarray:
                if val.size == 1:
                    AllFeatures[idx][key] = val.flatten()[0]
    tocfeatures = time.time()
    print("Elapsed Time Loading Features: ", tocfeatures-ticfeatures)
    stdout.flush()

    K = 20
    NIters = 3
    Ds = {'SNF':np.zeros((thisM, thisN))}
    for Feature in CSMTypes.keys():
        Ds[Feature] = np.zeros((thisM, thisN))
    for i in range(thisM):
        print("i = %i"%i)
        stdout.flush()
        thisi = i + idxs[0]
        Features1 = AllFeatures[thisi]
        for j in range(thisN):
            thisj = j + idxs[2]
            if thisj < thisi:
                #Only compute upper triangular part since it's symmetric
                continue
            Features2 = AllFeatures[thisj]
            #Compare all tempo levels
            for a in range(Features1['NTempos']):
                O1 = {'ChromaMean':Features1['ChromaMean%i'%a].flatten()}
                for b in range(Features2['NTempos']):
                    O2 = {'ChromaMean':Features2['ChromaMean%i'%b].flatten()}
                    Ws = []
                    OtherCSMs = {}
                    #Compute all W matrices
                    (M, N) = (0, 0)
                    for F in CSMTypes.keys():
                        CSMAB = getCSMType(Features1['%s%i'%(F, a)], O1, Features2['%s%i'%(F, b)], O2, CSMTypes[F])
                        OtherCSMs[F] = CSMAB
                        (M, N) = (CSMAB.shape[0], CSMAB.shape[1])
                        k1 = int(0.5*Kappa*M)
                        k2 = int(0.5*Kappa*N)
                        WCSMAB = getWCSM(CSMAB, k1, k2)
                        WSSMA = Features1['W%s%i'%(F, a)]
                        WSSMB = Features2['W%s%i'%(F, b)]
                        Ws.append(setupWCSMSSM(WSSMA, WSSMB, WCSMAB))
                    #Do Similarity Fusion
                    D = doSimilarityFusionWs(Ws, K, NIters, 1)
                    #Extract CSM Part
                    CSM = D[0:M, M::] + D[M::, 0:M].T
                    DBinary = CSMToBinaryMutual(np.exp(-CSM), Kappa)
                    score = SAC.swalignimpconstrained(DBinary)
                    Ds['SNF'][i, j] = max(score, Ds['SNF'][i, j])
                    #In addition to fusion, compute scores for individual
                    #features to be used with the fusion later
                    for Feature in OtherCSMs:
                        DBinary = CSMToBinaryMutual(OtherCSMs[Feature], Kappa)
                        score = SAC.swalignimpconstrained(DBinary)
                        Ds[Feature][i, j] = max(Ds[Feature][i, j], score)
    toc = time.time()
    print("Elapsed Time Block: ", toc-tic)
    stdout.flush()
    sio.savemat(DsFilename, Ds)
    return Ds

    Process a rectangular block of the all pairs score matrix
    between all of the songs.  Return score matrices for each
    individual type of feature, in addition to one for early
    similarity network fusion
    :param idxs: [start1, end1, start2, end2] range of rectangular
        block of songs to compare
    :param Kappa: Percent nearest neighbors to use both for
        binary cross-similarity and similarity network fusion
    :param CSMTypes: Dictionary of types of features and
        associated cross-similarity comparisons to do
    :param allFiles: List of all files that are being compared
        from which this block is drawn
    :param scratchDir: Path to directory for storing block results
    
Process a rectangular block of the all pairs score matrix
between all of the songs.  Return score matrices for each
individual type of feature, in addition to one for early
similarity network fusion
Process a rectangular block of the all pairs score matrix between all of the songs . Return score matrices for each individual type of feature in addition to one for early similarity network fusion
def compareBatchBlock(args):
    (idxs, Kappa, CSMTypes, allFiles, scratchDir) = args
    DsFilename = "%s/D%i_%i_%i_%i.mat"%(scratchDir, idxs[0], idxs[1], idxs[2], idxs[3])
    if os.path.exists(DsFilename):
        return sio.loadmat(DsFilename)
    thisM = idxs[1] - idxs[0]
    thisN = idxs[3] - idxs[2]
    D = np.zeros((thisM, thisN))
    AllFeatures = {}
    tic = time.time()
    allidxs = [i + idxs[0] for i in range(thisM)]
    allidxs += [j + idxs[2] for j in range(thisN)]
    allidxs = np.unique(np.array(allidxs))
    ticfeatures = time.time()
    count = 1
    for idx in allidxs:
        filename = getMatFilename(scratchDir, allFiles[idx])
        AllFeatures[idx] = sio.loadmat(filename)
        for key, val in AllFeatures[idx].items():
            if type(val) is np.ndarray:
                if val.size == 1:
                    AllFeatures[idx][key] = val.flatten()[0]
    tocfeatures = time.time()
    print("Elapsed Time Loading Features: ", tocfeatures-ticfeatures)
    stdout.flush()
    K = 20
    NIters = 3
    Ds = {'SNF':np.zeros((thisM, thisN))}
    for Feature in CSMTypes.keys():
        Ds[Feature] = np.zeros((thisM, thisN))
    for i in range(thisM):
        print("i = %i"%i)
        stdout.flush()
        thisi = i + idxs[0]
        Features1 = AllFeatures[thisi]
        for j in range(thisN):
            thisj = j + idxs[2]
            if thisj < thisi:
                continue
            Features2 = AllFeatures[thisj]
            for a in range(Features1['NTempos']):
                O1 = {'ChromaMean':Features1['ChromaMean%i'%a].flatten()}
                for b in range(Features2['NTempos']):
                    O2 = {'ChromaMean':Features2['ChromaMean%i'%b].flatten()}
                    Ws = []
                    OtherCSMs = {}
                    (M, N) = (0, 0)
                    for F in CSMTypes.keys():
                        CSMAB = getCSMType(Features1['%s%i'%(F, a)], O1, Features2['%s%i'%(F, b)], O2, CSMTypes[F])
                        OtherCSMs[F] = CSMAB
                        (M, N) = (CSMAB.shape[0], CSMAB.shape[1])
                        k1 = int(0.5*Kappa*M)
                        k2 = int(0.5*Kappa*N)
                        WCSMAB = getWCSM(CSMAB, k1, k2)
                        WSSMA = Features1['W%s%i'%(F, a)]
                        WSSMB = Features2['W%s%i'%(F, b)]
                        Ws.append(setupWCSMSSM(WSSMA, WSSMB, WCSMAB))
                    D = doSimilarityFusionWs(Ws, K, NIters, 1)
                    CSM = D[0:M, M::] + D[M::, 0:M].T
                    DBinary = CSMToBinaryMutual(np.exp(-CSM), Kappa)
                    score = SAC.swalignimpconstrained(DBinary)
                    Ds['SNF'][i, j] = max(score, Ds['SNF'][i, j])
                    for Feature in OtherCSMs:
                        DBinary = CSMToBinaryMutual(OtherCSMs[Feature], Kappa)
                        score = SAC.swalignimpconstrained(DBinary)
                        Ds[Feature][i, j] = max(Ds[Feature][i, j], score)
    toc = time.time()
    print("Elapsed Time Block: ", toc-tic)
    stdout.flush()
    sio.savemat(DsFilename, Ds)
    return Ds
def compareBatchBlock ( args ) : ( idxs , Kappa , CSMTypes , allFiles , scratchDir ) = args DsFilename = "%s/D%i_%i_%i_%i.mat" % ( scratchDir , idxs [ 0 ] , idxs [ 1 ] , idxs [ 2 ] , idxs [ 3 ] ) if os . path . exists ( DsFilename ) : return sio . loadmat ( DsFilename ) thisM = idxs [ 1 ] - idxs [ 0 ] thisN = idxs [ 3 ] - idxs [ 2 ] D = np . zeros ( ( thisM , thisN ) ) AllFeatures = { } tic = time . time ( ) allidxs = [ i + idxs [ 0 ] for i in range ( thisM ) ] allidxs += [ j + idxs [ 2 ] for j in range ( thisN ) ] allidxs = np . unique ( np . array ( allidxs ) ) ticfeatures = time . time ( ) count = 1 for idx in allidxs : filename = getMatFilename ( scratchDir , allFiles [ idx ] ) AllFeatures [ idx ] = sio . loadmat ( filename ) for key , val in AllFeatures [ idx ] . items ( ) : if type ( val ) is np . ndarray : if val . size == 1 : AllFeatures [ idx ] [ key ] = val . flatten ( ) [ 0 ] tocfeatures = time . time ( ) print ( "Elapsed Time Loading Features: " , tocfeatures - ticfeatures ) stdout . flush ( ) K = 20 NIters = 3 Ds = { 'SNF' : np . zeros ( ( thisM , thisN ) ) } for Feature in CSMTypes . keys ( ) : Ds [ Feature ] = np . zeros ( ( thisM , thisN ) ) for i in range ( thisM ) : print ( "i = %i" % i ) stdout . flush ( ) thisi = i + idxs [ 0 ] Features1 = AllFeatures [ thisi ] for j in range ( thisN ) : thisj = j + idxs [ 2 ] if thisj < thisi : continue Features2 = AllFeatures [ thisj ] for a in range ( Features1 [ 'NTempos' ] ) : O1 = { 'ChromaMean' : Features1 [ 'ChromaMean%i' % a ] . flatten ( ) } for b in range ( Features2 [ 'NTempos' ] ) : O2 = { 'ChromaMean' : Features2 [ 'ChromaMean%i' % b ] . flatten ( ) } Ws = [ ] OtherCSMs = { } ( M , N ) = ( 0 , 0 ) for F in CSMTypes . keys ( ) : CSMAB = getCSMType ( Features1 [ '%s%i' % ( F , a ) ] , O1 , Features2 [ '%s%i' % ( F , b ) ] , O2 , CSMTypes [ F ] ) OtherCSMs [ F ] = CSMAB ( M , N ) = ( CSMAB . shape [ 0 ] , CSMAB . shape [ 1 ] ) k1 = int ( 0.5 * Kappa * M ) k2 = int ( 0.5 * Kappa * N ) WCSMAB = getWCSM ( CSMAB , k1 , k2 ) WSSMA = Features1 [ 'W%s%i' % ( F , a ) ] WSSMB = Features2 [ 'W%s%i' % ( F , b ) ] Ws . append ( setupWCSMSSM ( WSSMA , WSSMB , WCSMAB ) ) D = doSimilarityFusionWs ( Ws , K , NIters , 1 ) CSM = D [ 0 : M , M : : ] + D [ M : : , 0 : M ] . T DBinary = CSMToBinaryMutual ( np . exp ( - CSM ) , Kappa ) score = SAC . swalignimpconstrained ( DBinary ) Ds [ 'SNF' ] [ i , j ] = max ( score , Ds [ 'SNF' ] [ i , j ] ) for Feature in OtherCSMs : DBinary = CSMToBinaryMutual ( OtherCSMs [ Feature ] , Kappa ) score = SAC . swalignimpconstrained ( DBinary ) Ds [ Feature ] [ i , j ] = max ( Ds [ Feature ] [ i , j ] , score ) toc = time . time ( ) print ( "Elapsed Time Block: " , toc - tic ) stdout . flush ( ) sio . savemat ( DsFilename , Ds ) return Ds
Process a rectangular block of the all pairs score matrix
between all of the songs.
Process a rectangular block of the all pairs score matrix between all of the songs .
"""
    Process a rectangular block of the all pairs score matrix
    between all of the songs.  Return score matrices for each
    individual type of feature, in addition to one for early
    similarity network fusion
    :param idxs: [start1, end1, start2, end2] range of rectangular
        block of songs to compare
    :param Kappa: Percent nearest neighbors to use both for
        binary cross-similarity and similarity network fusion
    :param CSMTypes: Dictionary of types of features and
        associated cross-similarity comparisons to do
    :param allFiles: List of all files that are being compared
        from which this block is drawn
    :param scratchDir: Path to directory for storing block results
    """ #Figure out block size thisM x thisN #Preload features and Ws for SSM parts #Only compute upper triangular part since it's symmetric #Compare all tempo levels #Compute all W matrices #Do Similarity Fusion #Extract CSM Part #In addition to fusion, compute scores for individual #features to be used with the fusion later
args
idxs
[start1, end1, start2, end2] range of rectangular
block of songs to compare
[ start1 end1 start2 end2 ] range of rectangular block of songs to compare
Kappa
Percent nearest neighbors to use both for
binary cross-similarity and similarity network fusion
Percent nearest neighbors to use both for binary cross - similarity and similarity network fusion
CSMTypes
Dictionary of types of features and
associated cross-similarity comparisons to do
Dictionary of types of features and associated cross - similarity comparisons to do
allFiles
List of all files that are being compared
from which this block is drawn
List of all files that are being compared from which this block is drawn
scratchDir
Path to directory for storing block results
Path to directory for storing block results