splearn/decomposition/truncated_svd.py
def svd(blocked_rdd, k):
    """
    Calculate the SVD of a blocked RDD directly, returning only the leading k
    singular vectors. Assumes n rows and d columns, efficient when n >> d
    Must be able to fit d^2 within the memory of a single machine.
    Parameters
    ----------
    blocked_rdd : RDD
        RDD with data points in numpy array blocks
    k : Int
        Number of singular vectors to return
    Returns
    ----------
    u : RDD of blocks
        Left eigenvectors
    s : numpy array
        Singular values
    v : numpy array
        Right eigenvectors
    """

    # compute the covariance matrix (without mean subtraction)
    # TODO use one func for this (with mean subtraction as an option?)
    c = blocked_rdd.map(lambda x: (x.T.dot(x), x.shape[0]))
    prod, n = c.reduce(lambda x, y: (x[0] + y[0], x[1] + y[1]))

    # do local eigendecomposition
    w, v = ln.eig(prod / n)
    w = np.real(w)
    v = np.real(v)
    inds = np.argsort(w)[::-1]
    s = np.sqrt(w[inds[0:k]]) * np.sqrt(n)
    v = v[:, inds[0:k]].T

    # project back into data, normalize by singular values
    u = blocked_rdd.map(lambda x: np.inner(x, v) / s)

    return u, s, v
def svd(blocked_rdd, k):
    """
    Calculate the SVD of a blocked RDD directly, returning only the leading k
    singular vectors. Assumes n rows and d columns, efficient when n >> d
    Must be able to fit d^2 within the memory of a single machine.
    Parameters
    ----------
    blocked_rdd : RDD
        RDD with data points in numpy array blocks
    k : Int
        Number of singular vectors to return
    Returns
    ----------
    u : RDD of blocks
        Left eigenvectors
    s : numpy array
        Singular values
    v : numpy array
        Right eigenvectors
    """

    # compute the covariance matrix (without mean subtraction)
    # TODO use one func for this (with mean subtraction as an option?)
    c = blocked_rdd.map(lambda x: (x.T.dot(x), x.shape[0]))
    prod, n = c.reduce(lambda x, y: (x[0] + y[0], x[1] + y[1]))

    # do local eigendecomposition
    w, v = ln.eig(prod / n)
    w = np.real(w)
    v = np.real(v)
    inds = np.argsort(w)[::-1]
    s = np.sqrt(w[inds[0:k]]) * np.sqrt(n)
    v = v[:, inds[0:k]].T

    # project back into data, normalize by singular values
    u = blocked_rdd.map(lambda x: np.inner(x, v) / s)

    return u, s, v
def svd ( blocked_rdd , k ) : # compute the covariance matrix (without mean subtraction) # TODO use one func for this (with mean subtraction as an option?) c = blocked_rdd . map ( lambda x : ( x . T . dot ( x ) , x . shape [ 0 ] ) ) prod , n = c . reduce ( lambda x , y : ( x [ 0 ] + y [ 0 ] , x [ 1 ] + y [ 1 ] ) ) # do local eigendecomposition w , v = ln . eig ( prod / n ) w = np . real ( w ) v = np . real ( v ) inds = np . argsort ( w ) [ : : - 1 ] s = np . sqrt ( w [ inds [ 0 : k ] ] ) * np . sqrt ( n ) v = v [ : , inds [ 0 : k ] ] . T # project back into data, normalize by singular values u = blocked_rdd . map ( lambda x : np . inner ( x , v ) / s ) return u , s , v
Calculate the SVD of a blocked RDD directly, returning only the leading k
    singular vectors. Assumes n rows and d columns, efficient when n >> d
    Must be able to fit d^2 within the memory of a single machine.
    Parameters
    ----------
    blocked_rdd : RDD
        RDD with data points in numpy array blocks
    k : Int
        Number of singular vectors to return
    Returns
    ----------
    u : RDD of blocks
        Left eigenvectors
    s : numpy array
        Singular values
    v : numpy array
        Right eigenvectors
Calculate the SVD of a blocked RDD directly returning only the leading k singular vectors . Assumes n rows and d columns efficient when n >> d Must be able to fit d^2 within the memory of a single machine . Parameters ---------- blocked_rdd : RDD RDD with data points in numpy array blocks k : Int Number of singular vectors to return Returns ---------- u : RDD of blocks Left eigenvectors s : numpy array Singular values v : numpy array Right eigenvectors