splearn/preprocessing/data.py
def fit(self, Z):
        """Compute the mean and std to be used for later scaling.
        Parameters
        ----------
        Z : DictRDD containing (X, y) pairs
            X - Training vector.
                {array-like, sparse matrix}, shape [n_samples, n_features]
                The data used to compute the mean and standard deviation
                used for later scaling along the features axis.
            y - Target labels
                Passthrough for ``Pipeline`` compatibility.
        """

        # Reset internal state before fitting
        self._reset()
        X = Z[:, 'X'] if isinstance(Z, DictRDD) else Z
        check_rdd(X, (np.ndarray, sp.spmatrix))

        def mapper(X):
            """Calculate statistics for every numpy or scipy blocks."""
            X = check_array(X, ('csr', 'csc'), dtype=np.float64)
            if hasattr(X, "toarray"):   # sparse matrix
                mean, var = mean_variance_axis(X, axis=0)
            else:
                mean, var = np.mean(X, axis=0), np.var(X, axis=0)
            return X.shape[0], mean, var

        def reducer(a, b):
            """Calculate the combined statistics."""
            n_a, mean_a, var_a = a
            n_b, mean_b, var_b = b
            n_ab = n_a + n_b
            mean_ab = ((mean_a * n_a) + (mean_b * n_b)) / n_ab
            var_ab = (((n_a * var_a) + (n_b * var_b)) / n_ab) + \
                     ((n_a * n_b) * ((mean_b - mean_a) / n_ab) ** 2)
            return (n_ab, mean_ab, var_ab)

        if check_rdd_dtype(X, (sp.spmatrix)):
            if self.with_mean:
                raise ValueError(
                    "Cannot center sparse matrices: pass `with_mean=False` "
                    "instead. See docstring for motivation and alternatives.")
        self.n_samples_seen_, self.mean_, self.var_ = X.map(mapper).treeReduce(reducer)

        if self.with_std:
            self.scale_ = _handle_zeros_in_scale(np.sqrt(self.var_))
        else:
            self.scale_ = None

        return self
def fit(self, Z):
        """Compute the mean and std to be used for later scaling.
        Parameters
        ----------
        Z : DictRDD containing (X, y) pairs
            X - Training vector.
                {array-like, sparse matrix}, shape [n_samples, n_features]
                The data used to compute the mean and standard deviation
                used for later scaling along the features axis.
            y - Target labels
                Passthrough for ``Pipeline`` compatibility.
        """

        # Reset internal state before fitting
        self._reset()
        X = Z[:, 'X'] if isinstance(Z, DictRDD) else Z
        check_rdd(X, (np.ndarray, sp.spmatrix))

        def mapper(X):
            """Calculate statistics for every numpy or scipy blocks."""
            X = check_array(X, ('csr', 'csc'), dtype=np.float64)
            if hasattr(X, "toarray"):   # sparse matrix
                mean, var = mean_variance_axis(X, axis=0)
            else:
                mean, var = np.mean(X, axis=0), np.var(X, axis=0)
            return X.shape[0], mean, var

        def reducer(a, b):
            """Calculate the combined statistics."""
            n_a, mean_a, var_a = a
            n_b, mean_b, var_b = b
            n_ab = n_a + n_b
            mean_ab = ((mean_a * n_a) + (mean_b * n_b)) / n_ab
            var_ab = (((n_a * var_a) + (n_b * var_b)) / n_ab) + \
                     ((n_a * n_b) * ((mean_b - mean_a) / n_ab) ** 2)
            return (n_ab, mean_ab, var_ab)

        if check_rdd_dtype(X, (sp.spmatrix)):
            if self.with_mean:
                raise ValueError(
                    "Cannot center sparse matrices: pass `with_mean=False` "
                    "instead. See docstring for motivation and alternatives.")
        self.n_samples_seen_, self.mean_, self.var_ = X.map(mapper).treeReduce(reducer)

        if self.with_std:
            self.scale_ = _handle_zeros_in_scale(np.sqrt(self.var_))
        else:
            self.scale_ = None

        return self
def fit ( self , Z ) : # Reset internal state before fitting self . _reset ( ) X = Z [ : , 'X' ] if isinstance ( Z , DictRDD ) else Z check_rdd ( X , ( np . ndarray , sp . spmatrix ) ) def mapper ( X ) : """Calculate statistics for every numpy or scipy blocks.""" X = check_array ( X , ( 'csr' , 'csc' ) , dtype = np . float64 ) if hasattr ( X , "toarray" ) : # sparse matrix mean , var = mean_variance_axis ( X , axis = 0 ) else : mean , var = np . mean ( X , axis = 0 ) , np . var ( X , axis = 0 ) return X . shape [ 0 ] , mean , var def reducer ( a , b ) : """Calculate the combined statistics.""" n_a , mean_a , var_a = a n_b , mean_b , var_b = b n_ab = n_a + n_b mean_ab = ( ( mean_a * n_a ) + ( mean_b * n_b ) ) / n_ab var_ab = ( ( ( n_a * var_a ) + ( n_b * var_b ) ) / n_ab ) + ( ( n_a * n_b ) * ( ( mean_b - mean_a ) / n_ab ) ** 2 ) return ( n_ab , mean_ab , var_ab ) if check_rdd_dtype ( X , ( sp . spmatrix ) ) : if self . with_mean : raise ValueError ( "Cannot center sparse matrices: pass `with_mean=False` " "instead. See docstring for motivation and alternatives." ) self . n_samples_seen_ , self . mean_ , self . var_ = X . map ( mapper ) . treeReduce ( reducer ) if self . with_std : self . scale_ = _handle_zeros_in_scale ( np . sqrt ( self . var_ ) ) else : self . scale_ = None return self
Compute the mean and std to be used for later scaling.
        Parameters
        ----------
        Z : DictRDD containing (X, y) pairs
            X - Training vector.
                {array-like, sparse matrix}, shape [n_samples, n_features]
                The data used to compute the mean and standard deviation
                used for later scaling along the features axis.
            y - Target labels
                Passthrough for ``Pipeline`` compatibility.
Compute the mean and std to be used for later scaling . Parameters ---------- Z : DictRDD containing ( X y ) pairs X - Training vector . { array - like sparse matrix } shape [ n_samples n_features ] The data used to compute the mean and standard deviation used for later scaling along the features axis . y - Target labels Passthrough for Pipeline compatibility .