bolt/spark/array.py
def _getadvanced(self, index):
        """
        Advanced indexing (for sets, lists, or ndarrays).
        """
        index = [asarray(i) for i in index]
        shape = index[0].shape
        if not all([i.shape == shape for i in index]):
            raise ValueError("shape mismatch: indexing arrays could not be broadcast "
                             "together with shapes " + ("%s " * self.ndim)
                             % tuple([i.shape for i in index]))

        index = tuple([listify(i, d) for (i, d) in zip(index, self.shape)])

        # build tuples with target indices
        key_tuples = list(zip(*index[0:self.split]))
        value_tuples = list(zip(*index[self.split:]))

        # build dictionary to look up targets in values
        d = {}
        for k, g in groupby(zip(value_tuples, key_tuples), lambda x: x[1]):
            d[k] = map(lambda x: x[0], list(g))

        def key_check(key):
            return key in key_tuples

        def key_func(key):
            return unravel_index(key, shape)

        # filter records based on key targets
        filtered = self._rdd.filter(lambda kv: key_check(kv[0]))

        # subselect and flatten records based on value targets (if they exist)
        if len(value_tuples) > 0:
            flattened = filtered.flatMap(lambda kv: [(kv[0], kv[1][i]) for i in d[kv[0]]])
        else:
            flattened = filtered

        # reindex
        indexed = flattened.zipWithIndex()
        rdd = indexed.map(lambda kkv: (key_func(kkv[1]), kkv[0][1]))
        split = len(shape)

        return rdd, shape, split
def _getadvanced(self, index):
        """
        Advanced indexing (for sets, lists, or ndarrays).
        """
        index = [asarray(i) for i in index]
        shape = index[0].shape
        if not all([i.shape == shape for i in index]):
            raise ValueError("shape mismatch: indexing arrays could not be broadcast "
                             "together with shapes " + ("%s " * self.ndim)
                             % tuple([i.shape for i in index]))

        index = tuple([listify(i, d) for (i, d) in zip(index, self.shape)])

        # build tuples with target indices
        key_tuples = list(zip(*index[0:self.split]))
        value_tuples = list(zip(*index[self.split:]))

        # build dictionary to look up targets in values
        d = {}
        for k, g in groupby(zip(value_tuples, key_tuples), lambda x: x[1]):
            d[k] = map(lambda x: x[0], list(g))

        def key_check(key):
            return key in key_tuples

        def key_func(key):
            return unravel_index(key, shape)

        # filter records based on key targets
        filtered = self._rdd.filter(lambda kv: key_check(kv[0]))

        # subselect and flatten records based on value targets (if they exist)
        if len(value_tuples) > 0:
            flattened = filtered.flatMap(lambda kv: [(kv[0], kv[1][i]) for i in d[kv[0]]])
        else:
            flattened = filtered

        # reindex
        indexed = flattened.zipWithIndex()
        rdd = indexed.map(lambda kkv: (key_func(kkv[1]), kkv[0][1]))
        split = len(shape)

        return rdd, shape, split
def _getadvanced ( self , index ) : index = [ asarray ( i ) for i in index ] shape = index [ 0 ] . shape if not all ( [ i . shape == shape for i in index ] ) : raise ValueError ( "shape mismatch: indexing arrays could not be broadcast " "together with shapes " + ( "%s " * self . ndim ) % tuple ( [ i . shape for i in index ] ) ) index = tuple ( [ listify ( i , d ) for ( i , d ) in zip ( index , self . shape ) ] ) # build tuples with target indices key_tuples = list ( zip ( * index [ 0 : self . split ] ) ) value_tuples = list ( zip ( * index [ self . split : ] ) ) # build dictionary to look up targets in values d = { } for k , g in groupby ( zip ( value_tuples , key_tuples ) , lambda x : x [ 1 ] ) : d [ k ] = map ( lambda x : x [ 0 ] , list ( g ) ) def key_check ( key ) : return key in key_tuples def key_func ( key ) : return unravel_index ( key , shape ) # filter records based on key targets filtered = self . _rdd . filter ( lambda kv : key_check ( kv [ 0 ] ) ) # subselect and flatten records based on value targets (if they exist) if len ( value_tuples ) > 0 : flattened = filtered . flatMap ( lambda kv : [ ( kv [ 0 ] , kv [ 1 ] [ i ] ) for i in d [ kv [ 0 ] ] ] ) else : flattened = filtered # reindex indexed = flattened . zipWithIndex ( ) rdd = indexed . map ( lambda kkv : ( key_func ( kkv [ 1 ] ) , kkv [ 0 ] [ 1 ] ) ) split = len ( shape ) return rdd , shape , split
Advanced indexing (for sets, lists, or ndarrays).
Advanced indexing ( for sets lists or ndarrays ) .