packages/vaex-core/vaex/dataframe.py
def percentile_approx(self, expression, percentage=50., binby=[], limits=None, shape=default_shape, percentile_shape=1024, percentile_limits="minmax", selection=False, delay=False):
        """Calculate the percentile given by percentage, possibly on a grid defined by binby.

        NOTE: this value is approximated by calculating the cumulative distribution on a grid defined by
        percentile_shape and percentile_limits.


        Example:

        >>> df.percentile_approx("x", 10), df.percentile_approx("x", 90)
        (array([-8.3220355]), array([ 7.92080358]))
        >>> df.percentile_approx("x", 50, binby="x", shape=5, limits=[-10, 10])
        array([[-7.56462982],
                   [-3.61036641],
                   [-0.01296306],
                   [ 3.56697863],
                   [ 7.45838367]])

        :param expression: {expression}
        :param binby: {binby}
        :param limits: {limits}
        :param shape: {shape}
        :param percentile_limits: {percentile_limits}
        :param percentile_shape: {percentile_shape}
        :param selection: {selection}
        :param delay: {delay}
        :return: {return_stat_scalar}
        """
        waslist, [expressions, ] = vaex.utils.listify(expression)
        if not isinstance(binby, (tuple, list)):
            binby = [binby]
        else:
            binby = binby

        @delayed
        def calculate(expression, shape, limits):
            # task =  TaskStatistic(self, [expression] + binby, shape, limits, op=OP_ADD1, selection=selection)
            # self.executor.schedule(task)
            # return task
            return self.count(binby=list(binby) + [expression], shape=shape, limits=limits, selection=selection, delay=True, edges=True)

        @delayed
        def finish(percentile_limits, counts_list):
            results = []
            for i, counts in enumerate(counts_list):
                counts = counts.astype(np.float)
                # remove the nan and boundary edges from the first dimension,
                nonnans = list([slice(2, -1, None) for k in range(len(counts.shape) - 1)])
                nonnans.append(slice(1, None, None))  # we're gonna get rid only of the nan's, and keep the overflow edges
                nonnans = tuple(nonnans)
                cumulative_grid = np.cumsum(counts.__getitem__(nonnans), -1)  # convert to cumulative grid

                totalcounts = np.sum(counts.__getitem__(nonnans), -1)
                empty = totalcounts == 0

                original_shape = counts.shape
                shape = cumulative_grid.shape  # + (original_shape[-1] - 1,) #

                counts = np.sum(counts, -1)
                edges_floor = np.zeros(shape[:-1] + (2,), dtype=np.int64)
                edges_ceil = np.zeros(shape[:-1] + (2,), dtype=np.int64)
                # if we have an off  # of elements, say, N=3, the center is at i=1=(N-1)/2
                # if we have an even # of elements, say, N=4, the center is between i=1=(N-2)/2 and i=2=(N/2)
                # index = (shape[-1] -1-3) * percentage/100. # the -3 is for the edges
                values = np.array((totalcounts + 1) * percentage / 100.)  # make sure it's an ndarray
                values[empty] = 0
                floor_values = np.array(np.floor(values))
                ceil_values = np.array(np.ceil(values))
                vaex.vaexfast.grid_find_edges(cumulative_grid, floor_values, edges_floor)
                vaex.vaexfast.grid_find_edges(cumulative_grid, ceil_values, edges_ceil)

                def index_choose(a, indices):
                    # alternative to np.choise, which doesn't like the last dim to be >= 32
                    # print(a, indices)
                    out = np.zeros(a.shape[:-1])
                    # print(out.shape)
                    for i in np.ndindex(out.shape):
                        # print(i, indices[i])
                        out[i] = a[i + (indices[i],)]
                    return out

                def calculate_x(edges, values):
                    left, right = edges[..., 0], edges[..., 1]
                    left_value = index_choose(cumulative_grid, left)
                    right_value = index_choose(cumulative_grid, right)
                    u = np.array((values - left_value) / (right_value - left_value))
                    # TODO: should it really be -3? not -2
                    xleft, xright = percentile_limits[i][0] + (left - 0.5) * (percentile_limits[i][1] - percentile_limits[i][0]) / (shape[-1] - 3),\
                        percentile_limits[i][0] + (right - 0.5) * (percentile_limits[i][1] - percentile_limits[i][0]) / (shape[-1] - 3)
                    x = xleft + (xright - xleft) * u  # /2
                    return x

                x1 = calculate_x(edges_floor, floor_values)
                x2 = calculate_x(edges_ceil, ceil_values)
                u = values - floor_values
                x = x1 + (x2 - x1) * u
                results.append(x)

            return results

        shape = _expand_shape(shape, len(binby))
        percentile_shapes = _expand_shape(percentile_shape, len(expressions))
        if percentile_limits:
            percentile_limits = _expand_limits(percentile_limits, len(expressions))
        limits = self.limits(binby, limits, selection=selection, delay=True)
        percentile_limits = self.limits(expressions, percentile_limits, selection=selection, delay=True)

        @delayed
        def calculation(limits, percentile_limits):
            # print(">>>", expressions, percentile_limits)
            # print(percentile_limits[0], list(percentile_limits[0]))
            # print(list(np.array(limits).tolist()) + list(percentile_limits[0]))
            # print("limits", limits, expressions, percentile_limits, ">>", list(limits) + [list(percentile_limits[0]))
            tasks = [calculate(expression, tuple(shape) + (percentile_shape, ), list(limits) + [list(percentile_limit)])
                     for percentile_shape, percentile_limit, expression
                     in zip(percentile_shapes, percentile_limits, expressions)]
            return finish(percentile_limits, delayed_args(*tasks))
            # return tasks
        result = calculation(limits, percentile_limits)

        @delayed
        def finish2(grid):
            value = vaex.utils.unlistify(waslist, np.array(grid))
            return value
        return self._delay(delay, finish2(result))
def percentile_approx(self, expression, percentage=50., binby=[], limits=None, shape=default_shape, percentile_shape=1024, percentile_limits="minmax", selection=False, delay=False):
        """Calculate the percentile given by percentage, possibly on a grid defined by binby.

        NOTE: this value is approximated by calculating the cumulative distribution on a grid defined by
        percentile_shape and percentile_limits.


        Example:

        >>> df.percentile_approx("x", 10), df.percentile_approx("x", 90)
        (array([-8.3220355]), array([ 7.92080358]))
        >>> df.percentile_approx("x", 50, binby="x", shape=5, limits=[-10, 10])
        array([[-7.56462982],
                   [-3.61036641],
                   [-0.01296306],
                   [ 3.56697863],
                   [ 7.45838367]])

        :param expression: {expression}
        :param binby: {binby}
        :param limits: {limits}
        :param shape: {shape}
        :param percentile_limits: {percentile_limits}
        :param percentile_shape: {percentile_shape}
        :param selection: {selection}
        :param delay: {delay}
        :return: {return_stat_scalar}
        """
        waslist, [expressions, ] = vaex.utils.listify(expression)
        if not isinstance(binby, (tuple, list)):
            binby = [binby]
        else:
            binby = binby

        @delayed
        def calculate(expression, shape, limits):
            # task =  TaskStatistic(self, [expression] + binby, shape, limits, op=OP_ADD1, selection=selection)
            # self.executor.schedule(task)
            # return task
            return self.count(binby=list(binby) + [expression], shape=shape, limits=limits, selection=selection, delay=True, edges=True)

        @delayed
        def finish(percentile_limits, counts_list):
            results = []
            for i, counts in enumerate(counts_list):
                counts = counts.astype(np.float)
                # remove the nan and boundary edges from the first dimension,
                nonnans = list([slice(2, -1, None) for k in range(len(counts.shape) - 1)])
                nonnans.append(slice(1, None, None))  # we're gonna get rid only of the nan's, and keep the overflow edges
                nonnans = tuple(nonnans)
                cumulative_grid = np.cumsum(counts.__getitem__(nonnans), -1)  # convert to cumulative grid

                totalcounts = np.sum(counts.__getitem__(nonnans), -1)
                empty = totalcounts == 0

                original_shape = counts.shape
                shape = cumulative_grid.shape  # + (original_shape[-1] - 1,) #

                counts = np.sum(counts, -1)
                edges_floor = np.zeros(shape[:-1] + (2,), dtype=np.int64)
                edges_ceil = np.zeros(shape[:-1] + (2,), dtype=np.int64)
                # if we have an off  # of elements, say, N=3, the center is at i=1=(N-1)/2
                # if we have an even # of elements, say, N=4, the center is between i=1=(N-2)/2 and i=2=(N/2)
                # index = (shape[-1] -1-3) * percentage/100. # the -3 is for the edges
                values = np.array((totalcounts + 1) * percentage / 100.)  # make sure it's an ndarray
                values[empty] = 0
                floor_values = np.array(np.floor(values))
                ceil_values = np.array(np.ceil(values))
                vaex.vaexfast.grid_find_edges(cumulative_grid, floor_values, edges_floor)
                vaex.vaexfast.grid_find_edges(cumulative_grid, ceil_values, edges_ceil)

                def index_choose(a, indices):
                    # alternative to np.choise, which doesn't like the last dim to be >= 32
                    # print(a, indices)
                    out = np.zeros(a.shape[:-1])
                    # print(out.shape)
                    for i in np.ndindex(out.shape):
                        # print(i, indices[i])
                        out[i] = a[i + (indices[i],)]
                    return out

                def calculate_x(edges, values):
                    left, right = edges[..., 0], edges[..., 1]
                    left_value = index_choose(cumulative_grid, left)
                    right_value = index_choose(cumulative_grid, right)
                    u = np.array((values - left_value) / (right_value - left_value))
                    # TODO: should it really be -3? not -2
                    xleft, xright = percentile_limits[i][0] + (left - 0.5) * (percentile_limits[i][1] - percentile_limits[i][0]) / (shape[-1] - 3),\
                        percentile_limits[i][0] + (right - 0.5) * (percentile_limits[i][1] - percentile_limits[i][0]) / (shape[-1] - 3)
                    x = xleft + (xright - xleft) * u  # /2
                    return x

                x1 = calculate_x(edges_floor, floor_values)
                x2 = calculate_x(edges_ceil, ceil_values)
                u = values - floor_values
                x = x1 + (x2 - x1) * u
                results.append(x)

            return results

        shape = _expand_shape(shape, len(binby))
        percentile_shapes = _expand_shape(percentile_shape, len(expressions))
        if percentile_limits:
            percentile_limits = _expand_limits(percentile_limits, len(expressions))
        limits = self.limits(binby, limits, selection=selection, delay=True)
        percentile_limits = self.limits(expressions, percentile_limits, selection=selection, delay=True)

        @delayed
        def calculation(limits, percentile_limits):
            # print(">>>", expressions, percentile_limits)
            # print(percentile_limits[0], list(percentile_limits[0]))
            # print(list(np.array(limits).tolist()) + list(percentile_limits[0]))
            # print("limits", limits, expressions, percentile_limits, ">>", list(limits) + [list(percentile_limits[0]))
            tasks = [calculate(expression, tuple(shape) + (percentile_shape, ), list(limits) + [list(percentile_limit)])
                     for percentile_shape, percentile_limit, expression
                     in zip(percentile_shapes, percentile_limits, expressions)]
            return finish(percentile_limits, delayed_args(*tasks))
            # return tasks
        result = calculation(limits, percentile_limits)

        @delayed
        def finish2(grid):
            value = vaex.utils.unlistify(waslist, np.array(grid))
            return value
        return self._delay(delay, finish2(result))
def percentile_approx ( self , expression , percentage = 50. , binby = [ ] , limits = None , shape = default_shape , percentile_shape = 1024 , percentile_limits = "minmax" , selection = False , delay = False ) : waslist , [ expressions , ] = vaex . utils . listify ( expression ) if not isinstance ( binby , ( tuple , list ) ) : binby = [ binby ] else : binby = binby @ delayed def calculate ( expression , shape , limits ) : # task =  TaskStatistic(self, [expression] + binby, shape, limits, op=OP_ADD1, selection=selection) # self.executor.schedule(task) # return task return self . count ( binby = list ( binby ) + [ expression ] , shape = shape , limits = limits , selection = selection , delay = True , edges = True ) @ delayed def finish ( percentile_limits , counts_list ) : results = [ ] for i , counts in enumerate ( counts_list ) : counts = counts . astype ( np . float ) # remove the nan and boundary edges from the first dimension, nonnans = list ( [ slice ( 2 , - 1 , None ) for k in range ( len ( counts . shape ) - 1 ) ] ) nonnans . append ( slice ( 1 , None , None ) ) # we're gonna get rid only of the nan's, and keep the overflow edges nonnans = tuple ( nonnans ) cumulative_grid = np . cumsum ( counts . __getitem__ ( nonnans ) , - 1 ) # convert to cumulative grid totalcounts = np . sum ( counts . __getitem__ ( nonnans ) , - 1 ) empty = totalcounts == 0 original_shape = counts . shape shape = cumulative_grid . shape # + (original_shape[-1] - 1,) # counts = np . sum ( counts , - 1 ) edges_floor = np . zeros ( shape [ : - 1 ] + ( 2 , ) , dtype = np . int64 ) edges_ceil = np . zeros ( shape [ : - 1 ] + ( 2 , ) , dtype = np . int64 ) # if we have an off  # of elements, say, N=3, the center is at i=1=(N-1)/2 # if we have an even # of elements, say, N=4, the center is between i=1=(N-2)/2 and i=2=(N/2) # index = (shape[-1] -1-3) * percentage/100. # the -3 is for the edges values = np . array ( ( totalcounts + 1 ) * percentage / 100. ) # make sure it's an ndarray values [ empty ] = 0 floor_values = np . array ( np . floor ( values ) ) ceil_values = np . array ( np . ceil ( values ) ) vaex . vaexfast . grid_find_edges ( cumulative_grid , floor_values , edges_floor ) vaex . vaexfast . grid_find_edges ( cumulative_grid , ceil_values , edges_ceil ) def index_choose ( a , indices ) : # alternative to np.choise, which doesn't like the last dim to be >= 32 # print(a, indices) out = np . zeros ( a . shape [ : - 1 ] ) # print(out.shape) for i in np . ndindex ( out . shape ) : # print(i, indices[i]) out [ i ] = a [ i + ( indices [ i ] , ) ] return out def calculate_x ( edges , values ) : left , right = edges [ ... , 0 ] , edges [ ... , 1 ] left_value = index_choose ( cumulative_grid , left ) right_value = index_choose ( cumulative_grid , right ) u = np . array ( ( values - left_value ) / ( right_value - left_value ) ) # TODO: should it really be -3? not -2 xleft , xright = percentile_limits [ i ] [ 0 ] + ( left - 0.5 ) * ( percentile_limits [ i ] [ 1 ] - percentile_limits [ i ] [ 0 ] ) / ( shape [ - 1 ] - 3 ) , percentile_limits [ i ] [ 0 ] + ( right - 0.5 ) * ( percentile_limits [ i ] [ 1 ] - percentile_limits [ i ] [ 0 ] ) / ( shape [ - 1 ] - 3 ) x = xleft + ( xright - xleft ) * u # /2 return x x1 = calculate_x ( edges_floor , floor_values ) x2 = calculate_x ( edges_ceil , ceil_values ) u = values - floor_values x = x1 + ( x2 - x1 ) * u results . append ( x ) return results shape = _expand_shape ( shape , len ( binby ) ) percentile_shapes = _expand_shape ( percentile_shape , len ( expressions ) ) if percentile_limits : percentile_limits = _expand_limits ( percentile_limits , len ( expressions ) ) limits = self . limits ( binby , limits , selection = selection , delay = True ) percentile_limits = self . limits ( expressions , percentile_limits , selection = selection , delay = True ) @ delayed def calculation ( limits , percentile_limits ) : # print(">>>", expressions, percentile_limits) # print(percentile_limits[0], list(percentile_limits[0])) # print(list(np.array(limits).tolist()) + list(percentile_limits[0])) # print("limits", limits, expressions, percentile_limits, ">>", list(limits) + [list(percentile_limits[0])) tasks = [ calculate ( expression , tuple ( shape ) + ( percentile_shape , ) , list ( limits ) + [ list ( percentile_limit ) ] ) for percentile_shape , percentile_limit , expression in zip ( percentile_shapes , percentile_limits , expressions ) ] return finish ( percentile_limits , delayed_args ( * tasks ) ) # return tasks result = calculation ( limits , percentile_limits ) @ delayed def finish2 ( grid ) : value = vaex . utils . unlistify ( waslist , np . array ( grid ) ) return value return self . _delay ( delay , finish2 ( result ) )
Calculate the percentile given by percentage, possibly on a grid defined by binby.

        NOTE: this value is approximated by calculating the cumulative distribution on a grid defined by
        percentile_shape and percentile_limits.


        Example:

        >>> df.percentile_approx("x", 10), df.percentile_approx("x", 90)
        (array([-8.3220355]), array([ 7.92080358]))
        >>> df.percentile_approx("x", 50, binby="x", shape=5, limits=[-10, 10])
        array([[-7.56462982],
                   [-3.61036641],
                   [-0.01296306],
                   [ 3.56697863],
                   [ 7.45838367]])

        :param expression: {expression}
        :param binby: {binby}
        :param limits: {limits}
        :param shape: {shape}
        :param percentile_limits: {percentile_limits}
        :param percentile_shape: {percentile_shape}
        :param selection: {selection}
        :param delay: {delay}
        :return: {return_stat_scalar}
Calculate the percentile given by percentage possibly on a grid defined by binby .