packages/vaex-core/vaex/export.py
def _export(dataset_input, dataset_output, random_index_column, path, column_names=None, byteorder="=", shuffle=False, selection=False, progress=None, virtual=True, sort=None, ascending=True):
    """
    :param DatasetLocal dataset: dataset to export
    :param str path: path for file
    :param lis[str] column_names: list of column names to export or None for all columns
    :param str byteorder: = for native, < for little endian and > for big endian
    :param bool shuffle: export rows in random order
    :param bool selection: export selection or not
    :param progress: progress callback that gets a progress fraction as argument and should return True to continue
    :return:
    """

    if selection:
        if selection == True:  # easier to work with the name
            selection = "default"

    N = len(dataset_input) if not selection else dataset_input.selected_length(selection)
    if N == 0:
        raise ValueError("Cannot export empty table")

    if shuffle and sort:
        raise ValueError("Cannot shuffle and sort at the same time")

    if shuffle:
        shuffle_array = dataset_output.columns[random_index_column]

    partial_shuffle = shuffle and len(dataset_input) != N

    order_array = None
    order_array_inverse = None

    # for strings we also need the inverse order_array, keep track of that
    has_strings = any([dataset_input.dtype(k) == str_type for k in column_names])

    if partial_shuffle:
        # if we only export a portion, we need to create the full length random_index array, and
        shuffle_array_full = np.random.choice(len(dataset_input), len(dataset_input), replace=False)
        # then take a section of it
        shuffle_array[:] = shuffle_array_full[shuffle_array_full < N]
        del shuffle_array_full
        order_array = shuffle_array
    elif shuffle:
        # better to do this in memory
        shuffle_array_memory = np.random.choice(N, N, replace=False)
        shuffle_array[:] = shuffle_array_memory
        order_array = shuffle_array
    if order_array is not None:
        indices_r = np.zeros_like(order_array)
        indices_r[order_array] = np.arange(len(order_array))
        order_array_inverse = indices_r
        del indices_r

    if sort:
        if selection:
            raise ValueError("sorting selections not yet supported")
        # these indices sort the input array, but we evaluate the input in sequential order and write it out in sorted order
        # e.g., not b[:] = a[indices]
        # but b[indices_r] = a
        logger.info("sorting...")
        indices = np.argsort(dataset_input.evaluate(sort))
        indices_r = np.zeros_like(indices)
        indices_r[indices] = np.arange(len(indices))
        if has_strings:
            # in this case we already have the inverse ready
            order_array_inverse = indices if ascending else indices[:--1]
        else:
            del indices
        order_array = indices_r if ascending else indices_r[::-1]
        logger.info("sorting done")

    if progress == True:
        progress = vaex.utils.progressbar_callable(title="exporting")
    progress = progress or (lambda value: True)
    progress_total = len(column_names) * len(dataset_input)
    progress_status = ProgressStatus()
    progress_status.cancelled = False
    progress_status.value = 0
    if selection:
        full_mask = dataset_input.evaluate_selection_mask(selection)
    else:
        full_mask = None

    sparse_groups = collections.defaultdict(list)
    sparse_matrices = {}  # alternative to a set of matrices, since they are not hashable
    string_columns = []
    futures = []

    thread_pool = concurrent.futures.ThreadPoolExecutor(max_workers=1) 
    if True:
        for column_name in column_names:
            sparse_matrix = dataset_output._sparse_matrix(column_name)
            if sparse_matrix is not None:
                # sparse columns are written differently
                sparse_groups[id(sparse_matrix)].append(column_name)
                sparse_matrices[id(sparse_matrix)] = sparse_matrix
                continue
            logger.debug("  exporting column: %s " % column_name)
            future = thread_pool.submit(_export_column, dataset_input, dataset_output, column_name, full_mask,
                shuffle, sort, selection, N, order_array, order_array_inverse, progress_status)
            futures.append(future)

    done = False
    while not done:
        done = True
        for future in futures:
            try:
                future.result(0.1/4)
            except concurrent.futures.TimeoutError:
                done = False
                break
        if not done:
            if not progress(progress_status.value / float(progress_total)):
                progress_status.cancelled = True

    for sparse_matrix_id, column_names in sparse_groups.items():
        sparse_matrix = sparse_matrices[sparse_matrix_id]
        for column_name in column_names:
            assert not shuffle
            assert selection in [None, False]
            column = dataset_output.columns[column_name]
            column.matrix.data[:] = dataset_input.columns[column_name].matrix.data
            column.matrix.indptr[:] = dataset_input.columns[column_name].matrix.indptr
            column.matrix.indices[:] = dataset_input.columns[column_name].matrix.indices
    return column_names
def _export(dataset_input, dataset_output, random_index_column, path, column_names=None, byteorder="=", shuffle=False, selection=False, progress=None, virtual=True, sort=None, ascending=True):
    """
    :param DatasetLocal dataset: dataset to export
    :param str path: path for file
    :param lis[str] column_names: list of column names to export or None for all columns
    :param str byteorder: = for native, < for little endian and > for big endian
    :param bool shuffle: export rows in random order
    :param bool selection: export selection or not
    :param progress: progress callback that gets a progress fraction as argument and should return True to continue
    :return:
    """

    if selection:
        if selection == True:  # easier to work with the name
            selection = "default"

    N = len(dataset_input) if not selection else dataset_input.selected_length(selection)
    if N == 0:
        raise ValueError("Cannot export empty table")

    if shuffle and sort:
        raise ValueError("Cannot shuffle and sort at the same time")

    if shuffle:
        shuffle_array = dataset_output.columns[random_index_column]

    partial_shuffle = shuffle and len(dataset_input) != N

    order_array = None
    order_array_inverse = None

    # for strings we also need the inverse order_array, keep track of that
    has_strings = any([dataset_input.dtype(k) == str_type for k in column_names])

    if partial_shuffle:
        # if we only export a portion, we need to create the full length random_index array, and
        shuffle_array_full = np.random.choice(len(dataset_input), len(dataset_input), replace=False)
        # then take a section of it
        shuffle_array[:] = shuffle_array_full[shuffle_array_full < N]
        del shuffle_array_full
        order_array = shuffle_array
    elif shuffle:
        # better to do this in memory
        shuffle_array_memory = np.random.choice(N, N, replace=False)
        shuffle_array[:] = shuffle_array_memory
        order_array = shuffle_array
    if order_array is not None:
        indices_r = np.zeros_like(order_array)
        indices_r[order_array] = np.arange(len(order_array))
        order_array_inverse = indices_r
        del indices_r

    if sort:
        if selection:
            raise ValueError("sorting selections not yet supported")
        # these indices sort the input array, but we evaluate the input in sequential order and write it out in sorted order
        # e.g., not b[:] = a[indices]
        # but b[indices_r] = a
        logger.info("sorting...")
        indices = np.argsort(dataset_input.evaluate(sort))
        indices_r = np.zeros_like(indices)
        indices_r[indices] = np.arange(len(indices))
        if has_strings:
            # in this case we already have the inverse ready
            order_array_inverse = indices if ascending else indices[:--1]
        else:
            del indices
        order_array = indices_r if ascending else indices_r[::-1]
        logger.info("sorting done")

    if progress == True:
        progress = vaex.utils.progressbar_callable(title="exporting")
    progress = progress or (lambda value: True)
    progress_total = len(column_names) * len(dataset_input)
    progress_status = ProgressStatus()
    progress_status.cancelled = False
    progress_status.value = 0
    if selection:
        full_mask = dataset_input.evaluate_selection_mask(selection)
    else:
        full_mask = None

    sparse_groups = collections.defaultdict(list)
    sparse_matrices = {}  # alternative to a set of matrices, since they are not hashable
    string_columns = []
    futures = []

    thread_pool = concurrent.futures.ThreadPoolExecutor(max_workers=1) 
    if True:
        for column_name in column_names:
            sparse_matrix = dataset_output._sparse_matrix(column_name)
            if sparse_matrix is not None:
                # sparse columns are written differently
                sparse_groups[id(sparse_matrix)].append(column_name)
                sparse_matrices[id(sparse_matrix)] = sparse_matrix
                continue
            logger.debug("  exporting column: %s " % column_name)
            future = thread_pool.submit(_export_column, dataset_input, dataset_output, column_name, full_mask,
                shuffle, sort, selection, N, order_array, order_array_inverse, progress_status)
            futures.append(future)

    done = False
    while not done:
        done = True
        for future in futures:
            try:
                future.result(0.1/4)
            except concurrent.futures.TimeoutError:
                done = False
                break
        if not done:
            if not progress(progress_status.value / float(progress_total)):
                progress_status.cancelled = True

    for sparse_matrix_id, column_names in sparse_groups.items():
        sparse_matrix = sparse_matrices[sparse_matrix_id]
        for column_name in column_names:
            assert not shuffle
            assert selection in [None, False]
            column = dataset_output.columns[column_name]
            column.matrix.data[:] = dataset_input.columns[column_name].matrix.data
            column.matrix.indptr[:] = dataset_input.columns[column_name].matrix.indptr
            column.matrix.indices[:] = dataset_input.columns[column_name].matrix.indices
    return column_names
def _export ( dataset_input , dataset_output , random_index_column , path , column_names = None , byteorder = "=" , shuffle = False , selection = False , progress = None , virtual = True , sort = None , ascending = True ) : if selection : if selection == True : # easier to work with the name selection = "default" N = len ( dataset_input ) if not selection else dataset_input . selected_length ( selection ) if N == 0 : raise ValueError ( "Cannot export empty table" ) if shuffle and sort : raise ValueError ( "Cannot shuffle and sort at the same time" ) if shuffle : shuffle_array = dataset_output . columns [ random_index_column ] partial_shuffle = shuffle and len ( dataset_input ) != N order_array = None order_array_inverse = None # for strings we also need the inverse order_array, keep track of that has_strings = any ( [ dataset_input . dtype ( k ) == str_type for k in column_names ] ) if partial_shuffle : # if we only export a portion, we need to create the full length random_index array, and shuffle_array_full = np . random . choice ( len ( dataset_input ) , len ( dataset_input ) , replace = False ) # then take a section of it shuffle_array [ : ] = shuffle_array_full [ shuffle_array_full < N ] del shuffle_array_full order_array = shuffle_array elif shuffle : # better to do this in memory shuffle_array_memory = np . random . choice ( N , N , replace = False ) shuffle_array [ : ] = shuffle_array_memory order_array = shuffle_array if order_array is not None : indices_r = np . zeros_like ( order_array ) indices_r [ order_array ] = np . arange ( len ( order_array ) ) order_array_inverse = indices_r del indices_r if sort : if selection : raise ValueError ( "sorting selections not yet supported" ) # these indices sort the input array, but we evaluate the input in sequential order and write it out in sorted order # e.g., not b[:] = a[indices] # but b[indices_r] = a logger . info ( "sorting..." ) indices = np . argsort ( dataset_input . evaluate ( sort ) ) indices_r = np . zeros_like ( indices ) indices_r [ indices ] = np . arange ( len ( indices ) ) if has_strings : # in this case we already have the inverse ready order_array_inverse = indices if ascending else indices [ : - - 1 ] else : del indices order_array = indices_r if ascending else indices_r [ : : - 1 ] logger . info ( "sorting done" ) if progress == True : progress = vaex . utils . progressbar_callable ( title = "exporting" ) progress = progress or ( lambda value : True ) progress_total = len ( column_names ) * len ( dataset_input ) progress_status = ProgressStatus ( ) progress_status . cancelled = False progress_status . value = 0 if selection : full_mask = dataset_input . evaluate_selection_mask ( selection ) else : full_mask = None sparse_groups = collections . defaultdict ( list ) sparse_matrices = { } # alternative to a set of matrices, since they are not hashable string_columns = [ ] futures = [ ] thread_pool = concurrent . futures . ThreadPoolExecutor ( max_workers = 1 ) if True : for column_name in column_names : sparse_matrix = dataset_output . _sparse_matrix ( column_name ) if sparse_matrix is not None : # sparse columns are written differently sparse_groups [ id ( sparse_matrix ) ] . append ( column_name ) sparse_matrices [ id ( sparse_matrix ) ] = sparse_matrix continue logger . debug ( "  exporting column: %s " % column_name ) future = thread_pool . submit ( _export_column , dataset_input , dataset_output , column_name , full_mask , shuffle , sort , selection , N , order_array , order_array_inverse , progress_status ) futures . append ( future ) done = False while not done : done = True for future in futures : try : future . result ( 0.1 / 4 ) except concurrent . futures . TimeoutError : done = False break if not done : if not progress ( progress_status . value / float ( progress_total ) ) : progress_status . cancelled = True for sparse_matrix_id , column_names in sparse_groups . items ( ) : sparse_matrix = sparse_matrices [ sparse_matrix_id ] for column_name in column_names : assert not shuffle assert selection in [ None , False ] column = dataset_output . columns [ column_name ] column . matrix . data [ : ] = dataset_input . columns [ column_name ] . matrix . data column . matrix . indptr [ : ] = dataset_input . columns [ column_name ] . matrix . indptr column . matrix . indices [ : ] = dataset_input . columns [ column_name ] . matrix . indices return column_names
:param DatasetLocal dataset: dataset to export
    :param str path: path for file
    :param lis[str] column_names: list of column names to export or None for all columns
    :param str byteorder: = for native, < for little endian and > for big endian
    :param bool shuffle: export rows in random order
    :param bool selection: export selection or not
    :param progress: progress callback that gets a progress fraction as argument and should return True to continue
    :return:
: param DatasetLocal dataset : dataset to export : param str path : path for file : param lis [ str ] column_names : list of column names to export or None for all columns : param str byteorder : = for native < for little endian and > for big endian : param bool shuffle : export rows in random order : param bool selection : export selection or not : param progress : progress callback that gets a progress fraction as argument and should return True to continue : return :