packages/vaex-core/vaex/__init__.py
def open(path, convert=False, shuffle=False, copy_index=True, *args, **kwargs):
    """Open a DataFrame from file given by path.

    Example:

    >>> ds = vaex.open('sometable.hdf5')
    >>> ds = vaex.open('somedata*.csv', convert='bigdata.hdf5')

    :param str path: local or absolute path to file, or glob string
    :param convert: convert files to an hdf5 file for optimization, can also be a path
    :param bool shuffle: shuffle converted DataFrame or not
    :param args: extra arguments for file readers that need it
    :param kwargs: extra keyword arguments
    :param bool copy_index: copy index when source is read via pandas
    :return: return a DataFrame on succes, otherwise None
    :rtype: DataFrame

    """
    import vaex
    try:
        if path in aliases:
            path = aliases[path]
        if path.startswith("http://") or path.startswith("ws://"):  # TODO: think about https and wss
            server, DataFrame = path.rsplit("/", 1)
            server = vaex.server(server, **kwargs)
            DataFrames = server.DataFrames(as_dict=True)
            if DataFrame not in DataFrames:
                raise KeyError("no such DataFrame '%s' at server, possible DataFrame names: %s" % (DataFrame, " ".join(DataFrames.keys())))
            return DataFrames[DataFrame]
        if path.startswith("cluster"):
            import vaex.distributed
            return vaex.distributed.open(path, *args, **kwargs)
        else:
            import vaex.file
            import glob
            # sort to get predicatable behaviour (useful for testing)
            filenames = list(sorted(glob.glob(path)))
            ds = None
            if len(filenames) == 0:
                raise IOError('Could not open file: {}, it does not exist'.format(path))
            filename_hdf5 = _convert_name(filenames, shuffle=shuffle)
            filename_hdf5_noshuffle = _convert_name(filenames, shuffle=False)
            if len(filenames) == 1:
                path = filenames[0]
                ext = os.path.splitext(path)[1]
                if os.path.exists(filename_hdf5) and convert:  # also check mtime?
                    if convert:
                        ds = vaex.file.open(filename_hdf5)
                    else:
                        ds = vaex.file.open(filename_hdf5, *args, **kwargs)
                else:
                    if ext == '.csv':  # special support for csv.. should probably approach it a different way
                        ds = from_csv(path, copy_index=copy_index, **kwargs)
                    else:
                        ds = vaex.file.open(path, *args, **kwargs)
                    if convert:
                        ds.export_hdf5(filename_hdf5, shuffle=shuffle)
                        ds = vaex.file.open(filename_hdf5) # argument were meant for pandas?
                if ds is None:
                    if os.path.exists(path):
                        raise IOError('Could not open file: {}, did you install vaex-hdf5?'.format(path))
                    if os.path.exists(path):
                        raise IOError('Could not open file: {}, it does not exist?'.format(path))
            elif len(filenames) > 1:
                if convert not in [True, False]:
                    filename_hdf5 = convert
                else:
                    filename_hdf5 = _convert_name(filenames, shuffle=shuffle)
                if os.path.exists(filename_hdf5) and convert:  # also check mtime
                    ds = open(filename_hdf5)
                else:
                    # with ProcessPoolExecutor() as executor:
                    # executor.submit(read_csv_and_convert, filenames, shuffle=shuffle, **kwargs)
                    DataFrames = []
                    for filename in filenames:
                        DataFrames.append(open(filename, convert=bool(convert), shuffle=shuffle, **kwargs))
                    ds = vaex.dataframe.DataFrameConcatenated(DataFrames)
                if convert:
                    ds.export_hdf5(filename_hdf5, shuffle=shuffle)
                    ds = vaex.file.open(filename_hdf5, *args, **kwargs)

        if ds is None:
            raise IOError('Unknown error opening: {}'.format(path))
        return ds
    except:
        logging.getLogger("vaex").error("error opening %r" % path)
        raise
def open(path, convert=False, shuffle=False, copy_index=True, *args, **kwargs):
    """Open a DataFrame from file given by path.

    Example:

    >>> ds = vaex.open('sometable.hdf5')
    >>> ds = vaex.open('somedata*.csv', convert='bigdata.hdf5')

    :param str path: local or absolute path to file, or glob string
    :param convert: convert files to an hdf5 file for optimization, can also be a path
    :param bool shuffle: shuffle converted DataFrame or not
    :param args: extra arguments for file readers that need it
    :param kwargs: extra keyword arguments
    :param bool copy_index: copy index when source is read via pandas
    :return: return a DataFrame on succes, otherwise None
    :rtype: DataFrame

    """
    import vaex
    try:
        if path in aliases:
            path = aliases[path]
        if path.startswith("http://") or path.startswith("ws://"):  # TODO: think about https and wss
            server, DataFrame = path.rsplit("/", 1)
            server = vaex.server(server, **kwargs)
            DataFrames = server.DataFrames(as_dict=True)
            if DataFrame not in DataFrames:
                raise KeyError("no such DataFrame '%s' at server, possible DataFrame names: %s" % (DataFrame, " ".join(DataFrames.keys())))
            return DataFrames[DataFrame]
        if path.startswith("cluster"):
            import vaex.distributed
            return vaex.distributed.open(path, *args, **kwargs)
        else:
            import vaex.file
            import glob
            # sort to get predicatable behaviour (useful for testing)
            filenames = list(sorted(glob.glob(path)))
            ds = None
            if len(filenames) == 0:
                raise IOError('Could not open file: {}, it does not exist'.format(path))
            filename_hdf5 = _convert_name(filenames, shuffle=shuffle)
            filename_hdf5_noshuffle = _convert_name(filenames, shuffle=False)
            if len(filenames) == 1:
                path = filenames[0]
                ext = os.path.splitext(path)[1]
                if os.path.exists(filename_hdf5) and convert:  # also check mtime?
                    if convert:
                        ds = vaex.file.open(filename_hdf5)
                    else:
                        ds = vaex.file.open(filename_hdf5, *args, **kwargs)
                else:
                    if ext == '.csv':  # special support for csv.. should probably approach it a different way
                        ds = from_csv(path, copy_index=copy_index, **kwargs)
                    else:
                        ds = vaex.file.open(path, *args, **kwargs)
                    if convert:
                        ds.export_hdf5(filename_hdf5, shuffle=shuffle)
                        ds = vaex.file.open(filename_hdf5) # argument were meant for pandas?
                if ds is None:
                    if os.path.exists(path):
                        raise IOError('Could not open file: {}, did you install vaex-hdf5?'.format(path))
                    if os.path.exists(path):
                        raise IOError('Could not open file: {}, it does not exist?'.format(path))
            elif len(filenames) > 1:
                if convert not in [True, False]:
                    filename_hdf5 = convert
                else:
                    filename_hdf5 = _convert_name(filenames, shuffle=shuffle)
                if os.path.exists(filename_hdf5) and convert:  # also check mtime
                    ds = open(filename_hdf5)
                else:
                    # with ProcessPoolExecutor() as executor:
                    # executor.submit(read_csv_and_convert, filenames, shuffle=shuffle, **kwargs)
                    DataFrames = []
                    for filename in filenames:
                        DataFrames.append(open(filename, convert=bool(convert), shuffle=shuffle, **kwargs))
                    ds = vaex.dataframe.DataFrameConcatenated(DataFrames)
                if convert:
                    ds.export_hdf5(filename_hdf5, shuffle=shuffle)
                    ds = vaex.file.open(filename_hdf5, *args, **kwargs)

        if ds is None:
            raise IOError('Unknown error opening: {}'.format(path))
        return ds
    except:
        logging.getLogger("vaex").error("error opening %r" % path)
        raise
def open ( path , convert = False , shuffle = False , copy_index = True , * args , * * kwargs ) : import vaex try : if path in aliases : path = aliases [ path ] if path . startswith ( "http://" ) or path . startswith ( "ws://" ) : # TODO: think about https and wss server , DataFrame = path . rsplit ( "/" , 1 ) server = vaex . server ( server , * * kwargs ) DataFrames = server . DataFrames ( as_dict = True ) if DataFrame not in DataFrames : raise KeyError ( "no such DataFrame '%s' at server, possible DataFrame names: %s" % ( DataFrame , " " . join ( DataFrames . keys ( ) ) ) ) return DataFrames [ DataFrame ] if path . startswith ( "cluster" ) : import vaex . distributed return vaex . distributed . open ( path , * args , * * kwargs ) else : import vaex . file import glob # sort to get predicatable behaviour (useful for testing) filenames = list ( sorted ( glob . glob ( path ) ) ) ds = None if len ( filenames ) == 0 : raise IOError ( 'Could not open file: {}, it does not exist' . format ( path ) ) filename_hdf5 = _convert_name ( filenames , shuffle = shuffle ) filename_hdf5_noshuffle = _convert_name ( filenames , shuffle = False ) if len ( filenames ) == 1 : path = filenames [ 0 ] ext = os . path . splitext ( path ) [ 1 ] if os . path . exists ( filename_hdf5 ) and convert : # also check mtime? if convert : ds = vaex . file . open ( filename_hdf5 ) else : ds = vaex . file . open ( filename_hdf5 , * args , * * kwargs ) else : if ext == '.csv' : # special support for csv.. should probably approach it a different way ds = from_csv ( path , copy_index = copy_index , * * kwargs ) else : ds = vaex . file . open ( path , * args , * * kwargs ) if convert : ds . export_hdf5 ( filename_hdf5 , shuffle = shuffle ) ds = vaex . file . open ( filename_hdf5 ) # argument were meant for pandas? if ds is None : if os . path . exists ( path ) : raise IOError ( 'Could not open file: {}, did you install vaex-hdf5?' . format ( path ) ) if os . path . exists ( path ) : raise IOError ( 'Could not open file: {}, it does not exist?' . format ( path ) ) elif len ( filenames ) > 1 : if convert not in [ True , False ] : filename_hdf5 = convert else : filename_hdf5 = _convert_name ( filenames , shuffle = shuffle ) if os . path . exists ( filename_hdf5 ) and convert : # also check mtime ds = open ( filename_hdf5 ) else : # with ProcessPoolExecutor() as executor: # executor.submit(read_csv_and_convert, filenames, shuffle=shuffle, **kwargs) DataFrames = [ ] for filename in filenames : DataFrames . append ( open ( filename , convert = bool ( convert ) , shuffle = shuffle , * * kwargs ) ) ds = vaex . dataframe . DataFrameConcatenated ( DataFrames ) if convert : ds . export_hdf5 ( filename_hdf5 , shuffle = shuffle ) ds = vaex . file . open ( filename_hdf5 , * args , * * kwargs ) if ds is None : raise IOError ( 'Unknown error opening: {}' . format ( path ) ) return ds except : logging . getLogger ( "vaex" ) . error ( "error opening %r" % path ) raise
Open a DataFrame from file given by path.

    Example:

    >>> ds = vaex.open('sometable.hdf5')
    >>> ds = vaex.open('somedata*.csv', convert='bigdata.hdf5')

    :param str path: local or absolute path to file, or glob string
    :param convert: convert files to an hdf5 file for optimization, can also be a path
    :param bool shuffle: shuffle converted DataFrame or not
    :param args: extra arguments for file readers that need it
    :param kwargs: extra keyword arguments
    :param bool copy_index: copy index when source is read via pandas
    :return: return a DataFrame on succes, otherwise None
    :rtype: DataFrame
Open a DataFrame from file given by path .