h2o-py/h2o/h2o.py
def parse_setup(raw_frames, destination_frame=None, header=0, separator=None, column_names=None,
                column_types=None, na_strings=None, skipped_columns=None, custom_non_data_line_markers=None):
    """
    Retrieve H2O's best guess as to what the structure of the data file is.

    During parse setup, the H2O cluster will make several guesses about the attributes of
    the data. This method allows a user to perform corrective measures by updating the
    returning dictionary from this method. This dictionary is then fed into `parse_raw` to
    produce the H2OFrame instance.

    :param raw_frames: a collection of imported file frames
    :param destination_frame: The unique hex key assigned to the imported file. If none is given, a key will
        automatically be generated.
    :param header: -1 means the first line is data, 0 means guess, 1 means first line is header.
    :param separator: The field separator character. Values on each line of the file are separated by
        this character. If not provided, the parser will automatically detect the separator.
    :param column_names: A list of column names for the file. If skipped_columns are specified, only list column names
         of columns that are not skipped.
    :param column_types: A list of types or a dictionary of column names to types to specify whether columns
        should be forced to a certain type upon import parsing. If a list, the types for elements that are
        one will be guessed. If skipped_columns are specified, only list column types of columns that are not skipped.
        The possible types a column may have are:

        - "unknown" - this will force the column to be parsed as all NA
        - "uuid"    - the values in the column must be true UUID or will be parsed as NA
        - "string"  - force the column to be parsed as a string
        - "numeric" - force the column to be parsed as numeric. H2O will handle the compression of the numeric
          data in the optimal manner.
        - "enum"    - force the column to be parsed as a categorical column.
        - "time"    - force the column to be parsed as a time column. H2O will attempt to parse the following
          list of date time formats: (date) "yyyy-MM-dd", "yyyy MM dd", "dd-MMM-yy", "dd MMM yy", (time)
          "HH:mm:ss", "HH:mm:ss:SSS", "HH:mm:ss:SSSnnnnnn", "HH.mm.ss" "HH.mm.ss.SSS", "HH.mm.ss.SSSnnnnnn".
          Times can also contain "AM" or "PM".

    :param na_strings: A list of strings, or a list of lists of strings (one list per column), or a dictionary
        of column names to strings which are to be interpreted as missing values.
    :param skipped_columns: an integer lists of column indices to skip and not parsed into the final frame from the import file.
    :param custom_non_data_line_markers: If a line in imported file starts with any character in given string it will NOT be imported. Empty string means all lines are imported, None means that default behaviour for given format will be used

    :returns: a dictionary containing parse parameters guessed by the H2O backend.
    """
    coltype = U(None, "unknown", "uuid", "string", "float", "real", "double", "int", "numeric",
                "categorical", "factor", "enum", "time")
    natype = U(str, [str])
    assert_is_type(raw_frames, str, [str])
    assert_is_type(destination_frame, None, str)
    assert_is_type(header, -1, 0, 1)
    assert_is_type(separator, None, I(str, lambda s: len(s) == 1))
    assert_is_type(column_names, [str], None)
    assert_is_type(column_types, [coltype], {str: coltype}, None)
    assert_is_type(na_strings, [natype], {str: natype}, None)
    check_frame_id(destination_frame)

    # The H2O backend only accepts things that are quoted
    if is_type(raw_frames, str): raw_frames = [raw_frames]

    # temporary dictionary just to pass the following information to the parser: header, separator
    kwargs = {"check_header": header, "source_frames": [quoted(frame_id) for frame_id in raw_frames]}
    if separator:
        kwargs["separator"] = ord(separator)
    
    if custom_non_data_line_markers is not None:
        kwargs["custom_non_data_line_markers"] = custom_non_data_line_markers;

    j = api("POST /3/ParseSetup", data=kwargs)
    if "warnings" in j and j["warnings"]:
        for w in j["warnings"]:
            warnings.warn(w)
    # TODO: really should be url encoding...
    if destination_frame:
        j["destination_frame"] = destination_frame

    parse_column_len = len(j["column_types"]) if skipped_columns is None else (len(j["column_types"])-len(skipped_columns))
    tempColumnNames = j["column_names"] if j["column_names"] is not None else gen_header(j["number_columns"])
    useType = [True]*len(tempColumnNames)
    if skipped_columns is not None:
        useType = [True]*len(tempColumnNames)

        for ind in range(len(tempColumnNames)):
            if ind in skipped_columns:
                useType[ind]=False

    if column_names is not None:
        if not isinstance(column_names, list): raise ValueError("col_names should be a list")
        if (skipped_columns is not None) and len(skipped_columns)>0:
            if (len(column_names)) != parse_column_len:
                raise ValueError(
                    "length of col_names should be equal to the number of columns parsed: %d vs %d"
                    % (len(column_names), parse_column_len))
        else:
            if len(column_names) != len(j["column_types"]): raise ValueError(
                "length of col_names should be equal to the number of columns: %d vs %d"
                % (len(column_names), len(j["column_types"])))
        j["column_names"] = column_names
        counter = 0
        for ind in range(len(tempColumnNames)):
            if useType[ind]:
                tempColumnNames[ind]=column_names[counter]
                counter=counter+1

    if (column_types is not None): # keep the column types to include all columns
        if isinstance(column_types, dict):
            # overwrite dictionary to ordered list of column types. if user didn't specify column type for all names,
            # use type provided by backend
            if j["column_names"] is None:  # no colnames discovered! (C1, C2, ...)
                j["column_names"] = gen_header(j["number_columns"])
            if not set(column_types.keys()).issubset(set(j["column_names"])): raise ValueError(
                "names specified in col_types is not a subset of the column names")
            idx = 0
            column_types_list = []

            for name in tempColumnNames: # column_names may have already been changed
                if name in column_types:
                    column_types_list.append(column_types[name])
                else:
                    column_types_list.append(j["column_types"][idx])
                idx += 1

            column_types = column_types_list

        elif isinstance(column_types, list):
            if len(column_types) != parse_column_len: raise ValueError(
                "length of col_types should be equal to the number of parsed columns")
            # need to expand it out to all columns, not just the parsed ones
            column_types_list = j["column_types"]
            counter = 0
            for ind in range(len(j["column_types"])):
                if useType[ind] and (column_types[counter]!=None):
                    column_types_list[ind]=column_types[counter]
                    counter=counter+1

            column_types = column_types_list
        else:  # not dictionary or list
            raise ValueError("col_types should be a list of types or a dictionary of column names to types")
        j["column_types"] = column_types
    if na_strings is not None:
        if isinstance(na_strings, dict):
            # overwrite dictionary to ordered list of lists of na_strings
            if not j["column_names"]: raise ValueError("column names should be specified")
            if not set(na_strings.keys()).issubset(set(j["column_names"])): raise ValueError(
                "names specified in na_strings is not a subset of the column names")
            j["na_strings"] = [[] for _ in range(len(j["column_names"]))]
            for name, na in na_strings.items():
                idx = j["column_names"].index(name)
                if is_type(na, str): na = [na]
                for n in na: j["na_strings"][idx].append(quoted(n))
        elif is_type(na_strings, [[str]]):
            if len(na_strings) != len(j["column_types"]):
                raise ValueError("length of na_strings should be equal to the number of columns")
            j["na_strings"] = [[quoted(na) for na in col] if col is not None else [] for col in na_strings]
        elif isinstance(na_strings, list):
            j["na_strings"] = [[quoted(na) for na in na_strings]] * len(j["column_types"])
        else:  # not a dictionary or list
            raise ValueError(
                "na_strings should be a list, a list of lists (one list per column), or a dictionary of column "
                "names to strings which are to be interpreted as missing values")

    if skipped_columns is not None:
        if isinstance(skipped_columns, list):
            j["skipped_columns"] = []
            for colidx in skipped_columns:
                if (colidx < 0): raise ValueError("skipped column index cannot be negative")
                j["skipped_columns"].append(colidx)
    

    # quote column names and column types also when not specified by user
    if j["column_names"]: j["column_names"] = list(map(quoted, j["column_names"]))
    j["column_types"] = list(map(quoted, j["column_types"]))
    return j
def parse_setup(raw_frames, destination_frame=None, header=0, separator=None, column_names=None,
                column_types=None, na_strings=None, skipped_columns=None, custom_non_data_line_markers=None):
    """
    Retrieve H2O's best guess as to what the structure of the data file is.

    During parse setup, the H2O cluster will make several guesses about the attributes of
    the data. This method allows a user to perform corrective measures by updating the
    returning dictionary from this method. This dictionary is then fed into `parse_raw` to
    produce the H2OFrame instance.

    :param raw_frames: a collection of imported file frames
    :param destination_frame: The unique hex key assigned to the imported file. If none is given, a key will
        automatically be generated.
    :param header: -1 means the first line is data, 0 means guess, 1 means first line is header.
    :param separator: The field separator character. Values on each line of the file are separated by
        this character. If not provided, the parser will automatically detect the separator.
    :param column_names: A list of column names for the file. If skipped_columns are specified, only list column names
         of columns that are not skipped.
    :param column_types: A list of types or a dictionary of column names to types to specify whether columns
        should be forced to a certain type upon import parsing. If a list, the types for elements that are
        one will be guessed. If skipped_columns are specified, only list column types of columns that are not skipped.
        The possible types a column may have are:

        - "unknown" - this will force the column to be parsed as all NA
        - "uuid"    - the values in the column must be true UUID or will be parsed as NA
        - "string"  - force the column to be parsed as a string
        - "numeric" - force the column to be parsed as numeric. H2O will handle the compression of the numeric
          data in the optimal manner.
        - "enum"    - force the column to be parsed as a categorical column.
        - "time"    - force the column to be parsed as a time column. H2O will attempt to parse the following
          list of date time formats: (date) "yyyy-MM-dd", "yyyy MM dd", "dd-MMM-yy", "dd MMM yy", (time)
          "HH:mm:ss", "HH:mm:ss:SSS", "HH:mm:ss:SSSnnnnnn", "HH.mm.ss" "HH.mm.ss.SSS", "HH.mm.ss.SSSnnnnnn".
          Times can also contain "AM" or "PM".

    :param na_strings: A list of strings, or a list of lists of strings (one list per column), or a dictionary
        of column names to strings which are to be interpreted as missing values.
    :param skipped_columns: an integer lists of column indices to skip and not parsed into the final frame from the import file.
    :param custom_non_data_line_markers: If a line in imported file starts with any character in given string it will NOT be imported. Empty string means all lines are imported, None means that default behaviour for given format will be used

    :returns: a dictionary containing parse parameters guessed by the H2O backend.
    """
    coltype = U(None, "unknown", "uuid", "string", "float", "real", "double", "int", "numeric",
                "categorical", "factor", "enum", "time")
    natype = U(str, [str])
    assert_is_type(raw_frames, str, [str])
    assert_is_type(destination_frame, None, str)
    assert_is_type(header, -1, 0, 1)
    assert_is_type(separator, None, I(str, lambda s: len(s) == 1))
    assert_is_type(column_names, [str], None)
    assert_is_type(column_types, [coltype], {str: coltype}, None)
    assert_is_type(na_strings, [natype], {str: natype}, None)
    check_frame_id(destination_frame)

    # The H2O backend only accepts things that are quoted
    if is_type(raw_frames, str): raw_frames = [raw_frames]

    # temporary dictionary just to pass the following information to the parser: header, separator
    kwargs = {"check_header": header, "source_frames": [quoted(frame_id) for frame_id in raw_frames]}
    if separator:
        kwargs["separator"] = ord(separator)
    
    if custom_non_data_line_markers is not None:
        kwargs["custom_non_data_line_markers"] = custom_non_data_line_markers;

    j = api("POST /3/ParseSetup", data=kwargs)
    if "warnings" in j and j["warnings"]:
        for w in j["warnings"]:
            warnings.warn(w)
    # TODO: really should be url encoding...
    if destination_frame:
        j["destination_frame"] = destination_frame

    parse_column_len = len(j["column_types"]) if skipped_columns is None else (len(j["column_types"])-len(skipped_columns))
    tempColumnNames = j["column_names"] if j["column_names"] is not None else gen_header(j["number_columns"])
    useType = [True]*len(tempColumnNames)
    if skipped_columns is not None:
        useType = [True]*len(tempColumnNames)

        for ind in range(len(tempColumnNames)):
            if ind in skipped_columns:
                useType[ind]=False

    if column_names is not None:
        if not isinstance(column_names, list): raise ValueError("col_names should be a list")
        if (skipped_columns is not None) and len(skipped_columns)>0:
            if (len(column_names)) != parse_column_len:
                raise ValueError(
                    "length of col_names should be equal to the number of columns parsed: %d vs %d"
                    % (len(column_names), parse_column_len))
        else:
            if len(column_names) != len(j["column_types"]): raise ValueError(
                "length of col_names should be equal to the number of columns: %d vs %d"
                % (len(column_names), len(j["column_types"])))
        j["column_names"] = column_names
        counter = 0
        for ind in range(len(tempColumnNames)):
            if useType[ind]:
                tempColumnNames[ind]=column_names[counter]
                counter=counter+1

    if (column_types is not None): # keep the column types to include all columns
        if isinstance(column_types, dict):
            # overwrite dictionary to ordered list of column types. if user didn't specify column type for all names,
            # use type provided by backend
            if j["column_names"] is None:  # no colnames discovered! (C1, C2, ...)
                j["column_names"] = gen_header(j["number_columns"])
            if not set(column_types.keys()).issubset(set(j["column_names"])): raise ValueError(
                "names specified in col_types is not a subset of the column names")
            idx = 0
            column_types_list = []

            for name in tempColumnNames: # column_names may have already been changed
                if name in column_types:
                    column_types_list.append(column_types[name])
                else:
                    column_types_list.append(j["column_types"][idx])
                idx += 1

            column_types = column_types_list

        elif isinstance(column_types, list):
            if len(column_types) != parse_column_len: raise ValueError(
                "length of col_types should be equal to the number of parsed columns")
            # need to expand it out to all columns, not just the parsed ones
            column_types_list = j["column_types"]
            counter = 0
            for ind in range(len(j["column_types"])):
                if useType[ind] and (column_types[counter]!=None):
                    column_types_list[ind]=column_types[counter]
                    counter=counter+1

            column_types = column_types_list
        else:  # not dictionary or list
            raise ValueError("col_types should be a list of types or a dictionary of column names to types")
        j["column_types"] = column_types
    if na_strings is not None:
        if isinstance(na_strings, dict):
            # overwrite dictionary to ordered list of lists of na_strings
            if not j["column_names"]: raise ValueError("column names should be specified")
            if not set(na_strings.keys()).issubset(set(j["column_names"])): raise ValueError(
                "names specified in na_strings is not a subset of the column names")
            j["na_strings"] = [[] for _ in range(len(j["column_names"]))]
            for name, na in na_strings.items():
                idx = j["column_names"].index(name)
                if is_type(na, str): na = [na]
                for n in na: j["na_strings"][idx].append(quoted(n))
        elif is_type(na_strings, [[str]]):
            if len(na_strings) != len(j["column_types"]):
                raise ValueError("length of na_strings should be equal to the number of columns")
            j["na_strings"] = [[quoted(na) for na in col] if col is not None else [] for col in na_strings]
        elif isinstance(na_strings, list):
            j["na_strings"] = [[quoted(na) for na in na_strings]] * len(j["column_types"])
        else:  # not a dictionary or list
            raise ValueError(
                "na_strings should be a list, a list of lists (one list per column), or a dictionary of column "
                "names to strings which are to be interpreted as missing values")

    if skipped_columns is not None:
        if isinstance(skipped_columns, list):
            j["skipped_columns"] = []
            for colidx in skipped_columns:
                if (colidx < 0): raise ValueError("skipped column index cannot be negative")
                j["skipped_columns"].append(colidx)
    

    # quote column names and column types also when not specified by user
    if j["column_names"]: j["column_names"] = list(map(quoted, j["column_names"]))
    j["column_types"] = list(map(quoted, j["column_types"]))
    return j
def parse_setup ( raw_frames , destination_frame = None , header = 0 , separator = None , column_names = None , column_types = None , na_strings = None , skipped_columns = None , custom_non_data_line_markers = None ) : coltype = U ( None , "unknown" , "uuid" , "string" , "float" , "real" , "double" , "int" , "numeric" , "categorical" , "factor" , "enum" , "time" ) natype = U ( str , [ str ] ) assert_is_type ( raw_frames , str , [ str ] ) assert_is_type ( destination_frame , None , str ) assert_is_type ( header , - 1 , 0 , 1 ) assert_is_type ( separator , None , I ( str , lambda s : len ( s ) == 1 ) ) assert_is_type ( column_names , [ str ] , None ) assert_is_type ( column_types , [ coltype ] , { str : coltype } , None ) assert_is_type ( na_strings , [ natype ] , { str : natype } , None ) check_frame_id ( destination_frame ) # The H2O backend only accepts things that are quoted if is_type ( raw_frames , str ) : raw_frames = [ raw_frames ] # temporary dictionary just to pass the following information to the parser: header, separator kwargs = { "check_header" : header , "source_frames" : [ quoted ( frame_id ) for frame_id in raw_frames ] } if separator : kwargs [ "separator" ] = ord ( separator ) if custom_non_data_line_markers is not None : kwargs [ "custom_non_data_line_markers" ] = custom_non_data_line_markers j = api ( "POST /3/ParseSetup" , data = kwargs ) if "warnings" in j and j [ "warnings" ] : for w in j [ "warnings" ] : warnings . warn ( w ) # TODO: really should be url encoding... if destination_frame : j [ "destination_frame" ] = destination_frame parse_column_len = len ( j [ "column_types" ] ) if skipped_columns is None else ( len ( j [ "column_types" ] ) - len ( skipped_columns ) ) tempColumnNames = j [ "column_names" ] if j [ "column_names" ] is not None else gen_header ( j [ "number_columns" ] ) useType = [ True ] * len ( tempColumnNames ) if skipped_columns is not None : useType = [ True ] * len ( tempColumnNames ) for ind in range ( len ( tempColumnNames ) ) : if ind in skipped_columns : useType [ ind ] = False if column_names is not None : if not isinstance ( column_names , list ) : raise ValueError ( "col_names should be a list" ) if ( skipped_columns is not None ) and len ( skipped_columns ) > 0 : if ( len ( column_names ) ) != parse_column_len : raise ValueError ( "length of col_names should be equal to the number of columns parsed: %d vs %d" % ( len ( column_names ) , parse_column_len ) ) else : if len ( column_names ) != len ( j [ "column_types" ] ) : raise ValueError ( "length of col_names should be equal to the number of columns: %d vs %d" % ( len ( column_names ) , len ( j [ "column_types" ] ) ) ) j [ "column_names" ] = column_names counter = 0 for ind in range ( len ( tempColumnNames ) ) : if useType [ ind ] : tempColumnNames [ ind ] = column_names [ counter ] counter = counter + 1 if ( column_types is not None ) : # keep the column types to include all columns if isinstance ( column_types , dict ) : # overwrite dictionary to ordered list of column types. if user didn't specify column type for all names, # use type provided by backend if j [ "column_names" ] is None : # no colnames discovered! (C1, C2, ...) j [ "column_names" ] = gen_header ( j [ "number_columns" ] ) if not set ( column_types . keys ( ) ) . issubset ( set ( j [ "column_names" ] ) ) : raise ValueError ( "names specified in col_types is not a subset of the column names" ) idx = 0 column_types_list = [ ] for name in tempColumnNames : # column_names may have already been changed if name in column_types : column_types_list . append ( column_types [ name ] ) else : column_types_list . append ( j [ "column_types" ] [ idx ] ) idx += 1 column_types = column_types_list elif isinstance ( column_types , list ) : if len ( column_types ) != parse_column_len : raise ValueError ( "length of col_types should be equal to the number of parsed columns" ) # need to expand it out to all columns, not just the parsed ones column_types_list = j [ "column_types" ] counter = 0 for ind in range ( len ( j [ "column_types" ] ) ) : if useType [ ind ] and ( column_types [ counter ] != None ) : column_types_list [ ind ] = column_types [ counter ] counter = counter + 1 column_types = column_types_list else : # not dictionary or list raise ValueError ( "col_types should be a list of types or a dictionary of column names to types" ) j [ "column_types" ] = column_types if na_strings is not None : if isinstance ( na_strings , dict ) : # overwrite dictionary to ordered list of lists of na_strings if not j [ "column_names" ] : raise ValueError ( "column names should be specified" ) if not set ( na_strings . keys ( ) ) . issubset ( set ( j [ "column_names" ] ) ) : raise ValueError ( "names specified in na_strings is not a subset of the column names" ) j [ "na_strings" ] = [ [ ] for _ in range ( len ( j [ "column_names" ] ) ) ] for name , na in na_strings . items ( ) : idx = j [ "column_names" ] . index ( name ) if is_type ( na , str ) : na = [ na ] for n in na : j [ "na_strings" ] [ idx ] . append ( quoted ( n ) ) elif is_type ( na_strings , [ [ str ] ] ) : if len ( na_strings ) != len ( j [ "column_types" ] ) : raise ValueError ( "length of na_strings should be equal to the number of columns" ) j [ "na_strings" ] = [ [ quoted ( na ) for na in col ] if col is not None else [ ] for col in na_strings ] elif isinstance ( na_strings , list ) : j [ "na_strings" ] = [ [ quoted ( na ) for na in na_strings ] ] * len ( j [ "column_types" ] ) else : # not a dictionary or list raise ValueError ( "na_strings should be a list, a list of lists (one list per column), or a dictionary of column " "names to strings which are to be interpreted as missing values" ) if skipped_columns is not None : if isinstance ( skipped_columns , list ) : j [ "skipped_columns" ] = [ ] for colidx in skipped_columns : if ( colidx < 0 ) : raise ValueError ( "skipped column index cannot be negative" ) j [ "skipped_columns" ] . append ( colidx ) # quote column names and column types also when not specified by user if j [ "column_names" ] : j [ "column_names" ] = list ( map ( quoted , j [ "column_names" ] ) ) j [ "column_types" ] = list ( map ( quoted , j [ "column_types" ] ) ) return j
Retrieve H2O's best guess as to what the structure of the data file is.

    During parse setup, the H2O cluster will make several guesses about the attributes of
    the data. This method allows a user to perform corrective measures by updating the
    returning dictionary from this method. This dictionary is then fed into `parse_raw` to
    produce the H2OFrame instance.

    :param raw_frames: a collection of imported file frames
    :param destination_frame: The unique hex key assigned to the imported file. If none is given, a key will
        automatically be generated.
    :param header: -1 means the first line is data, 0 means guess, 1 means first line is header.
    :param separator: The field separator character. Values on each line of the file are separated by
        this character. If not provided, the parser will automatically detect the separator.
    :param column_names: A list of column names for the file. If skipped_columns are specified, only list column names
         of columns that are not skipped.
    :param column_types: A list of types or a dictionary of column names to types to specify whether columns
        should be forced to a certain type upon import parsing. If a list, the types for elements that are
        one will be guessed. If skipped_columns are specified, only list column types of columns that are not skipped.
        The possible types a column may have are:

        - "unknown" - this will force the column to be parsed as all NA
        - "uuid"    - the values in the column must be true UUID or will be parsed as NA
        - "string"  - force the column to be parsed as a string
        - "numeric" - force the column to be parsed as numeric. H2O will handle the compression of the numeric
          data in the optimal manner.
        - "enum"    - force the column to be parsed as a categorical column.
        - "time"    - force the column to be parsed as a time column. H2O will attempt to parse the following
          list of date time formats: (date) "yyyy-MM-dd", "yyyy MM dd", "dd-MMM-yy", "dd MMM yy", (time)
          "HH:mm:ss", "HH:mm:ss:SSS", "HH:mm:ss:SSSnnnnnn", "HH.mm.ss" "HH.mm.ss.SSS", "HH.mm.ss.SSSnnnnnn".
          Times can also contain "AM" or "PM".

    :param na_strings: A list of strings, or a list of lists of strings (one list per column), or a dictionary
        of column names to strings which are to be interpreted as missing values.
    :param skipped_columns: an integer lists of column indices to skip and not parsed into the final frame from the import file.
    :param custom_non_data_line_markers: If a line in imported file starts with any character in given string it will NOT be imported. Empty string means all lines are imported, None means that default behaviour for given format will be used

    :returns: a dictionary containing parse parameters guessed by the H2O backend.
Retrieve H2O s best guess as to what the structure of the data file is .