py2/h2o_ray.py
def parse(self, key, hex_key=None, columnTypeDict=None,
          timeoutSecs=300, retryDelaySecs=0.2, initialDelaySecs=None, pollTimeoutSecs=180,
          noise=None, benchmarkLogging=None, noPoll=False, intermediateResults=False, **kwargs):
    '''
    Parse an imported raw file or files into a Frame.
    '''
    # these should override what parse setup gets below
    params_dict = {
        'source_frames': None,
        'destination_frame': hex_key, 
        'parse_type': None, # file type 
        'separator': None,
        'single_quotes': None,
        'check_header': None, # forces first line to be seen as column names 
        'number_columns': None,
        'column_names': None, # a list
        'column_types': None, # a list. or can use columnTypeDict param (see below)
        'na_strings' : None, # a list
        'chunk_size': None,
        # are these two no longer supported?
        'delete_on_done': None,
        'blocking': None,
    }
        
    # if key is a list, create a comma separated string
    # list or tuple but not string
    if not isinstance(key, basestring):
        # it's a list of some kind (tuple ok?)
        # if len(key) > 1:
        #     print "I noticed you're giving me a list of > 1 keys %s to parse:" % len(key), key

        # len 1 is ok here. 0 not. what if None or [None] here
        if not key:
            raise Exception("key seems to be bad in parse. Should be list or string. %s" % key)
        # have to put double quotes around the individual list items (single not legal)
        source_frames = "[" + ",".join(map((lambda x: '"' + x + '"'), key)) + "]"

    else:
        # what if None here
        source_frames = '["' + key + '"]' # quotes required on key

    params_dict['source_frames'] = source_frames

    # merge kwargs into params_dict
    # =None overwrites params_dict

    # columnTypeDict not used here
    h2o_methods.check_params_update_kwargs(params_dict, kwargs, 'parse before setup merge', print_params=False)
    # Call ParseSetup?source_frames=[keys] . . .

    # if benchmarkLogging:
    #     cloudPerfH2O.get_log_save(initOnly=True)

    # h2o_methods.check_params_update_kwargs(params_dict, kwargs, 'parse_setup', print_params=True)
    params_setup = {'source_frames': source_frames}
    setup_result = self.do_json_request(jsonRequest="3/ParseSetup.json", cmd='post', timeout=timeoutSecs, postData=params_setup)
    h2o_sandbox.check_sandbox_for_errors()
    verboseprint("ParseSetup result:", dump_json(setup_result))

    # this should match what we gave as input?
    if setup_result['source_frames']:
        # should these be quoted?
        source_framesStr = "[" + ",".join([('"%s"' % src['name']) for src in setup_result['source_frames'] ]) + "]"
    else:
        source_framesStr = None

    # I suppose we need a way for parameters to parse() to override these
    # should it be an array or a dict?
    if setup_result['column_names']:
        # single quotes not legal..need double quotes
        columnNamesStr = "[" + ",".join(map((lambda x: '"' + x + '"'), setup_result['column_names'])) + "]"
    else:
        columnNamesStr = None

    columnTypes = setup_result['column_types']
    assert columnTypes is not None, "%s %s" % ("column_types:", columnTypes)

    if setup_result['na_strings']:
        # single quotes not legal..need double quotes
        naStrings = "[" + ",".join(map((lambda x: '"' + x + '"' if x != None else '""'), setup_result['na_strings'])) + "]"
    else:
        naStrings = None

    # dict parameter to update columnTypeDict?
    # but we don't pass columnNames like this?
    ct = setup_result['column_types']
    columnNames = setup_result['column_names']
    if columnTypeDict: 
        for k,v in columnTypeDict.iteritems():
            if isinstance(k, int):
                # if a column index
                if k>=0 and k<len(ct):
                    ct[k] = v
                else:
                    raise Exception("bad col index %s in columnTypeDict param %s" % (k, columnTypeDict))
            # if a column name
            elif isinstance(k, basestring):
                # find the index
                if k not in columnNames:
                    raise Exception("bad col name %s in columnTypeDict param %s. columnNames: %s" % (k, columnTypeDict, columnNames))
                ci = columnNames.index(k)
                ct[ci] = v
            else:
                raise Exception("%s %s should be int or string" % (k, type(k)))

    columnTypesStr = "[" + ",".join(map((lambda x: '"' + x + '"'), ct)) + "]"


    parse_params = {
        'source_frames': source_framesStr,
        'destination_frame': setup_result['destination_frame'],
        'parse_type': setup_result['parse_type'],
        'separator': setup_result['separator'],
        'single_quotes': setup_result['single_quotes'],
        'check_header': setup_result['check_header'],
        'number_columns': setup_result['number_columns'],
        'column_names': columnNamesStr,
        'column_types': columnTypesStr,
        'na_strings': naStrings, 
        'chunk_size': setup_result['chunk_size'],
        # No longer supported? how come these aren't in setup_result?
        'delete_on_done': params_dict['delete_on_done'],
        'blocking': params_dict['blocking'],
    }
    # HACK: if there are too many column names..don't print! it is crazy output
    # just check the output of parse setup. Don't worry about columnNames passed as params here. 
    tooManyColNamesToPrint = setup_result['column_names'] and len(setup_result['column_names']) > 2000
    if tooManyColNamesToPrint:
        h2p.yellow_print("Not printing the parameters to Parse because the columnNames are too lengthy.") 
        h2p.yellow_print("See sandbox/commands.log")

    # merge params_dict into parse_params
    # don't want =None to overwrite parse_params
    h2o_methods.check_params_update_kwargs(parse_params, params_dict, 'parse after merge into parse setup', 
        print_params=not tooManyColNamesToPrint, ignoreNone=True)

    print "parse source_frames is length:", len(parse_params['source_frames'])
    # This can be null now? parseSetup doesn't return default colnames?
    # print "parse column_names is length:", len(parse_params['column_names'])

    # none of the kwargs passed to here!
    parse_result = self.do_json_request( jsonRequest="3/Parse.json", cmd='post', postData=parse_params, timeout=timeoutSecs)
    verboseprint("Parse result:", dump_json(parse_result))

    job_key = parse_result['job']['key']['name']
    hex_key = parse_params['destination_frame']

    # TODO: dislike having different shapes for noPoll and poll
    if noPoll:
        # ??
        h2o_sandbox.check_sandbox_for_errors()
        # return self.jobs(job_key)
        return parse_result

    # does Frame also, while polling
    if intermediateResults:
        key = hex_key
    else:
        key = None

    job_result = self.poll_job(job_key, timeoutSecs=timeoutSecs, key=key)

    if job_result:
        jobs = job_result['jobs'][0]
        description = jobs['description']
        dest = jobs['dest']
        msec = jobs['msec']
        status = jobs['status']
        progress = jobs['progress']
        dest_key = dest['name']

        # can condition this with a parameter if some FAILED are expected by tests.
        if status=='FAILED':
            print dump_json(job_result)
            raise Exception("Taking exception on parse job status: %s %s %s %s %s" % \
                (status, progress, msec, dest_key, description))

        return self.frames(dest_key)
    else:
        # ? we should always get a job_json result
        raise Exception("parse didn't get a job_result when it expected one")
def parse(self, key, hex_key=None, columnTypeDict=None,
          timeoutSecs=300, retryDelaySecs=0.2, initialDelaySecs=None, pollTimeoutSecs=180,
          noise=None, benchmarkLogging=None, noPoll=False, intermediateResults=False, **kwargs):
    '''
    Parse an imported raw file or files into a Frame.
    '''
    # these should override what parse setup gets below
    params_dict = {
        'source_frames': None,
        'destination_frame': hex_key, 
        'parse_type': None, # file type 
        'separator': None,
        'single_quotes': None,
        'check_header': None, # forces first line to be seen as column names 
        'number_columns': None,
        'column_names': None, # a list
        'column_types': None, # a list. or can use columnTypeDict param (see below)
        'na_strings' : None, # a list
        'chunk_size': None,
        # are these two no longer supported?
        'delete_on_done': None,
        'blocking': None,
    }
        
    # if key is a list, create a comma separated string
    # list or tuple but not string
    if not isinstance(key, basestring):
        # it's a list of some kind (tuple ok?)
        # if len(key) > 1:
        #     print "I noticed you're giving me a list of > 1 keys %s to parse:" % len(key), key

        # len 1 is ok here. 0 not. what if None or [None] here
        if not key:
            raise Exception("key seems to be bad in parse. Should be list or string. %s" % key)
        # have to put double quotes around the individual list items (single not legal)
        source_frames = "[" + ",".join(map((lambda x: '"' + x + '"'), key)) + "]"

    else:
        # what if None here
        source_frames = '["' + key + '"]' # quotes required on key

    params_dict['source_frames'] = source_frames

    # merge kwargs into params_dict
    # =None overwrites params_dict

    # columnTypeDict not used here
    h2o_methods.check_params_update_kwargs(params_dict, kwargs, 'parse before setup merge', print_params=False)
    # Call ParseSetup?source_frames=[keys] . . .

    # if benchmarkLogging:
    #     cloudPerfH2O.get_log_save(initOnly=True)

    # h2o_methods.check_params_update_kwargs(params_dict, kwargs, 'parse_setup', print_params=True)
    params_setup = {'source_frames': source_frames}
    setup_result = self.do_json_request(jsonRequest="3/ParseSetup.json", cmd='post', timeout=timeoutSecs, postData=params_setup)
    h2o_sandbox.check_sandbox_for_errors()
    verboseprint("ParseSetup result:", dump_json(setup_result))

    # this should match what we gave as input?
    if setup_result['source_frames']:
        # should these be quoted?
        source_framesStr = "[" + ",".join([('"%s"' % src['name']) for src in setup_result['source_frames'] ]) + "]"
    else:
        source_framesStr = None

    # I suppose we need a way for parameters to parse() to override these
    # should it be an array or a dict?
    if setup_result['column_names']:
        # single quotes not legal..need double quotes
        columnNamesStr = "[" + ",".join(map((lambda x: '"' + x + '"'), setup_result['column_names'])) + "]"
    else:
        columnNamesStr = None

    columnTypes = setup_result['column_types']
    assert columnTypes is not None, "%s %s" % ("column_types:", columnTypes)

    if setup_result['na_strings']:
        # single quotes not legal..need double quotes
        naStrings = "[" + ",".join(map((lambda x: '"' + x + '"' if x != None else '""'), setup_result['na_strings'])) + "]"
    else:
        naStrings = None

    # dict parameter to update columnTypeDict?
    # but we don't pass columnNames like this?
    ct = setup_result['column_types']
    columnNames = setup_result['column_names']
    if columnTypeDict: 
        for k,v in columnTypeDict.iteritems():
            if isinstance(k, int):
                # if a column index
                if k>=0 and k<len(ct):
                    ct[k] = v
                else:
                    raise Exception("bad col index %s in columnTypeDict param %s" % (k, columnTypeDict))
            # if a column name
            elif isinstance(k, basestring):
                # find the index
                if k not in columnNames:
                    raise Exception("bad col name %s in columnTypeDict param %s. columnNames: %s" % (k, columnTypeDict, columnNames))
                ci = columnNames.index(k)
                ct[ci] = v
            else:
                raise Exception("%s %s should be int or string" % (k, type(k)))

    columnTypesStr = "[" + ",".join(map((lambda x: '"' + x + '"'), ct)) + "]"


    parse_params = {
        'source_frames': source_framesStr,
        'destination_frame': setup_result['destination_frame'],
        'parse_type': setup_result['parse_type'],
        'separator': setup_result['separator'],
        'single_quotes': setup_result['single_quotes'],
        'check_header': setup_result['check_header'],
        'number_columns': setup_result['number_columns'],
        'column_names': columnNamesStr,
        'column_types': columnTypesStr,
        'na_strings': naStrings, 
        'chunk_size': setup_result['chunk_size'],
        # No longer supported? how come these aren't in setup_result?
        'delete_on_done': params_dict['delete_on_done'],
        'blocking': params_dict['blocking'],
    }
    # HACK: if there are too many column names..don't print! it is crazy output
    # just check the output of parse setup. Don't worry about columnNames passed as params here. 
    tooManyColNamesToPrint = setup_result['column_names'] and len(setup_result['column_names']) > 2000
    if tooManyColNamesToPrint:
        h2p.yellow_print("Not printing the parameters to Parse because the columnNames are too lengthy.") 
        h2p.yellow_print("See sandbox/commands.log")

    # merge params_dict into parse_params
    # don't want =None to overwrite parse_params
    h2o_methods.check_params_update_kwargs(parse_params, params_dict, 'parse after merge into parse setup', 
        print_params=not tooManyColNamesToPrint, ignoreNone=True)

    print "parse source_frames is length:", len(parse_params['source_frames'])
    # This can be null now? parseSetup doesn't return default colnames?
    # print "parse column_names is length:", len(parse_params['column_names'])

    # none of the kwargs passed to here!
    parse_result = self.do_json_request( jsonRequest="3/Parse.json", cmd='post', postData=parse_params, timeout=timeoutSecs)
    verboseprint("Parse result:", dump_json(parse_result))

    job_key = parse_result['job']['key']['name']
    hex_key = parse_params['destination_frame']

    # TODO: dislike having different shapes for noPoll and poll
    if noPoll:
        # ??
        h2o_sandbox.check_sandbox_for_errors()
        # return self.jobs(job_key)
        return parse_result

    # does Frame also, while polling
    if intermediateResults:
        key = hex_key
    else:
        key = None

    job_result = self.poll_job(job_key, timeoutSecs=timeoutSecs, key=key)

    if job_result:
        jobs = job_result['jobs'][0]
        description = jobs['description']
        dest = jobs['dest']
        msec = jobs['msec']
        status = jobs['status']
        progress = jobs['progress']
        dest_key = dest['name']

        # can condition this with a parameter if some FAILED are expected by tests.
        if status=='FAILED':
            print dump_json(job_result)
            raise Exception("Taking exception on parse job status: %s %s %s %s %s" % \
                (status, progress, msec, dest_key, description))

        return self.frames(dest_key)
    else:
        # ? we should always get a job_json result
        raise Exception("parse didn't get a job_result when it expected one")
def parse ( self , key , hex_key = None , columnTypeDict = None , timeoutSecs = 300 , retryDelaySecs = 0.2 , initialDelaySecs = None , pollTimeoutSecs = 180 , noise = None , benchmarkLogging = None , noPoll = False , intermediateResults = False , * * kwargs ) : # these should override what parse setup gets below params_dict = { 'source_frames' : None , 'destination_frame' : hex_key , 'parse_type' : None , # file type  'separator' : None , 'single_quotes' : None , 'check_header' : None , # forces first line to be seen as column names  'number_columns' : None , 'column_names' : None , # a list 'column_types' : None , # a list. or can use columnTypeDict param (see below) 'na_strings' : None , # a list 'chunk_size' : None , # are these two no longer supported? 'delete_on_done' : None , 'blocking' : None , } # if key is a list, create a comma separated string # list or tuple but not string if not isinstance ( key , basestring ) : # it's a list of some kind (tuple ok?) # if len(key) > 1: #     print "I noticed you're giving me a list of > 1 keys %s to parse:" % len(key), key # len 1 is ok here. 0 not. what if None or [None] here if not key : raise Exception ( "key seems to be bad in parse. Should be list or string. %s" % key ) # have to put double quotes around the individual list items (single not legal) source_frames = "[" + "," . join ( map ( ( lambda x : '"' + x + '"' ) , key ) ) + "]" else : # what if None here source_frames = '["' + key + '"]' # quotes required on key params_dict [ 'source_frames' ] = source_frames # merge kwargs into params_dict # =None overwrites params_dict # columnTypeDict not used here h2o_methods . check_params_update_kwargs ( params_dict , kwargs , 'parse before setup merge' , print_params = False ) # Call ParseSetup?source_frames=[keys] . . . # if benchmarkLogging: #     cloudPerfH2O.get_log_save(initOnly=True) # h2o_methods.check_params_update_kwargs(params_dict, kwargs, 'parse_setup', print_params=True) params_setup = { 'source_frames' : source_frames } setup_result = self . do_json_request ( jsonRequest = "3/ParseSetup.json" , cmd = 'post' , timeout = timeoutSecs , postData = params_setup ) h2o_sandbox . check_sandbox_for_errors ( ) verboseprint ( "ParseSetup result:" , dump_json ( setup_result ) ) # this should match what we gave as input? if setup_result [ 'source_frames' ] : # should these be quoted? source_framesStr = "[" + "," . join ( [ ( '"%s"' % src [ 'name' ] ) for src in setup_result [ 'source_frames' ] ] ) + "]" else : source_framesStr = None # I suppose we need a way for parameters to parse() to override these # should it be an array or a dict? if setup_result [ 'column_names' ] : # single quotes not legal..need double quotes columnNamesStr = "[" + "," . join ( map ( ( lambda x : '"' + x + '"' ) , setup_result [ 'column_names' ] ) ) + "]" else : columnNamesStr = None columnTypes = setup_result [ 'column_types' ] assert columnTypes is not None , "%s %s" % ( "column_types:" , columnTypes ) if setup_result [ 'na_strings' ] : # single quotes not legal..need double quotes naStrings = "[" + "," . join ( map ( ( lambda x : '"' + x + '"' if x != None else '""' ) , setup_result [ 'na_strings' ] ) ) + "]" else : naStrings = None # dict parameter to update columnTypeDict? # but we don't pass columnNames like this? ct = setup_result [ 'column_types' ] columnNames = setup_result [ 'column_names' ] if columnTypeDict : for k , v in columnTypeDict . iteritems ( ) : if isinstance ( k , int ) : # if a column index if k >= 0 and k < len ( ct ) : ct [ k ] = v else : raise Exception ( "bad col index %s in columnTypeDict param %s" % ( k , columnTypeDict ) ) # if a column name elif isinstance ( k , basestring ) : # find the index if k not in columnNames : raise Exception ( "bad col name %s in columnTypeDict param %s. columnNames: %s" % ( k , columnTypeDict , columnNames ) ) ci = columnNames . index ( k ) ct [ ci ] = v else : raise Exception ( "%s %s should be int or string" % ( k , type ( k ) ) ) columnTypesStr = "[" + "," . join ( map ( ( lambda x : '"' + x + '"' ) , ct ) ) + "]" parse_params = { 'source_frames' : source_framesStr , 'destination_frame' : setup_result [ 'destination_frame' ] , 'parse_type' : setup_result [ 'parse_type' ] , 'separator' : setup_result [ 'separator' ] , 'single_quotes' : setup_result [ 'single_quotes' ] , 'check_header' : setup_result [ 'check_header' ] , 'number_columns' : setup_result [ 'number_columns' ] , 'column_names' : columnNamesStr , 'column_types' : columnTypesStr , 'na_strings' : naStrings , 'chunk_size' : setup_result [ 'chunk_size' ] , # No longer supported? how come these aren't in setup_result? 'delete_on_done' : params_dict [ 'delete_on_done' ] , 'blocking' : params_dict [ 'blocking' ] , } # HACK: if there are too many column names..don't print! it is crazy output # just check the output of parse setup. Don't worry about columnNames passed as params here.  tooManyColNamesToPrint = setup_result [ 'column_names' ] and len ( setup_result [ 'column_names' ] ) > 2000 if tooManyColNamesToPrint : h2p . yellow_print ( "Not printing the parameters to Parse because the columnNames are too lengthy." ) h2p . yellow_print ( "See sandbox/commands.log" ) # merge params_dict into parse_params # don't want =None to overwrite parse_params h2o_methods . check_params_update_kwargs ( parse_params , params_dict , 'parse after merge into parse setup' , print_params = not tooManyColNamesToPrint , ignoreNone = True ) print "parse source_frames is length:" , len ( parse_params [ 'source_frames' ] ) # This can be null now? parseSetup doesn't return default colnames? # print "parse column_names is length:", len(parse_params['column_names']) # none of the kwargs passed to here! parse_result = self . do_json_request ( jsonRequest = "3/Parse.json" , cmd = 'post' , postData = parse_params , timeout = timeoutSecs ) verboseprint ( "Parse result:" , dump_json ( parse_result ) ) job_key = parse_result [ 'job' ] [ 'key' ] [ 'name' ] hex_key = parse_params [ 'destination_frame' ] # TODO: dislike having different shapes for noPoll and poll if noPoll : # ?? h2o_sandbox . check_sandbox_for_errors ( ) # return self.jobs(job_key) return parse_result # does Frame also, while polling if intermediateResults : key = hex_key else : key = None job_result = self . poll_job ( job_key , timeoutSecs = timeoutSecs , key = key ) if job_result : jobs = job_result [ 'jobs' ] [ 0 ] description = jobs [ 'description' ] dest = jobs [ 'dest' ] msec = jobs [ 'msec' ] status = jobs [ 'status' ] progress = jobs [ 'progress' ] dest_key = dest [ 'name' ] # can condition this with a parameter if some FAILED are expected by tests. if status == 'FAILED' : print dump_json ( job_result ) raise Exception ( "Taking exception on parse job status: %s %s %s %s %s" % ( status , progress , msec , dest_key , description ) ) return self . frames ( dest_key ) else : # ? we should always get a job_json result raise Exception ( "parse didn't get a job_result when it expected one" )
Parse an imported raw file or files into a Frame.
Parse an imported raw file or files into a Frame .