complex_model/GulcalcToBin.py
def gulcalc_sqlite_fp_to_bin(working_dir, db_fp, output, num_sample, stream_id=DEFAULT_GUL_STREAM,
                             oasis_event_batch=None):
    """This transforms a sqlite result table (rf format) into oasis loss binary stream

    :param working_dir: working directory
    :param db_fp: path to the sqlite database
    :param output: output stream where results will be written to
    :param num_sample: number of samples in result
    :param stream_id: item, coverage or loss stream id
    :param oasis_event_batch: event batch id attached to this process
    :return: OASIS compliant item or coverage binary stream
    """
    start = time.time()
    logging.info("STARTED: Transforming sqlite losses into gulcalc item/loss binary stream for oasis_event_batch "
                 + str(oasis_event_batch))
    gulcalc_create_header(output, num_sample, stream_id)

    con = sqlite3.connect(db_fp)
    cur = con.cursor()
    cur.execute("SELECT count(*) FROM event_batches")
    num_partial = cur.fetchone()[0]
    cur.execute("SELECT batch_id FROM event_batches ORDER BY batch_id")
    add_first_separator = False
    rc = 0
    for batch_id in cur.fetchall():
        logging.info("RUNNING: Streaming partial loss " + str(1 + int(batch_id[0])) + "/" + str(num_partial)
                     + " for oasis_event_batch " + str(oasis_event_batch))
        batch_res_fp = os.path.join(working_dir, "oasis_loss_{0}.db".format(batch_id[0]))
        batch_res_con = sqlite3.connect(batch_res_fp)
        rc = rc + gulcalc_sqlite_to_bin(batch_res_con, output, add_first_separator)
        add_first_separator = True
    con.close()

    hours, rem = divmod(time.time() - start, 3600)
    minutes, seconds = divmod(rem, 60)
    exec_time = "{:0>2}:{:0>2}:{:05.2f}".format(int(hours), int(minutes), seconds)
    logging.info("COMPLETED: Successfully generated losses as gulcalc binary stream for event batch "
                 + str(oasis_event_batch) + ": " + str(rc) + " rows were streamed in " + exec_time)
This transforms a sqlite result table (rf format) into oasis loss binary stream

    :param working_dir: working directory
    :param db_fp: path to the sqlite database
    :param output: output stream where results will be written to
    :param num_sample: number of samples in result
    :param stream_id: item, coverage or loss stream id
    :param oasis_event_batch: event batch id attached to this process
    :return: OASIS compliant item or coverage binary stream
    
This transforms a sqlite result table (rf format) into oasis loss binary stream
This transforms a sqlite result table ( rf format ) into oasis loss binary stream
def gulcalc_sqlite_fp_to_bin(working_dir, db_fp, output, num_sample, stream_id=DEFAULT_GUL_STREAM,
                             oasis_event_batch=None):
    start = time.time()
    logging.info("STARTED: Transforming sqlite losses into gulcalc item/loss binary stream for oasis_event_batch "
                 + str(oasis_event_batch))
    gulcalc_create_header(output, num_sample, stream_id)
    con = sqlite3.connect(db_fp)
    cur = con.cursor()
    cur.execute("SELECT count(*) FROM event_batches")
    num_partial = cur.fetchone()[0]
    cur.execute("SELECT batch_id FROM event_batches ORDER BY batch_id")
    add_first_separator = False
    rc = 0
    for batch_id in cur.fetchall():
        logging.info("RUNNING: Streaming partial loss " + str(1 + int(batch_id[0])) + "/" + str(num_partial)
                     + " for oasis_event_batch " + str(oasis_event_batch))
        batch_res_fp = os.path.join(working_dir, "oasis_loss_{0}.db".format(batch_id[0]))
        batch_res_con = sqlite3.connect(batch_res_fp)
        rc = rc + gulcalc_sqlite_to_bin(batch_res_con, output, add_first_separator)
        add_first_separator = True
    con.close()
    hours, rem = divmod(time.time() - start, 3600)
    minutes, seconds = divmod(rem, 60)
    exec_time = "{:0>2}:{:0>2}:{:05.2f}".format(int(hours), int(minutes), seconds)
    logging.info("COMPLETED: Successfully generated losses as gulcalc binary stream for event batch "
                 + str(oasis_event_batch) + ": " + str(rc) + " rows were streamed in " + exec_time)
def gulcalc_sqlite_fp_to_bin ( working_dir , db_fp , output , num_sample , stream_id = DEFAULT_GUL_STREAM , oasis_event_batch = None ) : start = time . time ( ) logging . info ( "STARTED: Transforming sqlite losses into gulcalc item/loss binary stream for oasis_event_batch " + str ( oasis_event_batch ) ) gulcalc_create_header ( output , num_sample , stream_id ) con = sqlite3 . connect ( db_fp ) cur = con . cursor ( ) cur . execute ( "SELECT count(*) FROM event_batches" ) num_partial = cur . fetchone ( ) [ 0 ] cur . execute ( "SELECT batch_id FROM event_batches ORDER BY batch_id" ) add_first_separator = False rc = 0 for batch_id in cur . fetchall ( ) : logging . info ( "RUNNING: Streaming partial loss " + str ( 1 + int ( batch_id [ 0 ] ) ) + "/" + str ( num_partial ) + " for oasis_event_batch " + str ( oasis_event_batch ) ) batch_res_fp = os . path . join ( working_dir , "oasis_loss_{0}.db" . format ( batch_id [ 0 ] ) ) batch_res_con = sqlite3 . connect ( batch_res_fp ) rc = rc + gulcalc_sqlite_to_bin ( batch_res_con , output , add_first_separator ) add_first_separator = True con . close ( ) hours , rem = divmod ( time . time ( ) - start , 3600 ) minutes , seconds = divmod ( rem , 60 ) exec_time = "{:0>2}:{:0>2}:{:05.2f}" . format ( int ( hours ) , int ( minutes ) , seconds ) logging . info ( "COMPLETED: Successfully generated losses as gulcalc binary stream for event batch " + str ( oasis_event_batch ) + ": " + str ( rc ) + " rows were streamed in " + exec_time )
This transforms a sqlite result table (rf format) into oasis loss binary stream
This transforms a sqlite result table ( rf format ) into oasis loss binary stream
"""This transforms a sqlite result table (rf format) into oasis loss binary stream

    :param working_dir: working directory
    :param db_fp: path to the sqlite database
    :param output: output stream where results will be written to
    :param num_sample: number of samples in result
    :param stream_id: item, coverage or loss stream id
    :param oasis_event_batch: event batch id attached to this process
    :return: OASIS compliant item or coverage binary stream
    """
OASIS compliant item or coverage binary stream
OASIS compliant item or coverage binary stream
working_dir
None
db_fp
path to the sqlite database
path to the sqlite database
output
output stream where results will be written to
output stream where results will be written to
num_sample
number of samples in result
number of samples in result
stream_id
item, coverage or loss stream id
item coverage or loss stream id
oasis_event_batch
event batch id attached to this process
event batch id attached to this process