def parser_main(): 
    (bytes_to_process,
     files_to_process,
     files_to_ignore) = find_files_to_process()

    for path in files_to_ignore:
        print "Unparseable filename: {0}".format(os.path.basename(path))

    print "Files to process: {0}".format(len(files_to_process))
    print "Bytes to process: {0}".format(pretty_bytes(bytes_to_process))
    print "Continue?"
    user_input = raw_input()
    if not 'yes'.startswith(user_input.lower()):
       return

    transactions = {}

    failed_lines = file(os.path.join(DATA_DIR, 'failed_lines.out'), 'w')
    failed_files = file(os.path.join(DATA_DIR, 'failed_files.out'), 'w')
    
    begin_time = time.time()
    for files_processed, ((filepath, import_date, filesize), bytes_processed) in enumerate(files_to_process, start=1):
        try:
            print
            print "Parsing {0}".format(os.path.basename(filepath))
            file_transactions = parse_file(filepath, import_date)
            for (award_id, t) in file_transactions:
                if award_id not in transactions:
                    transactions[award_id] = t

        except UnicodeDecodeError, error:
            log_error(db, filepath, "Unable to parse file: {0}".format(unicode(error)))

        except __HOLE__:
            break

        now_time = time.time()
        bytes_per_second = bytes_processed / max(now_time - begin_time, 1)
        bytes_processed_pct = bytes_processed * 100 / bytes_to_process
        eta_seconds = (bytes_to_process - bytes_processed) / max(bytes_per_second, 1)
        print "{0}/{1} ({2}%), {3}/s, ETA {4}".format(
            pretty_bytes(bytes_processed),
            pretty_bytes(bytes_to_process),
            bytes_processed_pct,
            pretty_bytes(bytes_per_second),
            pretty_seconds(eta_seconds))

    failed_lines.close()
    failed_files.close()
   
    print "Dumping awards dictionary..."
    with file(os.path.join(DATA_DIR, 'cfda_awards.out.bin'), 'wb') as outf:
        pickle.dump(transactions, outf)
dataset/ETHPy150Open sunlightlabs/clearspending/timeliness/parser.py/parser_main