def main():
    ''' map() a function against data as given in an input pickle file, saving result to a pickle.'''
    import sys, os, optparse

    usage = "%prog [options] inputPickle outputPickle \n" + main.__doc__
    parser = optparse.OptionParser(usage)
    parser.add_option('--mode', dest='mode', type='string',
                      help="""set the mode of operation.  Should be one of ['map', 'reduce']""")
    opts, args = parser.parse_args()
    inputPickleName, outputPickleName = args
    with open(inputPickleName, 'rb') as inputPickleFile:
        with open(outputPickleName, 'wb') as outputPickleFile:
            #os.environ['PYTHONPATH'] = pickle.load(inputPickleFile)
            try:
                initializer = pickle.load(inputPickleFile)
                if opts.mode == 'map':
                    initargs = pickle.load(inputPickleFile)
                else:
                    initargs = None
            except Exception as e:
                # save and return the exception, to be debugged upstream
                pickle.dump(type(e)(('Error Loading initializer from pickle file',) + e.args), outputPickleFile, -1)
                raise
            else:
                if initializer is not None:
                    if initargs is not None:
                        initializer(initargs)
                    else:
                        initializer()
            try:
                func = pickle.load(inputPickleFile)
            except __HOLE__ as e:
                pickle.dump(e, outputPickleFile, -1)  # Error loading
                raise
            else:
                # If I didn't want error checking on each step, I'd do this at the top level:
                ## result = reduce(func, (data if type(data) != Job else pickle.load(data.outputFile) for data in curData), initializer)
                moreData = True
                curData = []
                while moreData:
                    try:
                        data = pickle.load(inputPickleFile)
                    except EOFError:
                        moreData = False
                    else:
                        if data.__class__ == Job:  # load the results from the previous Job
                            try:
                                data = pickle.load(data.outputFile)
                            except Exception as e:
                                pickle.dump(type(e)(("Error loading previous Job's results! ",) + e.args), outputPickleFile, -1)
                                raise
                        curData.append(data)
                        if opts.mode == 'map':
                            try:
                                #print 'mapping', func, 'on', curData[0]
                                result = func(curData.pop(0))
                            except Exception as e:
                                # save and return the exception, to be debugged upstream
                                pickle.dump(type(e)(('Error in map function: ',) + e.args), outputPickleFile, -1)
                                raise
                            else:
                                print 'saving', result
                                pickle.dump(result, outputPickleFile, -1)
                        else:
                            # reduce curData
                            result = reduce(func, curData[:2], initializer)
                            initializer = None
                            del curData[:2]
                            curData.insert(0, result)
dataset/ETHPy150Open uci-cbcl/tree-hmm/treehmm/sge.py/main