Python - multiprocessing while writing to a single result file
Question: I am really new to the multiprocessing package and I am failing to get the
task done.

I have lots of calculations to do on a list of objects.

The results I need to write down are saved in those objects, too.

The results should be written in a single file as soon as the process finished
the calculations (the way I got it at least working, waits until all
calculations are done).

    
    
    import multiprocessing 
    import time
    import csv
    
    class simpl():
        def __init__(self, name, val):
            self.name = name
            self.val = val
    
    def pot_val(inpt):
        print("Process %s\t ..." % (inpt.name))
        old_v = inpt.val
        inpt.val *= inpt.val
        if old_v != 8:
                time.sleep(old_v)
        print("Process %s\t ... Done" % (inpt.name))
    
    
    def mp_worker(inpt):
        pot_val(inpt)
        return inpt
    
    def mp_handler(data_list):
        p = multiprocessing.Pool(4)
        with open('results.csv', 'a') as f:
            res = p.map_async(mp_worker, data_list)
            results = (res.get())
            for result in results:
                print("Writing result for ",result.name)
                writer= csv.writer(f, lineterminator = '\n', delimiter=";")
                writer.writerow((result.name, result.val))
    if __name__=='__main__':
        data = []
        counter=0
        for i in range(10):
            data.append(simpl("name"+str(counter),counter))
            counter += 1    
    
        for d in data:
            print(d.name, d.val)
        mp_handler(data)
    

How to write the results from the calculations simultaneously to one single
file, without having to wait for all processes to finish?


Answer: You can use
[imap_unordered](https://docs.python.org/3/library/multiprocessing.html#multiprocessing.pool.Pool.imap_unordered)

    
    
    def mp_handler(data_list):
        p = multiprocessing.Pool(4)
        with open('results.csv', 'a') as f:
            writer= csv.writer(f, lineterminator = '\n', delimiter=";")
            for result in p.imap_unordered(mp_worker, data_list):
                print("Writing result for ",result.name)
                writer.writerow((result.name, result.val))
    

With Python 3.3+ better do

    
    
    def mp_handler(data_list):
        with multiprocessing.Pool(4) as p:
            with open('results.csv', 'a') as f:
                writer= csv.writer(f, lineterminator = '\n', delimiter=";")
                for result in p.imap_unordered(mp_worker, data_list):
                    print("Writing result for ",result.name)
                    writer.writerow((result.name, result.val))
    

