How can I parallel parsing in python?
Question: I have the following code which converts graph from edges list to adjacency
matrix:

    
    
    for line in open('graph.txt'):
        converted = [sparse_to_dense.get(int(ID)) for ID in line.split()]
        i = converted[0]
        j = converted[1]
        I.append(i)
        J.append(j)
    n = max([max(I), max(J)]) + 1
    data = [1]*len(I)
    return coo_matrix((data, (I,J)), shape=(n,n), dtype='i1')
    

This code is awfully slow -- on may machine conversion of 500k edges takes
hours. On the other hand i/o is obviously is not bottleneck (I can read full
file in memory almost instantaneously) so I think there is a room for
parallelism. But I'm not sure how to proceed: should I read file in parallel
or something?


Answer: Use multiprocessing one way to do it is this. I did not check and could be
further improved

    
    
    import multiprocessing
    
    
    class Worker(multiprocessing.Process):
    
        def __init__(self, queue, results):
             multiprocessing.Process.__init__(self):
             self.q = queue
             self.results = results
    
        def run(self):
            while True:
                try:
                    lineno, linecontents = self.q.get(block=False)
                except Queue.Empty:
                    break
                converted = [sparse_to_dense.get(int(ID)) for ID in line.split()]
                 i = converted[0]
                 j = converted[1]
                self.results.put((i, j))
    
    
    def main():
        q = multiprocessing.Queue()
        results = multiprocessing.JoinableQueue()
    
        for i, l in open(fname):
            q.put((i, l))
    
        for _ in xrange(4):
            w = Worker(q, results)
            w.start()
    
        I, J = []
        while True:
            try:
                i, j = results.get(block=False)
            except Queue.Empty:
            break
        I.append(i)
        J.append(j)
        results.task_done()
    
        results.join()
    
        n = max([max(I), max(J)]) + 1
        data = [1]*len(I)
        coo = coo_matrix((data, (I,J)), shape=(n,n), dtype='i1')
    

