Class with multiple workers I am building a tool that interacts with a batched stream of incoming data. This data needs to be processed and the result returned. To split up the work I have created a class that has inbound (_in) and outbound (out) queues and workers that are getting, processing, and depositing the work.
This example takes an iterable of numbers (in pass_data) and multiplies them by f.
import queue, random, time
from multiprocessing import Process, Queue

def _worker(_in, out, f):
    &quot;&quot;&quot;Get work from _in and output processed data to out&quot;&quot;&quot;
    while True:
        try:
            work = _in.get()
        except queue.Empty:
            continue
        # simulate blocking for some time
        time.sleep(random.uniform(0.01, 0.5))
        out.put(work * f)

class C:
    def __init__(self, f, threads=2):
        self.f = f
        self.threads = threads
        self._in, self.out = Queue(), Queue()
        self.args = (self._in, self.out, self.f)
        self.workers = [
            Process(target=_worker, args=self.args) for _ in range(self.threads)
        ]

    def __repr__(self):
        return f&quot;{self.__class__.__name__}(threads={self.threads})&quot;

    def start(self):
        &quot;&quot;&quot;Start all workers&quot;&quot;&quot;
        for worker in self.workers:
            worker.start()

    def terminate(self):
        &quot;&quot;&quot;Terminate all workers&quot;&quot;&quot;
        for worker in self.workers:
            worker.terminate()

    def pass_data(self, data):
        &quot;&quot;&quot;Pass data to the queue to be processed&quot;&quot;&quot;
        for rec in data:
            self._in.put(rec)

    def get_completed(self):
        &quot;&quot;&quot;Return a list of processed data&quot;&quot;&quot;
        items = []
        while True:
            try:
                items.append(self.out.get_nowait())
            except queue.Empty:
                break
        return items

if __name__ == &quot;__main__&quot;:
    c = C(f=12, threads=2)
    c.start()

    for i in range(5):
        s = 0
        n = random.randint(1, 20)
        c.pass_data(list(range(n)))
        print(f&quot;sent: {n}&quot;)
        while s &lt; n:
            r = c.get_completed()
            s += len(r)
            if r:
                print(len(r), end=&quot;, &quot;)
            time.sleep(random.uniform(0.01, 0.4))
        print()
    c.terminate()


This is, at the moment, a proof of concept. Are there any pitfalls to this method? Is there a better way to do this already?!
Aspects that I intend to address:

queue size limits
thread number limits
Use multiprocessing.Pool
The multiprocessing library already has a worker pool implementation ready to be used. Your code could be rewritten as:
import time
from multiprocessing import Pool

def f(x):
    time.sleep(random.uniform(0.01, 0.5))
    return x * 12

if __name__ == &quot;__main__&quot;:
    c = Pool(2)

    for i in range(5):
        n = random.randint(1, 20)
        r = c.map_async(f, list(range(n)))
        print(f&quot;sent: {n}&quot;)
        print(f&quot;got: {len(r.get())}&quot;)

While multiprocessing.Pool allows you to check if the results are ready by using .ready() on the result of an apply_async() or map_async() call, you can't get a partial result from map_async(). However, if you do want to process individual results as soon as they are ready, you can consider calling apply_async() with a callback function that handles the result.