title: Python 3.x multiprocess TypeError: can't pickle _thread.lock objects
body: <p>I am testing python multiprocess. I use pymongo to manage queue, my code and error is like below. I can't solve the issue and I don't know the root cause, please help me, thank you very much. I know multithreading could work, everything else works too, I had a test line saying:
  process_crawler(seed_url, scrape_callback=scrape_callback, cache=cache, max_threads=max_threads, timeout=10)</p>
  
  <pre><code>import time
  import threading
  from mongo_queue import MongoQueue
  from downloader import Downloader
  import multiprocessing
  
  
  SLEEP_TIME = 1
  
  def threaded_crawler(seed_url, delay=5, cache=None, scrape_callback=None, user_agent='wswp', proxies=None, num_retries=1, max_threads=10, timeout=60):
      """Crawl a website in multiple threads"""
      # url queues to be crawled
      crawl_queue = MongoQueue()
      crawl_queue.clear()
      crawl_queue.push(seed_url)
      downloader = Downloader(delay=delay, user_agent=user_agent, proxies=proxies, num_retries=num_retries, cache=cache, timeout=timeout)
  
      def process_queue():
          while True:
              try:
                  url = crawl_queue.pop()
              except KeyError:
                  #crawl queue is empty
                  break
              else:
                  html = downloader(url)
                  if scrape_callback:
                      try:
                          links = scrape_callback(url, html) or []
                      except Exception as e:
                          print('Error in call back for %s, %s' % (url, e))
                      else:
                          for link in links:
                              crawl_queue.push(link)
  
      threads = []
      while threads or crawl_queue:
          # the craw is still active
          for thread in threads:
              if not thread.is_alive():
                  threads.remove(thread)
          while len(threads) &lt; max_threads and crawl_queue.peek():
              # can start some more threads
              thread = threading.Thread(target=process_queue)
              thread.setDaemon(True)
              thread.start()
              threads.append(thread)
          time.sleep(SLEEP_TIME)
  
  
  def process_crawler(args, **kwargs):
      num_cpus = multiprocessing.cpu_count()
      print('Starting Multiprocessing.... CPU Number is ', num_cpus)
      processes = []
      for i in range(num_cpus):
          p = multiprocessing.Process(target=threaded_crawler, args=[args], kwargs=kwargs)
          p.start()
          processes.append(p)
      for p in processes:
          p.join()
  
  
  
  Traceback (most recent call last):
  Starting Multiprocessing.... CPU Number is  8
    File "C:/Users/Michael Qian/Desktop/Python/MyScraper/process_test.py", line 15, in &lt;module&gt;
      test(1)
    File "C:/Users/Michael Qian/Desktop/Python/MyScraper/process_test.py", line 10, in test
      process_crawler(scrape_callback.seed_url, scrape_callback=scrape_callback, cache=cache, max_threads=max_threads, timeout=10)
    File "C:\Users\Michael Qian\Desktop\Python\MyScraper\process_crawler.py", line 58, in process_crawler
      p.start()
    File "C:\Program Files\Python35\lib\multiprocessing\process.py", line 105, in start
      self._popen = self._Popen(self)
    File "C:\Program Files\Python35\lib\multiprocessing\context.py", line 212, in _Popen
      return _default_context.get_context().Process._Popen(process_obj)
    File "C:\Program Files\Python35\lib\multiprocessing\context.py", line 313, in _Popen
      return Popen(process_obj)
    File "C:\Program Files\Python35\lib\multiprocessing\popen_spawn_win32.py", line 66, in __init__
      reduction.dump(process_obj, to_child)
    File "C:\Program Files\Python35\lib\multiprocessing\reduction.py", line 59, in dump
      ForkingPickler(file, protocol).dump(obj)
  TypeError: can't pickle _thread.lock objects
  Traceback (most recent call last):
    File "&lt;string&gt;", line 1, in &lt;module&gt;
    File "C:\Program Files\Python35\lib\multiprocessing\spawn.py", line 106, in spawn_main
      exitcode = _main(fd)
    File "C:\Program Files\Python35\lib\multiprocessing\spawn.py", line 116, in _main
      self = pickle.load(from_parent)
  EOFError: Ran out of input
  </code></pre>