title: Why is Pymongo shutting down the connection mid-process?
body: <p>I am running a python script in Jupyter Notebook that connects to a mongo db to process text documents (news articles). The script runs fine for the first few batches of data, but then terminates with the following error:</p>
  
  <p><strong>AutoReconnect: localhost:27017: [WinError 10054] An existing connection was forcibly closed by the remote host</strong> (<em>full error below</em>).</p>
  
  <p>Have already tried deleting the <code>mongod.lock</code> file as prescribed here(<a href="https://stackoverflow.com/questions/7744147/pymongo-keeps-refusing-the-connection-at-27017">Pymongo keeps refusing the connection at 27017</a>), however, this hasn't solved the issue.</p>
  
  <p>This is my function (with a few sub-functions as well not included here):</p>
  
  <pre><code>    data_list = []
      for collection in mongo_collections_dir:
          mongo_collection = mongo_collections_dir[collection]
          filter_dict = {"file_info.source": source}
          if filter_year:
              filter_dict["extracted.publication_date.year"] = filter_year
          elif min_year:
              print('filter year not found') # - note added by dror for debug 24/10/2019
              filter_dict["extracted.publication_date.year"] = {"$gt": min_year}
          source_count = mongo_collection.count(filter_dict)
          print("{} articles found in collection {} {}".format(source_count, collection, filter_year))
          if source_count == "0":
              continue
          docs = mongo_collection.find(filter_dict, no_cursor_timeout=True)
          if not docs:
              print("source {} was not found in collection {}".format(source, collection))
              continue
          for pos, doc in enumerate(docs):
              if pos % 100000 == 0:
                  print("processed {} articles out of {} from {}".format(pos, source_count, source))
              try:
                  text = doc["body"]["content"]
              except KeyError:
  #                 print('no body')
                  continue  
              if clean_text:
                  clean_text = mpd_clean_text(text, stop_words)
              else:
                  clean_text = ''
              try:
                  title = doc['body']['head']['hedline']
  #                 author = doc['body']['head']['byline']
                  temp_dir = {"collection": mongo_collection.name, "source": doc["file_info"]["source"], 
                              "urn": doc["urn"],  "title": title,
                              'unit_text': text, 'clean text': clean_text, 
                             }
              except KeyError:
                  temp_dir = {"collection": mongo_collection.name, "source": doc["file_info"]["source"], "urn": doc["urn"],
                             'unit_text': text, 'clean text': clean_text}  
  
              try:
                  publication_date = get_dt(doc["extracted"]["publication_date"])
                  temp_dir['publication_date'] = publication_date
              except KeyError:
                  print('no extracted')
              try:
                  temp_dir['section'] = doc['extracted']['section']
              except KeyError:
                  pass
              try:
                  temp_dir['publication_name'] = doc['extracted']['publication_name']
              except KeyError:
                  pass
  
              if temp_dir:
  #                 temp_dir['section'] = section
                  data_list.append(temp_dir)
  #     df = pd.DataFrame(data_list)
  #     df['section'] = section
      return pd.DataFrame(data_list)
  </code></pre>
  
  <p>Full error:</p>
  
  <pre><code>~\Anaconda3\envs\py35\lib\site-packages\pymongo\pool.py in command(self, dbname, spec, slave_ok, read_preference, codec_options, check, allowable_errors, check_keys, read_concern, write_concern, parse_write_concern_error, collation, session, client, retryable_write, publish_events)
      578                            use_op_msg=self.op_msg_enabled,
  --&gt; 579                            unacknowledged=unacknowledged)
      580         except OperationFailure:
  
  ~\Anaconda3\envs\py35\lib\site-packages\pymongo\network.py in command(sock, dbname, spec, slave_ok, is_mongos, read_preference, codec_options, session, client, check, allowable_errors, address, check_keys, listeners, max_bson_size, read_concern, parse_write_concern_error, collation, compression_ctx, use_op_msg, unacknowledged)
      140         else:
  --&gt; 141             reply = receive_message(sock, request_id)
      142             unpacked_docs = reply.unpack_response(codec_options=codec_options)
  
  ~\Anaconda3\envs\py35\lib\site-packages\pymongo\network.py in receive_message(sock, request_id, max_message_size)
      172     length, _, response_to, op_code = _UNPACK_HEADER(
  --&gt; 173         _receive_data_on_socket(sock, 16))
      174     # No request_id for exhaust cursor "getMore".
  
  ~\Anaconda3\envs\py35\lib\site-packages\pymongo\network.py in _receive_data_on_socket(sock, length)
      231             try:
  --&gt; 232                 chunk_length = sock.recv_into(mv[bytes_read:])
      233             except (IOError, OSError) as exc:
  
  ConnectionResetError: [WinError 10054] An existing connection was forcibly closed by the remote host
  
  During handling of the above exception, another exception occurred:
  
  AutoReconnect                             Traceback (most recent call last)
  &lt;ipython-input-14-37a7effc9859&gt; in &lt;module&gt;
  ----&gt; 1 collect_df_by_sources(collections_dir, sources, newspapers_df, filter_year='2008')
  
  &lt;ipython-input-8-bbd9e966e95c&gt; in collect_df_by_sources(collections_dir, sources, newspapers_df, return_df, filter_year, min_year, override)
       20 
       21         print("collect articles from: {}, {}".format(source_name, source_id))
  ---&gt; 22         source_df = collect_df_by_source(collections_dir, source_id, filter_year=filter_year, min_year=min_year)
       23         source_df.to_csv(source_path, encoding='utf8')
       24         if return_df:
  
  &lt;ipython-input-7-82d129ff329c&gt; in collect_df_by_source(mongo_collections_dir, source, clean_text, filter_year, min_year)
        9             print('filter year not found') # - note added by dror for debug 24/10/2019
       10             filter_dict["extracted.publication_date.year"] = {"$gt": min_year}
  ---&gt; 11         source_count = mongo_collection.count(filter_dict)
       12         print("{} articles found in collection {} {}".format(source_count, collection, filter_year))
       13         if source_count == "0":
  
  ~\Anaconda3\envs\py35\lib\site-packages\pymongo\collection.py in count(self, filter, session, **kwargs)
     1764         collation = validate_collation_or_none(kwargs.pop('collation', None))
     1765         cmd.update(kwargs)
  -&gt; 1766         return self._count(cmd, collation, session)
     1767 
     1768     def create_indexes(self, indexes, session=None, **kwargs):
  
  ~\Anaconda3\envs\py35\lib\site-packages\pymongo\collection.py in _count(self, cmd, collation, session)
     1570                 read_concern=self.read_concern,
     1571                 collation=collation,
  -&gt; 1572                 session=session)
     1573         if res.get("errmsg", "") == "ns missing":
     1574             return 0
  
  ~\Anaconda3\envs\py35\lib\site-packages\pymongo\collection.py in _command(self, sock_info, command, slave_ok, read_preference, codec_options, check, allowable_errors, read_concern, write_concern, collation, session, retryable_write)
      242                 session=s,
      243                 client=self.__database.client,
  --&gt; 244                 retryable_write=retryable_write)
      245 
      246     def __create(self, options, collation, session):
  
  ~\Anaconda3\envs\py35\lib\site-packages\pymongo\pool.py in command(self, dbname, spec, slave_ok, read_preference, codec_options, check, allowable_errors, check_keys, read_concern, write_concern, parse_write_concern_error, collation, session, client, retryable_write, publish_events)
      582         # Catch socket.error, KeyboardInterrupt, etc. and close ourselves.
      583         except BaseException as error:
  --&gt; 584             self._raise_connection_failure(error)
      585 
      586     def send_message(self, message, max_doc_size):
  
  ~\Anaconda3\envs\py35\lib\site-packages\pymongo\pool.py in _raise_connection_failure(self, error)
      741         self.close()
      742         if isinstance(error, socket.error):
  --&gt; 743             _raise_connection_failure(self.address, error)
      744         else:
      745             raise error
  
  ~\Anaconda3\envs\py35\lib\site-packages\pymongo\pool.py in _raise_connection_failure(address, error, msg_prefix)
      281         raise NetworkTimeout(msg)
      282     else:
  --&gt; 283         raise AutoReconnect(msg)
      284 
      285 
  
  AutoReconnect: localhost:27017: [WinError 10054] An existing connection was forcibly closed by the remote host```
  
  
  </code></pre>