title: Reading lots of files using producer causes CPU usage to be 100%
body: <p>I wrote a simple consumer-producer pattern to help me achieve the following task:</p>
  
  <ol>
  <li>Read a file from a directory that contain ~500,000 TSV (tab-separated) files.</li>
  <li>Manipulate each file into a data structure and put it in blocking queue.</li>
  <li>Consume the queue using consumer and query DB.</li>
  <li>Compare both hash-maps and if there's a difference, print difference to file.</li>
  </ol>
  
  <p>When I run the program, even with 5 threads, my CPU consumption skyrockets to 100%. Could this be because I'm using a single producer to read the files?</p>
  
  <p><strong>File example (tab separated)</strong></p>
  
  <pre><code>Column1   Column2   Column3   Column 4   Column5
  A         1         *         -          -
  B         1         *         -          -
  C         1         %         -          -
  </code></pre>
  
  <p><strong>Producer</strong></p>
  
  <pre><code>public class Producer implements Runnable{
  private BlockingQueue&lt;Map&lt;String, Map&lt;String, String&gt;&gt;&gt; m_Queue;
  private String m_Directory;
  
  public Producer(BlockingQueue&lt;Map&lt;String, Map&lt;String, String&gt;&gt;&gt; i_Queue, String i_Directory)
  {
      m_Queue = i_Queue;
      m_Directory = i_Directory;
  }
  
  @Override
  public void run()
  {
      if (Files.exists(Paths.get(m_Directory)))
      {
          File[] files = new File(m_Directory).listFiles();
  
          if (files != null)
          {
              for (File file : files)
              {
                  Map&lt;String, String&gt; map = new HashMap&lt;&gt;();
                  try (BufferedReader reader = new BufferedReader(new FileReader(file)))
                  {
                      String line, lastcolumn3 = "", column1 = "", column2 = "", column3 = "";
                      while ((line = reader.readLine()) != null)
                      {
                          //Skip column header
                          if (!Character.isLetter(line.charAt(0)))
                          {
                              String[] splitLine = line.split("\t");
  
                              column1 = splitLine[0].replace("\"", "");
                              column2 = splitLine[1].replace("\"", "");
                              column3 = splitLine[2].replace("\"", "");
  
                              if (!lastcolumn3.equals(column3))
                              {
                                  map.put(column3, column1);
                                  lastcolumn3 = column3;
                              }
                          }
                      }
  
                      map.put(column3, column1);
  
                      //Column 1 is always the same per file, it'll be the key. Column2 and Column3 are stored as the value (as a key-value pair)
                      Map&lt;String, Map&lt;String, String&gt;&gt; mapPerFile = new HashMap&lt;&gt;();
                      mapPerFile.put(column2, map);
  
                      m_Queue.put(mapPerFile);
                  }
                  catch (IOException | InterruptedException e)
                  {
                      System.out.println(file);
                      e.printStackTrace();
                  }
              }
          }
      }
  }}
  </code></pre>
  
  <p><strong>Consumer</strong></p>
  
  <pre><code>public class Consumer implements Runnable{
  private HashMap&lt;String, String&gt; m_DBResults;
  private BlockingQueue&lt;Map&lt;String, Map&lt;String, String&gt;&gt;&gt; m_Queue;
  private Map&lt;String, Map&lt;String, String&gt;&gt; m_DBResultsPerFile;
  private String m_Column1;
  private int m_ThreadID;
  
  public Consumer(BlockingQueue&lt;Map&lt;String, Map&lt;String, String&gt;&gt;&gt; i_Queue, int i_ThreadID)
  {
      m_Queue = i_Queue;
      m_ThreadID = i_ThreadID;
  }
  
  @Override
  public void run()
  {
      try
      {
          while ((m_DBResultsPerFile = m_Queue.poll()) != null)
          {
              //Column1 is always the same, only need the first entry.
              m_Column1 = m_DBResultsPerFile.keySet().toArray()[0].toString();
  
              //Queries DB and puts returned data into m_DBResults
              queryDB(m_Column1);
  
              //Write the difference, if any, per thread into a file.
              writeDifference();
          }
      }
      catch (Exception e)
      {
          e.printStackTrace();
      }
  }
  
  private void writeDifference()
  {
      MapDifference&lt;String, String&gt; difference = Maps.difference(m_DBResultsPerFile.get(m_Column1), m_DBResults);
  
      if (difference.entriesOnlyOnLeft().size() &gt; 0 || difference.entriesOnlyOnRight().size() &gt; 0)
      {
          try (BufferedWriter writer = new BufferedWriter(new FileWriter(String.format("thread_%d.tsv", m_ThreadID), true)))
          {
              if (difference.entriesOnlyOnLeft().size() &gt; 0)
              {
                  writer.write(String.format("%s\t%s\t", "Missing", m_Column1));
                  for (Map.Entry&lt;String, String&gt; entry : difference.entriesOnlyOnLeft().entrySet())
                  {
                      writer.write(String.format("[%s,%s]; ", entry.getKey(), entry.getValue()));
                  }
  
                  writer.write("\n");
              }
              if (difference.entriesOnlyOnRight().size() &gt; 0)
              {
                  writer.write(String.format("%s\t%s\t", "Extra", m_Column1));
                  for (Map.Entry&lt;String, String&gt; entry : difference.entriesOnlyOnRight().entrySet())
                  {
                      writer.write(String.format("[%s,%s]; ", entry.getKey(), entry.getValue()));
                  }
  
                  writer.write("\n");
              }
          }
          catch (IOException e)
          {
              e.printStackTrace();
          }
      }
  }}
  </code></pre>
  
  <p><strong>Main</strong></p>
  
  <pre><code>public static void main(String[]args) {
  BlockingQueue&lt;Map&lt;String, Map&lt;String,String&gt;&gt;&gt; queue = new LinkedBlockingQueue &lt;&gt; ();
  
  //Start the reader thread.
  threadPool.execute(new Producer(queue, args[0]));
  
  //Create configurable threads.
  for (int i = 0; i &lt; 10; i++) {
      threadPool.execute(new Consumer(queue, i + 1));
  }
  
  threadPool.shutdown();
  System.out.println("INFO: Shutting down threads.");
  
  try {
      threadPool.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
      System.out.println("INFO: Threadpool terminated successfully.");
  } catch (InterruptedException e) {
      e.printStackTrace();
  }}
  </code></pre>