title: Why does my NodeJS script bog down during fs.readFile and fs.appendFile handling large numbers of files.
body: <p>I have a folder with about 120k HTML pages that I need to open (each file is about 70kb), parse some data out with xPath and append that data to a .csv file.</p>
  
  <p>Below is my code:</p>
  
  <p>It should read a list of files from the parseFolder, iterate through each file name opening it with fs.readFile then parsing the data out with jsdom and xpath and saving it to the csv file using fs.appendFile.</p>
  
  <p>It seems to do well for the first 100 or so files but after that will gradually slow down, consume memory and cpu and eventually stall. I have 16 gigs of memory and it seems to hit some limit when my memory usage is up to about 7gigs.</p>
  
  <p>I'm new to JS and Node, any help pointing out what I'm missing would be very much appreciated.</p>
  
  <pre><code>var fs = require('fs');
  var jsdom = require('jsdom').jsdom;
  var xpath = require('xpath');
  var S = require('string');
  var os = require('os');
  
  ParserRules = {
      saveFile: 'output.csv',
      parseFolder: '/a/folder/with/120k/HTML/files',
      fields: {
          "field1": "//div[@class='field1']/text()",
      }
  };
  
  start();
  
  function start() {
      console.log('Starting...');
      fs.readdir(ParserRules.parseFolder, iterateFiles);
  }
  
  function iterateFiles(err, filesToParse) {
      for (var i = 0; i &lt; filesToParse.length; i++) {
          file = ParserRules.parseFolder + '/' + filesToParse[i];
          console.log('Beginning read of ' + file);
          fs.readFile(file, {encoding: 'utf8'}, parseFile);
      }
  }
  
  function parseFile(err, data) {
      if (err == null) {
          var jsdomDocument = jsdom(data);
          var document = jsdomDocument.parentWindow.document;
          getContent(document);
      }
  }
  
  function getContent(document) {
      fields = ParserRules.fields;
      var csvRow = [];
      for (var field in fields) {
          try {
              console.log('Looking for ' + field);
              var nodes = xpath.select(fields[field], document);
              for (var i = 0; i &lt; nodes.length; i++) {
                  csvRow.push(getValue(nodes[i]));
              }
          } catch (err) {
              console.log(err);
          }
      }
      saveToCsv(csvRow, ParserRules.saveFile);
  }
  
  function getValue(node) {
      if(node.nodeValue != null) {
          toReturn = node.nodeValue;
      } else {
          newNode = $(node);
          toReturn = newNode.html();
      }
      return toReturn;
  }
  
  function saveToCsv(object, filePath) {
      console.log('Saving...');
      if(object.length &gt; 0) {
          console.log('Row Exists, Saving...');
          toString = S(object).toCSV().s + os.EOL;
          fs.appendFile(filePath, toString, {encoding: 'utf8', flag: 'a'}, function(err){
              if (err) {
                  console.log('Write Error: ' + err);
              } else {
                  console.log('Saved ' + object);
              }
          });
      }
  }
  </code></pre>