 DIR_BY_URL_PATH = Path.join config.DATA_DIR, 'content-by-url'
 
 DIR_PATH_FROM_URL_TO_ID = Path.relative DIR_BY_URL_PATH, DIR_BY_ID_PATH
+
+# How many files to process in parallel:
+NUM_PARALLEL = 100
 
 getFilePathForId = (id) ->
     # NOTE: Since file systems are typically case-insensitive (even though

    path = path.replace /_([A-Z])/g, '$1'
    Path.basename path, Path.extname path

# Now query all the IDs we have:
echo 'Querying content files...'
idFileNames = FS.readdir DIR_BY_ID_PATH, _
totalNumIds = idFileNames.length
echo "#{totalNumIds} files found."

# Now go through each ID,and process it, outputting percentage every minute:
echo 'Processing...'
nextLogTime = Date.now() + 1000 * 60

for idFileName, i in idFileNames
    # Skip non-content files:
    continue if (Path.extname idFileName) isnt '.json'

    id = getIdForFilePath idFileName
    idPath = getFilePathForId id  # full path, not just file name

config = require '../config'
crypto = require 'crypto'
echo = console.log
flows = require 'streamline/lib/util/flows'
FS = require 'fs'
Path = require 'path'

