 DEBUG_updateRowsLimit =  null   # set for debugging
 TEMP_DIR = "data/imports/temp"
 OMIT_NULL_VALUE_FIELDS_IN_COMMITMENTS_JSON = false
-AIDDATA_TEMP_FILE = TEMP_DIR + "/_aiddata.json" 
+
+AIDDATA_TEMP_FILE = TEMP_DIR + "/aiddata.json"
+LOCATIONS_TEMP_FILE = TEMP_DIR + "/locations.json"
+
+USE_MONGOIMPORT = false
 
 mongodb = pgclient = null
-aiddataColl = null
+
+locationNameToCode = {}
+
+
+
+importCollectionToMongo = (collection, file, upsertFields, callWhenEnded) ->
+
+  if USE_MONGOIMPORT   # faster
+
+    os.run "/usr/bin/mongoimport " +
+                "-d aiddata -c #{collection} --upsert --upsertFields #{upsertFields.join(',')} " +
+                "-u #{dbconf.mongodb.user} -p #{dbconf.mongodb.password} "+ 
+                "#{file}",
+      (err) ->
+        unless err?
+          fs.unlink(AIDDATA_TEMP_FILE)
+
+        callWhenEnded(err)
+
+  else
+
+    
+    mongodb.collection collection, (err, coll) =>
+      if err? then callWhenEnded(err)
+      else
+
+        reader = linereader.open file
+
+        nextRow = ->
+          if reader.hasNextLine()
+            JSON.parse reader.nextLine()
+          else
+            null
+
+
+        # update synchronously
+        upsert = do -> 
+          upsertCount = 0
+          (row) ->
+            unless row?
+              callWhenEnded()
+              console.log "Upserted #{upsertCount} documents"
+            else
+              criteria = {}
+              for f in upsertFields
+                criteria[f] = row[f]
+
+              coll.update(
+                criteria, row, { safe:true, upsert:true }, 
+                (err, docs) ->
+                  if err?
+                    console.log "Upserted #{upsertCount} documents"
+                    console.warn "Problem upserting aiddata_id: #{row.aiddata_id}: #{err}"
+                    callWhenEnded(err)
+                  else
+                    upsertCount++
+                    if ((upsertCount % 10000) is 0) then console.log "Upsert count: #{upsertCount}" 
+                    upsert nextRow()   # proceed to next row              
+                , true
+              )
+        upsert nextRow()
+
+
 
 
 queue = require('queue-async')

    pgclient.connect()
    callWhenEnded()

  (callWhenEnded) ->
    console.log "> Ensure aiddata_id index"
  
    aiddataColl.ensureIndex { aiddata_id : 1 }, (err) -> 
      if err? then callWhenEnded(err)
      else 
        console.log "Index for aiddata_id ensured"
        callWhenEnded()



  (callWhenEnded) ->
    console.log "> Export AidData commitments from PostgreSQL to a temporary file"

    os.mkdir TEMP_DIR
    fd = fs.openSync(AIDDATA_TEMP_FILE, 'w')

      if err? then callWhenEnded(err)
      else
        mongodb = db
        callWhenEnded()







  (callWhenEnded) ->
    console.log "> Connecting to PostgreSQL"
