 checkIfFileHasBeenProccessed = (project_id, callback)->
 	exec "grep #{project_id} #{finished_projects_path}", (error, results) ->
 		hasBeenProcessed = _.include(results, project_id)
-		console.log hasBeenProcessed, project_id
+		#console.log hasBeenProcessed, project_id
 		callback(null, hasBeenProcessed)
 
+loadProjectIds = (callback)->
+	fs.readFile all_projects_path, "utf-8", (err, data)->
+		console.log data.length
+		ids = data.split("\n")
+		console.log ids.length
+		callback err, ids
+
+getAndWriteProjectids = (callback)->
+	console.log "finding all project id's - #{new Date().toString()}"
+	db.projects.find {}, {_id:1}, (err, ids)->
+		console.log "total found projects in mongo #{ids.length} - #{new Date().toString()}"
+		ids = _.pluck ids, '_id'
+		ids = _.filter ids, (id)-> id?
+		fileData = ids.join("\n")
+		fs.writeFile all_projects_path, fileData, ->
+			callback(err, ids)
+
 getProjectIds = (callback)->
-	console.log "finding all project id's - #{new Date().toString()}"
-	db.projects.find {}, {_id:1}, (err,ids)->
-		console.log "total found projects in mongo #{ids.length} - #{new Date().toString()}"
-		ids = _.map ids, (id)-> return id._id.toString()
-		callback(err, ids)
+	exists = fs.existsSync all_projects_path
+	if exists 
+		loadProjectIds callback
+	else
+		getAndWriteProjectids callback
 
 markProjectAsProcessed = (project_id, callback)->
 	fs.appendFile finished_projects_path, "#{project_id}\n", callback

exec = require("child_process").exec

finished_projects_path = "/tmp/finished-projects"


printProgress = ->
	exec "wc #{finished_projects_path}", (error, results) ->



processNext = (project_id, callback)->
	checkIfFileHasBeenProccessed project_id, (err, hasBeenProcessed)->
		if hasBeenProcessed
			console.log "#{project_id} already procssed, skipping"
			return callback()
		getAllDocs project_id, (err, docs)->
			if err?
