 
 
 processNext = (project_id, callback)->
-	#console.log("starting to process #{project_id} - #{new Date().toString()}")
 	checkIfFileHasBeenProccessed project_id, (err, hasBeenProcessed)->
 		if hasBeenProcessed
+			console.log "#{project_id} already procssed, skipping"
 			return callback()
 		getAllDocs project_id, (err, docs)->
 			if err?

checkIfFileHasBeenProccessed = (project_id, callback)->
	exec "grep #{project_id} #{finished_projects_path}", (error, results) ->
		hasBeenProcessed = _.include(results, project_id)
		console.log hasBeenProcessed, project_id
		callback(null, hasBeenProcessed)

getProjectIds = (callback)->
	console.log "finding all project id's - #{new Date().toString()}"
	db.projects.find {}, {_id:1}, (err,ids)->
		console.log "total found projects in mongo #{ids.length} - #{new Date().toString()}"
		ids = _.map ids, (id)-> return id._id.toString()
		callback(err, ids)

markProjectAsProcessed = (project_id, callback)->
	fs.appendFile finished_projects_path, "#{project_id}\n", callback

getAllDocs = (project_id, callback = (error, docs) ->) ->
	db.projects.findOne _id:ObjectId(project_id), (error, project) ->
		return callback(error) if error?
		if !project?
			console.error("No such project: #{project_id}")
			return callback("no such project #{project_id}")
		findAllDocsInProject project, (error, docs) ->
			return callback(error) if error?
			return callback null, docs
