Commit 2bd88e53 authored by Henry Oswald's avatar Henry Oswald
Browse files

added some null checks and more persistance

parent 59307aa9
......@@ -8,7 +8,7 @@ async = require("async")
exec = require("child_process").exec
finished_projects_path = "/tmp/finished-projects"
all_projects_path = "/tmp/all-projects"
printProgress = ->
exec "wc #{finished_projects_path}", (error, results) ->
......@@ -18,15 +18,32 @@ printProgress = ->
checkIfFileHasBeenProccessed = (project_id, callback)->
exec "grep #{project_id} #{finished_projects_path}", (error, results) ->
hasBeenProcessed = _.include(results, project_id)
console.log hasBeenProcessed, project_id
#console.log hasBeenProcessed, project_id
callback(null, hasBeenProcessed)
getProjectIds = (callback)->
loadProjectIds = (callback)->
fs.readFile all_projects_path, "utf-8", (err, data)->
console.log data.length
ids = data.split("\n")
console.log ids.length
callback err, ids
getAndWriteProjectids = (callback)->
console.log "finding all project id's - #{new Date().toString()}"
db.projects.find {}, {_id:1}, (err,ids)->
db.projects.find {}, {_id:1}, (err, ids)->
console.log "total found projects in mongo #{ids.length} - #{new Date().toString()}"
ids = _.map ids, (id)-> return id._id.toString()
callback(err, ids)
ids = _.pluck ids, '_id'
ids = _.filter ids, (id)-> id?
fileData = ids.join("\n")
fs.writeFile all_projects_path, fileData, ->
callback(err, ids)
getProjectIds = (callback)->
exists = fs.existsSync all_projects_path
if exists
loadProjectIds callback
else
getAndWriteProjectids callback
markProjectAsProcessed = (project_id, callback)->
fs.appendFile finished_projects_path, "#{project_id}\n", callback
......@@ -34,7 +51,9 @@ markProjectAsProcessed = (project_id, callback)->
getAllDocs = (project_id, callback = (error, docs) ->) ->
db.projects.findOne _id:ObjectId(project_id), (error, project) ->
return callback(error) if error?
return callback new Errors.NotFoundError("No such project: #{project_id}") if !project?
if !project?
console.error("No such project: #{project_id}")
return callback("no such project #{project_id}")
findAllDocsInProject project, (error, docs) ->
return callback(error) if error?
return callback null, docs
......@@ -78,9 +97,9 @@ saveDocsIntoMongo = (project_id, docs, callback)->
processNext = (project_id, callback)->
#console.log("starting to process #{project_id} - #{new Date().toString()}")
checkIfFileHasBeenProccessed project_id, (err, hasBeenProcessed)->
if hasBeenProcessed
console.log "#{project_id} already procssed, skipping"
return callback()
getAllDocs project_id, (err, docs)->
if err?
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment