chore(cluster): disconnect worker on error, then kill after 30 sec

(cc @lefnire): This disconnects the process, sets up a kill timer, and
then sends error emails and responses back to the client. This should
stop new clients from connecting to the dead worker, allow connections
already established to finish, and cause the master to fork a new worker
to pick up the slack immediately.

src/server.js

@@ -12,9 +12,9 @@ if (cluster.isMaster && (isDev || isProd)) {
   // Fork workers.
   _.times(require('os').cpus().length, function(){
     cluster.fork();
-  })
+  });
 
-  cluster.on('exit', function(worker, code, signal) {
+  cluster.on('disconnect', function(worker, code, signal) {
     var w = cluster.fork(); // replace the dead worker
     console.error('[%s] [master:%s] worker:%s disconnect! new worker:%s fork', new Date(), process.pid, worker.process.pid, w.process.pid);
   });
@@ -135,6 +135,7 @@ if (cluster.isMaster && (isDev || isProd)) {
   app.use('/api/v1', require('./routes/apiv1').middleware);
   app.use('/export', require('./routes/dataexport').middleware);
 
+  app.use(utils.crashWorker(server));
   app.use(utils.errorHandler);
 
   require('./routes/apiv2.coffee')(swagger, v2);
@@ -145,4 +146,4 @@ if (cluster.isMaster && (isDev || isProd)) {
   });
 
   module.exports = server;
-}
+}

src/utils.js

@@ -2,6 +2,7 @@ var nodemailer = require('nodemailer');
 var nconf = require('nconf');
 var crypto = require('crypto');
 var path = require("path");
+var cluster = require("cluster");
 
 module.exports.sendEmail = function(mailData) {
   var smtpTransport = nodemailer.createTransport("SMTP",{
@@ -59,6 +60,23 @@ module.exports.setupConfig = function(){
   if (nconf.get('NODE_ENV') === 'production') require('newrelic');
 };
 
+module.exports.crashWorker = function(server) {
+    return function(err, req, res, next) {
+        if (!cluster.isMaster) {
+            // make sure we close down within 30 seconds
+            var killtimer = setTimeout(function() {
+                process.exit(1);
+            }, 30000);
+            // But don't keep the process open just for that!
+            killtimer.unref();
+            // stop taking new requests.
+            server.close();
+            cluster.worker.disconnect();
+        }
+        next(err);
+    };
+}
+
 
 module.exports.errorHandler = function(err, req, res, next) {
   // when we hit an error, send it to admin as an email. If no ADMIN_EMAIL is present, just send it to yourself (SMTP_USER)
@@ -79,5 +97,4 @@ module.exports.errorHandler = function(err, req, res, next) {
   var message = err.message ? err.message : err;
   message =  (message.length < 200) ? message : message.substring(0,100) + message.substring(message.length-100,message.length);
   res.json(500,{err:message}); //res.end(err.message);
-  process.exit(0);
-}
+}