From 85eb0aa38a472633d32aa00bbe6202c0efa03faa Mon Sep 17 00:00:00 2001
From: Cole Gleason <colewashere@gmail.com>
Date: Sun, 2 Feb 2014 17:41:29 -0600
Subject: [PATCH] chore(cluster): disconnect worker on error, then kill after
 30 sec

(cc @lefnire): This disconnects the process, sets up a kill timer, and
then sends error emails and responses back to the client. This should
stop new clients from connecting to the dead worker, allow connections
already established to finish, and cause the master to fork a new worker
to pick up the slack immediately.
---
 src/server.js |  7 ++++---
 src/utils.js  | 21 +++++++++++++++++++--
 2 files changed, 23 insertions(+), 5 deletions(-)

diff --git a/src/server.js b/src/server.js
index 8d95852ad0..54971e719e 100644
--- a/src/server.js
+++ b/src/server.js
@@ -12,9 +12,9 @@ if (cluster.isMaster && (isDev || isProd)) {
   // Fork workers.
   _.times(require('os').cpus().length, function(){
     cluster.fork();
-  })
+  });
 
-  cluster.on('exit', function(worker, code, signal) {
+  cluster.on('disconnect', function(worker, code, signal) {
     var w = cluster.fork(); // replace the dead worker
     console.error('[%s] [master:%s] worker:%s disconnect! new worker:%s fork', new Date(), process.pid, worker.process.pid, w.process.pid);
   });
@@ -135,6 +135,7 @@ if (cluster.isMaster && (isDev || isProd)) {
   app.use('/api/v1', require('./routes/apiv1').middleware);
   app.use('/export', require('./routes/dataexport').middleware);
 
+  app.use(utils.crashWorker(server));
   app.use(utils.errorHandler);
 
   require('./routes/apiv2.coffee')(swagger, v2);
@@ -145,4 +146,4 @@ if (cluster.isMaster && (isDev || isProd)) {
   });
 
   module.exports = server;
-}
\ No newline at end of file
+}
diff --git a/src/utils.js b/src/utils.js
index eacb9e7c59..dc99aafd23 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -2,6 +2,7 @@ var nodemailer = require('nodemailer');
 var nconf = require('nconf');
 var crypto = require('crypto');
 var path = require("path");
+var cluster = require("cluster");
 
 module.exports.sendEmail = function(mailData) {
   var smtpTransport = nodemailer.createTransport("SMTP",{
@@ -59,6 +60,23 @@ module.exports.setupConfig = function(){
   if (nconf.get('NODE_ENV') === 'production') require('newrelic');
 };
 
+module.exports.crashWorker = function(server) {
+    return function(err, req, res, next) {
+        if (!cluster.isMaster) {
+            // make sure we close down within 30 seconds
+            var killtimer = setTimeout(function() {
+                process.exit(1);
+            }, 30000);
+            // But don't keep the process open just for that!
+            killtimer.unref();
+            // stop taking new requests.
+            server.close();
+            cluster.worker.disconnect();
+        }
+        next(err);
+    };
+}
+
 
 module.exports.errorHandler = function(err, req, res, next) {
   // when we hit an error, send it to admin as an email. If no ADMIN_EMAIL is present, just send it to yourself (SMTP_USER)
@@ -79,5 +97,4 @@ module.exports.errorHandler = function(err, req, res, next) {
   var message = err.message ? err.message : err;
   message =  (message.length < 200) ? message : message.substring(0,100) + message.substring(message.length-100,message.length);
   res.json(500,{err:message}); //res.end(err.message);
-  process.exit(0);
-}
\ No newline at end of file
+}