Merge pull request #10761 from alphagov/unicorn-reloader

Add a script to monitor unicornherder reloads

hieradata_aws/common.yaml

@@ -1184,6 +1184,8 @@ govuk_sudo::sudo_conf:
     content: 'deploy ALL=NOPASSWD:/etc/init.d/varnish'
   deploy_varnishadm:
     content: 'deploy ALL=NOPASSWD:/usr/bin/varnishadm'
+  deploy_govuk_unicorn_reload:
+    content: 'deploy ALL=NOPASSWD:/usr/local/bin/govuk_unicorn_reload'
   icinga_init_ctl:
     content: 'nagios ALL=NOPASSWD:/sbin/initctl reload *'
   icinga_initctl_restart:

modules/govuk/files/usr/local/bin/govuk_unicorn_reload

@@ -0,0 +1,78 @@
+#!/bin/bash
+
+# This script is a wrapper around the upstart reload command intended for a
+# an application that is run via unicornherder.
+#
+# This script supervises the termination and creation of the processes
+# unicornherder manages. This is so that we can know when a deployment of
+# an application has flipped from the previous version to the new version,
+# which can be used to determine when it is safe to test a new deployment
+# of an app.
+
+if [ "$#" -ne "1" ]; then
+  echo "Usage: sudo govuk_unicorn_reload <APP_NAME>"
+  exit 1
+fi
+
+set -eu
+
+status () {
+  echo "---> ${@}"
+}
+
+error () {
+  echo "ERROR: ${@}" >&2
+  exit 1
+}
+
+pid_running () {
+  ps -p ${@} &> /dev/null
+}
+
+read_pid () {
+  cat /var/run/${@}/app.pid
+}
+
+app=$1
+
+status "Checking ${app} is running"
+
+if [ -e "/var/run/${app}/app.pid" ]; then
+  old_pid=$(read_pid $app)
+else
+  error "No pid file at /var/run/${app}/app.pid"
+fi
+
+pid_running $old_pid || error "Process ${old_pid} doesn't appear to be running"
+
+status "${app} is running with process ${old_pid}. Reloading..."
+initctl reload $app
+
+waiting_for=0
+new_pid=""
+
+while true; do
+  sleep 10
+  waiting_for=$(($waiting_for+10))
+  current_pid=$(read_pid $app) || error "Couldn't establish the current pid"
+
+  if [ -z $new_pid ] && [ $old_pid -ne $current_pid ]; then
+    new_pid=$current_pid
+    status "New process ${new_pid} started"
+  fi
+
+  if ! $(pid_running $old_pid); then
+    status "Process ${old_pid} has exited"
+    break
+  elif [ $waiting_for -gt 180 ]; then
+    error "Process ${old_pid} is still running after 3 minutes, something has gone wrong"
+  else
+    status "Waiting for processes to swap over"
+  fi
+done
+
+if $(pid_running $new_pid); then
+  status "Successfully reloaded ${app}"
+else
+  error "The new process for ${app} doesn't appear to be running"
+fi

modules/govuk/manifests/deploy/config.pp

@@ -78,6 +78,16 @@
     mode   => '0755',
   }
 
+  # govuk_unicorn_reload is a wrapper around the upstart command to reload
+  # unicorn following a deploy. It supervises the reloading to delay confirming
+  # a reload as complete until the new process is running and the old one is
+  # stopped.
+  file { '/usr/local/bin/govuk_unicorn_reload':
+    ensure => present,
+    source => 'puppet:///modules/govuk/usr/local/bin/govuk_unicorn_reload',
+    mode   => '0755',
+  }
+
   # govuk_setenv is a simple script that loads the environment for a GOV.UK
   # application and execs its arguments
   # daemontools provides envdir, used by govuk_setenv