move monit-driven http healthchecking into custom script

* ancient bosh monit will continuously restart us at 5+1 monit cycles
  whenever CC might be having a hard time getting up within 10 seconds,
  so this produces more predictable behavior where it only restarts
  every 5 failed curls. it also doesn't interfere with initial startup.

[fixes #125]
[fixes #164263056]

Co-authored-by: Connor Braa <cbraa@pivotal.io>
Co-authored-by: Tim Downey <tdowney@pivotal.io>

Author: tcdowney

jobs/cloud_controller_ng/monit

@@ -1,41 +1,22 @@
-<%
-  def discover_external_ip
-    networks = spec.networks.marshal_dump
-
-    _, network = networks.find do |_name, network_spec|
-      network_spec.default
-    end
-
-    if !network
-      _, network = networks.first
-    end
-
-    if !network
-      raise "Could not determine IP via network spec: #{networks}"
-    end
-
-    network.ip
-  end
-%>
 
 <% if p("bpm.enabled") %>
 
 check process cloud_controller_ng
   with pidfile /var/vcap/sys/run/bpm/cloud_controller_ng/cloud_controller_ng.pid
   start program "/var/vcap/jobs/bpm/bin/bpm start cloud_controller_ng"
   stop program "/var/vcap/jobs/bpm/bin/bpm stop cloud_controller_ng"
+  depends on ccng_monit_http_healthcheck
   group vcap
   if totalmem > <%= p("cc.thresholds.api.alert_if_above_mb") %> Mb for 3 cycles then alert
   if totalmem > <%= p("cc.thresholds.api.restart_if_consistently_above_mb") %> Mb for 15 cycles then exec "/var/vcap/jobs/cloud_controller_ng/bin/restart_drain"
   if totalmem > <%= p("cc.thresholds.api.restart_if_above_mb") %> Mb for 3 cycles then exec "/var/vcap/jobs/cloud_controller_ng/bin/restart_drain"
-<% if p('cc.nginx.ip').empty? %>
-  if failed host <%= discover_external_ip %> port <%= p("cc.external_port") %> protocol http
-<% else %>
-  if failed host <%= p('cc.nginx.ip') %> port <%= p("cc.external_port") %> protocol http
-<% end %>
-      and request '/v2/info'
-      with timeout 60 seconds for 5 cycles
-  then restart
+
+
+check process ccng_monit_http_healthcheck
+    with pidfile /var/vcap/sys/run/bpm/cloud_controller_ng/ccng_monit_http_healthcheck.pid
+    start program "/var/vcap/jobs/bpm/bin/bpm start cloud_controller_ng -p ccng_monit_http_healthcheck"
+    stop program "/var/vcap/jobs/bpm/bin/bpm stop cloud_controller_ng -p ccng_monit_http_healthcheck"
+    group vcap
 
 <% (1..(p("cc.jobs.local.number_of_workers"))).each do |index| %>
 check process cloud_controller_worker_local_<%= index %>
@@ -75,14 +56,6 @@ check process cloud_controller_ng
   if totalmem > <%= p("cc.thresholds.api.alert_if_above_mb") %> Mb for 3 cycles then alert
   if totalmem > <%= p("cc.thresholds.api.restart_if_consistently_above_mb") %> Mb for 15 cycles then exec "/var/vcap/jobs/cloud_controller_ng/bin/restart_drain"
   if totalmem > <%= p("cc.thresholds.api.restart_if_above_mb") %> Mb for 3 cycles then exec "/var/vcap/jobs/cloud_controller_ng/bin/restart_drain"
-<% if p('cc.nginx.ip').empty? %>
-  if failed host <%= discover_external_ip %> port <%= p("cc.external_port") %> protocol http
-<% else %>
-  if failed host <%= p('cc.nginx.ip') %> port <%= p("cc.external_port") %> protocol http
-<% end %>
-      and request '/v2/info'
-      with timeout 60 seconds for 5 cycles
-  then restart
 
 <% (1..(p("cc.jobs.local.number_of_workers"))).each do |index| %>
 check process cloud_controller_worker_local_<%= index %>

jobs/cloud_controller_ng/spec

@@ -20,6 +20,7 @@ templates:
   cloud_controller_api_ctl.erb: bin/cloud_controller_ng_ctl
   cloud_controller_api_health_check.erb: bin/cloud_controller_ng_health_check
   cloud_controller_api_worker_ctl.erb: bin/cloud_controller_worker_ctl
+  ccng_monit_http_healthcheck.sh.erb: bin/ccng_monit_http_healthcheck
   console.erb: bin/console
   dns_health_check.erb: bin/dns_health_check
   drain.sh.erb: bin/drain

jobs/cloud_controller_ng/templates/bpm.yml.erb

@@ -70,12 +70,18 @@ nginx_maintenance_config = {
   "ephemeral_disk" => true,
 }
 
+ccng_monit_http_healthcheck_config = {
+  "name" => "ccng_monit_http_healthcheck",
+  "executable" => "/var/vcap/jobs/cloud_controller_ng/bin/ccng_monit_http_healthcheck",
+ }
+
 config = {
   "processes" => [
     cloud_controller_ng_config,
     nginx_config,
     nginx_newrelic_plugin_config,
     nginx_maintenance_config,
+    ccng_monit_http_healthcheck_config,
   ]
 }
 

jobs/cloud_controller_ng/templates/ccng_monit_http_healthcheck.sh.erb

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+set -e
+
+<%
+  def discover_external_ip
+    networks = spec.networks.marshal_dump
+
+    _, network = networks.find do |_name, network_spec|
+      network_spec.default
+    end
+
+    if !network
+      _, network = networks.first
+    end
+
+    if !network
+      raise "Could not determine IP via network spec: #{networks}"
+    end
+
+    network.ip
+  end
+%>
+
+function log_failure {
+  echo "$(date --rfc-3339=seconds) :: Healthcheck failed consistently, restarting CC"
+}
+
+HOST=<%= p('cc.nginx.ip').empty? ? discover_external_ip : p('cc.nginx.ip') %>
+PORT=<%= p("cc.external_port") %>
+URL="http://${HOST}:${PORT}/v2/info"
+
+source /var/vcap/packages/capi_utils/monit_utils.sh
+
+echo 'Waiting for Cloud Controller to initially become healthy'
+
+wait_for_server_to_become_healthy "${URL}" "<%= p("cc.api_post_start_healthcheck_timeout_in_seconds") %>"
+
+echo 'Initial check passed, will now restart CC over on repeated failures'
+
+trap log_failure EXIT
+
+# if we fail to curl it 5 times in a row across 50 seconds, die so monit will restart us
+set -e
+while true; do
+  curl \
+    -sS \
+    --max-time <%= p('cc.api_health_check_timeout_per_retry') %> \
+    --retry 5 \
+    --retry-delay 10 \
+    -A "ccng_monit_http_healthcheck" \
+    "${URL}" > /dev/null
+  sleep 10
+done

jobs/cloud_controller_ng/templates/cloud_controller_api_health_check.erb

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+# this script is ran by the route-registrar
+
 curl \
   --max-time <%= p('cc.api_health_check_timeout_per_retry') %> \
   --retry-max-time <%= p('cc.api_health_check_total_timeout') %> \