diff --git a/builder/run.sh b/builder/run.sh index 0c986c4..0957b80 100755 --- a/builder/run.sh +++ b/builder/run.sh @@ -4,4 +4,4 @@ sed -i "s/^.*Endpoint\": \"\(http:\/\/haproxy-ip-address:8000\)\".*$/ \"EndPo ${CONFIG_PATH:=config/production.example.json} fi haproxy -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid -/usr/bin/supervisord +exec /usr/bin/supervisord diff --git a/builder/supervisord.conf b/builder/supervisord.conf index ae1733d..53d1148 100644 --- a/builder/supervisord.conf +++ b/builder/supervisord.conf @@ -4,5 +4,5 @@ loglevel=debug [program:bamboo] redirect_stderr=true -command=/bin/bash -c "MARATHON_ENDPOINT=${MARATHON_ENDPOINT}; BAMBOO_ENDPOINT=${BAMBOO_ENDPOINT}; BAMBOO_ZK_HOST=${BAMBOO_ZK_HOST}; BAMBOO_ZK_PATH=${BAMBOO_ZK_PATH}; /var/bamboo/bamboo -bind="${BIND:-:8000}" -config=${CONFIG_PATH:-config/production.example.json}" - +command=/bin/bash -c "MARATHON_ENDPOINT=${MARATHON_ENDPOINT}; BAMBOO_ENDPOINT=${BAMBOO_ENDPOINT}; BAMBOO_ZK_HOST=${BAMBOO_ZK_HOST}; BAMBOO_ZK_PATH=${BAMBOO_ZK_PATH}; exec /var/bamboo/bamboo -bind="${BIND:-:8000}" -config=${CONFIG_PATH:-config/production.example.json}" +stopwaitsecs=60 diff --git a/builder/supervisord.conf.prod b/builder/supervisord.conf.prod index d9863d5..41f004e 100644 --- a/builder/supervisord.conf.prod +++ b/builder/supervisord.conf.prod @@ -2,5 +2,5 @@ nodaemon=true [program:bamboo] -command=/bin/bash -c "MARATHON_ENDPOINT=${MARATHON_ENDPOINT}; BAMBOO_ENDPOINT=${BAMBOO_ENDPOINT}; BAMBOO_ZK_HOST=${BAMBOO_ZK_HOST}; BAMBOO_ZK_PATH=${BAMBOO_ZK_PATH}; /var/bamboo/bamboo -bind="${BIND:-:8000}" -config=${CONFIG_PATH:-config/production.example.json}" - +command=/bin/bash -c "MARATHON_ENDPOINT=${MARATHON_ENDPOINT}; BAMBOO_ENDPOINT=${BAMBOO_ENDPOINT}; BAMBOO_ZK_HOST=${BAMBOO_ZK_HOST}; BAMBOO_ZK_PATH=${BAMBOO_ZK_PATH}; exec /var/bamboo/bamboo -bind="${BIND:-:8000}" -config=${CONFIG_PATH:-config/production.example.json}" +stopwaitsecs=60 diff --git a/config/development.json b/config/development.json index 8b58998..5a1d183 100644 --- a/config/development.json +++ b/config/development.json @@ -15,7 +15,9 @@ "HAProxy": { "TemplatePath": "config/haproxy_template.cfg", "OutputPath": "/etc/haproxy/haproxy.cfg", - "ReloadCommand": "haproxy -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid -D -sf $(cat /var/run/haproxy.pid)" + "ReloadCommand": "haproxy -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid -D -sf $(cat /var/run/haproxy.pid)", + "ShutdownCommand": "PID=$(cat /var/run/haproxy.pid); kill -USR1 $PID; while kill -0 $PID; do sleep 0.1; done", + "GraceSeconds": 5 }, "StatsD": { diff --git a/config/haproxy_template.cfg b/config/haproxy_template.cfg index d2405ed..6a367bd 100644 --- a/config/haproxy_template.cfg +++ b/config/haproxy_template.cfg @@ -25,6 +25,8 @@ defaults timeout client 50000 timeout server 50000 + grace {{ .GraceSeconds }}s + errorfile 400 /etc/haproxy/errors/400.http errorfile 403 /etc/haproxy/errors/403.http errorfile 408 /etc/haproxy/errors/408.http @@ -34,6 +36,15 @@ defaults errorfile 504 /etc/haproxy/errors/504.http +frontend health + bind *:2000 + + # This returns 503 once we receive USR1 signal + acl shutting_down stopping + monitor-uri /health + monitor fail if shutting_down + + # Template Customization frontend http-in bind *:80 diff --git a/config/production.example.json b/config/production.example.json index 4d9b035..4ca272d 100644 --- a/config/production.example.json +++ b/config/production.example.json @@ -15,7 +15,9 @@ "HAProxy": { "TemplatePath": "config/haproxy_template.cfg", "OutputPath": "/etc/haproxy/haproxy.cfg", - "ReloadCommand": "haproxy -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid -D -sf $(cat /var/run/haproxy.pid)" + "ReloadCommand": "haproxy -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid -D -sf $(cat /var/run/haproxy.pid)", + "ShutdownCommand": "PID=$(cat /var/run/haproxy.pid); kill -USR1 $PID; while kill -0 $PID; do sleep 0.1; done", + "GraceSeconds": 5 }, "StatsD": { diff --git a/configuration/configuration.go b/configuration/configuration.go index 75d0839..b593784 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -55,6 +55,8 @@ func FromFile(filePath string) (Configuration, error) { setValueFromEnv(&conf.HAProxy.TemplatePath, "HAPROXY_TEMPLATE_PATH") setValueFromEnv(&conf.HAProxy.OutputPath, "HAPROXY_OUTPUT_PATH") setValueFromEnv(&conf.HAProxy.ReloadCommand, "HAPROXY_RELOAD_CMD") + setValueFromEnv(&conf.HAProxy.ShutdownCommand, "HAPROXY_SHUTDOWN_CMD") + setIntValueFromEnv(&conf.HAProxy.GraceSeconds, "HAPROXY_GRACE_SECONDS") setValueFromEnv(&conf.StatsD.Host, "STATSD_HOST") setValueFromEnv(&conf.StatsD.Prefix, "STATSD_PREFIX") setBoolValueFromEnv(&conf.StatsD.Enabled, "STATSD_ENABLED") @@ -69,6 +71,21 @@ func setValueFromEnv(field *string, envVar string) { } } +func setIntValueFromEnv(field *int, envVar string) { + env := os.Getenv(envVar) + if len(env) > 0 { + log.Printf("Using environment override %s=%t", envVar, env) + x, err := strconv.Atoi(env) + if err != nil { + log.Printf("Error converting int value: %s\n", err) + } + *field = x + } else { + log.Printf("Environment variable not set: %s", envVar) + } +} + + func setBoolValueFromEnv(field *bool, envVar string) { env := os.Getenv(envVar) if len(env) > 0 { diff --git a/configuration/haproxy.go b/configuration/haproxy.go index 4a1be6c..a351604 100644 --- a/configuration/haproxy.go +++ b/configuration/haproxy.go @@ -1,7 +1,9 @@ package configuration type HAProxy struct { - TemplatePath string - OutputPath string - ReloadCommand string + TemplatePath string + OutputPath string + ReloadCommand string + ShutdownCommand string + GraceSeconds int } diff --git a/main/bamboo/bamboo.go b/main/bamboo/bamboo.go index 808896b..07fdcf9 100644 --- a/main/bamboo/bamboo.go +++ b/main/bamboo/bamboo.go @@ -74,7 +74,7 @@ func main() { eventBus.Publish(event_bus.MarathonEvent{EventType: "bamboo_startup", Timestamp: time.Now().Format(time.RFC3339)}) // Handle gracefully exit - registerOSSignals() + registerOSSignals(&conf) // Start server initServer(&conf, zkConn, eventBus) @@ -187,12 +187,23 @@ func configureLog() { } } -func registerOSSignals() { - c := make(chan os.Signal, 1) - signal.Notify(c, os.Interrupt) +func registerOSSignals(conf *configuration.Configuration) { + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, os.Interrupt, syscall.SIGTERM) go func() { - for _ = range c { - log.Println("Server Stopped") + sig := <-sigs + if sig == os.Interrupt { + log.Println("Server stopping now") + os.Exit(0) + } else if sig == syscall.SIGTERM { + log.Println("Server gracefully exiting...") + err := event_bus.ExecCommand(conf.HAProxy.ShutdownCommand) + if err != nil { + log.Fatalf("HAProxy: graceful shutdown failed\n") + } else { + log.Println("HAProxy: graceful shutdown complete") + } + log.Println("Server stopping after graceful haproxy shutdown") os.Exit(0) } }() diff --git a/services/event_bus/event_handler.go b/services/event_bus/event_handler.go index 70d9f59..2427d64 100644 --- a/services/event_bus/event_handler.go +++ b/services/event_bus/event_handler.go @@ -98,7 +98,7 @@ func handleHAPUpdate(conf *configuration.Configuration, conn *zk.Conn) bool { log.Fatalf("Failed to write template on path: %s", err) } - err = execCommand(conf.HAProxy.ReloadCommand) + err = ExecCommand(conf.HAProxy.ReloadCommand) if err != nil { log.Fatalf("HAProxy: update failed\n") } else { @@ -112,7 +112,7 @@ func handleHAPUpdate(conf *configuration.Configuration, conn *zk.Conn) bool { } } -func execCommand(cmd string) error { +func ExecCommand(cmd string) error { log.Printf("Exec cmd: %s \n", cmd) output, err := exec.Command("sh", "-c", cmd).CombinedOutput() if err != nil { diff --git a/services/haproxy/haproxy.go b/services/haproxy/haproxy.go index 125650e..85d22c6 100644 --- a/services/haproxy/haproxy.go +++ b/services/haproxy/haproxy.go @@ -10,6 +10,7 @@ import ( type templateData struct { Apps marathon.AppList Services map[string]service.Service + GraceSeconds int } func GetTemplateData(config *conf.Configuration, conn *zk.Conn) (interface{}, error) { @@ -26,5 +27,7 @@ func GetTemplateData(config *conf.Configuration, conn *zk.Conn) (interface{}, er return nil, err } - return templateData{apps, services}, nil + graceSeconds := config.HAProxy.GraceSeconds + + return templateData{apps, services, graceSeconds}, nil }