Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support graceful shutdown of haproxy #156

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

Support graceful shutdown of haproxy #156

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
73eb565
Support graceful shutdown of haproxy
drewrobb Aug 25, 2015
bfc8add
Parse settings from ENV
drewrobb Aug 25, 2015
1a678c1
Stopwaitsecs
drewrobb Aug 25, 2015
6c30dcf
Simpler healthcheck
drewrobb Aug 27, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion builder/run.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@ sed -i "s/^.*Endpoint\": \"\(http:\/\/haproxy-ip-address:8000\)\".*$/ \"EndPo
${CONFIG_PATH:=config/production.example.json}
fi
haproxy -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid
/usr/bin/supervisord
exec /usr/bin/supervisord
3 changes: 1 addition & 2 deletions builder/supervisord.conf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,4 @@ loglevel=debug

[program:bamboo]
redirect_stderr=true
command=/bin/bash -c "MARATHON_ENDPOINT=${MARATHON_ENDPOINT}; BAMBOO_ENDPOINT=${BAMBOO_ENDPOINT}; BAMBOO_ZK_HOST=${BAMBOO_ZK_HOST}; BAMBOO_ZK_PATH=${BAMBOO_ZK_PATH}; /var/bamboo/bamboo -bind="${BIND:-:8000}" -config=${CONFIG_PATH:-config/production.example.json}"

command=/bin/bash -c "MARATHON_ENDPOINT=${MARATHON_ENDPOINT}; BAMBOO_ENDPOINT=${BAMBOO_ENDPOINT}; BAMBOO_ZK_HOST=${BAMBOO_ZK_HOST}; BAMBOO_ZK_PATH=${BAMBOO_ZK_PATH}; exec /var/bamboo/bamboo -bind="${BIND:-:8000}" -config=${CONFIG_PATH:-config/production.example.json}"
3 changes: 1 addition & 2 deletions builder/supervisord.conf.prod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,4 @@
nodaemon=true

[program:bamboo]
command=/bin/bash -c "MARATHON_ENDPOINT=${MARATHON_ENDPOINT}; BAMBOO_ENDPOINT=${BAMBOO_ENDPOINT}; BAMBOO_ZK_HOST=${BAMBOO_ZK_HOST}; BAMBOO_ZK_PATH=${BAMBOO_ZK_PATH}; /var/bamboo/bamboo -bind="${BIND:-:8000}" -config=${CONFIG_PATH:-config/production.example.json}"

command=/bin/bash -c "MARATHON_ENDPOINT=${MARATHON_ENDPOINT}; BAMBOO_ENDPOINT=${BAMBOO_ENDPOINT}; BAMBOO_ZK_HOST=${BAMBOO_ZK_HOST}; BAMBOO_ZK_PATH=${BAMBOO_ZK_PATH}; exec /var/bamboo/bamboo -bind="${BIND:-:8000}" -config=${CONFIG_PATH:-config/production.example.json}"
4 changes: 3 additions & 1 deletion config/development.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,9 @@
"HAProxy": {
"TemplatePath": "config/haproxy_template.cfg",
"OutputPath": "/etc/haproxy/haproxy.cfg",
"ReloadCommand": "haproxy -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid -D -sf $(cat /var/run/haproxy.pid)"
"ReloadCommand": "haproxy -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid -D -sf $(cat /var/run/haproxy.pid)",
"ShutdownCommand": "PID=$(cat /var/run/haproxy.pid); kill -USR1 $PID; while kill -0 $PID; do sleep 0.1; done",
"GraceSeconds": 5
},

"StatsD": {
Expand Down
20 changes: 20 additions & 0 deletions config/haproxy_template.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ defaults
timeout client 50000
timeout server 50000

grace {{ .GraceSeconds }}s

errorfile 400 /etc/haproxy/errors/400.http
errorfile 403 /etc/haproxy/errors/403.http
errorfile 408 /etc/haproxy/errors/408.http
Expand All @@ -34,6 +36,24 @@ defaults
errorfile 504 /etc/haproxy/errors/504.http


frontend graceful_stop_check
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I couldn't think of a way to get rid of this extra frontend just for checking if shutdown is happening

bind *:2001
# Close port immediately on a SIGUSR1.
grace 0s

backend graceful_stop_detector
description See if we are shutting down gracefully
server healthcheck-frontend *:2001 check inter 1s fall 1

frontend health
bind *:2000

# This returns 503 once we receive USR1 signal
acl shutting_down nbsrv(graceful_stop_detector) eq 0
monitor-uri /health
monitor fail if shutting_down


# Template Customization
frontend http-in
bind *:80
Expand Down
4 changes: 3 additions & 1 deletion config/production.example.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,9 @@
"HAProxy": {
"TemplatePath": "config/haproxy_template.cfg",
"OutputPath": "/etc/haproxy/haproxy.cfg",
"ReloadCommand": "haproxy -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid -D -sf $(cat /var/run/haproxy.pid)"
"ReloadCommand": "haproxy -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid -D -sf $(cat /var/run/haproxy.pid)",
"ShutdownCommand": "PID=$(cat /var/run/haproxy.pid); kill -USR1 $PID; while kill -0 $PID; do sleep 0.1; done",
"GraceSeconds": 5
},

"StatsD": {
Expand Down
8 changes: 5 additions & 3 deletions configuration/haproxy.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
package configuration

type HAProxy struct {
TemplatePath string
OutputPath string
ReloadCommand string
TemplatePath string
OutputPath string
ReloadCommand string
ShutdownCommand string
GraceSeconds int
}
23 changes: 17 additions & 6 deletions main/bamboo/bamboo.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ func main() {
eventBus.Publish(event_bus.MarathonEvent{EventType: "bamboo_startup", Timestamp: time.Now().Format(time.RFC3339)})

// Handle gracefully exit
registerOSSignals()
registerOSSignals(&conf)

// Start server
initServer(&conf, zkConn, eventBus)
Expand Down Expand Up @@ -187,12 +187,23 @@ func configureLog() {
}
}

func registerOSSignals() {
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt)
func registerOSSignals(conf *configuration.Configuration) {
sigs := make(chan os.Signal, 1)
signal.Notify(sigs, os.Interrupt, syscall.SIGTERM)
go func() {
for _ = range c {
log.Println("Server Stopped")
sig := <-sigs
if sig == os.Interrupt {
log.Println("Server stopping now")
os.Exit(0)
} else if sig == syscall.SIGTERM {
log.Println("Server gracefully exiting...")
err := event_bus.ExecCommand(conf.HAProxy.ShutdownCommand)
if err != nil {
log.Fatalf("HAProxy: graceful shutdown failed\n")
} else {
log.Println("HAProxy: graceful shutdown complete")
}
log.Println("Server stopping after graceful haproxy shutdown")
os.Exit(0)
}
}()
Expand Down
4 changes: 2 additions & 2 deletions services/event_bus/event_handler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ func handleHAPUpdate(conf *configuration.Configuration, conn *zk.Conn) bool {
log.Fatalf("Failed to write template on path: %s", err)
}

err = execCommand(conf.HAProxy.ReloadCommand)
err = ExecCommand(conf.HAProxy.ReloadCommand)
if err != nil {
log.Fatalf("HAProxy: update failed\n")
} else {
Expand All @@ -112,7 +112,7 @@ func handleHAPUpdate(conf *configuration.Configuration, conn *zk.Conn) bool {
}
}

func execCommand(cmd string) error {
func ExecCommand(cmd string) error {
log.Printf("Exec cmd: %s \n", cmd)
output, err := exec.Command("sh", "-c", cmd).CombinedOutput()
if err != nil {
Expand Down
5 changes: 4 additions & 1 deletion services/haproxy/haproxy.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
type templateData struct {
Apps marathon.AppList
Services map[string]service.Service
GraceSeconds int
}

func GetTemplateData(config *conf.Configuration, conn *zk.Conn) (interface{}, error) {
Expand All @@ -26,5 +27,7 @@ func GetTemplateData(config *conf.Configuration, conn *zk.Conn) (interface{}, er
return nil, err
}

return templateData{apps, services}, nil
graceSeconds := config.HAProxy.GraceSeconds

return templateData{apps, services, graceSeconds}, nil
}