Add request accepting grace period delaying graceful shutdown.

The new grace period defines how long Traefik should continue accepting
requests prior to shutting down all connection listeners. It preceeds
the existing (HTTP shutdown) grace period and allows for a full-circle
graceful termination by expecting downstream load-balancers to take
Traefik out of rotation in time, without the need for rather complex and
external request draining procedures.

Immediate beneficiaries are providers such as Kubernetes and Marathon
which expect in-cluster applications to exhibit the described
termination behavior.

docs/toml.md

@@ -9,6 +9,20 @@
 # Global configuration
 ################################################################
 
+# Duration to keep accepting requests prior to initiating the graceful
+# termination period (as defined by the `graceTimeOut` option). This
+# option is meant to give downstream load-balancer sufficient time to
+# take Traefik out of rotation.
+# Can be provided in a format supported by [time.ParseDuration](https://golang.org/pkg/time/#ParseDuration) or as raw values (digits).
+# If no units are provided, the value is parsed assuming seconds.
+# The zero duration disables the request accepting grace period, i.e.,
+# Traefik will immediately proceed to the grace period.
+#
+# Optional
+# Default: 0
+#
+# reqAcceptGraceTimeOut = "10s"
+
 # Duration to give active requests a chance to finish before Traefik stops.
 # Can be provided in a format supported by [time.ParseDuration](https://golang.org/pkg/time/#ParseDuration) or as raw values (digits).
 # If no units are provided, the value is parsed assuming seconds.

integration/basic_test.go

@@ -3,7 +3,9 @@ package integration
 import (
 	"fmt"
 	"net/http"
+	"os"
 	"strings"
+	"syscall"
 	"time"
 
 	"github.com/containous/traefik/integration/try"
@@ -101,3 +103,67 @@ func (s *SimpleSuite) TestPrintHelp(c *check.C) {
 	})
 	c.Assert(err, checker.IsNil)
 }
+
+func (s *SimpleSuite) TestReqAcceptGraceTimeout(c *check.C) {
+	s.createComposeProject(c, "reqacceptgrace")
+	s.composeProject.Start(c)
+
+	whoami := "http://" + s.composeProject.Container(c, "whoami").NetworkSettings.IPAddress + ":80"
+
+	file := s.adaptFile(c, "fixtures/reqacceptgrace.toml", struct {
+		Server string
+	}{whoami})
+	defer os.Remove(file)
+	cmd, output := s.cmdTraefik(withConfigFile(file))
+	err := cmd.Start()
+	c.Assert(err, checker.IsNil)
+	defer cmd.Process.Kill()
+
+	// Wait for Traefik to turn ready.
+	err = try.GetRequest("http://127.0.0.1:8000/", 2*time.Second, try.StatusCodeIs(http.StatusNotFound))
+	c.Assert(err, checker.IsNil)
+
+	// Show the Traefik log if any assertion fails. If the entire test runs
+	// to a successful completion, we flip the flag at the very end and don't
+	// display anything.
+	showTraefikLog := true
+	defer func() {
+		if showTraefikLog {
+			s.displayTraefikLog(c, output)
+		}
+	}()
+
+	// Make sure exposed service is ready.
+	err = try.GetRequest("http://127.0.0.1:8000/service", 3*time.Second, try.StatusCodeIs(http.StatusOK))
+	c.Assert(err, checker.IsNil)
+
+	// Send SIGTERM to Traefik.
+	proc, err := os.FindProcess(cmd.Process.Pid)
+	c.Assert(err, checker.IsNil)
+	err = proc.Signal(syscall.SIGTERM)
+	c.Assert(err, checker.IsNil)
+
+	// Give Traefik time to process the SIGTERM and send a request half-way
+	// into the request accepting grace period, by which requests should
+	// still get served.
+	time.Sleep(5 * time.Second)
+	resp, err := http.Get("http://127.0.0.1:8000/service")
+	c.Assert(err, checker.IsNil)
+	defer resp.Body.Close()
+	c.Assert(resp.StatusCode, checker.Equals, http.StatusOK)
+
+	// Expect Traefik to shut down gracefully once the request accepting grace
+	// period has elapsed.
+	waitErr := make(chan error)
+	go func() {
+		waitErr <- cmd.Wait()
+	}()
+
+	select {
+	case err := <-waitErr:
+		c.Assert(err, checker.IsNil)
+	case <-time.After(10 * time.Second):
+		c.Fatal("Traefik did not terminate in time")
+	}
+	showTraefikLog = false
+}

integration/fixtures/reqacceptgrace.toml

@@ -0,0 +1,21 @@
+defaultEntryPoints = ["http"]
+
+reqAcceptGraceTimeOut = "10s"
+
+logLevel = "DEBUG"
+
+[entryPoints]
+  [entryPoints.http]
+  address = ":8000"
+
+[file]
+[backends]
+  [backends.backend]
+    [backends.backend.servers.server]
+    url = "{{.Server}}"
+
+[frontends]
+  [frontends.frontend]
+  backend = "backend"
+    [frontends.frontend.routes.service]
+    rule = "Path:/service"

integration/resources/compose/reqacceptgrace.yml

@@ -0,0 +1,2 @@
+whoami:
+  image: emilevauge/whoami

server/configuration.go

@@ -47,6 +47,7 @@ type TraefikConfiguration struct {
 // GlobalConfiguration holds global configuration (with providers, etc.).
 // It's populated from the traefik configuration file passed as an argument to the binary.
 type GlobalConfiguration struct {
+	ReqAcceptGraceTimeOut     flaeg.Duration          `description:"Duration to keep accepting requests before Traefik initiates the graceful shutdown procedure"`
 	GraceTimeOut              flaeg.Duration          `short:"g" description:"Duration to give active requests a chance to finish before Traefik stops"`
 	Debug                     bool                    `short:"d" description:"Enable debug mode"`
 	CheckNewVersion           bool                    `description:"Periodically check if a new version has been released"`
@@ -586,6 +587,7 @@ func NewTraefikDefaultPointersConfiguration() *TraefikConfiguration {
 func NewTraefikConfiguration() *TraefikConfiguration {
 	return &TraefikConfiguration{
 		GlobalConfiguration: GlobalConfiguration{
+			ReqAcceptGraceTimeOut:     flaeg.Duration(0 * time.Second),
 			GraceTimeOut:              flaeg.Duration(10 * time.Second),
 			AccessLogsFile:            "",
 			TraefikLogsFile:           "",

server/server_signals.go

@@ -5,6 +5,7 @@ package server
 import (
 	"os/signal"
 	"syscall"
+	"time"
 
 	"github.com/containous/traefik/log"
 )
@@ -31,7 +32,12 @@ func (server *Server) listenSignals() {
 			}
 		default:
 			log.Infof("I have to go... %+v", sig)
-			log.Info("Stopping server")
+			reqTermGraceTimeOut := time.Duration(server.globalConfiguration.ReqAcceptGraceTimeOut)
+			if reqTermGraceTimeOut > 0 && sig == syscall.SIGTERM {
+				log.Infof("Waiting %s for incoming requests to cease", reqTermGraceTimeOut)
+				time.Sleep(reqTermGraceTimeOut)
+			}
+			log.Info("Stopping server gracefully")
 			server.Stop()
 		}
 	}

traefik.sample.toml

@@ -2,6 +2,20 @@
 # Global configuration
 ################################################################
 
+# Duration to keep accepting requests prior to initiating the graceful
+# termination period (as defined by the `graceTimeOut` option). This
+# option is meant to give downstream load-balancer sufficient time to
+# take Traefik out of rotation.
+# Can be provided in a format supported by [time.ParseDuration](https://golang.org/pkg/time/#ParseDuration) or as raw values (digits).
+# If no units are provided, the value is parsed assuming seconds.
+# The zero duration disables the request accepting grace period, i.e.,
+# Traefik will immediately proceed to the grace period.
+#
+# Optional
+# Default: 0
+#
+# reqAcceptGraceTimeOut = "10s"
+
 # Duration to give active requests a chance to finish before Traefik stops.
 # Can be provided in a format supported by [time.ParseDuration](https://golang.org/pkg/time/#ParseDuration) or as raw values (digits).
 # If no units are provided, the value is parsed assuming seconds.