Adding retries and backoff. Fixes #1

cmd/root.go

@@ -36,6 +36,8 @@ var (
 	namespace     string
 	multicluster  bool
 	demoCount     int
+	demoDelay     int
+	demoDuration  int
 )
 
 func init() {
@@ -44,12 +46,14 @@ func init() {
 	rootCmd.PersistentFlags().StringVar(&caCertFile, "ca-cert", "", "The path the CA cert file in PEM format")
 	rootCmd.PersistentFlags().StringVarP(&host, "host", "", "", "The hostname or IP address of the allocator server")
 	rootCmd.PersistentFlags().StringVarP(&namespace, "namespace", "n", "", "The namespace of gameservers to request from")
-	rootCmd.PersistentFlags().BoolVarP(&multicluster, "multicluster", "", false, "If true, multicluster allocation will be requested")
+	rootCmd.PersistentFlags().BoolVarP(&multicluster, "multicluster", "m", false, "If true, multicluster allocation will be requested")
 
 	rootCmd.AddCommand(allocateCmd)
-	rootCmd.AddCommand(udpDemoCmd)
+	rootCmd.AddCommand(loadTestCmd)
 
-	udpDemoCmd.PersistentFlags().IntVar(&demoCount, "demo-count", 10, "The number of connections to make during the demo.")
+	loadTestCmd.PersistentFlags().IntVarP(&demoCount, "count", "c", 10, "The number of connections to make during the demo.")
+	loadTestCmd.PersistentFlags().IntVar(&demoDelay, "delay", 2, "The number of seconds to wait between connections")
+	loadTestCmd.PersistentFlags().IntVarP(&demoDuration, "duration", "d", 10, "The number of seconds to leave each connection open.")
 
 	klog.InitFlags(nil)
 	flag.Parse()
@@ -113,17 +117,17 @@ var allocateCmd = &cobra.Command{
 	},
 }
 
-var udpDemoCmd = &cobra.Command{
-	Use:     "udp-demo",
-	Short:   "udp-demo",
-	Long:    `Allocates a server, communicates with it, and then closes the connection.`,
+var loadTestCmd = &cobra.Command{
+	Use:     "load-test",
+	Short:   "load-test",
+	Long:    `Allocates a set of servers, communicates with them, and then closes the connection.`,
 	PreRunE: argsValidator,
 	Run: func(cmd *cobra.Command, args []string) {
 		allocatorClient, err := allocator.NewClient(keyFile, certFile, caCertFile, host, namespace, multicluster)
 		if err != nil {
 			klog.Error(err)
 		}
-		err = allocatorClient.RunUDPDemo(demoCount)
+		err = allocatorClient.RunUDPLoad(demoCount, demoDelay, demoDuration)
 	},
 }
 

go.mod

@@ -4,6 +4,7 @@ go 1.14
 
 require (
 	agones.dev/agones v1.6.0
+	github.com/cenkalti/backoff/v4 v4.0.2
 	github.com/google/go-cmp v0.4.0 // indirect
 	github.com/markbates/pkger v0.17.0
 	github.com/pkg/errors v0.9.1

go.sum

@@ -28,6 +28,10 @@ github.com/aws/aws-sdk-go v1.16.20/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpi
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/cenkalti/backoff v1.1.0 h1:QnvVp8ikKCDWOsFheytRCoYWYPO/ObCTBGxT19Hc+yE=
+github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
+github.com/cenkalti/backoff/v4 v4.0.2 h1:JIufpQLbh4DkbQoii76ItQIUFzevQSqOLZca4eamEDs=
+github.com/cenkalti/backoff/v4 v4.0.2/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=

pkg/allocator/udp.go

@@ -7,36 +7,61 @@ import (
 	"sync"
 	"time"
 
+	backoff "github.com/cenkalti/backoff/v4"
+
 	"k8s.io/klog"
 )
 
-// RunUDPDemo runs many concurrent game connections
-func (c *Client) RunUDPDemo(count int) error {
+// RunUDPLoad runs many concurrent game connections on a simple UDP server
+// This is designed to test the allocator service and autoscaling of the game servers.
+func (c *Client) RunUDPLoad(count int, delay int, duration int) error {
 	var wg sync.WaitGroup
 	for i := 0; i < count; i++ {
 		wg.Add(1)
-		go c.testUDP(i, &wg)
-		time.Sleep(3 * time.Second)
+		go c.testUDP(i, &wg, duration)
+		time.Sleep(time.Duration(delay) * time.Second)
 	}
 	wg.Wait()
 	return nil
 }
 
-func (c *Client) testUDP(id int, wg *sync.WaitGroup) {
+func (c *Client) testUDP(id int, wg *sync.WaitGroup, duration int) {
 	defer wg.Done()
-	allocation, err := c.AllocateGameserver()
-	if err != nil {
-		klog.Error(err)
+	maxRetries := 10
+
+	var a *Allocation
+	var err error
+
+	b := backoff.NewExponentialBackOff()
+	b.InitialInterval = time.Duration(1 * time.Second)
+
+	i := 0
+	for {
+		if i == maxRetries {
+			klog.Errorf("max retries (%d) reached", maxRetries)
+			return
+		}
+		i++
+		delay := b.NextBackOff()
+		a, err = c.AllocateGameserver()
+		if err != nil {
+			klog.Infof("%d %s - retrying in %fs", id, err.Error(), delay.Seconds())
+			time.Sleep(delay)
+			continue
+		} else {
+			break
+		}
 	}
-	fmt.Printf("%d - got allocation %s %d. Proceeding to connection...\n", id, allocation.Address, allocation.Port)
-	err = allocation.testUDP(id)
+
+	klog.Infof("%d - got allocation %s %d. Proceeding to connection...\n", id, a.Address, a.Port)
+	err = a.testUDP(id, duration)
 	if err != nil {
 		klog.Error(err)
 	}
 }
 
 // testUDP tests a series of connections to the simple-udp server gameserver example
-func (a *Allocation) testUDP(id int) error {
+func (a *Allocation) testUDP(id int, duration int) error {
 	endpoint := fmt.Sprintf("%s:%d", a.Address, a.Port)
 
 	conn, err := net.ListenPacket("udp", ":0")
@@ -52,12 +77,13 @@ func (a *Allocation) testUDP(id int) error {
 
 	klog.Infof("%d - connected to gameserver and sending hello", id)
 
-	_, err = conn.WriteTo([]byte("Hello Gameserver!"), dst)
+	msg := fmt.Sprintf("Hello from process %d!", id)
+	_, err = conn.WriteTo([]byte(msg), dst)
 	if err != nil {
 		return err
 	}
-	klog.V(2).Infof("%d - sleeping 10 seconds to view logs", id)
-	time.Sleep(10 * time.Second)
+	klog.Infof("%d - sleeping %d seconds to view logs", id, duration)
+	time.Sleep(time.Duration(duration) * time.Second)
 	klog.Infof("%d - sending EXIT command", id)
 
 	_, err = conn.WriteTo([]byte("EXIT"), dst)