Ipxe hot fix

README.md

@@ -97,27 +97,31 @@ FLAGS
   -backend-kube-namespace             [backend] an optional Kubernetes namespace override to query hardware data from, kube backend only
   -dhcp-addr                          [dhcp] local IP:Port to listen on for DHCP requests (default "0.0.0.0:67")
   -dhcp-enabled                       [dhcp] enable DHCP server (default "true")
-  -dhcp-http-ipxe-binary-url          [dhcp] HTTP iPXE binaries URL to use in DHCP packets (default "http://172.17.0.2:8080/ipxe/")
+  -dhcp-http-ipxe-binary-url          [dhcp] HTTP iPXE binaries URL to use in DHCP packets (default "http://172.17.0.3:8080/ipxe/")
   -dhcp-http-ipxe-script-prepend-mac  [dhcp] prepend the hardware MAC address to iPXE script URL base, http://1.2.3.4/auto.ipxe -> http://1.2.3.4/40:15:ff:89:cc:0e/auto.ipxe (default "true")
-  -dhcp-http-ipxe-script-url          [dhcp] HTTP iPXE script URL to use in DHCP packets (default "http://172.17.0.2/auto.ipxe")
+  -dhcp-http-ipxe-script-url          [dhcp] HTTP iPXE script URL to use in DHCP packets (default "http://172.17.0.3/auto.ipxe")
   -dhcp-iface                         [dhcp] interface to bind to for DHCP requests
-  -dhcp-ip-for-packet                 [dhcp] IP address to use in DHCP packets (opt 54, etc) (default "172.17.0.2")
+  -dhcp-ip-for-packet                 [dhcp] IP address to use in DHCP packets (opt 54, etc) (default "172.17.0.3")
   -dhcp-mode                          [dhcp] DHCP mode (reservation, proxy) (default "reservation")
-  -dhcp-syslog-ip                     [dhcp] Syslog server IP address to use in DHCP packets (opt 7) (default "172.17.0.2")
-  -dhcp-tftp-ip                       [dhcp] TFTP server IP address to use in DHCP packets (opt 66, etc) (default "172.17.0.2:69")
+  -dhcp-syslog-ip                     [dhcp] Syslog server IP address to use in DHCP packets (opt 7) (default "172.17.0.3")
+  -dhcp-tftp-ip                       [dhcp] TFTP server IP address to use in DHCP packets (opt 66, etc) (default "172.17.0.3:69")
   -disable-discover-trusted-proxies   [http] disable discovery of trusted proxies from Kubernetes, only available for the Kubernetes backend (default "false")
   -extra-kernel-args                  [http] extra set of kernel args (k=v k=v) that are appended to the kernel cmdline iPXE script
-  -http-addr                          [http] local IP:Port to listen on for iPXE HTTP script requests (default "172.17.0.2:80")
+  -http-addr                          [http] local IP:Port to listen on for iPXE HTTP script requests (default "172.17.0.3:80")
   -http-ipxe-binary-enabled           [http] enable iPXE HTTP binary server (default "true")
   -http-ipxe-script-enabled           [http] enable iPXE HTTP script server (default "true")
+  -ipxe-script-retries                [http] number of retries to attempt when fetching kernel and initrd files in the iPXE script (default "0")
+  -ipxe-script-retry-delay            [http] delay (in seconds) between retries when fetching kernel and initrd files in the iPXE script (default "2")
   -osie-url                           [http] URL where OSIE (HookOS) images are located
   -tink-server                        [http] IP:Port for the Tink server
   -tink-server-tls                    [http] use TLS for Tink server (default "false")
   -trusted-proxies                    [http] comma separated list of trusted proxies in CIDR notation
-  -syslog-addr                        [syslog] local IP:Port to listen on for Syslog messages (default "172.17.0.2:514")
+  -otel-endpoint                      [otel] OpenTelemetry collector endpoint
+  -otel-insecure                      [otel] OpenTelemetry collector insecure (default "true")
+  -syslog-addr                        [syslog] local IP:Port to listen on for Syslog messages (default "172.17.0.3:514")
   -syslog-enabled                     [syslog] enable Syslog server(receiver) (default "true")
   -ipxe-script-patch                  [tftp/http] iPXE script fragment to patch into served iPXE binaries served via TFTP or HTTP
-  -tftp-addr                          [tftp] local IP:Port to listen on for iPXE TFTP binary requests (default "172.17.0.2:69")
+  -tftp-addr                          [tftp] local IP:Port to listen on for iPXE TFTP binary requests (default "172.17.0.3:69")
   -tftp-block-size                    [tftp] TFTP block size a value between 512 (the default block size for TFTP) and 65456 (the max size a UDP packet payload can be) (default "512")
   -tftp-enabled                       [tftp] enable iPXE TFTP binary server) (default "true")
   -tftp-timeout                       [tftp] iPXE TFTP binary server requests timeout (default "5s")

cmd/smee/flag.go

@@ -107,6 +107,7 @@ func ipxeHTTPScriptFlags(c *config, fs *flag.FlagSet) {
 	fs.StringVar(&c.ipxeHTTPScript.tinkServer, "tink-server", "", "[http] IP:Port for the Tink server")
 	fs.BoolVar(&c.ipxeHTTPScript.tinkServerUseTLS, "tink-server-tls", false, "[http] use TLS for Tink server")
 	fs.IntVar(&c.ipxeHTTPScript.retries, "ipxe-script-retries", 0, "[http] number of retries to attempt when fetching kernel and initrd files in the iPXE script")
+	fs.IntVar(&c.ipxeHTTPScript.retryDelay, "ipxe-script-retry-delay", 2, "[http] delay (in seconds) between retries when fetching kernel and initrd files in the iPXE script")
 }
 
 func dhcpFlags(c *config, fs *flag.FlagSet) {

cmd/smee/flag_test.go

@@ -25,8 +25,9 @@ func TestParser(t *testing.T) {
 			enabled: true,
 		},
 		ipxeHTTPScript: ipxeHTTPScript{
-			enabled:  true,
-			bindAddr: "192.168.2.4:80",
+			enabled:    true,
+			bindAddr:   "192.168.2.4:80",
+			retryDelay: 2,
 		},
 		dhcp: dhcpConfig{
 			enabled:           true,
@@ -112,6 +113,7 @@ FLAGS
   -http-ipxe-binary-enabled           [http] enable iPXE HTTP binary server (default "true")
   -http-ipxe-script-enabled           [http] enable iPXE HTTP script server (default "true")
   -ipxe-script-retries                [http] number of retries to attempt when fetching kernel and initrd files in the iPXE script (default "0")
+  -ipxe-script-retry-delay            [http] delay (in seconds) between retries when fetching kernel and initrd files in the iPXE script (default "2")
   -osie-url                           [http] URL where OSIE (HookOS) images are located
   -tink-server                        [http] IP:Port for the Tink server
   -tink-server-tls                    [http] use TLS for Tink server (default "false")

cmd/smee/main.go

@@ -84,6 +84,7 @@
 	trustedProxies                string
 	disableDiscoverTrustedProxies bool
 	retries                       int
+	retryDelay                    int
 }
 
 type dhcpConfig struct {
@@ -213,14 +214,15 @@
 		}
 
 		jh := script.Handler{
-			Logger:             log,
-			Backend:            br,
-			OSIEURL:            cfg.ipxeHTTPScript.hookURL,
-			ExtraKernelParams:  strings.Split(cfg.ipxeHTTPScript.extraKernelArgs, " "),
-			PublicSyslogFQDN:   cfg.dhcp.syslogIP,
-			TinkServerTLS:      cfg.ipxeHTTPScript.tinkServerUseTLS,
-			TinkServerGRPCAddr: cfg.ipxeHTTPScript.tinkServer,
-			IPXEScriptRetries:  cfg.ipxeHTTPScript.retries,
+			Logger:               log,
+			Backend:              br,
+			OSIEURL:              cfg.ipxeHTTPScript.hookURL,
+			ExtraKernelParams:    strings.Split(cfg.ipxeHTTPScript.extraKernelArgs, " "),
+			PublicSyslogFQDN:     cfg.dhcp.syslogIP,
+			TinkServerTLS:        cfg.ipxeHTTPScript.tinkServerUseTLS,
+			TinkServerGRPCAddr:   cfg.ipxeHTTPScript.tinkServer,
+			IPXEScriptRetries:    cfg.ipxeHTTPScript.retries,
+			IPXEScriptRetryDelay: cfg.ipxeHTTPScript.retryDelay,
 		}
 		// serve ipxe script from the "/" URI.
 		handlers["/"] = jh.HandlerFunc()

internal/ipxe/script/auto_test.go

@@ -25,6 +25,7 @@ func TestGenerateTemplate(t *testing.T) {
 				ExtraKernelParams: []string{"tink_worker_image=quay.io/tinkerbell/tink-worker:v0.8.0", "tinkerbell=packet"},
 				HWAddr:            "3c:ec:ef:4c:4f:54",
 				Retries:           10,
+				RetryDelay:        3,
 			},
 			script: HookScript,
 			want: `#!ipxe
@@ -34,31 +35,37 @@ echo Loading the Tinkerbell Hook iPXE script...
 set arch x86_64
 set download-url http://location:8080/to/kernel/and/initrd
 set retries:int32 10
+set retry_delay:int32 3
 
 set idx:int32 0
 :retry_kernel
 kernel ${download-url}/vmlinuz-${arch} tink_worker_image=quay.io/tinkerbell/tink-worker:v0.8.0 tinkerbell=packet \
 facility=onprem syslog_host=1.2.3.4 grpc_authority=1.2.3.4:42113 tinkerbell_tls=false worker_id=3c:ec:ef:4c:4f:54 hw_addr=3c:ec:ef:4c:4f:54 \
-modules=loop,squashfs,sd-mod,usb-storage intel_iommu=on iommu=pt initrd=initramfs-${arch} console=tty0 console=ttyS1,115200 || iseq ${idx} ${retries} && goto kernel-error || inc idx && goto retry_kernel
+modules=loop,squashfs,sd-mod,usb-storage intel_iommu=on iommu=pt initrd=initramfs-${arch} console=tty0 console=ttyS1,115200 && goto download_initrd || iseq ${idx} ${retries} && goto kernel-error || inc idx && echo retry in ${retry_delay} seconds ; sleep ${retry_delay} ; goto retry_kernel
 
+:download_initrd
 set idx:int32 0
 :retry_initrd
-initrd ${download-url}/initramfs-${arch} || iseq ${idx} ${retries} && goto initrd-error || inc idx && goto retry_initrd
+initrd ${download-url}/initramfs-${arch} && goto boot || iseq ${idx} ${retries} && goto initrd-error || inc idx && echo retry in ${retry_delay} seconds ; sleep ${retry_delay} ; goto retry_initrd
 
+:boot
 set idx:int32 0
 :retry_boot
-boot || iseq ${idx} ${retries} && goto boot-error || inc idx && goto retry_boot
+boot || iseq ${idx} ${retries} && goto boot-error || inc idx && echo retry in ${retry_delay} seconds ; sleep ${retry_delay} ; goto retry_boot
 
 :kernel-error
 echo Failed to load kernel
+imgfree
 exit
 
 :initrd-error
 echo Failed to load initrd
+imgfree
 exit
 
 :boot-error
 echo Failed to boot
+imgfree
 exit
 `,
 		},
@@ -75,6 +82,7 @@ exit
 				HWAddr:            "3c:ec:ef:4c:4f:54",
 				VLANID:            "16",
 				Retries:           10,
+				RetryDelay:        3,
 			},
 			script: HookScript,
 			want: `#!ipxe
@@ -84,31 +92,37 @@ echo Loading the Tinkerbell Hook iPXE script...
 set arch x86_64
 set download-url http://location:8080/to/kernel/and/initrd
 set retries:int32 10
+set retry_delay:int32 3
 
 set idx:int32 0
 :retry_kernel
 kernel ${download-url}/vmlinuz-${arch} vlan_id=16 tink_worker_image=quay.io/tinkerbell/tink-worker:v0.8.0 tinkerbell=packet \
 facility=onprem syslog_host=1.2.3.4 grpc_authority=1.2.3.4:42113 tinkerbell_tls=false worker_id=3c:ec:ef:4c:4f:54 hw_addr=3c:ec:ef:4c:4f:54 \
-modules=loop,squashfs,sd-mod,usb-storage intel_iommu=on iommu=pt initrd=initramfs-${arch} console=tty0 console=ttyS1,115200 || iseq ${idx} ${retries} && goto kernel-error || inc idx && goto retry_kernel
+modules=loop,squashfs,sd-mod,usb-storage intel_iommu=on iommu=pt initrd=initramfs-${arch} console=tty0 console=ttyS1,115200 && goto download_initrd || iseq ${idx} ${retries} && goto kernel-error || inc idx && echo retry in ${retry_delay} seconds ; sleep ${retry_delay} ; goto retry_kernel
 
+:download_initrd
 set idx:int32 0
 :retry_initrd
-initrd ${download-url}/initramfs-${arch} || iseq ${idx} ${retries} && goto initrd-error || inc idx && goto retry_initrd
+initrd ${download-url}/initramfs-${arch} && goto boot || iseq ${idx} ${retries} && goto initrd-error || inc idx && echo retry in ${retry_delay} seconds ; sleep ${retry_delay} ; goto retry_initrd
 
+:boot
 set idx:int32 0
 :retry_boot
-boot || iseq ${idx} ${retries} && goto boot-error || inc idx && goto retry_boot
+boot || iseq ${idx} ${retries} && goto boot-error || inc idx && echo retry in ${retry_delay} seconds ; sleep ${retry_delay} ; goto retry_boot
 
 :kernel-error
 echo Failed to load kernel
+imgfree
 exit
 
 :initrd-error
 echo Failed to load initrd
+imgfree
 exit
 
 :boot-error
 echo Failed to boot
+imgfree
 exit
 `,
 		},

internal/ipxe/script/hook.go

@@ -11,31 +11,37 @@ echo Debug TraceID: {{ .TraceID }}
 set arch {{ .Arch }}
 set download-url {{ .DownloadURL }}
 set retries:int32 {{ .Retries }}
+set retry_delay:int32 {{ .RetryDelay }}
 
 set idx:int32 0
 :retry_kernel
 kernel ${download-url}/vmlinuz-${arch} {{- if ne .VLANID "" }} vlan_id={{ .VLANID }} {{- end }} {{- range .ExtraKernelParams}} {{.}} {{- end}} \
 facility={{ .Facility }} syslog_host={{ .SyslogHost }} grpc_authority={{ .TinkGRPCAuthority }} tinkerbell_tls={{ .TinkerbellTLS }} worker_id={{ .WorkerID }} hw_addr={{ .HWAddr }} \
-modules=loop,squashfs,sd-mod,usb-storage intel_iommu=on iommu=pt initrd=initramfs-${arch} console=tty0 console=ttyS1,115200 || iseq ${idx} ${retries} && goto kernel-error || inc idx && goto retry_kernel
+modules=loop,squashfs,sd-mod,usb-storage intel_iommu=on iommu=pt initrd=initramfs-${arch} console=tty0 console=ttyS1,115200 && goto download_initrd || iseq ${idx} ${retries} && goto kernel-error || inc idx && echo retry in ${retry_delay} seconds ; sleep ${retry_delay} ; goto retry_kernel
 
+:download_initrd
 set idx:int32 0
 :retry_initrd
-initrd ${download-url}/initramfs-${arch} || iseq ${idx} ${retries} && goto initrd-error || inc idx && goto retry_initrd
+initrd ${download-url}/initramfs-${arch} && goto boot || iseq ${idx} ${retries} && goto initrd-error || inc idx && echo retry in ${retry_delay} seconds ; sleep ${retry_delay} ; goto retry_initrd
 
+:boot
 set idx:int32 0
 :retry_boot
-boot || iseq ${idx} ${retries} && goto boot-error || inc idx && goto retry_boot
+boot || iseq ${idx} ${retries} && goto boot-error || inc idx && echo retry in ${retry_delay} seconds ; sleep ${retry_delay} ; goto retry_boot
 
 :kernel-error
 echo Failed to load kernel
+imgfree
 exit
 
 :initrd-error
 echo Failed to load initrd
+imgfree
 exit
 
 :boot-error
 echo Failed to boot
+imgfree
 exit
 `
 
@@ -54,4 +60,5 @@ type Hook struct {
 	VLANID            string // string number between 1-4095
 	WorkerID          string // example 3c:ec:ef:4c:4f:54 or worker1
 	Retries           int    // number of retries to attempt when fetching kernel and initrd files
+	RetryDelay        int    // number of seconds to wait between retries
 }

internal/ipxe/script/ipxe.go

@@ -19,14 +19,15 @@ import (
 )
 
 type Handler struct {
-	Logger             logr.Logger
-	Backend            handler.BackendReader
-	OSIEURL            string
-	ExtraKernelParams  []string
-	PublicSyslogFQDN   string
-	TinkServerTLS      bool
-	TinkServerGRPCAddr string
-	IPXEScriptRetries  int
+	Logger               logr.Logger
+	Backend              handler.BackendReader
+	OSIEURL              string
+	ExtraKernelParams    []string
+	PublicSyslogFQDN     string
+	TinkServerTLS        bool
+	TinkServerGRPCAddr   string
+	IPXEScriptRetries    int
+	IPXEScriptRetryDelay int
 }
 
 type data struct {
@@ -229,6 +230,7 @@ func (h *Handler) defaultScript(span trace.Span, hw data) (string, error) {
 		VLANID:            hw.VLANID,
 		WorkerID:          wID,
 		Retries:           h.IPXEScriptRetries,
+		RetryDelay:        h.IPXEScriptRetryDelay,
 	}
 	if sc := span.SpanContext(); sc.IsSampled() {
 		auto.TraceID = sc.TraceID().String()

internal/ipxe/script/ipxe_test.go

@@ -51,31 +51,37 @@ echo Loading the Tinkerbell Hook iPXE script...
 set arch x86_64
 set download-url http://127.1.1.1
 set retries:int32 10
+set retry_delay:int32 3
 
 set idx:int32 0
 :retry_kernel
 kernel ${download-url}/vmlinuz-${arch} vlan_id=1234 \
 facility=onprem syslog_host= grpc_authority= tinkerbell_tls=false worker_id=00:01:02:03:04:05 hw_addr=00:01:02:03:04:05 \
-modules=loop,squashfs,sd-mod,usb-storage intel_iommu=on iommu=pt initrd=initramfs-${arch} console=tty0 console=ttyS1,115200 || iseq ${idx} ${retries} && goto kernel-error || inc idx && goto retry_kernel
+modules=loop,squashfs,sd-mod,usb-storage intel_iommu=on iommu=pt initrd=initramfs-${arch} console=tty0 console=ttyS1,115200 && goto download_initrd || iseq ${idx} ${retries} && goto kernel-error || inc idx && echo retry in ${retry_delay} seconds ; sleep ${retry_delay} ; goto retry_kernel
 
+:download_initrd
 set idx:int32 0
 :retry_initrd
-initrd ${download-url}/initramfs-${arch} || iseq ${idx} ${retries} && goto initrd-error || inc idx && goto retry_initrd
+initrd ${download-url}/initramfs-${arch} && goto boot || iseq ${idx} ${retries} && goto initrd-error || inc idx && echo retry in ${retry_delay} seconds ; sleep ${retry_delay} ; goto retry_initrd
 
+:boot
 set idx:int32 0
 :retry_boot
-boot || iseq ${idx} ${retries} && goto boot-error || inc idx && goto retry_boot
+boot || iseq ${idx} ${retries} && goto boot-error || inc idx && echo retry in ${retry_delay} seconds ; sleep ${retry_delay} ; goto retry_boot
 
 :kernel-error
 echo Failed to load kernel
+imgfree
 exit
 
 :initrd-error
 echo Failed to load initrd
+imgfree
 exit
 
 :boot-error
 echo Failed to boot
+imgfree
 exit
 `
 	tests := map[string]struct {
@@ -86,8 +92,9 @@ exit
 	for name, tt := range tests {
 		t.Run(name, func(t *testing.T) {
 			h := &Handler{
-				OSIEURL:           "http://127.1.1.1",
-				IPXEScriptRetries: 10,
+				OSIEURL:              "http://127.1.1.1",
+				IPXEScriptRetries:    10,
+				IPXEScriptRetryDelay: 3,
 			}
 			d := data{MACAddress: net.HardwareAddr{0x00, 0x01, 0x02, 0x03, 0x04, 0x05}, VLANID: "1234", Facility: "onprem", Arch: "x86_64"}
 			sp := trace.SpanFromContext(context.Background())