[Release-1.32] 2025 January Backports

.github/workflows/e2e.yaml

@@ -80,6 +80,13 @@ jobs:
           chmod +x ./dist/artifacts/k3s
           cd tests/e2e/${{ matrix.etest }}
           go test -v -timeout=45m ./${{ matrix.etest}}_test.go -ci -local
+      - name: On Failure, Upload Journald Logs
+        uses: actions/upload-artifact@v4
+        if: ${{ failure() }}
+        with:
+          name: ${{ matrix.etest}}-journald-logs
+          path: tests/e2e/${{ matrix.etest }}/*-jlog.txt
+          retention-days: 30
       - name: On Failure, Launch Debug Session
         uses: lhotari/action-upterm@v1
         if: ${{ failure() }}
@@ -159,6 +166,9 @@ jobs:
           fi
         elif [ -z "${{ github.base_ref }}" ]; then
           # We are in a fork, and need some git history to determine the branch name
+          # For some reason, the first fetch doesn't always get the full history, so we sleep and fetch again
+          git fetch origin --depth=100 +refs/heads/*:refs/remotes/origin/*
+          sleep 5
           git fetch origin --depth=100 +refs/heads/*:refs/remotes/origin/*
           BRANCH_NAME=$(git show-branch -a 2> /dev/null  | grep '\*' | grep -v `git rev-parse --abbrev-ref HEAD` | head -n1 |  sed 's/.*\[\(.*\/\)\(.*\)\].*/\2/' | sed 's/[\^~].*//')
         else
@@ -171,6 +181,7 @@ jobs:
       run: |
         if [[ ! ${{ steps.branch_step.outputs.branch_name }} =~ ^(master|release-[0-9]+\.[0-9]+)$ ]]; then
           echo "Branch name ${{ steps.branch_step.outputs.branch_name }} does not match pattern"
+          echo "If this is a PR/fork, ensure you have recently rebased off master/release-1.XX branch"
           exit 1
         fi
 

pkg/cli/cert/cert.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"text/tabwriter"
 	"time"
 
 	"github.com/k3s-io/k3s/pkg/agent/util"
@@ -92,27 +93,60 @@ func check(app *cli.Context, cfg *cmds.Server) error {
 
 	now := time.Now()
 	warn := now.Add(time.Hour * 24 * config.CertificateRenewDays)
-
-	for service, files := range fileMap {
-		logrus.Info("Checking certificates for " + service)
-		for _, file := range files {
-			// ignore errors, as some files may not exist, or may not contain certs.
-			// Only check whatever exists and has certs.
-			certs, _ := certutil.CertsFromFile(file)
-			for _, cert := range certs {
-				if now.Before(cert.NotBefore) {
-					logrus.Errorf("%s: certificate %s is not valid before %s", file, cert.Subject, cert.NotBefore.Format(time.RFC3339))
-				} else if now.After(cert.NotAfter) {
-					logrus.Errorf("%s: certificate %s expired at %s", file, cert.Subject, cert.NotAfter.Format(time.RFC3339))
-				} else if warn.After(cert.NotAfter) {
-					logrus.Warnf("%s: certificate %s will expire within %d days at %s", file, cert.Subject, config.CertificateRenewDays, cert.NotAfter.Format(time.RFC3339))
-				} else {
-					logrus.Infof("%s: certificate %s is ok, expires at %s", file, cert.Subject, cert.NotAfter.Format(time.RFC3339))
+	outFmt := app.String("output")
+	switch outFmt {
+	case "text":
+		for service, files := range fileMap {
+			logrus.Info("Checking certificates for " + service)
+			for _, file := range files {
+				// ignore errors, as some files may not exist, or may not contain certs.
+				// Only check whatever exists and has certs.
+				certs, _ := certutil.CertsFromFile(file)
+				for _, cert := range certs {
+					if now.Before(cert.NotBefore) {
+						logrus.Errorf("%s: certificate %s is not valid before %s", file, cert.Subject, cert.NotBefore.Format(time.RFC3339))
+					} else if now.After(cert.NotAfter) {
+						logrus.Errorf("%s: certificate %s expired at %s", file, cert.Subject, cert.NotAfter.Format(time.RFC3339))
+					} else if warn.After(cert.NotAfter) {
+						logrus.Warnf("%s: certificate %s will expire within %d days at %s", file, cert.Subject, config.CertificateRenewDays, cert.NotAfter.Format(time.RFC3339))
+					} else {
+						logrus.Infof("%s: certificate %s is ok, expires at %s", file, cert.Subject, cert.NotAfter.Format(time.RFC3339))
+					}
 				}
 			}
 		}
+	case "table":
+		var tabBuffer bytes.Buffer
+		w := tabwriter.NewWriter(&tabBuffer, 0, 0, 2, ' ', 0)
+		fmt.Fprintf(w, "\n")
+		fmt.Fprintf(w, "CERTIFICATE\tSUBJECT\tSTATUS\tEXPIRES\n")
+		fmt.Fprintf(w, "-----------\t-------\t------\t-------")
+		for _, files := range fileMap {
+			for _, file := range files {
+				certs, _ := certutil.CertsFromFile(file)
+				for _, cert := range certs {
+					baseName := filepath.Base(file)
+					var status string
+					expiration := cert.NotAfter.Format(time.RFC3339)
+					if now.Before(cert.NotBefore) {
+						status = "NOT YET VALID"
+						expiration = cert.NotBefore.Format(time.RFC3339)
+					} else if now.After(cert.NotAfter) {
+						status = "EXPIRED"
+					} else if warn.After(cert.NotAfter) {
+						status = "WARNING"
+					} else {
+						status = "OK"
+					}
+					fmt.Fprintf(w, "\n%s\t%s\t%s\t%s", baseName, cert.Subject, status, expiration)
+				}
+			}
+		}
+		w.Flush()
+		fmt.Println(tabBuffer.String())
+	default:
+		return fmt.Errorf("invalid output format %s", outFmt)
 	}
-
 	return nil
 }
 

pkg/cli/cmds/certs.go

@@ -63,7 +63,11 @@ func NewCertCommands(check, rotate, rotateCA func(ctx *cli.Context) error) cli.C
 				SkipFlagParsing: false,
 				SkipArgReorder:  true,
 				Action:          check,
-				Flags:           CertRotateCommandFlags,
+				Flags: append(CertRotateCommandFlags, &cli.StringFlag{
+					Name:  "output,o",
+					Usage: "Format output. Options: text, table",
+					Value: "text",
+				}),
 			},
 			{
 				Name:            "rotate",

tests/e2e/btrfs/btrfs_test.go

@@ -90,6 +90,9 @@ var _ = AfterEach(func() {
 })
 
 var _ = AfterSuite(func() {
+	if failed {
+		Expect(e2e.SaveJournalLogs(serverNodeNames)).To(Succeed())
+	}
 	if !failed || *ci {
 		Expect(e2e.DestroyCluster()).To(Succeed())
 		Expect(os.Remove(kubeConfigFile)).To(Succeed())

tests/e2e/embeddedmirror/embeddedmirror_test.go

@@ -147,7 +147,7 @@ var _ = AfterEach(func() {
 
 var _ = AfterSuite(func() {
 	if failed {
-		AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...)))
+		Expect(e2e.SaveJournalLogs(append(serverNodeNames, agentNodeNames...))).To(Succeed())
 	} else {
 		Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed())
 	}

tests/e2e/externalip/externalip_test.go

@@ -166,7 +166,7 @@ var _ = AfterEach(func() {
 
 var _ = AfterSuite(func() {
 	if failed {
-		AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...)))
+		Expect(e2e.SaveJournalLogs(append(serverNodeNames, agentNodeNames...))).To(Succeed())
 	} else {
 		Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed())
 	}

tests/e2e/privateregistry/privateregistry_test.go

@@ -150,7 +150,7 @@ var _ = AfterEach(func() {
 
 var _ = AfterSuite(func() {
 	if failed {
-		AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...)))
+		Expect(e2e.SaveJournalLogs(append(serverNodeNames, agentNodeNames...))).To(Succeed())
 	} else {
 		Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed())
 	}

tests/e2e/s3/s3_test.go

@@ -178,7 +178,7 @@ var _ = AfterEach(func() {
 
 var _ = AfterSuite(func() {
 	if failed {
-		AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...)))
+		Expect(e2e.SaveJournalLogs(append(serverNodeNames, agentNodeNames...))).To(Succeed())
 	} else {
 		Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed())
 	}

tests/e2e/startup/startup_test.go

@@ -454,7 +454,7 @@ var _ = AfterEach(func() {
 var _ = AfterSuite(func() {
 	if failed {
 		AddReportEntry("config", e2e.GetConfig(append(serverNodeNames, agentNodeNames...)))
-		AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...)))
+		Expect(e2e.SaveJournalLogs(append(serverNodeNames, agentNodeNames...))).To(Succeed())
 	} else {
 		Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed())
 	}

tests/e2e/testutils.go

@@ -377,6 +377,26 @@ func TailJournalLogs(lines int, nodes []string) string {
 	return logs.String()
 }
 
+// SaveJournalLogs saves the journal logs of each node to a <NAME>-jlog.txt file.
+// When used in GHA CI, the logs are uploaded as an artifact on failure.
+func SaveJournalLogs(nodeNames []string) error {
+	for _, node := range nodeNames {
+		lf, err := os.Create(node + "-jlog.txt")
+		if err != nil {
+			return err
+		}
+		defer lf.Close()
+		logs, err := GetJournalLogs(node)
+		if err != nil {
+			return err
+		}
+		if _, err := lf.Write([]byte(logs)); err != nil {
+			return fmt.Errorf("failed to write %s node logs: %v", node, err)
+		}
+	}
+	return nil
+}
+
 func GetConfig(nodes []string) string {
 	config := &strings.Builder{}
 	for _, node := range nodes {

tests/e2e/wasm/wasm_test.go

@@ -136,7 +136,7 @@ var _ = AfterEach(func() {
 
 var _ = AfterSuite(func() {
 	if failed {
-		AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...)))
+		Expect(e2e.SaveJournalLogs(append(serverNodeNames, agentNodeNames...))).To(Succeed())
 	} else {
 		Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed())
 	}

tests/integration/certrotation/certrotation_int_test.go

@@ -46,15 +46,15 @@ var _ = Describe("certificate rotation", Ordered, func() {
 			certHash, err = testutil.RunCommand("md5sum " + tmpdDataDir + "/server/tls/serving-kube-apiserver.crt | cut -f 1 -d' '")
 			Expect(err).ToNot(HaveOccurred())
 		})
-		It("stop k3s", func() {
+		It("stops k3s", func() {
 			Expect(testutil.K3sKillServer(server)).To(Succeed())
 		})
-		It("certificate rotate", func() {
+		It("rotates certificates", func() {
 			_, err := testutil.K3sCmd("certificate", "rotate", "-d", tmpdDataDir)
 			Expect(err).ToNot(HaveOccurred())
 
 		})
-		It("start k3s server", func() {
+		It("starts k3s server", func() {
 			var err error
 			server2, err = testutil.K3sStartServer(serverArgs...)
 			Expect(err).ToNot(HaveOccurred())
@@ -64,7 +64,18 @@ var _ = Describe("certificate rotation", Ordered, func() {
 				return testutil.K3sDefaultDeployments()
 			}, "360s", "5s").Should(Succeed())
 		})
-		It("get certificate hash", func() {
+		It("checks the certificate status", func() {
+			res, err := testutil.K3sCmd("certificate", "check", "-d", tmpdDataDir)
+			Expect(err).ToNot(HaveOccurred())
+			for i, line := range strings.Split(res, "\n") {
+				// First line is just server info
+				if i == 0 || line == "" {
+					continue
+				}
+				Expect(line).To(MatchRegexp("certificate.*is ok|Checking certificates"), res)
+			}
+		})
+		It("gets certificate hash", func() {
 			// get md5sum of the CA certs
 			var err error
 			caCertHashAfter, err := testutil.RunCommand("md5sum " + tmpdDataDir + "/server/tls/client-ca.crt | cut -f 1 -d' '")