diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 2ba4b3e64e31..d52cb0feb5fc 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -80,6 +80,13 @@ jobs: chmod +x ./dist/artifacts/k3s cd tests/e2e/${{ matrix.etest }} go test -v -timeout=45m ./${{ matrix.etest}}_test.go -ci -local + - name: On Failure, Upload Journald Logs + uses: actions/upload-artifact@v4 + if: ${{ failure() }} + with: + name: ${{ matrix.etest}}-journald-logs + path: tests/e2e/${{ matrix.etest }}/*-jlog.txt + retention-days: 30 - name: On Failure, Launch Debug Session uses: lhotari/action-upterm@v1 if: ${{ failure() }} @@ -159,6 +166,9 @@ jobs: fi elif [ -z "${{ github.base_ref }}" ]; then # We are in a fork, and need some git history to determine the branch name + # For some reason, the first fetch doesn't always get the full history, so we sleep and fetch again + git fetch origin --depth=100 +refs/heads/*:refs/remotes/origin/* + sleep 5 git fetch origin --depth=100 +refs/heads/*:refs/remotes/origin/* BRANCH_NAME=$(git show-branch -a 2> /dev/null | grep '\*' | grep -v `git rev-parse --abbrev-ref HEAD` | head -n1 | sed 's/.*\[\(.*\/\)\(.*\)\].*/\2/' | sed 's/[\^~].*//') else @@ -171,6 +181,7 @@ jobs: run: | if [[ ! ${{ steps.branch_step.outputs.branch_name }} =~ ^(master|release-[0-9]+\.[0-9]+)$ ]]; then echo "Branch name ${{ steps.branch_step.outputs.branch_name }} does not match pattern" + echo "If this is a PR/fork, ensure you have recently rebased off master/release-1.XX branch" exit 1 fi diff --git a/pkg/cli/cert/cert.go b/pkg/cli/cert/cert.go index d5b0b96b8ae5..5d97ccd8c82d 100644 --- a/pkg/cli/cert/cert.go +++ b/pkg/cli/cert/cert.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "strings" + "text/tabwriter" "time" "github.com/k3s-io/k3s/pkg/agent/util" @@ -92,27 +93,60 @@ func check(app *cli.Context, cfg *cmds.Server) error { now := time.Now() warn := now.Add(time.Hour * 24 * config.CertificateRenewDays) - - for service, files := range fileMap { - logrus.Info("Checking certificates for " + service) - for _, file := range files { - // ignore errors, as some files may not exist, or may not contain certs. - // Only check whatever exists and has certs. - certs, _ := certutil.CertsFromFile(file) - for _, cert := range certs { - if now.Before(cert.NotBefore) { - logrus.Errorf("%s: certificate %s is not valid before %s", file, cert.Subject, cert.NotBefore.Format(time.RFC3339)) - } else if now.After(cert.NotAfter) { - logrus.Errorf("%s: certificate %s expired at %s", file, cert.Subject, cert.NotAfter.Format(time.RFC3339)) - } else if warn.After(cert.NotAfter) { - logrus.Warnf("%s: certificate %s will expire within %d days at %s", file, cert.Subject, config.CertificateRenewDays, cert.NotAfter.Format(time.RFC3339)) - } else { - logrus.Infof("%s: certificate %s is ok, expires at %s", file, cert.Subject, cert.NotAfter.Format(time.RFC3339)) + outFmt := app.String("output") + switch outFmt { + case "text": + for service, files := range fileMap { + logrus.Info("Checking certificates for " + service) + for _, file := range files { + // ignore errors, as some files may not exist, or may not contain certs. + // Only check whatever exists and has certs. + certs, _ := certutil.CertsFromFile(file) + for _, cert := range certs { + if now.Before(cert.NotBefore) { + logrus.Errorf("%s: certificate %s is not valid before %s", file, cert.Subject, cert.NotBefore.Format(time.RFC3339)) + } else if now.After(cert.NotAfter) { + logrus.Errorf("%s: certificate %s expired at %s", file, cert.Subject, cert.NotAfter.Format(time.RFC3339)) + } else if warn.After(cert.NotAfter) { + logrus.Warnf("%s: certificate %s will expire within %d days at %s", file, cert.Subject, config.CertificateRenewDays, cert.NotAfter.Format(time.RFC3339)) + } else { + logrus.Infof("%s: certificate %s is ok, expires at %s", file, cert.Subject, cert.NotAfter.Format(time.RFC3339)) + } } } } + case "table": + var tabBuffer bytes.Buffer + w := tabwriter.NewWriter(&tabBuffer, 0, 0, 2, ' ', 0) + fmt.Fprintf(w, "\n") + fmt.Fprintf(w, "CERTIFICATE\tSUBJECT\tSTATUS\tEXPIRES\n") + fmt.Fprintf(w, "-----------\t-------\t------\t-------") + for _, files := range fileMap { + for _, file := range files { + certs, _ := certutil.CertsFromFile(file) + for _, cert := range certs { + baseName := filepath.Base(file) + var status string + expiration := cert.NotAfter.Format(time.RFC3339) + if now.Before(cert.NotBefore) { + status = "NOT YET VALID" + expiration = cert.NotBefore.Format(time.RFC3339) + } else if now.After(cert.NotAfter) { + status = "EXPIRED" + } else if warn.After(cert.NotAfter) { + status = "WARNING" + } else { + status = "OK" + } + fmt.Fprintf(w, "\n%s\t%s\t%s\t%s", baseName, cert.Subject, status, expiration) + } + } + } + w.Flush() + fmt.Println(tabBuffer.String()) + default: + return fmt.Errorf("invalid output format %s", outFmt) } - return nil } diff --git a/pkg/cli/cmds/certs.go b/pkg/cli/cmds/certs.go index 0c3ee9c7026d..44440a040db8 100644 --- a/pkg/cli/cmds/certs.go +++ b/pkg/cli/cmds/certs.go @@ -63,7 +63,11 @@ func NewCertCommands(check, rotate, rotateCA func(ctx *cli.Context) error) cli.C SkipFlagParsing: false, SkipArgReorder: true, Action: check, - Flags: CertRotateCommandFlags, + Flags: append(CertRotateCommandFlags, &cli.StringFlag{ + Name: "output,o", + Usage: "Format output. Options: text, table", + Value: "text", + }), }, { Name: "rotate", diff --git a/tests/e2e/btrfs/btrfs_test.go b/tests/e2e/btrfs/btrfs_test.go index a592a2e78823..a99211a1634c 100644 --- a/tests/e2e/btrfs/btrfs_test.go +++ b/tests/e2e/btrfs/btrfs_test.go @@ -90,6 +90,9 @@ var _ = AfterEach(func() { }) var _ = AfterSuite(func() { + if failed { + Expect(e2e.SaveJournalLogs(serverNodeNames)).To(Succeed()) + } if !failed || *ci { Expect(e2e.DestroyCluster()).To(Succeed()) Expect(os.Remove(kubeConfigFile)).To(Succeed()) diff --git a/tests/e2e/embeddedmirror/embeddedmirror_test.go b/tests/e2e/embeddedmirror/embeddedmirror_test.go index 089fb465277b..d753528bf616 100644 --- a/tests/e2e/embeddedmirror/embeddedmirror_test.go +++ b/tests/e2e/embeddedmirror/embeddedmirror_test.go @@ -147,7 +147,7 @@ var _ = AfterEach(func() { var _ = AfterSuite(func() { if failed { - AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...))) + Expect(e2e.SaveJournalLogs(append(serverNodeNames, agentNodeNames...))).To(Succeed()) } else { Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed()) } diff --git a/tests/e2e/externalip/externalip_test.go b/tests/e2e/externalip/externalip_test.go index 9d2150991924..f8acbc9bce41 100644 --- a/tests/e2e/externalip/externalip_test.go +++ b/tests/e2e/externalip/externalip_test.go @@ -166,7 +166,7 @@ var _ = AfterEach(func() { var _ = AfterSuite(func() { if failed { - AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...))) + Expect(e2e.SaveJournalLogs(append(serverNodeNames, agentNodeNames...))).To(Succeed()) } else { Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed()) } diff --git a/tests/e2e/privateregistry/privateregistry_test.go b/tests/e2e/privateregistry/privateregistry_test.go index fe25a94e2181..e7ab9187f870 100644 --- a/tests/e2e/privateregistry/privateregistry_test.go +++ b/tests/e2e/privateregistry/privateregistry_test.go @@ -150,7 +150,7 @@ var _ = AfterEach(func() { var _ = AfterSuite(func() { if failed { - AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...))) + Expect(e2e.SaveJournalLogs(append(serverNodeNames, agentNodeNames...))).To(Succeed()) } else { Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed()) } diff --git a/tests/e2e/s3/s3_test.go b/tests/e2e/s3/s3_test.go index 29c9d43cad0d..64066cf45572 100644 --- a/tests/e2e/s3/s3_test.go +++ b/tests/e2e/s3/s3_test.go @@ -178,7 +178,7 @@ var _ = AfterEach(func() { var _ = AfterSuite(func() { if failed { - AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...))) + Expect(e2e.SaveJournalLogs(append(serverNodeNames, agentNodeNames...))).To(Succeed()) } else { Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed()) } diff --git a/tests/e2e/startup/startup_test.go b/tests/e2e/startup/startup_test.go index a92cff67cf8a..5323c44e5ba8 100644 --- a/tests/e2e/startup/startup_test.go +++ b/tests/e2e/startup/startup_test.go @@ -454,7 +454,7 @@ var _ = AfterEach(func() { var _ = AfterSuite(func() { if failed { AddReportEntry("config", e2e.GetConfig(append(serverNodeNames, agentNodeNames...))) - AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...))) + Expect(e2e.SaveJournalLogs(append(serverNodeNames, agentNodeNames...))).To(Succeed()) } else { Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed()) } diff --git a/tests/e2e/testutils.go b/tests/e2e/testutils.go index 2d2cb12071b0..c50fb0c398d9 100644 --- a/tests/e2e/testutils.go +++ b/tests/e2e/testutils.go @@ -377,6 +377,26 @@ func TailJournalLogs(lines int, nodes []string) string { return logs.String() } +// SaveJournalLogs saves the journal logs of each node to a -jlog.txt file. +// When used in GHA CI, the logs are uploaded as an artifact on failure. +func SaveJournalLogs(nodeNames []string) error { + for _, node := range nodeNames { + lf, err := os.Create(node + "-jlog.txt") + if err != nil { + return err + } + defer lf.Close() + logs, err := GetJournalLogs(node) + if err != nil { + return err + } + if _, err := lf.Write([]byte(logs)); err != nil { + return fmt.Errorf("failed to write %s node logs: %v", node, err) + } + } + return nil +} + func GetConfig(nodes []string) string { config := &strings.Builder{} for _, node := range nodes { diff --git a/tests/e2e/wasm/wasm_test.go b/tests/e2e/wasm/wasm_test.go index 7fa216088b35..237bc440d613 100644 --- a/tests/e2e/wasm/wasm_test.go +++ b/tests/e2e/wasm/wasm_test.go @@ -136,7 +136,7 @@ var _ = AfterEach(func() { var _ = AfterSuite(func() { if failed { - AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...))) + Expect(e2e.SaveJournalLogs(append(serverNodeNames, agentNodeNames...))).To(Succeed()) } else { Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed()) } diff --git a/tests/integration/certrotation/certrotation_int_test.go b/tests/integration/certrotation/certrotation_int_test.go index ffd7fad53a4f..9574a6b02ec3 100644 --- a/tests/integration/certrotation/certrotation_int_test.go +++ b/tests/integration/certrotation/certrotation_int_test.go @@ -46,15 +46,15 @@ var _ = Describe("certificate rotation", Ordered, func() { certHash, err = testutil.RunCommand("md5sum " + tmpdDataDir + "/server/tls/serving-kube-apiserver.crt | cut -f 1 -d' '") Expect(err).ToNot(HaveOccurred()) }) - It("stop k3s", func() { + It("stops k3s", func() { Expect(testutil.K3sKillServer(server)).To(Succeed()) }) - It("certificate rotate", func() { + It("rotates certificates", func() { _, err := testutil.K3sCmd("certificate", "rotate", "-d", tmpdDataDir) Expect(err).ToNot(HaveOccurred()) }) - It("start k3s server", func() { + It("starts k3s server", func() { var err error server2, err = testutil.K3sStartServer(serverArgs...) Expect(err).ToNot(HaveOccurred()) @@ -64,7 +64,18 @@ var _ = Describe("certificate rotation", Ordered, func() { return testutil.K3sDefaultDeployments() }, "360s", "5s").Should(Succeed()) }) - It("get certificate hash", func() { + It("checks the certificate status", func() { + res, err := testutil.K3sCmd("certificate", "check", "-d", tmpdDataDir) + Expect(err).ToNot(HaveOccurred()) + for i, line := range strings.Split(res, "\n") { + // First line is just server info + if i == 0 || line == "" { + continue + } + Expect(line).To(MatchRegexp("certificate.*is ok|Checking certificates"), res) + } + }) + It("gets certificate hash", func() { // get md5sum of the CA certs var err error caCertHashAfter, err := testutil.RunCommand("md5sum " + tmpdDataDir + "/server/tls/client-ca.crt | cut -f 1 -d' '")