feat: support replicated architecture w/ sentinel

.github/workflows/test.yaml

@@ -61,4 +61,6 @@ jobs:
       upgrade-flavors: ${{ needs.check-flavor.outputs.upgrade-flavors }}
       flavor: ${{ matrix.flavor }}
       type: ${{ matrix.type }}
+      timeout: 60
+      runsOn: uds-swf-ubuntu-big-boy-8-core
     secrets: inherit # Inherits all secrets from the parent workflow.

bundle/uds-bundle.yaml

@@ -18,24 +18,77 @@ packages:
     overrides:
       valkey:
         uds-valkey-config:
+          namespace: valkey-standalone
           values:
             - path: custom
               value:
                 - direction: Ingress
                   selector:
                     app.kubernetes.io/name: valkey
-                  remoteNamespace: valkey-cli
+                  remoteNamespace: valkey-test
                   remoteSelector:
-                    app: valkey-cli
+                    app: valkey-test
                   port: 6379
                   description: "Ingress from Valkey CLI (for tests)"
             - path: copyPassword
               value:
                 enabled: true
-                namespace: valkey-cli
-                secretName: valkey
-                secretKey: valkey-password
+                namespace: valkey-test
+                secretName: valkey-standalone
+                secretKey: REDISCLI_AUTH  # This allows us to mount it in as an env var and the valkey-cli picks it right up
         valkey:
+          namespace: valkey-standalone
+          variables:
+            - name: VALKEY_RESOURCES
+              path: "master.resources"
+              default:
+                limits:
+                  cpu: 100m
+                  memory: 300Mi
+                requests:
+                  cpu: 100m
+                  memory: 300Mi
+  - name: valkey
+    path: ../
+    # x-release-please-start-version
+    ref: 8.0.1-uds.0
+    # x-release-please-end
+    overrides:
+      valkey:
+        uds-valkey-config:
+          namespace: valkey-replicated-w-sentinel
+          values:
+            - path: custom
+              value:
+                - direction: Ingress
+                  selector:
+                    app.kubernetes.io/name: valkey
+                  remoteNamespace: valkey-test
+                  remoteSelector:
+                    app: valkey-test
+                  port: 6379
+                  description: "Ingress from Valkey CLI (for tests)"
+                - direction: Ingress
+                  selector:
+                    app.kubernetes.io/name: valkey
+                  remoteNamespace: valkey-test
+                  remoteSelector:
+                    app: valkey-test
+                  port: 26379
+                  description: "Ingress from Valkey CLI (for tests) sentinel"
+            - path: copyPassword
+              value:
+                enabled: true
+                namespace: valkey-test
+                secretName: valkey-replicated-w-sentinel
+                secretKey: REDISCLI_AUTH
+        valkey:
+          namespace: valkey-replicated-w-sentinel
+          values:
+            - path: architecture
+              value: replication
+            - path: sentinel.enabled  # /~https://github.com/bitnami/charts/blob/main/bitnami/valkey/values.yaml#L1143
+              value: true
           variables:
             - name: VALKEY_RESOURCES
               path: "master.resources"

chart/templates/networking.yaml

@@ -0,0 +1,37 @@
+# Copyright 2024 Defense Unicorns
+# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
+
+# This removes the NR filter_not_found error that you get when you try to access
+# the primary node.
+apiVersion: networking.istio.io/v1beta1
+kind: ServiceEntry
+metadata:
+  name: valkey-headless-entry
+  namespace:  {{ .Release.Namespace }}
+spec:
+  hosts:
+  - "*.valkey-headless.{{ .Release.Namespace }}.svc.cluster.local" # Matches pod-specific DNS names
+  ports:
+  - number: 6379
+    name: redis
+    protocol: TCP
+  location: MESH_INTERNAL
+  resolution: NONE
+---
+# This enables comms that are IP based: /~https://github.com/istio/istio/issues/37423
+# The read replicas use each-other's IPs to talk once the sentinels tell them what
+# the IPs are.
+apiVersion: security.istio.io/v1beta1
+kind: PeerAuthentication
+metadata:
+  name: valkey-clustering-exception
+  namespace: {{ .Release.Namespace }}
+spec:
+  mtls:
+    mode: STRICT
+  portLevelMtls:
+    6379:
+      mode: PERMISSIVE
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: valkey

chart/templates/uds-package.yaml

@@ -13,13 +13,10 @@ spec:
       targetPort: 9121
       portName: http-metrics
       description: Metrics
-
   network:
     allow:
       - direction: Ingress
         remoteGenerated: IntraNamespace
-      - direction: Egress
-        remoteGenerated: IntraNamespace
 
       # Custom rules to allow clients to connect
     {{- range .Values.custom }}

common/zarf.yaml

@@ -21,22 +21,3 @@ components:
         url: oci://registry-1.docker.io/bitnamicharts/valkey
         valuesFiles:
           - ../values/values.yaml
-    actions:
-      onDeploy:
-        after:
-          - description: Validate Valkey Package
-            maxTotalSeconds: 300
-            wait:
-              cluster:
-                kind: packages.uds.dev
-                name: valkey
-                namespace: valkey
-                condition: "'{.status.phase}'=Ready"
-          - description: Valkey to be Healthy
-            maxTotalSeconds: 90
-            wait:
-              cluster:
-                kind: pod
-                name: app.kubernetes.io/name=valkey
-                namespace: valkey
-                condition: Ready

tasks.yaml

@@ -32,9 +32,14 @@ tasks:
   - name: create-deploy-test-bundle
     description: Test and validate cluster is deployed with Valkey
     actions:
+      - task: create-dev-package
       - task: create:test-bundle
       - task: deploy:test-bundle
-      - task: setup:create-doug-user
+      - task: test:all
+
+  - name: test-bundle
+    description: Test the deployed test bundle vai the test package
+    actions:
       - task: test:all
 
   - name: dev
@@ -60,7 +65,6 @@ tasks:
       - task: upgrade:create-latest-tag-bundle
       - task: setup:k3d-test-cluster
       - task: deploy:test-bundle
-      - task: setup:create-doug-user
       - task: compliance:validate
       - task: create-dev-package
       - task: create-deploy-test-bundle

tasks/test.yaml

@@ -4,22 +4,19 @@
 tasks:
   - name: all
     actions:
-      - task: health-check
-      - task: setup-data-stores
+      - task: create-test-package
+      - task: test-valkey
 
-  - name: setup-data-stores
+  - name: create-test-package
+    description: Create the test package to confirm Valkey is working
+    inputs:
+      architecture:
+        description: The architecture of the package to create
+        default: ${UDS_ARCH}
     actions:
-      - description: Create the data store test package for the Valkey instance
-        cmd: uds zarf package create tests --confirm --no-progress --architecture="${UDS_ARCH}" --skip-sbom --no-progress
-      - description: Deploy the test package into the cluster
-        cmd: uds zarf package deploy "zarf-package-valkey-test-${UDS_ARCH}-0.1.0.tar.zst" --confirm --no-progress
+      - cmd: uds zarf package create tests --confirm --architecture="${{ .inputs.architecture }}" --skip-sbom
 
-  - name: health-check
+  - name: test-valkey
     actions:
-      - description: Valkey Status
-        wait:
-          cluster:
-            kind: pod
-            name: app.kubernetes.io/name=valkey
-            namespace: valkey
-            condition: Ready
+      - description: Deploy the test package into the cluster
+        cmd: uds zarf package deploy "zarf-package-valkey-test-${UDS_ARCH}-0.1.0.tar.zst" --confirm

tests/valkey/test-job.yaml

@@ -0,0 +1,117 @@
+# Copyright 2024 Defense Unicorns
+# SPDX-License-Identifier: AGPL-3.0-or-later OR LicenseRef-Defense-Unicorns-Commercial
+
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: valkey-test-standalone
+  labels:
+    app: valkey-test
+spec:
+  template:
+    metadata:
+      labels:
+        app: valkey-test
+    spec:
+      containers:
+        - name: valkey-test
+          image: docker.io/bitnami/valkey:8.0.1-debian-12-r2
+          envFrom:
+            - secretRef:
+                name: valkey-standalone
+                optional: false
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              # Set the standalone Valkey host and port
+              HOST="valkey-master.valkey-standalone.svc.cluster.local"
+              PORT="6379"
+
+              # Check if the Valkey server responds to PING
+              PING_OUTPUT=$(echo "ping" | valkey-cli -h ${HOST} -p ${PORT})
+              echo "${PING_OUTPUT}" | grep PONG && echo "Valkey server ponged back" || { echo "Failed to contact Valkey. Output was: ${PING_OUTPUT}"; exit 1; }
+
+              # Try to set a key-value pair
+              SET_OUTPUT=$(echo "set TEST_name ${POD_NAME}" | valkey-cli -h ${HOST} -p ${PORT})
+              echo "${SET_OUTPUT}" | grep OK && echo "Set value via Valkey" || { echo "Failed to set value via Valkey. Output was: ${SET_OUTPUT}"; exit 1; }
+
+              # Try to get the value for the key
+              GET_OUTPUT=$(echo "get TEST_name" | valkey-cli -h ${HOST} -p ${PORT})
+              echo "${GET_OUTPUT}" | grep "${POD_NAME}" && echo "Retrieved value via Valkey" || { echo "Failed to retrieve value via Valkey. Output was: ${GET_OUTPUT}"; exit 1; }
+
+              echo "All checks passed."
+          resources: {}
+      restartPolicy: Never
+  backoffLimit: 0
+status: {}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  creationTimestamp: null
+  name: valkey-test-replicated-w-sentinel
+  labels:
+    app: valkey-test
+spec:
+  completions: 10 # I need to know I didn't accidentally hit the write pod.
+  parallelism: 1
+  template:
+    metadata:
+      labels:
+        app: valkey-test
+    spec:
+      containers:
+        - name: valkey-test
+          image: docker.io/bitnami/valkey:8.0.1-debian-12-r2
+          envFrom:
+            - secretRef:
+                name: valkey-replicated-w-sentinel
+                optional: false
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              # Ask the Sentinel which node is the primary node
+              PRIMARY_ADDR="$(echo 'SENTINEL GET-PRIMARY-ADDR-BY-NAME mymaster' | valkey-cli -h valkey.valkey-replicated-w-sentinel.svc.cluster.local -p 26379)"
+              echo "Primary ADDR is: ${PRIMARY_ADDR}"
+
+              # Extract HOST and PORT using sed
+              HOST=$(echo "${PRIMARY_ADDR}" | sed -n '1p')
+              PORT=$(echo "${PRIMARY_ADDR}" | sed -n '2p')
+              echo "Primary is ${HOST}:${PORT}"
+
+              # Check if primary responds to PING
+              PING_OUTPUT=$(echo "ping" | valkey-cli -h ${HOST} -p ${PORT})
+              echo "${PING_OUTPUT}" | grep PONG && echo "Primary ponged back" || { echo "Failed to contact primary. Output was: ${PING_OUTPUT}"; exit 1; }
+
+              # Try to set a value on the primary
+              SET_OUTPUT=$(echo "set TEST_name ${POD_NAME}" | valkey-cli -h ${HOST} -p ${PORT})
+              echo "${SET_OUTPUT}" | grep OK && echo "Set value via primary" || { echo "Failed to set value via primary. Output was: ${SET_OUTPUT}"; exit 1; }
+
+              # Ensure replicas return the value
+              GET_OUTPUT=$(echo "get TEST_name" | valkey-cli -h valkey.valkey-replicated-w-sentinel.svc.cluster.local)
+              echo "${GET_OUTPUT}" | grep "${POD_NAME}" && echo "Got value via replica" || { echo "Failed to get value via replica. Output was: ${GET_OUTPUT}"; exit 1; }
+
+              GET_OUTPUT=$(echo "get TEST_name" | valkey-cli -h valkey.valkey-replicated-w-sentinel.svc.cluster.local)
+              echo "${GET_OUTPUT}" | grep "${POD_NAME}" && echo "Got value via replica" || { echo "Failed to get value via replica. Output was: ${GET_OUTPUT}"; exit 1; }
+
+              GET_OUTPUT=$(echo "get TEST_name" | valkey-cli -h valkey.valkey-replicated-w-sentinel.svc.cluster.local)
+              echo "${GET_OUTPUT}" | grep "${POD_NAME}" && echo "Got value via replica" || { echo "Failed to get value via replica. Output was: ${GET_OUTPUT}"; exit 1; }
+
+              GET_OUTPUT=$(echo "get TEST_name" | valkey-cli -h valkey.valkey-replicated-w-sentinel.svc.cluster.local)
+              echo "${GET_OUTPUT}" | grep "${POD_NAME}" && echo "Got value via replica" || { echo "Failed to get value via replica. Output was: ${GET_OUTPUT}"; exit 1; }
+
+              echo "All checks passed."
+          resources: {}
+      restartPolicy: Never
+  backoffLimit: 0
+status: {}

tests/valkey/uds-package.yaml

@@ -4,16 +4,31 @@
 apiVersion: uds.dev/v1alpha1
 kind: Package
 metadata:
-  name: valkey-cli
-  namespace: valkey-cli
+  name: valkey-test
 spec:
   network:
     allow:
       - direction: Egress
         selector:
-          app: valkey-cli
-        remoteNamespace: valkey
+          app: valkey-test
+        remoteNamespace: valkey-standalone
         remoteSelector:
           app.kubernetes.io/name: valkey
         port: 6379
-        description: "Egress from Valkey CLI (for tests)"
+        description: "Egress from Valkey CLI (for tests of standalone instance)"
+      - direction: Egress
+        selector:
+          app: valkey-test
+        remoteNamespace: valkey-replicated-w-sentinel
+        remoteSelector:
+          app.kubernetes.io/name: valkey
+        port: 6379
+        description: "Egress from Valkey CLI (for tests of replicated w/ sentinel) to read-port"
+      - direction: Egress
+        selector:
+          app: valkey-test
+        remoteNamespace: valkey-replicated-w-sentinel
+        remoteSelector:
+          app.kubernetes.io/name: valkey
+        port: 26379
+        description: "Egress from Valkey CLI (for tests of replicated w/ sentinel) to sentinel-port"