add test workflow for upgrade

.github/actions/health-check/action.yml

@@ -0,0 +1,19 @@
+name: health-check
+description: "Check health of Nebari deployment"
+
+inputs:
+  domain:
+    description: Domain name
+    required: true
+
+runs:
+  using: composite
+
+  steps:
+    - name: List kubernetes components
+      shell: bash
+      run: kubectl get --all-namespaces all,cm,secret,pv,pvc,ing
+
+    - name: Check if JupyterHub login page is accessible
+      shell: bash
+      run: curl --insecure --include 'https://${{ inputs.domain }}/hub/home'

.github/actions/init-local/action.yml

@@ -0,0 +1,81 @@
+name: init-local
+description: "Initialize Nebari config for local deployment"
+
+inputs:
+  directory:
+    description: "Path to directory to initialize in"
+    required: false
+    default: './local-deployment'
+
+outputs:
+  directory:
+    description: "Path to config directory"
+    value: ${{ steps.metadata.outputs.directory }}
+  config:
+    description: "Path to Nebari config"
+    value: ${{ steps.metadata.outputs.config }}
+  project:
+    description: "Project name"
+    value: ${{ steps.metadata.outputs.project }}
+  domain:
+    description: "Domain name"
+    value: ${{ steps.metadata.outputs.domain }}
+
+runs:
+  using: composite
+
+  steps:
+    - shell: bash
+      id: metadata
+      run: |
+        # Setup metadata
+        DIRECTORY=$(realpath '${{ inputs.directory }}')
+        mkdir --parents "${DIRECTORY}"
+        echo "directory=${DIRECTORY}" | tee --append "${GITHUB_OUTPUT}"
+
+        CONFIG="${DIRECTORY}/nebari-config.yaml"
+        echo "config=${CONFIG}" | tee --append "${GITHUB_OUTPUT}"
+
+        PROJECT='github-actions'
+        echo "project=${PROJECT}" | tee --append "${GITHUB_OUTPUT}"
+
+        DOMAIN='github-actions.nebari.dev'
+        nslookup "${DOMAIN}"
+        echo "domain=${DOMAIN}" | tee --append "${GITHUB_OUTPUT}"
+
+    - shell: bash -l {0}
+      id: init
+      working-directory: ${{ steps.metadata.outputs.directory }}
+      run: |
+        nebari init local \
+          --project-name '${{ steps.metadata.outputs.project }}' \
+          --domain-name '${{ steps.metadata.outputs.domain }}' \
+          --auth-provider password \
+          --output '${{ steps.metadata.outputs.config }}'
+
+    - shell: bash
+      run: |
+        # Update nebari config for CI
+
+        # Change default JupyterLab theme
+        cat >> '${{ steps.metadata.outputs.config }}' <<- EOM
+        jupyterlab:
+          default_settings:
+            "@jupyterlab/apputils-extension:themes":
+              theme: JupyterLab Dark
+        EOM
+
+        # Change default value for minio persistence size
+        cat >> '${{ steps.metadata.outputs.config }}' <<- EOM
+        monitoring:
+          enabled: true
+          overrides:
+            minio:
+              persistence:
+                size: 1Gi
+        EOM
+
+    - shell: bash
+      run: |
+        # Display Nebari config
+        cat '${{ steps.metadata.outputs.config }}'

.github/actions/setup-local/action.yml

@@ -0,0 +1,30 @@
+name: setup-local
+description: "Setup runner for local deployment"
+
+inputs:
+  kubectl-version:
+    description: "Version of kubectl to install"
+    required: false
+    default: "1.19.16"
+
+runs:
+  using: composite
+
+  steps:
+    - uses: azure/setup-kubectl@v4
+      with:
+        version: v${{ inputs.kubectl-version }}
+
+    - shell: bash
+      run: |
+        # Enable docker permissions for user
+        sudo docker ps
+        sudo usermod -aG docker $USER && newgrp docker
+
+        docker info
+        docker ps
+
+    - shell: bash
+      run: |
+        # Get routing table for docker pods
+        ip route

.github/workflows/test_local_integration.yaml

@@ -53,6 +53,9 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: Setup runner for local deployment
+        uses: ./.github/actions/setup-local
+
       - name: Checkout the branch from the PR that triggered the job
         if: ${{ github.event_name == 'issue_comment' }}
         run: |
@@ -75,82 +78,29 @@ jobs:
           pip install .[dev]
           playwright install
 
-      - uses: azure/setup-kubectl@v4.0.0
-        with:
-          version: v1.19.16
-
-      - name: Enable docker permissions for user
-        run: |
-          sudo docker ps
-          sudo usermod -aG docker $USER && newgrp docker
-
-          docker info
-          docker ps
-
-      - name: Get routing table for docker pods
-        run: |
-          ip route
-
-      - name: Initialize Nebari Cloud
-        run: |
-          mkdir -p local-deployment
-          cd local-deployment
-          nebari init local --project=thisisatest --domain github-actions.nebari.dev --auth-provider=password
-
-          # Need smaller profiles on Local Kind
-          sed -i -E 's/(cpu_guarantee):\s+[0-9\.]+/\1: 0.25/g' "nebari-config.yaml"
-          sed -i -E 's/(mem_guarantee):\s+[A-Za-z0-9\.]+/\1: 0.25G/g' "nebari-config.yaml"
-
-          # Change default JupyterLab theme
-          cat >> nebari-config.yaml <<- EOM
-          jupyterlab:
-            default_settings:
-              "@jupyterlab/apputils-extension:themes":
-                theme: JupyterLab Dark
-          EOM
-
-          # Change default value for minio persistence size
-          cat >> nebari-config.yaml <<- EOM
-          monitoring:
-            enabled: true
-            overrides:
-              minio:
-                persistence:
-                  size: 1Gi
-          EOM
-
-          cat nebari-config.yaml
+      - name: Initialize Nebari config for local deployment
+        id: init
+        uses: ./.github/actions/init-local
 
       - name: Deploy Nebari
-        working-directory: local-deployment
-        run: |
-          nebari deploy --config nebari-config.yaml --disable-prompt
+        working-directory: ${{ steps.init.outputs.directory }}
+        run: nebari deploy --config ${{ steps.init.outputs.config }} --disable-prompt
 
-      - name: Basic kubectl checks after deployment
-        if: always()
-        run: |
-          kubectl get all,cm,secret,pv,pvc,ing -A
-
-      - name: Check github-actions.nebari.dev resolves
-        run: |
-          nslookup github-actions.nebari.dev
-
-      - name: Curl jupyterhub login page
-        run: |
-          curl -k https://github-actions.nebari.dev/hub/home -i
+      - name: Health check
+        uses: ./.github/actions/health-check
+        with:
+          domain: ${{ steps.init.outputs.domain }}
 
       - name: Create example-user
-        working-directory: local-deployment
+        working-directory: ${{ steps.init.outputs.directory }}
         run: |
-          nebari keycloak adduser --user "${TEST_USERNAME}" "${TEST_PASSWORD}" --config nebari-config.yaml
-          nebari keycloak listusers --config nebari-config.yaml
-
-      - name: Get nebari-config.yaml full path
-        run: echo "NEBARI_CONFIG_PATH=`realpath ./local-deployment/nebari-config.yaml`" >> "$GITHUB_ENV"
+          nebari keycloak adduser --user "${TEST_USERNAME}" "${TEST_PASSWORD}" --config ${{ steps.init.outputs.config }}
+          nebari keycloak listusers --config ${{ steps.init.outputs.config }}
 
       ### DEPLOYMENT TESTS
       - name: Deployment Pytests
         env:
+          NEBARI_CONFIG_PATH: ${{ steps.init.outputs.config }}
           KEYCLOAK_USERNAME: ${{ env.TEST_USERNAME }}
           KEYCLOAK_PASSWORD: ${{ env.TEST_PASSWORD }}
         run: |
@@ -165,7 +115,7 @@ jobs:
         env:
           KEYCLOAK_USERNAME: ${{ env.TEST_USERNAME }}
           KEYCLOAK_PASSWORD: ${{ env.TEST_PASSWORD }}
-          NEBARI_FULL_URL: https://github-actions.nebari.dev/
+          NEBARI_FULL_URL: "https://${{ steps.init.outputs.domain }}/"
         working-directory: tests/tests_e2e/playwright
         run: |
           # create environment file
@@ -187,6 +137,5 @@ jobs:
         # in the CI, it makes sense to only run on merge to main or workflow_dispatch to speed
         # up feedback cycle
         if: github.ref_name == 'main' || github.event_name == 'workflow_dispatch'
-        working-directory: local-deployment
-        run: |
-          nebari destroy --config nebari-config.yaml --disable-prompt
+        working-directory: ${{ steps.init.outputs.directory }}
+        run: nebari destroy --config ${{ steps.init.outputs.config }} --disable-prompt

.github/workflows/test_local_upgrade.yaml

@@ -0,0 +1,85 @@
+name: "Local Upgrade Tests"
+
+on:
+  pull_request:
+    paths:
+      - ".github/actions/**"
+      - ".github/workflows/test_local_upgrade.yaml"
+  release:
+    types:
+      - prereleased
+  workflow_dispatch:
+
+
+concurrency:
+  group: ${{ github.ref_name }}
+  cancel-in-progress: true
+
+env:
+  NEBARI_IMAGE_TAG: "main"
+
+jobs:
+  test-local-upgrade:
+    runs-on: "cirun-runner--${{ github.run_id }}"
+    defaults:
+      run:
+        shell: bash -l {0}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup runner for local deployment
+        uses: ./.github/actions/setup-local
+
+      - name: Setup Python
+        uses: conda-incubator/setup-miniconda@v3
+        env:
+          CONDA: /home/runnerx/miniconda3
+        with:
+          auto-update-conda: true
+          python-version: "3.11"
+          miniconda-version: "latest"
+
+      - name: Install latest stable Nebari release
+        run: pip install nebari
+
+      - name: Initialize Nebari config for local deployment
+        id: init
+        uses: ./.github/actions/init-local
+
+      - name: Extract old Nebari version
+        run: |
+          OLD_NEBARI_VERSION=$(grep 'nebari_version: ' ${{ steps.init.outputs.config }} | sed 's/nebari_version: //')
+          echo "OLD_NEBARI_VERSION=${OLD_NEBARI_VERSION}" | tee --append "${GITHUB_ENV}"
+
+      - name: Deploy Nebari
+        working-directory: ${{ steps.init.outputs.directory }}
+        run: nebari deploy --config ${{ steps.init.outputs.config }} --disable-prompt
+
+      - name: Health check before upgrade
+        id: health-check-before
+        uses: ./.github/actions/health-check
+        with:
+          domain: ${{ steps.init.outputs.domain }}
+
+      - name: Install current Nebari
+        run: pip install --upgrade .
+
+      - name: Upgrade Nebari config
+        run: |
+          git add --force ${{ steps.init.outputs.config }}
+          nebari upgrade --config ${{ steps.init.outputs.config }} --attempt-fixes
+          git diff
+          nebari validate --config ${{ steps.init.outputs.config }}
+
+      - name: Redeploy Nebari
+        working-directory: ${{ steps.init.outputs.directory }}
+        run: nebari deploy --config ${{ steps.init.outputs.config }} --disable-prompt
+
+      - name: Health check after upgrade
+        id: health-check-after
+        uses: ./.github/actions/health-check
+        with:
+          domain: ${{ steps.init.outputs.domain }}