long-running-azure.yaml

# ------------------------------------------------------------
# Copyright 2023 The Radius Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ------------------------------------------------------------

# This workflow performs functional tests every an hour on a pre-provisioned
# AKS (Azure Kubernetes Service) cluster.
#
# There are two types of tests in Radius: functional-test and e2e-azure-test.
# 'functional-test' checks the functionality of our application using a local
# Kubernetes cluster (kind), while 'e2e-azure-test' is executed on an AKS cluster
# focusing on performance and reliability.
#
# The test AKS cluster is pre-provisioned with various monitoring tools using the
# Bicep template in /test/infra/azure. Additionally, this cluster has a
# monitoring and alerting system in place, configured to notify the team of any
# abnormalities during the test.
#
# It's also important to note that this workflow does not build the Radius in every
# run. Instead, Radius CLI and container images are built from the main branch every
# 12 hours. We leverage the GitHub Actions cache to store the 'rad cli' and test
# information. On workflow initiation, it checks the last build time to determine if
# the previous build is still valid. If valid, the workflow skips the build steps
# and uses the cached 'rad cli' for testing.
#
# Grafana dashboard URL:  https://radlrtest00-dashboard-e4ffc0cwggchdhba.wus3.grafana.azure.com

name: Long-running test on Azure

permissions:
  id-token: write # Required for requesting the JWT
  contents: read # Required for actions/checkout
  packages: write # Required for uploading the package

on:
  # Enable manual trigger to deploy the latest changes from main.
  workflow_dispatch:
    inputs:
      skip-build:
        description: 'Skip build (true/false)'
        required: false
        default: 'true'
        type: string
  schedule:
    # Run every 2 hours
    - cron: "0 */2 * * *"

env:
  # Go version
  GOVER: "1.22.2"
  GOPROXY: https://proxy.golang.org

  # gotestsum version - see: /~https://github.com/gotestyourself/gotestsum
  GOTESTSUM_VER: 1.12.0

  # Container registry for storing container images
  CONTAINER_REGISTRY: ghcr.io/radius-project/dev
  # Container registry for storing Bicep recipe artifacts
  BICEP_RECIPE_REGISTRY: ghcr.io/radius-project/dev
  # The radius functional test timeout
  FUNCTIONALTEST_TIMEOUT: 60m
  # The Azure Location to store test resources
  AZURE_LOCATION: westus3
  # The base directory for storing test logs
  RADIUS_CONTAINER_LOG_BASE: dist/container_logs
  # The Radius helm chart location.
  RADIUS_CHART_LOCATION: deploy/Chart/
  # The region for AWS resources
  AWS_REGION: us-west-2
  # The AWS account ID
  AWS_ACCOUNT_ID: "${{ secrets.FUNCTEST_AWS_ACCOUNT_ID }}"

  # The valid radius build time window in seconds to rebuild radius. 24 hours = 24 * 60 * 60 = 86400
  VALID_RADIUS_BUILD_WINDOW: 86400

  # The functional test GitHub app id
  FUNCTIONAL_TEST_APP_ID: 425843

  # The AKS cluster name
  AKS_CLUSTER_NAME: "radlrtest00-aks"
  # The resource group for AKS_CLUSTER_NAME resource.
  AKS_RESOURCE_GROUP: "radlrtest00"

  # Server where terraform test modules are deployed
  TF_RECIPE_MODULE_SERVER_URL: "http://tf-module-server.radius-test-tf-module-server.svc.cluster.local"

  # Radius test environment name
  RADIUS_TEST_ENVIRONMENT_NAME: "kind-radius"

  # The current GitHub action link
  ACTION_LINK: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"

jobs:
  build:
    name: Build Radius for test
    runs-on: ubuntu-latest
    if: github.repository == 'radius-project/radius'
    outputs:
      SKIP_BUILD: ${{ steps.skip-build.outputs.SKIP_BUILD }}
      REL_VERSION: ${{ steps.gen-id.outputs.REL_VERSION }}
      UNIQUE_ID: ${{ steps.gen-id.outputs.UNIQUE_ID }}
      PR_NUMBER: ${{ steps.gen-id.outputs.PR_NUMBER }}
      CHECKOUT_REPO: ${{ steps.gen-id.outputs.CHECKOUT_REPO }}
      CHECKOUT_REF: ${{ steps.gen-id.outputs.CHECKOUT_REF }}
      RAD_CLI_ARTIFACT_NAME: ${{ steps.gen-id.outputs.RAD_CLI_ARTIFACT_NAME }}
    steps:
      - name: Restore the latest cached binaries
        uses: actions/cache/restore@v4
        with:
          path: ./dist/cache
          key: radius-test-latest-
      - name: Skip build if build is still valid
        if: github.event_name != 'pull_request'
        id: skip-build
        run: |
          # check if the last build time to see if we need to build again
          SKIP_BUILD="false"
          if [ -f ./dist/cache/.lastbuildtime ]; then
            lastbuild=$(cat ./dist/cache/.lastbuildtime)
            current_time=$(date +%s)
            if [ $((current_time-lastbuild)) -lt ${{ env.VALID_RADIUS_BUILD_WINDOW }} ]; then
              echo "Last build is still within valid window"   
              SKIP_BUILD="true"
            fi
          fi
          
          # Check override in workflow_dispatch mode
          if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "${{ github.event.inputs.skip-build }}" = "false" ]; then
            echo "Manual run with skip-build=false, forcing build"
            SKIP_BUILD="false"
          fi

          echo "SKIP_BUILD=${SKIP_BUILD}" >> $GITHUB_OUTPUT
      - name: Set up checkout target (scheduled)
        if: steps.skip-build.outputs.SKIP_BUILD != 'true' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')
        run: |
          echo "CHECKOUT_REPO=${{ github.repository }}" >> $GITHUB_ENV
          echo "CHECKOUT_REF=refs/heads/main" >> $GITHUB_ENV
      - name: Set up checkout target (pull_request)
        if: steps.skip-build.outputs.SKIP_BUILD != 'true' && github.event_name == 'pull_request'
        run: |
          echo "CHECKOUT_REPO=${{ github.event.pull_request.head.repo.full_name }}" >> $GITHUB_ENV
          echo "CHECKOUT_REF=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
          echo "PR_NUMBER=${{ github.event.pull_request.number }}" >> $GITHUB_ENV
      - name: Generate ID for release
        id: gen-id
        run: |
          if [[ "${{ steps.skip-build.outputs.SKIP_BUILD }}" != "true" ]]; then
            BASE_STR="RADIUS|${GITHUB_SHA}|${GITHUB_SERVER_URL}|${GITHUB_REPOSITORY}|${GITHUB_RUN_ID}|${GITHUB_RUN_ATTEMPT}"
            if [ "$GITHUB_EVENT_NAME" == "schedule" ] || [ "$GITHUB_EVENT_NAME" == "workflow_dispatch" ]; then
              # Add run number to randomize unique id for scheduled runs.
              BASE_STR="${GITHUB_RUN_NUMBER}|${BASE_STR}"
            fi
            UNIQUE_ID=longr$(echo $BASE_STR | sha1sum | head -c 10)
            echo "REL_VERSION=pr-${UNIQUE_ID}" >> $GITHUB_ENV
            
            # Set output variables to be used in the other jobs
            echo "REL_VERSION=pr-${UNIQUE_ID}" >> $GITHUB_OUTPUT
            echo "UNIQUE_ID=${UNIQUE_ID}" >> $GITHUB_OUTPUT
            echo "CHECKOUT_REPO=${{ env.CHECKOUT_REPO }}" >> $GITHUB_OUTPUT
            echo "CHECKOUT_REF=${{ env.CHECKOUT_REF }}" >> $GITHUB_OUTPUT
            echo "AZURE_TEST_RESOURCE_GROUP=radtest-${UNIQUE_ID}" >> $GITHUB_OUTPUT
            echo "RAD_CLI_ARTIFACT_NAME=rad_cli_linux_amd64" >> $GITHUB_OUTPUT
            echo "PR_NUMBER=${{ env.PR_NUMBER }}" >> $GITHUB_OUTPUT
          else
            cat ./dist/cache/.buildenv >> $GITHUB_OUTPUT
            echo "## Radius functional test environment variables" >> $GITHUB_STEP_SUMMARY
            echo "Use the previously built binaries for the current test run:" >> $GITHUB_STEP_SUMMARY
            echo "\`\`\`bash" >> $GITHUB_STEP_SUMMARY
            cat ./dist/cache/.buildenv >> $GITHUB_STEP_SUMMARY
            echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
          fi
      - name: Check out code
        if: steps.skip-build.outputs.SKIP_BUILD != 'true'
        uses: actions/checkout@v4
        with:
          repository: ${{ env.CHECKOUT_REPO }}
          ref: ${{ env.CHECKOUT_REF }}
      - name: Set up Go ${{ env.GOVER }}
        if: steps.skip-build.outputs.SKIP_BUILD != 'true'
        uses: actions/setup-go@v5
        with:
          go-version: ${{ env.GOVER }}
          cache: true
      - name: Log the summary of build info for new version.
        if: steps.skip-build.outputs.SKIP_BUILD != 'true'
        continue-on-error: true
        run: |
          cat <<EOF > summary.md
          ## Radius long-running test overview

          | Name | Value |
          |------|-------|
          |**Repository** | ${{ steps.gen-id.outputs.CHECKOUT_REPO }} |
          |**Commit ref** | ${{ steps.gen-id.outputs.CHECKOUT_REF }} |
          |**Unique ID** | ${{ steps.gen-id.outputs.UNIQUE_ID }} |
          |**Image tag** | ${{ steps.gen-id.outputs.REL_VERSION }} |

          <details>
          <summary> Click here to see the list of tools in the current test run</summary>

          * gotestsum ${{ env.GOTESTSUM_VER }}
          * Bicep recipe location `${{ env.BICEP_RECIPE_REGISTRY }}/test/testrecipes/test-bicep-recipes/<name>:${{ steps.gen-id.outputs.REL_VERSION }}`
          * Terraform recipe location `${{ env.TF_RECIPE_MODULE_SERVER_URL }}/<name>.zip` (in cluster)
          * applications-rp test image location: `${{ env.CONTAINER_REGISTRY }}/applications-rp:${{ steps.gen-id.outputs.REL_VERSION }}`
          * dynamic-rp test image location: `${{ env.CONTAINER_REGISTRY }}/dynamic-rp:${{ steps.gen-id.outputs.REL_VERSION }}`
          * controller test image location: `${{ env.CONTAINER_REGISTRY }}/controller:${{ steps.gen-id.outputs.REL_VERSION }}`
          * ucp test image location: `${{ env.CONTAINER_REGISTRY }}/ucpd:${{ steps.gen-id.outputs.REL_VERSION }}`

          </details>

          ## Test Status
          EOF
          cat summary.md >> $GITHUB_STEP_SUMMARY
      - name: Setup Azure CLI
        if: steps.skip-build.outputs.SKIP_BUILD != 'true'
        run: curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
      - name: Login to Azure
        if: steps.skip-build.outputs.SKIP_BUILD != 'true'
        uses: azure/login@v2
        with:
          client-id: ${{ secrets.AZURE_SP_TESTS_APPID }}
          tenant-id: ${{ secrets.AZURE_SP_TESTS_TENANTID }}
          subscription-id: ${{ secrets.AZURE_SUBSCRIPTIONID_TESTS }}
      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Build and Push container images
        if: steps.skip-build.outputs.SKIP_BUILD != 'true'
        run: |
          make build && make docker-build && make docker-push
        env:
          DOCKER_REGISTRY: ${{ env.CONTAINER_REGISTRY }}
          DOCKER_TAG_VERSION: ${{ env.REL_VERSION }}
      - name: Upload CLI binary
        if: steps.skip-build.outputs.SKIP_BUILD != 'true'
        uses: actions/upload-artifact@v4
        with:
          name: ${{ steps.gen-id.outputs.RAD_CLI_ARTIFACT_NAME }}
          path: |
            ./dist/linux_amd64/release/rad
      - name: Log the build result (success)
        if: steps.skip-build.outputs.SKIP_BUILD != 'true' && success()
        continue-on-error: true
        run: |
          echo ":white_check_mark: Container images build succeeded" >> $GITHUB_STEP_SUMMARY
      - name: Log the build result (failure)
        if: steps.skip-build.outputs.SKIP_BUILD != 'true' && failure()
        continue-on-error: true
        run: |
          echo ":x: Container images build failed" >> $GITHUB_STEP_SUMMARY
      - name: Log test Bicep recipe publish status
        if: steps.skip-build.outputs.SKIP_BUILD != 'true'
        continue-on-error: true
        run: |
          echo ":hourglass: Publishing Bicep Recipes for functional tests..." >> $GITHUB_STEP_SUMMARY
      - name: Move the latest binaries to cache
        if: steps.skip-build.outputs.SKIP_BUILD != 'true'
        run: |
          mkdir -p ./dist/cache
          cp ./dist/linux_amd64/release/rad ./dist/cache
          echo $(date +%s) > ./dist/cache/.lastbuildtime
          echo "UNIQUE_ID=${{ steps.gen-id.outputs.UNIQUE_ID }}" >> ./dist/cache/.buildenv
          echo "REL_VERSION=${{ steps.gen-id.outputs.REL_VERSION }}" >> ./dist/cache/.buildenv
          echo "CHECKOUT_REPO=${{ steps.gen-id.outputs.CHECKOUT_REPO }}" >> ./dist/cache/.buildenv
          echo "CHECKOUT_REF=$(git rev-parse HEAD)" >> ./dist/cache/.buildenv
          echo "PR_NUMBER=${{ steps.gen-id.outputs.PR_NUMBER }}" >> ./dist/cache/.buildenv
          echo "BICEP_RECIPE_TAG_VERSION=${{ steps.gen-id.outputs.REL_VERSION }}" >> ./dist/cache/.buildenv
      - name: Store the latest binaries into cache
        uses: actions/cache/save@v4
        if: steps.skip-build.outputs.SKIP_BUILD != 'true' && success()
        with:
          path: ./dist/cache
          key: radius-test-latest-${{ github.sha }}-${{ github.run_number }}
      - name: Publish Bicep Test Recipes
        if: steps.skip-build.outputs.SKIP_BUILD != 'true'
        run: |
          mkdir ./bin
          cp ./dist/linux_amd64/release/rad ./bin/rad
          chmod +x ./bin/rad
          export PATH=$GITHUB_WORKSPACE/bin:$PATH
          which rad || { echo "cannot find rad"; exit 1; }
          rad bicep download
          rad version
          make publish-test-bicep-recipes
        env:
          BICEP_RECIPE_REGISTRY: ${{ env.BICEP_RECIPE_REGISTRY }}
          BICEP_RECIPE_TAG_VERSION: ${{ env.REL_VERSION }}
      - name: Log Bicep recipe publish status (success)
        if: steps.skip-build.outputs.SKIP_BUILD != 'true' && success()
        run: |
          echo ":white_check_mark: Recipe publishing succeeded" >> $GITHUB_STEP_SUMMARY
      - name: Log recipe publish status (failure)
        if: steps.skip-build.outputs.SKIP_BUILD != 'true' && failure()
        run: |
          echo ":x: Test recipe publishing failed" >> $GITHUB_STEP_SUMMARY
  tests:
    name: Run functional tests
    needs: build
    runs-on: ubuntu-latest-m
    if: github.repository == 'radius-project/radius'
    env:
      SKIP_BUILD: ${{ needs.build.outputs.SKIP_BUILD }}
      UNIQUE_ID: ${{ needs.build.outputs.UNIQUE_ID }}
      REL_VERSION: ${{ needs.build.outputs.REL_VERSION }}
      CHECKOUT_REPO: ${{ needs.build.outputs.CHECKOUT_REPO }}
      CHECKOUT_REF: ${{ needs.build.outputs.CHECKOUT_REF }}
      PR_NUMBER: ${{ needs.build.outputs.PR_NUMBER }}
      AZURE_TEST_RESOURCE_GROUP: radtest-${{ needs.build.outputs.UNIQUE_ID }}-e2e-all
      RAD_CLI_ARTIFACT_NAME: ${{ needs.build.outputs.RAD_CLI_ARTIFACT_NAME }}
      BICEP_RECIPE_TAG_VERSION: ${{ needs.build.outputs.REL_VERSION }}
    steps:
      - name: Get GitHub app token
        uses: tibdex/github-app-token@v2
        id: get_installation_token
        with:
          app_id: ${{ env.FUNCTIONAL_TEST_APP_ID }}
          private_key: ${{ secrets.FUNCTIONAL_TEST_APP_PRIVATE_KEY }}
      - name: Checkout
        uses: actions/checkout@v4
        with:
          repository: ${{ env.CHECKOUT_REPO }}
          ref: ${{ env.CHECKOUT_REF }}
      - name: Checkout samples repo
        uses: actions/checkout@v4
        with:
          repository: radius-project/samples
          ref: refs/heads/edge
          token: ${{ secrets.GH_RAD_CI_BOT_PAT }}
          path: samples
      - name: Set up Go ${{ env.GOVER }}
        uses: actions/setup-go@v5
        with:
          go-version: ${{ env.GOVER }}
          cache: true
      - name: Download rad CLI
        if: env.SKIP_BUILD != 'true'
        uses: actions/download-artifact@v4
        with:
          name: ${{ env.RAD_CLI_ARTIFACT_NAME }}
          path: bin
      - name: Restore the latest cached binaries
        if: env.SKIP_BUILD == 'true'
        uses: actions/cache/restore@v4
        with:
          path: ./dist/cache
          key: radius-test-latest-
      - name: Install rad CLI in bin
        if: env.SKIP_BUILD == 'true'
        run: |
          mkdir -p ./bin
          mv ./dist/cache/rad ./bin/
          chmod +x ./bin/rad
      - name: Login to Azure
        uses: azure/login@v2
        with:
          client-id: ${{ secrets.AZURE_SP_TESTS_APPID }}
          tenant-id: ${{ secrets.AZURE_SP_TESTS_TENANTID }}
          subscription-id: ${{ secrets.AZURE_SUBSCRIPTIONID_TESTS }}
      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Create azure resource group - ${{ env.AZURE_TEST_RESOURCE_GROUP }}
        run: |
          current_time=$(date +%s)
          az group create \
            --location ${{ env.AZURE_LOCATION }} \
            --name $RESOURCE_GROUP \
            --subscription ${{ secrets.AZURE_SUBSCRIPTIONID_TESTS }} \
            --tags creationTime=$current_time
          while [ $(az group exists --name $RESOURCE_GROUP) = false ]; do sleep 2; done
        env:
          RESOURCE_GROUP: ${{ env.AZURE_TEST_RESOURCE_GROUP }}
      - name: Get kubeconf credential for AKS cluster
        run: |
          az aks get-credentials \
            --subscription ${{ secrets.AZURE_SUBSCRIPTIONID_TESTS }} \
            --resource-group ${{ env.AKS_RESOURCE_GROUP }} \
            --name ${{ env.AKS_CLUSTER_NAME }} --admin
        env:
          RESOURCE_GROUP: ${{ env.AZURE_TEST_RESOURCE_GROUP }}
      - name: Restore skip-delete-resources-list
        if: always()
        uses: actions/cache/restore@v4
        with:
          path: skip-delete-resources-list.txt
          key: skip-delete-resources-list-file    
      - name: Clean up cluster
        if: always()
        timeout-minutes: 60
        run: |
          if [ "${{ env.SKIP_BUILD }}" != "true" ] ; then
            echo "Running cleanup without skip delete resource list. This will delete all resources."
            ./.github/scripts/cleanup-long-running-cluster.sh
          else
            echo "Running cleanup with skip-delete-resources-list.txt. Built-In resources will not be deleted."
            ./.github/scripts/cleanup-long-running-cluster.sh skip-delete-resources-list.txt
          fi
      - name: Download Bicep
        run: |
          chmod +x ./bin/rad
          export PATH=$GITHUB_WORKSPACE/bin:$PATH
          which rad || { echo "cannot find rad"; exit 1; }
          rad bicep download
          rad version
      - name: Install gotestsum (test reporting tool)
        run: |
          go install gotest.tools/gotestsum@v${{ env.GOTESTSUM_VER }}
      - name: Install Radius
        if: env.SKIP_BUILD != 'true'
        run: |
          set -x
          export PATH=$GITHUB_WORKSPACE/bin:$PATH
          which rad || { echo "cannot find rad"; exit 1; }

          echo "*** Installing Radius to Kubernetes ***"
          rad install kubernetes --reinstall \
            --chart ${{ env.RADIUS_CHART_LOCATION }} \
            --set rp.image=${{ env.CONTAINER_REGISTRY }}/applications-rp,rp.tag=${{ env.REL_VERSION }},dynamicrp.image=${{ env.CONTAINER_REGISTRY }}/dynamic-rp,dynamicrp.tag=${{ env.REL_VERSION }},controller.image=${{ env.CONTAINER_REGISTRY }}/controller,controller.tag=${{ env.REL_VERSION }},ucp.image=${{ env.CONTAINER_REGISTRY }}/ucpd,ucp.tag=${{ env.REL_VERSION }}
      - name: Verify manifests are registered
        if: env.SKIP_BUILD != 'true'
        run: |
          rm -f registermanifest_logs.txt
          # Find the pod with container "ucp"
          POD_NAME=$(
            kubectl get pods -n radius-system \
              -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.spec.containers[*].name}{"\n"}{end}' \
            | grep "ucp" \
            | head -n1 \
            | cut -d" " -f1
          )
          echo "Found ucp pod: $POD_NAME"

          if [ -z "$POD_NAME" ]; then
            echo "No pod with container 'ucp' found in namespace radius-system."
            exit 1
          fi

          # Poll logs for up to 20 iterations, 30 seconds each (up to 10 minutes total)
          for i in {1..20}; do
            kubectl logs "$POD_NAME" -n radius-system | tee registermanifest_logs.txt > /dev/null

            # Exit on error
            if grep -qi "Service initializer terminated with error" registermanifest_logs.txt; then
              echo "Error found in ucp logs."
              grep -i "Service initializer terminated with error" registermanifest_logs.txt
              exit 1
            fi

            # Check for success
            if grep -q "Successfully registered manifests" registermanifest_logs.txt; then
              echo "Successfully registered manifests - message found."
              break
            fi

            echo "Logs not ready, waiting 30 seconds..."
            sleep 30
          done

          # Final check to ensure success message was found
          if ! grep -q "Successfully registered manifests" registermanifest_logs.txt; then
            echo "Manifests not registered after 10 minutes."
            exit 1
          fi
      - name: Create a list of resources not to be deleted
        if: env.SKIP_BUILD != 'true'
        run: |
          kubectl get resources.ucp.dev -n radius-system --no-headers -o custom-columns=":metadata.name" > skip-delete-resources-list.txt
      - name: Save list of resources not to be deleted
        if: env.SKIP_BUILD != 'true'
        uses: actions/cache/save@v4
        with:
          path: skip-delete-resources-list.txt
          key: skip-delete-resources-list-file
      - name: Configure Radius test workspace
        run: |
          set -x

          export PATH=$GITHUB_WORKSPACE/bin:$PATH
          which rad || { echo "cannot find rad"; exit 1; }

          echo "*** Create workspace, group and environment for test ***"
          rad workspace create kubernetes --force
          rad group create ${{ env.RADIUS_TEST_ENVIRONMENT_NAME }}
          rad group switch ${{ env.RADIUS_TEST_ENVIRONMENT_NAME }}

          # The functional test is designed to use default namespace. So you must create the environment for default namespace.
          rad env create ${{ env.RADIUS_TEST_ENVIRONMENT_NAME }} --namespace default
          rad env switch ${{ env.RADIUS_TEST_ENVIRONMENT_NAME }}

          echo "*** Configuring Azure provider ***"
          rad env update ${{ env.RADIUS_TEST_ENVIRONMENT_NAME }} --azure-subscription-id ${{ secrets.AZURE_SUBSCRIPTIONID_TESTS }} \
            --azure-resource-group ${{ env.AZURE_TEST_RESOURCE_GROUP }}
          rad credential register azure sp --client-id ${{ secrets.AZURE_SP_TESTS_APPID }} \
            --client-secret ${{ secrets.INTEGRATION_TEST_SP_PASSWORD }} \
            --tenant-id ${{ secrets.AZURE_SP_TESTS_TENANTID }}

          echo "*** Configuring AWS provider ***"
          rad env update ${{ env.RADIUS_TEST_ENVIRONMENT_NAME }} --aws-region ${{ env.AWS_REGION }} --aws-account-id ${{ secrets.FUNCTEST_AWS_ACCOUNT_ID }}
          rad credential register aws access-key \
            --access-key-id ${{ secrets.FUNCTEST_AWS_ACCESS_KEY_ID }} --secret-access-key ${{ secrets.FUNCTEST_AWS_SECRET_ACCESS_KEY }}
      - name: Log radius installation status (failure)
        if: failure()
        run: |
          echo ":x: Failed to install Radius for functional test." >> $GITHUB_STEP_SUMMARY
      - name: Publish Terraform test recipes
        run: |
          make publish-test-terraform-recipes
      - name: Get OIDC Issuer from AKS cluster
        run: |
          echo "FUNCTEST_OIDC_ISSUER=$(az aks show -n ${{ env.AKS_CLUSTER_NAME }} -g ${{ env.AKS_RESOURCE_GROUP }} --query "oidcIssuerProfile.issuerUrl" -otsv)" >> $GITHUB_ENV
      - name: Run functional tests
        run: |
          # Ensure rad cli is in path before running tests.
          export PATH=$GITHUB_WORKSPACE/bin:$PATH
          cd $GITHUB_WORKSPACE

          which rad || { echo "cannot find rad"; exit 1; }

          # Populate the following test environment variables from JSON secret.
          # AZURE_COSMOS_MONGODB_ACCOUNT_ID
          # AZURE_MSSQL_RESOURCE_ID
          # AZURE_MSSQL_USERNAME
          # AZURE_MSSQL_PASSWORD
          eval "export $(echo "${{ secrets.FUNCTEST_PREPROVISIONED_RESOURCE_JSON }}" | jq -r 'to_entries | map("\(.key)=\(.value)") | @sh')"

          make test-functional-all
        env:
          TEST_TIMEOUT: ${{ env.FUNCTIONALTEST_TIMEOUT }}
          RADIUS_CONTAINER_LOG_PATH: ${{ github.workspace }}/${{ env.RADIUS_CONTAINER_LOG_BASE }}
          AWS_ACCESS_KEY_ID: ${{ secrets.FUNCTEST_AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.FUNCTEST_AWS_SECRET_ACCESS_KEY }}
          AWS_REGION: ${{ env.AWS_REGION }}
          RADIUS_SAMPLES_REPO_ROOT: ${{ github.workspace }}/samples
          # Test_MongoDB_Recipe_Parameters is using the following environment variable.
          INTEGRATION_TEST_RESOURCE_GROUP_NAME: ${{ env.AZURE_TEST_RESOURCE_GROUP }}
          FUNC_TEST_OIDC_ISSUER: ${{ env.FUNCTEST_OIDC_ISSUER }}
          DOCKER_REGISTRY: ${{ env.CONTAINER_REGISTRY }}
          BICEP_RECIPE_REGISTRY: ${{ env.BICEP_RECIPE_REGISTRY }}
          BICEP_RECIPE_TAG_VERSION: ${{ env.BICEP_RECIPE_TAG_VERSION }}
          GH_TOKEN: ${{ steps.get_installation_token.outputs.token }}
      - name: Collect Pod details
        if: always()
        run: |
          POD_STATE_LOG_FILENAME='${{ env.RADIUS_CONTAINER_LOG_BASE }}/all-tests-pod-states.log'
          mkdir -p $(dirname $POD_STATE_LOG_FILENAME)
          echo "kubectl get pods -A" >> $POD_STATE_LOG_FILENAME
          kubectl get pods -A >> $POD_STATE_LOG_FILENAME
          echo "kubectl describe pods -A" >> $POD_STATE_LOG_FILENAME
          kubectl describe pods -A >> $POD_STATE_LOG_FILENAME
      - name: Upload container logs
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: all_container_logs
          path: ./${{ env.RADIUS_CONTAINER_LOG_BASE }}
      - name: Log radius e2e test status (success)
        if: success()
        run: |
          echo ":white_check_mark: functional tests succeeded" >> $GITHUB_STEP_SUMMARY
      - name: Log radius e2e test status (failure)
        if: failure()
        run: |
          echo ":x: functional test failed." >> $GITHUB_STEP_SUMMARY
      - name: Login to Azure
        if: always()
        uses: azure/login@v2
        with:
          client-id: ${{ secrets.AZURE_SP_TESTS_APPID }}
          tenant-id: ${{ secrets.AZURE_SP_TESTS_TENANTID }}
          subscription-id: ${{ secrets.AZURE_SUBSCRIPTIONID_TESTS }}
      - name: Delete azure resource group - ${{ env.AZURE_TEST_RESOURCE_GROUP }}
        if: always()
        run: |
          # if deletion fails, purge workflow will purge the resource group and its resources later.
          az group delete \
            --subscription ${{ secrets.AZURE_SUBSCRIPTIONID_TESTS }} \
            --name ${{ env.AZURE_TEST_RESOURCE_GROUP }} \
            --yes --verbose
      - name: Restore skip-delete-resources-list
        if: always()
        uses: actions/cache/restore@v4
        with:
          path: skip-delete-resources-list.txt
          key: skip-delete-resources-list-file
      - name: Clean up cluster
        if: always()
        timeout-minutes: 60
        run: |
          # Report if skip-delete-resources-list.txt is missing or empty
          if [ ! -s skip-delete-resources-list.txt ]; then
            echo "skip-delete-resources-list.txt does not exist or is empty. Proceeding without skip list..."
          else
            echo "Found skip-delete-resources-list.txt. Cleaning up resources..."
          fi
          ./.github/scripts/cleanup-long-running-cluster.sh skip-delete-resources-list.txt
  report-failure:
    name: Report test failure
    needs: [build, tests]
    runs-on: ubuntu-latest
    if: failure() && github.repository == 'radius-project/radius' && github.event_name == 'schedule'
    steps:
      - name: Create failure issue for failing long running test run
        uses: actions/github-script@v7
        with:
          github-token: ${{ secrets.GH_RAD_CI_BOT_PAT }}
          script: |
            github.rest.issues.create({
              ...context.repo,
              title: `Scheduled long running test failed - Run ID: ${context.runId}`,
              labels: ['bug', 'test-failure'],
              body: `## Bug information \n\nThis issue is automatically generated if the scheduled long running test fails. The Radius long running test operates on a schedule of every 2 hours everyday. It's important to understand that the test may fail due to workflow infrastructure issues, like network problems, rather than the flakiness of the test itself. For the further investigation, please visit [here](${process.env.ACTION_LINK}).`
            })