From 18b2c3d5e67969e31b6ec078c4ffebf249f926ff Mon Sep 17 00:00:00 2001 From: Marie Idleman Date: Thu, 12 Dec 2024 08:54:32 -0600 Subject: [PATCH] ci: improved tags in pull-request (#5706) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Summary Previously, re-running the `test-pull-request.yml` workflow failed to detect updated tags in the PR description because it relied on outdated data from the prior GitHub event. This issue has been resolved by using the GitHub API to fetch the latest PR information directly. In addition, a comment is now posted to the PR to confirm the detected tags and provide feedback whenever tags are updated in the PR description. ### QA Notes Confirmed I can re-run jobs after changing tags in PR description and it works as expected. **Feel free to edit/add tags and see the comment update on this PR.** @:data-explorer ### Screenshots Screenshot 2024-12-12 at 7 08 18 AM Note: The `?` in the comment links to the new section I added in the README. It won't work right now since that readme section is not available in `main` yet. --- .github/pull_request_template.md | 5 --- .github/workflows/pr-e2e-comment.yml | 31 +++++++++++++ .github/workflows/test-pull-request.yml | 7 ++- build/secrets/.secrets.baseline | 4 +- scripts/pr-e2e-comment.sh | 59 +++++++++++++++++++++++++ scripts/pr-tags-parse.sh | 36 +++++++++++---- test/e2e/README.md | 16 ++++++- test/e2e/helpers/test-tags.ts | 7 ++- 8 files changed, 143 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/pr-e2e-comment.yml create mode 100644 scripts/pr-e2e-comment.sh diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index a58c963e8d9..da9468e72d8 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -16,11 +16,6 @@ main branch by either pulling or rebasing. describe them here. --> -### Tests to Run - - -`@:critical` - ### QA Notes " "${{ env.tags }}" + env: + tags: ${{ steps.pr-tags.outputs.tags }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_EVENT_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} + diff --git a/.github/workflows/test-pull-request.yml b/.github/workflows/test-pull-request.yml index 4c4226d9530..2de5cdf9898 100644 --- a/.github/workflows/test-pull-request.yml +++ b/.github/workflows/test-pull-request.yml @@ -14,9 +14,14 @@ jobs: tags: ${{ steps.pr-tags.outputs.tags }} steps: - uses: actions/checkout@v4 + - name: Parse Tags from PR Body id: pr-tags - run: bash scripts/pr-tags-parse.sh "${{ github.event_path }}" + run: bash scripts/pr-tags-parse.sh + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_EVENT_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} e2e-electron: name: e2e diff --git a/build/secrets/.secrets.baseline b/build/secrets/.secrets.baseline index 79e1b3af552..03d175d25ef 100644 --- a/build/secrets/.secrets.baseline +++ b/build/secrets/.secrets.baseline @@ -157,7 +157,7 @@ "filename": ".github/workflows/test-pull-request.yml", "hashed_secret": "3e26d6750975d678acb8fa35a0f69237881576b0", "is_verified": false, - "line_number": 29 + "line_number": 39 } ], "build/azure-pipelines/alpine/product-build-alpine.yml": [ @@ -1933,5 +1933,5 @@ } ] }, - "generated_at": "2024-12-10T16:17:08Z" + "generated_at": "2024-12-12T00:45:45Z" } diff --git a/scripts/pr-e2e-comment.sh b/scripts/pr-e2e-comment.sh new file mode 100644 index 00000000000..0995b29bb58 --- /dev/null +++ b/scripts/pr-e2e-comment.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +# Script to update or create a PR comment with E2E test tags +# Usage: ./update_pr_comment.sh "" "" +# Example: ./update_pr_comment.sh "" "@critical,@quarto" + +set -e + +# Arguments +COMMENT_MARKER="$1" # e.g., "" +TAGS="$2" # e.g., "@critical,@quarto" + +# Ensure required arguments are provided +if [ -z "$COMMENT_MARKER" ] || [ -z "$TAGS" ]; then + echo "Usage: $0 \"\" \"\"" + exit 1 +fi + +# Fetch GitHub repository and PR number from the environment +REPO="${GITHUB_REPOSITORY}" # Automatically set by GitHub Actions +PR_NUMBER="${GITHUB_PR_NUMBER:-${GITHUB_EVENT_PULL_REQUEST_NUMBER}}" # Use the correct PR number env variable +GITHUB_TOKEN="${GITHUB_TOKEN}" # GitHub token for authentication + +if [ -z "$PR_NUMBER" ]; then + echo "Error: PR number not found in the environment. Ensure GITHUB_EVENT_PULL_REQUEST_NUMBER is set." + exit 1 +fi + +if [ -z "$GITHUB_TOKEN" ]; then + echo "Error: GITHUB_TOKEN is not set. Ensure you provide a GitHub token for authentication." + exit 1 +fi + +# Fetch existing comments on the PR +COMMENTS=$(gh api repos/${REPO}/issues/${PR_NUMBER}/comments --header "Authorization: token $GITHUB_TOKEN") + +# Check if a comment with the marker already exists +COMMENT_ID=$(echo "$COMMENTS" | jq -r ".[] | select(.body | contains(\"$COMMENT_MARKER\")) | .id") + +# Format the tags with individual backticks +FORMATTED_TAGS=$(echo "$TAGS" | sed 's/,/` `/g' | sed 's/^/`/' | sed 's/$/`/') + +# Build the new comment body with proper newlines +NEW_COMMENT=$(printf "${COMMENT_MARKER}\n\n**E2E Tests** 🚀  [?](/~https://github.com/posit-dev/positron/blob/main/test/e2e/README.md#pull-requests-and-test-tags)\nThis PR will run tests tagged with: %s" "$FORMATTED_TAGS") + +if [ -n "$COMMENT_ID" ]; then + # Update the existing comment + echo "Updating existing comment (ID: $COMMENT_ID)..." + gh api repos/${REPO}/issues/comments/$COMMENT_ID \ + -X PATCH \ + -F body="$NEW_COMMENT" \ + --header "Authorization: token $GITHUB_TOKEN" +else + # Create a new comment + echo "Creating a new comment..." + gh api repos/${REPO}/issues/${PR_NUMBER}/comments \ + -F body="$NEW_COMMENT" \ + --header "Authorization: token $GITHUB_TOKEN" +fi diff --git a/scripts/pr-tags-parse.sh b/scripts/pr-tags-parse.sh index 574f78bf30e..7665001386b 100644 --- a/scripts/pr-tags-parse.sh +++ b/scripts/pr-tags-parse.sh @@ -1,15 +1,33 @@ #!/bin/bash -# Usage: -# bash parse-pr-tags.sh "" -# Example: -# bash parse-pr-tags.sh "/path/to/event.json" +# Script to parse tags from a GitHub Pull Request body +# Usage: bash parse-pr-tags.sh -# Input: Path to the GitHub event JSON file -GITHUB_EVENT_PATH="$1" +set -e -# Extract the PR body from the event JSON -echo "Extracting PR body..." -PR_BODY=$(jq -r '.pull_request.body' "$GITHUB_EVENT_PATH" | tr '\n' ' ' | sed 's/"/\\"/g') +# Fetch GitHub repository and PR number from the environment +REPO="${GITHUB_REPOSITORY}" # Automatically set by GitHub Actions +PR_NUMBER="${GITHUB_PR_NUMBER:-${GITHUB_EVENT_PULL_REQUEST_NUMBER}}" # Use the correct PR number env variable +GITHUB_TOKEN="${GITHUB_TOKEN}" # GitHub token for authentication + +# Validate required environment variables +if [[ -z "$REPO" || -z "$PR_NUMBER" || -z "$GITHUB_TOKEN" ]]; then + echo "Error: Missing required environment variables." + echo "Ensure the following are set: GITHUB_REPOSITORY, GITHUB_PR_NUMBER or GITHUB_EVENT_PULL_REQUEST_NUMBER, GITHUB_TOKEN." + exit 1 +fi + +# Fetch the PR body using the GitHub CLI +echo "Fetching PR body for ${REPO} #${PR_NUMBER}..." +PULL_REQUEST_BODY=$(gh api repos/${REPO}/pulls/${PR_NUMBER} --header "Authorization: token $GITHUB_TOKEN" --jq '.body') + +# Handle empty PR body +if [[ -z "$PULL_REQUEST_BODY" ]]; then + echo "Error: PR body is empty or could not be fetched." + exit 1 +fi + +# Sanitize the PR BODY by removing newlines and escaping double quotes +PR_BODY=$(echo "$PULL_REQUEST_BODY" | tr '\n' ' ' | sed 's/"/\\"/g') echo "Parsing tags from PR body..." diff --git a/test/e2e/README.md b/test/e2e/README.md index 15290e35762..2bc253d7f71 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -193,9 +193,23 @@ For Python, add any package requirements to the `requirements.txt` file in the r For R, add any package requirements to the "imports" section of the `DESCRIPTION` file in the root of the [QA Content Examples](/~https://github.com/posit-dev/qa-example-content) repo. +## Pull Requests and Test Tags + +When you create a pull request, the test runner automatically scans the PR description for test tags to determine which tests to run. + +- **Always-on Tests:** Tests tagged with `@critical` always run, and you can’t opt out of them. +- **Custom Tags:** If your changes affect a specific feature, you can include additional tags in the PR description to trigger relevant tests. + +To add a test tag: + +1. Use the format `@:tag` in your PR description (e.g., `@:help`, `@:console`). +2. Once added, a comment will appear on your PR confirming that the tag was found and parsed correctly. + +From then on, all tests associated with the specified tag(s) will run during your test workflows. See [here](/~https://github.com/posit-dev/positron/blob/main/test/e2e/helpers/test-tags.ts) for a complete list of tags. + ## Running Tests in Github Actions -New tests are not complete until they run successfully across operating systems (Mac, Windows, & Ubuntu) and in [Github Actions](/~https://github.com/posit-dev/positron/actions/workflows/positron-full-test.yml). In Github Actions we use an Ubuntu instance to run the tests, so if you are developing your tests using a Mac or on Windows, this is an opportunity to test a different operating system. Also, you can easily run your new tests against a branch to verify them before merge. Simply pick the branch after you click on "Run Workflow". Note that you can also temporarily modify the workflow itself to get your new tests executed more quickly. To do this, skip the runs of the unit and integration tests. +New tests are not complete until they run successfully across operating systems (Mac, Windows, & Ubuntu) and in [Github Actions](/~https://github.com/posit-dev/positron/actions/workflows/test-full-suite.yml). In Github Actions we use an Ubuntu instance to run the tests, so if you are developing your tests using a Mac or on Windows, this is an opportunity to test a different operating system. Also, you can easily run your new tests against a branch to verify them before merge. Simply pick the branch after you click on "Run Workflow". Note that you can also temporarily modify the workflow itself to get your new tests executed more quickly. To do this, skip the runs of the unit and integration tests. ### Github Actions Test Artifacts diff --git a/test/e2e/helpers/test-tags.ts b/test/e2e/helpers/test-tags.ts index 2757f1b377b..0550dae130f 100644 --- a/test/e2e/helpers/test-tags.ts +++ b/test/e2e/helpers/test-tags.ts @@ -6,12 +6,11 @@ /** * Use the tags listed below to selectively run e2e tests against your PR. * - * - To run all e2e tests, include `@:all` in your PR description. - * - Avoid using `@:web` and `@:win` tags, as these are reserved for web and Windows platform-specific tests, - * which are not currently configured to run in PR workflows. - * * Each tag corresponds to a specific feature, functionality, or area of the application. Use them * thoughtfully to ensure your tests align with the intended scope. + * + * Avoid using `@:web` and `@:win` tags, as these are reserved for web and Windows platform-specific tests, + * which are not currently configured to run in PR workflows. */ export enum TestTags { APPS = '@apps',