feat: e2e reports in gh-pages (#16179)

.github/actions/yarn-install/action.yml

@@ -37,8 +37,6 @@ runs:
       with:
         path: ${{ steps.yarn-config.outputs.CACHE_FOLDER }}
         key: yarn-download-cache-${{ hashFiles('yarn.lock') }}
-        restore-keys: |
-          yarn-download-cache-
 
     # Invalidated on yarn.lock changes
     - name: Restore node_modules

.github/actions/yarn-playwright-install/action.yml

@@ -3,6 +3,10 @@ description: "Install playwright, cache and restore if necessary"
 runs:
   using: "composite"
   steps:
+    - name: Get installed Playwright version
+      shell: bash
+      id: playwright-version
+      run: echo "PLAYWRIGHT_VERSION=$(node -e "console.log(require('./package.json').devDependencies['@playwright/test'])")" >> $GITHUB_ENV
     - name: Cache playwright binaries
       id: playwright-cache
       uses: buildjet/cache@v4
@@ -11,9 +15,7 @@ runs:
           ~/Library/Caches/ms-playwright
           ~/.cache/ms-playwright
           ${{ github.workspace }}/node_modules/playwright
-        key: cache-playwright-${{ hashFiles('**/yarn.lock') }}
-        restore-keys: cache-playwright-
+        key: ${{ runner.os }}-playwright-${{ env.PLAYWRIGHT_VERSION }}
     - name: Yarn playwright install
       shell: bash
-      if: steps.playwright-cache.outputs.cache-hit != 'true'
-      run: yarn playwright install
+      run: yarn playwright install --with-deps

.github/workflows/cleanup.yml

@@ -0,0 +1,58 @@
+name: Delete
+
+on:
+  delete:
+    branches-ignore: [main, gh-pages]
+
+# ensures that currently running Playwright workflow of deleted branch gets cancelled
+concurrency:
+  group: ${{ github.event.ref }}
+  cancel-in-progress: true
+
+jobs:
+  delete_reports:
+    name: Delete Reports
+    runs-on: ubuntu-latest
+    env:
+      # Contains all reports for deleted branch
+      BRANCH_REPORTS_DIR: reports/${{ github.event.ref }}
+    steps:
+      - name: Checkout GitHub Pages Branch
+        uses: actions/checkout@v2
+        with:
+          ref: gh-pages
+      - name: Set Git User
+        # see: https://github.com/actions/checkout/issues/13#issuecomment-724415212
+        run: |
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
+      - name: Check for workflow reports
+        run: |
+          if [ -z "$(ls -A $BRANCH_REPORTS_DIR)" ]; then
+            echo "BRANCH_REPORTS_EXIST="false"" >> $GITHUB_ENV
+          else
+            echo "BRANCH_REPORTS_EXIST="true"" >> $GITHUB_ENV
+          fi
+      - name: Delete reports from repo for branch
+        if: ${{ env.BRANCH_REPORTS_EXIST == 'true' }}
+        timeout-minutes: 3
+        run: |
+          cd $BRANCH_REPORTS_DIR/..
+
+          rm -rf ${{ github.event.ref }}
+          git add .
+          git commit -m "workflow: remove all reports for branch ${{ github.event.ref }}"
+
+          while true; do
+            git pull --rebase
+            if [ $? -ne 0 ]; then
+              echo "Failed to rebase. Please review manually."
+              exit 1
+            fi
+
+            git push
+            if [ $? -eq 0 ]; then
+              echo "Successfully pushed HTML reports to repo."
+              exit 0
+            fi
+          done

.github/workflows/e2e.yml

@@ -89,8 +89,9 @@ jobs:
       - name: Run Tests
         run: yarn e2e --shard=${{ matrix.shard }}/${{ strategy.job-total }}
       - name: Upload Test Results
-        if: ${{ always() }}
         uses: actions/upload-artifact@v4
+        if: always()
         with:
-          name: test-results-${{ matrix.shard }}_${{ strategy.job-total }}
-          path: test-results
+          name: blob-report-${{ matrix.shard }}
+          path: blob-report
+          retention-days: 30

.github/workflows/merge-reports.yml

@@ -0,0 +1,25 @@
+# https://playwright.dev/docs/test-sharding#merging-reports-from-multiple-shards
+on:
+  workflow_call:
+jobs:
+  merge-reports:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/dangerous-git-checkout
+      - uses: ./.github/actions/yarn-install
+      - uses: ./.github/actions/yarn-playwright-install
+      - name: Download blob reports from GitHub Actions Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: all-blob-reports
+          pattern: blob-report-*
+          merge-multiple: true
+      - name: Merge into HTML Report
+        run: yarn playwright merge-reports --reporter html ./all-blob-reports
+      - name: Upload HTML report
+        uses: actions/upload-artifact@v4
+        with:
+          name: html-report--attempt-${{ github.run_number }}-${{ github.run_attempt }}
+          path: playwright-report
+          retention-days: 14

.github/workflows/pr.yml

@@ -91,44 +91,50 @@ jobs:
             console.log('Found the label?', labelFound);
             core.setOutput('run-e2e', labelFound);
 
+  deps:
+    name: Install dependencies
+    needs: [changes, check-label]
+    if: ${{ needs.changes.outputs.has-files-requiring-all-checks == 'true' }}
+    uses: ./.github/workflows/yarn-install.yml
+
   type-check:
     name: Type check
-    needs: [changes, check-label]
+    needs: [changes, check-label, deps]
     if: ${{ needs.changes.outputs.has-files-requiring-all-checks == 'true' }}
     uses: ./.github/workflows/check-types.yml
     secrets: inherit
 
   lint:
     name: Linters
-    needs: [changes, check-label]
+    needs: [changes, check-label, deps]
     if: ${{ needs.changes.outputs.has-files-requiring-all-checks == 'true' }}
     uses: ./.github/workflows/lint.yml
     secrets: inherit
 
   unit-test:
     name: Tests
-    needs: [changes, check-label]
+    needs: [changes, check-label, deps]
     if: ${{ needs.changes.outputs.has-files-requiring-all-checks == 'true' }}
     uses: ./.github/workflows/unit-tests.yml
     secrets: inherit
 
   build-api-v1:
     name: Production builds
-    needs: [changes, check-label]
+    needs: [changes, check-label, deps]
     if: ${{ needs.check-label.outputs.run-e2e == 'true' && needs.changes.outputs.has-files-requiring-all-checks == 'true' }}
     uses: ./.github/workflows/api-v1-production-build.yml
     secrets: inherit
 
   build-api-v2:
     name: Production builds
-    needs: [changes, check-label]
+    needs: [changes, check-label, deps]
     if: ${{ needs.check-label.outputs.run-e2e == 'true' && needs.changes.outputs.has-files-requiring-all-checks == 'true' }}
     uses: ./.github/workflows/api-v2-production-build.yml
     secrets: inherit
 
   build:
     name: Production builds
-    needs: [changes, check-label]
+    needs: [changes, check-label, deps]
     if: ${{ needs.check-label.outputs.run-e2e == 'true' && needs.changes.outputs.has-files-requiring-all-checks == 'true' }}
     uses: ./.github/workflows/production-build-without-database.yml
     secrets: inherit
@@ -181,8 +187,42 @@ jobs:
     uses: ./.github/workflows/nextjs-bundle-analysis.yml
     secrets: inherit
 
+  merge-reports:
+    name: Merge reports
+    if: ${{ !cancelled() }}
+    needs: [e2e]
+    uses: ./.github/workflows/merge-reports.yml
+    secrets: inherit
+
+  publish-report:
+    name: Publish HTML report
+    if: ${{ !cancelled() }}
+    permissions:
+      contents: write
+      issues: write
+      pull-requests: write
+    needs: [merge-reports]
+    uses: ./.github/workflows/publish-report.yml
+    secrets: inherit
+
   required:
-    needs: [changes, lint, type-check, unit-test, integration-test, check-label, build, build-api-v1, build-api-v2, e2e, e2e-api-v2, e2e-embed, e2e-embed-react, e2e-app-store]
+    needs:
+      [
+        changes,
+        lint,
+        type-check,
+        unit-test,
+        integration-test,
+        check-label,
+        build,
+        build-api-v1,
+        build-api-v2,
+        e2e,
+        e2e-api-v2,
+        e2e-embed,
+        e2e-embed-react,
+        e2e-app-store,
+      ]
     if: always()
     runs-on: buildjet-2vcpu-ubuntu-2204
     steps:

.github/workflows/publish-report.yml

@@ -0,0 +1,78 @@
+on:
+  workflow_call:
+permissions:
+  contents: write
+  issues: write
+  pull-requests: write
+jobs:
+  publish-report:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    env:
+      # Unique URL path for each workflow run attempt
+      HTML_REPORT_URL_PATH: reports/${{ github.ref_name }}/${{ github.run_id }}/${{ github.run_attempt }}
+    steps:
+      - name: Checkout GitHub Pages Branch
+        uses: actions/checkout@v4
+        with:
+          ref: gh-pages
+      - name: Set Git User
+        # see: https://github.com/actions/checkout/issues/13#issuecomment-724415212
+        run: |
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
+      - name: Download zipped HTML report
+        uses: actions/download-artifact@v4
+        with:
+          name: html-report--attempt-${{ github.run_number }}-${{ github.run_attempt }}
+          path: ${{ env.HTML_REPORT_URL_PATH }}
+      - name: Push HTML Report
+        timeout-minutes: 3
+        # commit report, then try push-rebase-loop until it's able to merge the HTML report to the gh-pages branch
+        # this is necessary when this job running at least twice at the same time (e.g. through two pushes at the same time)
+        run: |
+          git add .
+          git commit -m "workflow: add HTML report for run-id ${{ github.run_id }} (attempt:  ${{ github.run_attempt }})"
+
+          while true; do
+            git pull --rebase
+            if [ $? -ne 0 ]; then
+              echo "Failed to rebase. Please review manually."
+              exit 1
+            fi
+
+            git push
+            if [ $? -eq 0 ]; then
+              echo "Successfully pushed HTML report to repo."
+              exit 0
+            fi
+          done
+      - name: Output Report URL as Workflow Annotation
+        id: url
+        run: |
+          FULL_HTML_REPORT_URL=https://calcom.github.io/cal.com/$HTML_REPORT_URL_PATH
+          echo "::notice title=📋 Published Playwright Test Report::$FULL_HTML_REPORT_URL"
+          echo "link=$FULL_HTML_REPORT_URL" >> $GITHUB_OUTPUT
+      - name: Find Comment
+        uses: peter-evans/find-comment@v2
+        id: fc
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-author: "github-actions[bot]"
+          body-includes: <!-- GENERATED-E2E-RESULTS -->
+      - name: Create comment
+        if: steps.fc.outputs.comment-id == ''
+        uses: peter-evans/create-or-update-comment@v3
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          body: |
+            <!-- GENERATED-E2E-RESULTS -->
+            ## E2E results are ready!
+            - [Attempt #${{ github.run_number }}.${{ github.run_attempt }} results](${{ steps.url.outputs.link }})
+      - name: Update comment
+        if: steps.fc.outputs.comment-id != ''
+        uses: peter-evans/create-or-update-comment@v3
+        with:
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          body: |
+            - [Attempt #${{ github.run_number }}.${{ github.run_attempt }} results](${{ steps.url.outputs.link }})

.github/workflows/yarn-install.yml

@@ -15,3 +15,4 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/dangerous-git-checkout
       - uses: ./.github/actions/yarn-install
+      - uses: ./.github/actions/yarn-playwright-install

playwright.config.ts

@@ -85,7 +85,7 @@ const config: PlaywrightTestConfig = {
   maxFailures: headless ? 10 : undefined,
   fullyParallel: true,
   reporter: [
-    [process.env.CI ? "github" : "list"],
+    [process.env.CI ? "blob" : "list"],
     ["html", { outputFolder: "./test-results/reports/playwright-html-report", open: "never" }],
     ["junit", { outputFile: "./test-results/reports/results.xml" }],
   ],