From 966d4b5cc6644c65e3b02575caed556359600861 Mon Sep 17 00:00:00 2001 From: Keith Williams Date: Thu, 1 Jan 2026 23:17:01 -0300 Subject: [PATCH] fix: address flaky E2E tests (#26374) * fix: address flaky E2E tests - booking-pages.e2e.ts: Use selectFirstAvailableTimeSlotNextMonth helper instead of brittle nth(1) selector to avoid race condition where time slots become unavailable - fixtures/users.ts: Add retryOnNetworkError helper to handle transient ECONNRESET errors during apiLogin - lib/testUtils.ts: Add waitForLoadState('networkidle') to goToUrlWithErrorHandling to ensure page is fully loaded before checking URL - teams.e2e.ts: Add explicit wait for publish button visibility before clicking to avoid timeout - unpublished.e2e.ts: Change from parallel to serial mode to avoid database deadlocks from concurrent writes Co-Authored-By: keith@cal.com * fix: consolidate unpublished.e2e.ts tests to reduce concurrent DB writes Instead of using serial mode, consolidate related tests into single test functions that share setup data. This reduces concurrent users.create() and users.deleteAll() calls from 7 to 3, significantly reducing the chance of database deadlocks while maintaining parallel execution. Co-Authored-By: keith@cal.com * fix: only fail goToUrlWithErrorHandling on main navigation requests The previous fix was incorrectly resolving the promise when any request failed (like images, RSC requests, etc.), causing the URL check to fail. Now we only consider it a navigation failure if: - request.isNavigationRequest() is true - request.frame() === page.mainFrame() Also added a resolved flag to prevent multiple resolutions and removed the networkidle wait which was causing issues. Co-Authored-By: keith@cal.com --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- apps/web/playwright/booking-pages.e2e.ts | 13 +-- apps/web/playwright/fixtures/users.ts | 52 +++++++++-- apps/web/playwright/lib/testUtils.ts | 19 +++- apps/web/playwright/teams.e2e.ts | 7 +- apps/web/playwright/unpublished.e2e.ts | 105 ++++++++++------------- 5 files changed, 114 insertions(+), 82 deletions(-) diff --git a/apps/web/playwright/booking-pages.e2e.ts b/apps/web/playwright/booking-pages.e2e.ts index f38e672c50..aab886487c 100644 --- a/apps/web/playwright/booking-pages.e2e.ts +++ b/apps/web/playwright/booking-pages.e2e.ts @@ -577,16 +577,9 @@ test.describe("Booking on different layouts", () => { await page.click('[data-testid="toggle-group-item-column_view"]'); - await page.click('[data-testid="incrementMonth"]'); - - await page.waitForURL((url) => { - return url.searchParams.has("month"); - }) - - await page.reload(); - await page.waitForLoadState("networkidle"); - - await page.locator('[data-testid="time"]').nth(1).click(); + // Use the standard helper to select an available time slot next month + // This is more robust than manually clicking incrementMonth and reloading + await selectFirstAvailableTimeSlotNextMonth(page); // Fill what is this meeting about? name email and notes await page.locator('[name="name"]').fill("Test name"); diff --git a/apps/web/playwright/fixtures/users.ts b/apps/web/playwright/fixtures/users.ts index e530d1fd57..4a53966ec6 100644 --- a/apps/web/playwright/fixtures/users.ts +++ b/apps/web/playwright/fixtures/users.ts @@ -1106,17 +1106,49 @@ export async function login( await responsePromise; } +/** + * Helper to retry network requests that may fail with transient errors like ECONNRESET + */ +async function retryOnNetworkError( + fn: () => Promise, + maxRetries = 3, + delayMs = 500 +): Promise { + let lastError: Error | undefined; + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + return await fn(); + } catch (error) { + lastError = error as Error; + const errorMessage = lastError.message || ""; + // Only retry on transient network errors + const isRetryable = + errorMessage.includes("ECONNRESET") || + errorMessage.includes("ECONNREFUSED") || + errorMessage.includes("ETIMEDOUT") || + errorMessage.includes("socket hang up"); + + if (!isRetryable || attempt === maxRetries) { + throw lastError; + } + // Wait before retrying with exponential backoff + await new Promise((resolve) => setTimeout(resolve, delayMs * attempt)); + } + } + throw lastError; +} + export async function apiLogin( user: Pick & Partial> & { password: string | null }, page: Page, navigateToUrl?: string ) { - // Get CSRF token - const csrfToken = await page - .context() - .request.get("/api/auth/csrf") - .then((response) => response.json()) - .then((json) => json.csrfToken); + // Get CSRF token with retry for transient network errors + const csrfToken = await retryOnNetworkError(async () => { + const response = await page.context().request.get("/api/auth/csrf"); + const json = await response.json(); + return json.csrfToken; + }); // Make the login request const loginData = { @@ -1128,9 +1160,11 @@ export async function apiLogin( csrfToken, }; - const response = await page.context().request.post("/api/auth/callback/credentials", { - data: loginData, - }); + const response = await retryOnNetworkError(() => + page.context().request.post("/api/auth/callback/credentials", { + data: loginData, + }) + ); expect(response.status()).toBe(200); diff --git a/apps/web/playwright/lib/testUtils.ts b/apps/web/playwright/lib/testUtils.ts index 98f0a61d73..665edf3204 100644 --- a/apps/web/playwright/lib/testUtils.ts +++ b/apps/web/playwright/lib/testUtils.ts @@ -416,19 +416,32 @@ export async function fillStripeTestCheckout(page: Page) { export function goToUrlWithErrorHandling({ page, url }: { page: Page; url: string }) { return new Promise<{ success: boolean; url: string }>(async (resolve) => { + let resolved = false; const onRequestFailed = (request: PlaywrightRequest) => { + // Only consider it a navigation failure if it's the main document request + // Ignore failures for subresources like images, scripts, RSC requests, etc. + if (!request.isNavigationRequest() || request.frame() !== page.mainFrame()) { + const failedToLoadUrl = request.url(); + console.log("goToUrlWithErrorHandling: Failed to load URL:", failedToLoadUrl); + return; + } + if (resolved) return; + resolved = true; const failedToLoadUrl = request.url(); - console.log("goToUrlWithErrorHandling: Failed to load URL:", failedToLoadUrl); + console.log("goToUrlWithErrorHandling: Navigation failed for URL:", failedToLoadUrl); resolve({ success: false, url: failedToLoadUrl }); }; page.on("requestfailed", onRequestFailed); try { - await page.goto(url); + await page.goto(url, { waitUntil: "domcontentloaded" }); } catch { // do nothing } page.off("requestfailed", onRequestFailed); - resolve({ success: true, url: page.url() }); + if (!resolved) { + resolved = true; + resolve({ success: true, url: page.url() }); + } }); } diff --git a/apps/web/playwright/teams.e2e.ts b/apps/web/playwright/teams.e2e.ts index 6bc77519cd..376fe4ba78 100644 --- a/apps/web/playwright/teams.e2e.ts +++ b/apps/web/playwright/teams.e2e.ts @@ -171,8 +171,11 @@ test.describe("Teams - NonOrg", () => { // eslint-disable-next-line playwright/no-conditional-in-test if (IS_TEAM_BILLING_ENABLED) await fillStripeTestCheckout(page); await page.waitForURL(/\/settings\/teams\/(\d+)\/onboard-members.*$/i); - // Click text=Continue - await page.locator("[data-testid=publish-button]").click(); + // Wait for the page to fully load and the publish button to be visible + await page.waitForLoadState("networkidle"); + const publishButton = page.locator("[data-testid=publish-button]"); + await publishButton.waitFor({ state: "visible", timeout: 10000 }); + await publishButton.click(); await page.waitForURL(/\/settings\/teams\/(\d+)\/event-type*$/i); await page.locator("[data-testid=handle-later-button]").click(); await page.waitForURL(/\/settings\/teams\/(\d+)\/profile$/i); diff --git a/apps/web/playwright/unpublished.e2e.ts b/apps/web/playwright/unpublished.e2e.ts index f52fc5fd49..ef0bf00b09 100644 --- a/apps/web/playwright/unpublished.e2e.ts +++ b/apps/web/playwright/unpublished.e2e.ts @@ -4,16 +4,13 @@ import { SchedulingType } from "@calcom/prisma/enums"; import { test } from "./lib/fixtures"; +// Keep parallel mode - each test creates its own isolated data test.describe.configure({ mode: "parallel" }); const title = (name: string) => `${name} is unpublished`; const description = (entity: string) => `This ${entity} link is currently not available. Please contact the ${entity} owner or ask them to publish it.`; -test.afterEach(async ({ users }) => { - await users.deleteAll(); -}); - const assertChecks = async (page: any, entityName: string, entityType: string) => { await expect(page.locator('[data-testid="empty-screen"]')).toHaveCount(1); await expect(page.locator(`h2:has-text("${title(entityName)}")`)).toHaveCount(1); @@ -21,20 +18,13 @@ const assertChecks = async (page: any, entityName: string, entityType: string) = await expect(page.locator(`img`)).toHaveAttribute("src", /.*/); }; -test.describe("Unpublished", () => { - test("Regular team profile", async ({ page, users }) => { - const owner = await users.create(undefined, { hasTeam: true, isUnpublished: true }); - const { team } = await owner.getFirstTeamMembership(); - const { requestedSlug } = team.metadata as { requestedSlug: string }; - const prefixes = ["", "/en"]; - - for (const prefix of prefixes) { - await page.goto(`${prefix}/team/${requestedSlug}`); - await assertChecks(page, team.name, "team"); - } +// Group 1: Regular team tests - share setup data +test.describe("Unpublished - Regular team", () => { + test.afterEach(async ({ users }) => { + await users.deleteAll(); }); - test("Regular team event type", async ({ page, users }) => { + test("Regular team profile", async ({ page, users }) => { const owner = await users.create(undefined, { hasTeam: true, isUnpublished: true, @@ -42,46 +32,63 @@ test.describe("Unpublished", () => { }); const { team } = await owner.getFirstTeamMembership(); const { requestedSlug } = team.metadata as { requestedSlug: string }; - const { slug: teamEventSlug } = await owner.getFirstTeamEvent(team.id); + const teamEventSlug = (await owner.getFirstTeamEvent(team.id)).slug; const prefixes = ["", "/en"]; + // Test team profile + for (const prefix of prefixes) { + await page.goto(`${prefix}/team/${requestedSlug}`); + await assertChecks(page, team.name, "team"); + } + + // Test team event type (reuse same data) for (const prefix of prefixes) { await page.goto(`${prefix}/team/${requestedSlug}/${teamEventSlug}`); await assertChecks(page, team.name, "team"); } }); +}); - test("Organization profile", async ({ users, page }) => { +// Group 2: Organization tests - share setup data +test.describe("Unpublished - Organization", () => { + test.afterEach(async ({ users }) => { + await users.deleteAll(); + }); + + test("Organization profile and user", async ({ users, page }) => { const owner = await users.create(undefined, { hasTeam: true, isUnpublished: true, isOrg: true }); const { team: org } = await owner.getOrgMembership(); const { requestedSlug } = org.metadata as { requestedSlug: string }; + const [{ slug: ownerEventType }] = owner.eventTypes; const prefixes = ["", "/en"]; + // Test organization profile for (const prefix of prefixes) { await page.goto(`${prefix}/org/${requestedSlug}`); await assertChecks(page, org.name, "organization"); } - }); - - test("Organization sub-team", async ({ users, page }) => { - const owner = await users.create(undefined, { - hasTeam: true, - isUnpublished: true, - isOrg: true, - hasSubteam: true, - }); - const { team: org } = await owner.getOrgMembership(); - const { requestedSlug } = org.metadata as { requestedSlug: string }; - const [{ slug: subteamSlug }] = org.children as { slug: string }[]; - const prefixes = ["", "/en"]; + // Test organization user for (const prefix of prefixes) { - await page.goto(`${prefix}/org/${requestedSlug}/team/${subteamSlug}`); + await page.goto(`${prefix}/org/${requestedSlug}/${owner.username}`); + await assertChecks(page, org.name, "organization"); + } + + // Test organization user event-type + for (const prefix of prefixes) { + await page.goto(`${prefix}/org/${requestedSlug}/${owner.username}/${ownerEventType}`); await assertChecks(page, org.name, "organization"); } }); +}); - test("Organization sub-team event-type", async ({ users, page }) => { +// Group 3: Organization sub-team tests - share setup data +test.describe("Unpublished - Organization sub-team", () => { + test.afterEach(async ({ users }) => { + await users.deleteAll(); + }); + + test("Organization sub-team and event-type", async ({ users, page }) => { const owner = await users.create(undefined, { hasTeam: true, isUnpublished: true, @@ -94,34 +101,16 @@ test.describe("Unpublished", () => { const { slug: subteamEventSlug } = await owner.getFirstTeamEvent(subteamId); const prefixes = ["", "/en"]; + // Test organization sub-team + for (const prefix of prefixes) { + await page.goto(`${prefix}/org/${requestedSlug}/team/${subteamSlug}`); + await assertChecks(page, org.name, "organization"); + } + + // Test organization sub-team event-type for (const prefix of prefixes) { await page.goto(`${prefix}/org/${requestedSlug}/team/${subteamSlug}/${subteamEventSlug}`); await assertChecks(page, org.name, "organization"); } }); - - test("Organization user", async ({ users, page }) => { - const owner = await users.create(undefined, { hasTeam: true, isUnpublished: true, isOrg: true }); - const { team: org } = await owner.getOrgMembership(); - const { requestedSlug } = org.metadata as { requestedSlug: string }; - const prefixes = ["", "/en"]; - - for (const prefix of prefixes) { - await page.goto(`${prefix}/org/${requestedSlug}/${owner.username}`); - await assertChecks(page, org.name, "organization"); - } - }); - - test("Organization user event-type", async ({ users, page }) => { - const owner = await users.create(undefined, { hasTeam: true, isUnpublished: true, isOrg: true }); - const { team: org } = await owner.getOrgMembership(); - const { requestedSlug } = org.metadata as { requestedSlug: string }; - const [{ slug: ownerEventType }] = owner.eventTypes; - const prefixes = ["", "/en"]; - - for (const prefix of prefixes) { - await page.goto(`${prefix}/org/${requestedSlug}/${owner.username}/${ownerEventType}`); - await assertChecks(page, org.name, "organization"); - } - }); });