fix: improve E2E test stability and reduce flakiness (#25897)

* fix: improve E2E test stability and reduce flakiness

- Replace hardcoded waits with proper Playwright waitFor assertions in slot selection
- Add explicit waits for calendar and time slot elements to be visible before clicking
- Replace fixed 2s wait in gotoRoutingLink with networkidle wait
- Replace fixed 5s email wait with retry logic (10 retries, 500ms intervals)
- Use unique usernames with timestamps to avoid parallel test collisions
- Use features fixture instead of direct prisma calls for feature flag mutations
- Fix login.e2e.ts to use unique username instead of hardcoded 'pro'
- Fix signup.e2e.ts to use unique usernames and proper feature flag handling
- Remove unused 'users' parameter from tests that don't need it

Co-Authored-By: Volnei Munhoz <volnei.munhoz@gmail.com>

* perf: replace waitForTimeout with smart waits for faster E2E tests

- Replace 30 waitForTimeout calls with proper Playwright waits
- Use waitFor({ state: 'visible' }) for element visibility
- Use waitForLoadState('networkidle') for page load completion
- Use waitForFunction for localStorage state changes

Performance improvements:
- limit-tab.e2e.ts: 10s fixed wait -> element visibility wait
- booking-duplicate-api-calls.e2e.ts: 5s fixed wait -> networkidle
- change-theme.e2e.ts: 3s fixed wait -> localStorage state check
- team-invitation.e2e.ts: multiple 500ms-3s waits -> element waits
- booking-seats.e2e.ts: 2s waits -> dropdown visibility waits
- embed-code-generator.e2e.ts: 1s waits -> iframe visibility waits
- organization-privacy.e2e.ts: 500ms waits -> element/networkidle waits
- organization-invitation.e2e.ts: 500ms-1s waits -> element waits
- analyticsApps.e2e.ts: 1s wait -> networkidle
- integrations.e2e.ts: 1s wait -> calendar element wait
- fixtures/apps.ts: 1s waits -> element visibility waits

Co-Authored-By: Volnei Munhoz <volnei.munhoz@gmail.com>

---------

Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
Volnei Munhoz
2025-12-15 20:19:47 -03:00
committed by GitHub
co-authored by Volnei Munhoz Devin AI
parent 883ac103f8
commit b83f8dd6b8
14 changed files with 114 additions and 72 deletions
@@ -17,8 +17,8 @@ test.describe("check analytics Apps", () => {
await user.apiLogin();
await page.goto("apps/categories/analytics");
await appsPage.installAnalyticsAppSkipConfigure(app);
// eslint-disable-next-line playwright/no-wait-for-timeout
await page.waitForTimeout(1000); // waits for 1 second
// Wait for navigation to complete instead of fixed 1s wait
await page.waitForLoadState("networkidle");
await page.goto("/event-types");
await appsPage.goToEventType("30 min");
await appsPage.goToAppsTab();
@@ -27,7 +27,12 @@ async function testDuplicateAPICalls(
});
await page.goto(url);
await page.waitForTimeout(5000);
// Wait for the booking page to fully load and time slots to appear instead of fixed 5s wait
await page.waitForLoadState("networkidle");
// Also wait for the calendar to be visible as a more specific indicator
await page.locator('[data-testid="calendar"]').or(page.locator('[data-testid="time"]')).first().waitFor({ state: "visible", timeout: 10000 }).catch(() => {
// If neither element appears, the page might have a different structure - continue anyway
});
return {
totalCalls: trpcCalls.length + apiV2Calls.length,
+4 -2
View File
@@ -555,7 +555,8 @@ test.describe("Reschedule for booking with seats", () => {
await page.waitForSelector('[data-testid="bookings"]');
await page.locator('[data-testid="booking-actions-dropdown"]').nth(0).click();
await page.waitForTimeout(2000);
// Wait for the dropdown menu to appear instead of fixed 2s wait
await page.locator('[data-testid="reschedule"]').waitFor({ state: "visible" });
const href = await page.locator('[data-testid="reschedule"]').getAttribute("href");
const url = new URL(href!, page.url());
const seatReferenceUid = url.searchParams.get('seatReferenceUid');
@@ -563,7 +564,8 @@ test.describe("Reschedule for booking with seats", () => {
await page.reload();
await page.waitForSelector('[data-testid="bookings"]');
await page.locator('[data-testid="booking-actions-dropdown"]').nth(0).click();
await page.waitForTimeout(2000);
// Wait for the dropdown menu to appear instead of fixed 2s wait
await page.locator('[data-testid="reschedule"]').waitFor({ state: "visible" });
}
await page.locator('[data-testid="reschedule"]').click();
await expect(page.getByText("Seats available").first()).toBeVisible();
+2 -1
View File
@@ -55,7 +55,8 @@ test.describe("Change App Theme Test", () => {
const toast2 = await page.waitForSelector('[data-testid="toast-success"]');
expect(toast2).toBeTruthy();
await page.waitForTimeout(3000);
// Wait for localStorage to be updated after theme change instead of fixed 3s wait
await page.waitForFunction(() => localStorage.getItem("app-theme") !== null, { timeout: 5000 });
const themeValue = await page.evaluate(() => localStorage.getItem("app-theme"));
expect(themeValue).toBe("light");
+16 -16
View File
@@ -59,8 +59,8 @@ test.describe("Embed Code Generator Tests", () => {
orgSlug: null,
});
// To prevent early timeouts
await page.waitForTimeout(1000);
// Wait for the preview iframe to be ready instead of fixed 1s wait
await page.locator('iframe[data-testid="embed-preview"]').waitFor({ state: "visible", timeout: 5000 }).catch(() => {});
await expectToContainValidPreviewIframe(page, {
embedType: "inline",
calLink: `${pro.username}/multiple-duration`,
@@ -96,8 +96,8 @@ test.describe("Embed Code Generator Tests", () => {
orgSlug: null,
});
// To prevent early timeouts
await page.waitForTimeout(1000);
// Wait for the preview iframe to be ready instead of fixed 1s wait
await page.locator('iframe[data-testid="embed-preview"]').waitFor({ state: "visible", timeout: 5000 }).catch(() => {});
await expectToContainValidPreviewIframe(page, {
embedType: "floating-popup",
calLink: `${pro.username}/multiple-duration`,
@@ -133,8 +133,8 @@ test.describe("Embed Code Generator Tests", () => {
orgSlug: null,
});
// To prevent early timeouts
await page.waitForTimeout(1000);
// Wait for the preview iframe to be ready instead of fixed 1s wait
await page.locator('iframe[data-testid="embed-preview"]').waitFor({ state: "visible", timeout: 5000 }).catch(() => {});
await expectToContainValidPreviewIframe(page, {
embedType: "element-click",
calLink: `${pro.username}/multiple-duration`,
@@ -172,8 +172,8 @@ test.describe("Embed Code Generator Tests", () => {
orgSlug: null,
});
// To prevent early timeouts
await page.waitForTimeout(1000);
// Wait for the preview iframe to be ready instead of fixed 1s wait
await page.locator('iframe[data-testid="embed-preview"]').waitFor({ state: "visible", timeout: 5000 }).catch(() => {});
await expectToContainValidPreviewIframe(page, {
embedType: "inline",
calLink: decodeURIComponent(embedUrl),
@@ -229,8 +229,8 @@ test.describe("Embed Code Generator Tests", () => {
orgSlug: org.slug,
});
// To prevent early timeouts
await page.waitForTimeout(1000);
// Wait for the preview iframe to be ready instead of fixed 1s wait
await page.locator('iframe[data-testid="embed-preview"]').waitFor({ state: "visible", timeout: 5000 }).catch(() => {});
await expectToContainValidPreviewIframe(page, {
embedType: "inline",
calLink: `${user.username}/multiple-duration`,
@@ -269,8 +269,8 @@ test.describe("Embed Code Generator Tests", () => {
orgSlug: org.slug,
});
// To prevent early timeouts
await page.waitForTimeout(1000);
// Wait for the preview iframe to be ready instead of fixed 1s wait
await page.locator('iframe[data-testid="embed-preview"]').waitFor({ state: "visible", timeout: 5000 }).catch(() => {});
await expectToContainValidPreviewIframe(page, {
embedType: "floating-popup",
calLink: `${user.username}/multiple-duration`,
@@ -308,8 +308,8 @@ test.describe("Embed Code Generator Tests", () => {
orgSlug: org.slug,
});
// To prevent early timeouts
await page.waitForTimeout(1000);
// Wait for the preview iframe to be ready instead of fixed 1s wait
await page.locator('iframe[data-testid="embed-preview"]').waitFor({ state: "visible", timeout: 5000 }).catch(() => {});
await expectToContainValidPreviewIframe(page, {
embedType: "element-click",
calLink: `${user.username}/multiple-duration`,
@@ -326,8 +326,8 @@ function chooseEmbedType(page: Page, embedType: EmbedType) {
}
async function goToReactCodeTab(page: Page) {
// To prevent early timeo
await page.waitForTimeout(1000);
// Wait for the React tab to be visible instead of fixed 1s wait
await page.locator("[data-testid=horizontal-tab-react]").waitFor({ state: "visible" });
await page.locator("[data-testid=horizontal-tab-react]").click();
}
@@ -18,7 +18,8 @@ test.describe("Limits Tab - Event Type", () => {
await bookingPage.updateEventType();
const eventTypePage = await bookingPage.previewEventType();
await eventTypePage.waitForTimeout(10000);
// Wait for time slots to load instead of fixed 10s wait
await eventTypePage.getByTestId("time").first().waitFor({ state: "visible", timeout: 30000 });
const counter = await eventTypePage.getByTestId("time").count();
await bookingPage.checkTimeSlotsCount(eventTypePage, counter);
+4 -4
View File
@@ -33,8 +33,8 @@ export function createAppsFixture(page: Page) {
await page.click('[data-testid="install-app-button-personal"]');
await page.waitForURL(`apps/installation/event-types?slug=${app}`);
// eslint-disable-next-line playwright/no-wait-for-timeout
await page.waitForTimeout(1000);
// Wait for event type checkboxes to be visible instead of fixed 1s wait
await page.locator('[data-testid^="select-event-type-"]').first().waitFor({ state: "visible" });
for (const id of eventTypeIds) {
await page.click(`[data-testid="select-event-type-${id}"]`);
}
@@ -81,8 +81,8 @@ export function createAppsFixture(page: Page) {
await page.getByTestId("install-app-button").click();
await page.waitForURL(`apps/installation/event-types?slug=${app.slug}`);
// eslint-disable-next-line playwright/no-wait-for-timeout
await page.waitForTimeout(1000);
// Wait for event type checkboxes to be visible instead of fixed 1s wait
await page.locator('[data-testid^="select-event-type-"]').first().waitFor({ state: "visible" });
for (const id of eventTypeIds) {
await page.click(`[data-testid="select-event-type-${id}"]`);
}
+3 -6
View File
@@ -99,13 +99,10 @@ const addLocationIntegrationToFirstEvent = async function ({ user }: { user: { u
};
async function bookEvent(page: Page, calLink: string) {
// Let current month dates fully render.
// There is a bug where if we don't let current month fully render and quickly click go to next month, current month gets rendered
// This doesn't seem to be replicable with the speed of a person, only during automation.
// It would also allow correct snapshot to be taken for current month.
// eslint-disable-next-line playwright/no-wait-for-timeout
await page.waitForTimeout(1000);
// Navigate to the booking page and wait for calendar to be ready
await page.goto(`/${calLink}`);
// Wait for the calendar day elements to be visible instead of fixed 1s wait
await page.locator('[data-testid="day"][data-disabled="false"]').first().waitFor({ state: "visible" });
await page.locator('[data-testid="day"][data-disabled="false"]').nth(0).click();
page.locator('[data-testid="time"]').nth(0).click();
+34 -12
View File
@@ -109,19 +109,30 @@ export async function selectFirstAvailableTimeSlotNextMonth(page: Page | Frame)
// Let current month dates fully render.
await page.getByTestId("incrementMonth").click();
// Waiting for full month increment
await page.locator('[data-testid="day"][data-disabled="false"]').nth(0).click();
// Wait for the calendar to update after month increment before clicking
const availableDay = page.locator('[data-testid="day"][data-disabled="false"]').nth(0);
await availableDay.waitFor({ state: "visible" });
await availableDay.click();
await page.locator('[data-testid="time"]').nth(0).click();
// Wait for time slots to load after selecting a day
const timeSlot = page.locator('[data-testid="time"]').nth(0);
await timeSlot.waitFor({ state: "visible" });
await timeSlot.click();
}
export async function selectSecondAvailableTimeSlotNextMonth(page: Page) {
// Let current month dates fully render.
await page.getByTestId("incrementMonth").click();
await page.locator('[data-testid="day"][data-disabled="false"]').nth(1).click();
// Wait for the calendar to update after month increment before clicking
const availableDay = page.locator('[data-testid="day"][data-disabled="false"]').nth(1);
await availableDay.waitFor({ state: "visible" });
await availableDay.click();
await page.locator('[data-testid="time"]').nth(0).click();
// Wait for time slots to load after selecting a day
const timeSlot = page.locator('[data-testid="time"]').nth(0);
await timeSlot.waitFor({ state: "visible" });
await timeSlot.click();
}
export async function bookEventOnThisPage(page: Page) {
@@ -253,8 +264,9 @@ export async function gotoRoutingLink({
await page.goto(`${previewLink}${queryString ? `?${queryString}` : ""}`);
// HACK: There seems to be some issue with the inputs to the form getting reset if we don't wait.
await new Promise((resolve) => setTimeout(resolve, 2000));
// Wait for the form to be fully loaded and stable before interacting
// This replaces the previous hardcoded 2s wait with a proper Playwright wait
await page.waitForLoadState("networkidle");
}
export async function installAppleCalendar(page: Page) {
@@ -274,21 +286,31 @@ export async function getInviteLink(page: Page) {
export async function getEmailsReceivedByUser({
emails,
userEmail,
waitForEmailMs = 5000,
maxRetries = 10,
retryIntervalMs = 500,
}: {
emails?: ReturnType<typeof createEmailsFixture>;
userEmail: string;
waitForEmailMs?: number;
maxRetries?: number;
retryIntervalMs?: number;
}): Promise<Messages | null> {
if (!emails) return null;
// Wait for email to be sent/received
await new Promise((resolve) => setTimeout(resolve, waitForEmailMs));
// Use retry logic instead of a fixed wait to handle email delivery timing
for (let attempt = 0; attempt < maxRetries; attempt++) {
const matchingEmails = await emails.search(userEmail, "to");
if (matchingEmails?.total) {
return matchingEmails;
}
// Wait before retrying
await new Promise((resolve) => setTimeout(resolve, retryIntervalMs));
}
// Final attempt after all retries
const matchingEmails = await emails.search(userEmail, "to");
if (!matchingEmails?.total) {
console.log(
`No emails received by ${userEmail}. All emails sent to:`,
`No emails received by ${userEmail} after ${maxRetries} retries. All emails sent to:`,
(await emails.messages())?.items.map((e) => e.to)
);
}
+3 -2
View File
@@ -72,10 +72,11 @@ test.describe("Login and logout tests", () => {
test("Should warn when password is incorrect", async ({ page, users }) => {
const alertMessage = (await localize("en"))("incorrect_email_password");
// by default password===username with the users fixture
const pro = await users.create({ username: "pro" });
// Use unique username to avoid collisions with parallel tests
const user = await users.create();
// login with a wrong password
await login({ username: pro.username, password: "wrong" }, page);
await login({ username: user.username, password: "wrong" }, page);
// assert for the visibility of the localized alert message
await expect(page.locator(`text=${alertMessage}`)).toBeVisible();
@@ -582,7 +582,8 @@ async function expectUserToBeAMemberOfTeam({
// Check newly invited member is not pending anymore
await page.goto(`/settings/teams/${teamId}/members`);
await page.waitForLoadState("domcontentloaded");
await page.waitForTimeout(1000); // Add a small delay to ensure UI is fully loaded
// Wait for the member list to be ready instead of fixed 1s wait
await page.locator(`[data-testid="member-${username}"]`).waitFor({ state: "visible", timeout: 10000 });
expect(
(
await page
@@ -612,7 +613,8 @@ async function copyInviteLink(page: Page, teamPage?: boolean) {
if (teamIdMatch && teamIdMatch[1]) {
await page.goto(`/settings/teams/${teamIdMatch[1]}/members`);
await page.waitForLoadState("domcontentloaded");
await page.waitForTimeout(500); // Add a small delay to ensure UI is fully loaded
// Wait for the new member button to be visible instead of fixed 500ms wait
await page.getByTestId("new-member-button").waitFor({ state: "visible" });
}
await page.getByTestId("new-member-button").click();
} else {
@@ -128,7 +128,8 @@ test.describe("Organization - Privacy", () => {
await page.goto(`/settings/teams/${teamId}/members`);
await page.waitForLoadState("domcontentloaded");
await page.waitForTimeout(500); // Add a small delay to ensure UI is fully loaded
// Wait for the page content to be ready instead of fixed 500ms wait
await page.waitForLoadState("networkidle");
// As a user we can not see the member list when a team is private
const hiddenTableLocator = await page.getByTestId("team-member-list-container");
@@ -187,7 +188,8 @@ test.describe("Organization - Privacy", () => {
// 1) All team members can see members in team
await page.goto(`/settings/teams/${teamId}/members`);
await page.waitForLoadState("domcontentloaded");
await page.waitForTimeout(500);
// Wait for the member list to be ready instead of fixed 500ms wait
await page.getByTestId("team-member-list-container").waitFor({ state: "visible", timeout: 5000 }).catch(() => {});
const memberTableLocator = await page.getByTestId("team-member-list-container");
await expect(memberTableLocator).toBeVisible();
@@ -201,7 +203,8 @@ test.describe("Organization - Privacy", () => {
// 3) Admin/Owner can see members in team
await page.goto(`/settings/teams/${teamId}/members`);
await page.waitForLoadState("domcontentloaded");
await page.waitForTimeout(500);
// Wait for the member list to be ready instead of fixed 500ms wait
await page.getByTestId("team-member-list-container").waitFor({ state: "visible", timeout: 5000 }).catch(() => {});
const adminTableLocator = await page.getByTestId("team-member-list-container");
await expect(adminTableLocator).toBeVisible();
+14 -13
View File
@@ -182,7 +182,7 @@ test.describe("Email Signup Flow Test", async () => {
// Verify that the username is the same as the one provided and isn't accidentally changed to email derived username - That happens only for organization member signup
expect(dbUser?.username).toBe(userToCreate.username);
});
test("Signup fields prefilled with query params", async ({ page, users }) => {
test("Signup fields prefilled with query params", async ({ page }) => {
const signupUrlWithParams = "/signup?username=rick-jones&email=rick-jones%40example.com";
await page.goto(signupUrlWithParams);
await preventFlakyTest(page);
@@ -254,19 +254,18 @@ test.describe("Email Signup Flow Test", async () => {
await prisma.verificationToken.deleteMany({ where: { identifier: createdtoken.identifier } });
await prisma.team.deleteMany({ where: { id: createdtoken.teamId! } });
});
test("Email verification sent if enabled", async ({ page, prisma, emails, users, features }) => {
test("Email verification sent if enabled", async ({ page, emails, users, features }) => {
const EmailVerifyFlag = features.get("email-verification")?.enabled;
// eslint-disable-next-line playwright/no-skipped-test
test.skip(!EmailVerifyFlag || !IS_MAILHOG_ENABLED, "Skipping check - Email verify disabled");
// Ensure email verification before testing (TODO: this could break other tests but we can fix that later)
await prisma.feature.update({
where: { slug: "email-verification" },
data: { enabled: true },
});
// Use the features fixture to set the flag - it will be reset in beforeEach via features.reset()
await features.set("email-verification", true);
// Use unique username with timestamp to avoid collisions with parallel tests
const uniqueUsername = `email-verify-${Date.now()}`;
const userToCreate = users.buildForSignup({
email: users.trackEmail({ username: "email-verify", domain: "example.com" }),
username: "email-verify",
email: users.trackEmail({ username: uniqueUsername, domain: "example.com" }),
username: uniqueUsername,
password: "Password99!",
});
@@ -354,16 +353,18 @@ test.describe("Email Signup Flow Test", async () => {
});
});
test("Checkbox for cookie consent does not need to be checked", async ({ page, users }) => {
test("Checkbox for cookie consent does not need to be checked", async ({ page }) => {
await page.goto("/signup");
await preventFlakyTest(page);
// Navigate to email form
await page.getByTestId("continue-with-email-button").click();
// Fill form
await page.locator('input[name="username"]').fill("pro");
await page.locator('input[name="email"]').fill("pro@example.com");
// Fill form with unique username to avoid collisions with parallel tests
// Note: This test doesn't submit the form, so we don't need to create a real user
const uniqueUsername = `cookie-test-${Date.now()}`;
await page.locator('input[name="username"]').fill(uniqueUsername);
await page.locator('input[name="email"]').fill(`${uniqueUsername}@example.com`);
await page.locator('input[name="password"]').fill("Password99!");
const submitButton = page.getByTestId("signup-submit-button");
@@ -30,7 +30,8 @@ test.describe("Team", () => {
});
await page.goto(`/settings/teams/${team.id}/members`);
await page.waitForLoadState("domcontentloaded");
await page.waitForTimeout(500); // Add a small delay to ensure UI is fully loaded
// Wait for the new member button to be visible and clickable instead of fixed 500ms wait
await page.getByTestId("new-member-button").waitFor({ state: "visible" });
await page.getByTestId("new-member-button").click();
await page.locator('input[name="inviteUser"]').fill(invitedUserEmail);
await page.getByText(t("send_invite")).click();
@@ -82,14 +83,16 @@ test.describe("Team", () => {
await page.goto(`/settings/teams/${team.id}/members`);
await page.waitForLoadState("domcontentloaded");
await page.waitForTimeout(500); // Add a small delay to ensure UI is fully loaded
// Wait for the new member button to be visible and clickable instead of fixed 500ms wait
await page.getByTestId("new-member-button").waitFor({ state: "visible" });
await page.getByTestId("new-member-button").click();
const inviteLink = await getInviteLink(page);
const context = await browser.newContext();
const inviteLinkPage = await context.newPage();
await inviteLinkPage.goto(inviteLink);
await inviteLinkPage.waitForTimeout(3000);
// Wait for the form to be fully loaded instead of fixed 3s wait
await inviteLinkPage.locator("button[type=submit]").waitFor({ state: "visible" });
await inviteLinkPage.locator("button[type=submit]").click();
await expect(inviteLinkPage.locator('[data-testid="field-error"]')).toHaveCount(2);
@@ -116,7 +119,8 @@ test.describe("Team", () => {
});
await page.goto(`/settings/teams/${team.id}/members`);
await page.waitForLoadState("domcontentloaded");
await page.waitForTimeout(500); // Add a small delay to ensure UI is fully loaded
// Wait for the new member button to be visible and clickable instead of fixed 500ms wait
await page.getByTestId("new-member-button").waitFor({ state: "visible" });
await page.getByTestId("new-member-button").click();
await page.locator('input[name="inviteUser"]').fill(invitedUserEmail);
await page.getByText(t("send_invite")).click();
@@ -164,7 +168,8 @@ test.describe("Team", () => {
await page.goto(`/settings/teams/${team.id}/settings`);
await page.goto(`/settings/teams/${team.id}/members`);
await page.waitForLoadState("domcontentloaded");
await page.waitForTimeout(500); // Add a small delay to ensure UI is fully loaded
// Wait for the new member button to be visible and clickable instead of fixed 500ms wait
await page.getByTestId("new-member-button").waitFor({ state: "visible" });
await page.getByTestId("new-member-button").click();
await page.locator('input[name="inviteUser"]').fill(invitedMember.email);
await page.getByText(t("send_invite")).click();
@@ -208,7 +213,8 @@ test.describe("Team", () => {
await test.step("Send invitation to existing user", async () => {
await page.waitForLoadState("domcontentloaded");
await page.waitForTimeout(500);
// Wait for the new member button to be visible instead of fixed 500ms wait
await page.getByTestId("new-member-button").waitFor({ state: "visible" });
await page.getByTestId("new-member-button").click();
await page.locator('input[name="inviteUser"]').fill(invitedUser.email);
await page.getByText(t("send_invite")).click();
@@ -272,7 +278,8 @@ test.describe("Team", () => {
await test.step("Send invitation to specific user", async () => {
await page.waitForLoadState("domcontentloaded");
await page.waitForTimeout(500);
// Wait for the new member button to be visible instead of fixed 500ms wait
await page.getByTestId("new-member-button").waitFor({ state: "visible" });
await page.getByTestId("new-member-button").click();
await page.locator('input[name="inviteUser"]').fill(invitedUser.email);
await page.getByText(t("send_invite")).click();