diff --git a/.github/workflows/performance-tests.yml b/.github/workflows/performance-tests.yml new file mode 100644 index 0000000000..8e21180282 --- /dev/null +++ b/.github/workflows/performance-tests.yml @@ -0,0 +1,65 @@ +name: Performance Tests + +on: + release: + types: [created] + workflow_dispatch: + inputs: + base_url: + description: 'Base URL for performance tests' + required: false + default: 'https://cal.dev' + +jobs: + performance-tests: + runs-on: ubuntu-latest + env: + BASE_URL: ${{ github.event.inputs.base_url || 'https://cal.dev' }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup K6 + uses: grafana/setup-k6-action@v1 + with: + k6-version: '1.0.0' + + - name: Run Smoke Tests + uses: grafana/run-k6-action@v1 + env: + K6_CLOUD_TOKEN: ${{ secrets.K6_CLOUD_TOKEN }} + K6_CLOUD_PROJECT_ID: ${{ secrets.K6_CLOUD_PROJECT_ID }} + with: + path: tests/performance/smoke/*.js + cloud-run-locally: true + fail-fast: true + + - name: Run Load Tests + uses: grafana/run-k6-action@v1 + env: + K6_CLOUD_TOKEN: ${{ secrets.K6_CLOUD_TOKEN }} + K6_CLOUD_PROJECT_ID: ${{ secrets.K6_CLOUD_PROJECT_ID }} + with: + path: tests/performance/load/*.js + cloud-run-locally: true + fail-fast: true + + - name: Run Stress Tests + uses: grafana/run-k6-action@v1 + env: + K6_CLOUD_TOKEN: ${{ secrets.K6_CLOUD_TOKEN }} + K6_CLOUD_PROJECT_ID: ${{ secrets.K6_CLOUD_PROJECT_ID }} + with: + path: tests/performance/stress/*.js + cloud-run-locally: true + fail-fast: true + + - name: Run Spike Tests + uses: grafana/run-k6-action@v1 + env: + K6_CLOUD_TOKEN: ${{ secrets.K6_CLOUD_TOKEN }} + K6_CLOUD_PROJECT_ID: ${{ secrets.K6_CLOUD_PROJECT_ID }} + with: + path: tests/performance/spike/*.js + cloud-run-locally: true + fail-fast: true diff --git a/tests/performance/README.md b/tests/performance/README.md new file mode 100644 index 0000000000..42056b65c5 --- /dev/null +++ b/tests/performance/README.md @@ -0,0 +1,158 @@ +# Cal.com Booking Flow Performance Test Suite + +This directory contains performance tests for Cal.com's booking flow using [Grafana k6](https://k6.io/). The tests are designed to measure the performance of the booking flow under various load conditions, including high-volume scenarios with tens of thousands of requests per minute. + +## Prerequisites + +- [k6](https://k6.io/docs/getting-started/installation/) installed on your machine +- Cal.com running locally or a deployed instance to test against + +## Test Structure + +The test suite is organized into the following directories: + +- `smoke/`: Simple tests with minimal load to verify functionality +- `load/`: Tests that simulate expected normal load (thousands of requests per minute) +- `stress/`: Tests that simulate heavy load to find breaking points (tens of thousands of requests per minute) +- `spike/`: Tests that simulate sudden spikes in traffic (rapid increase to tens of thousands of requests per minute) +- `utils/`: Shared utilities and helper functions + +## Running Tests + +### Setting the Base URL + +By default, tests will run against `http://localhost:3000`. To test against a different environment, set the `BASE_URL` environment variable: + +```bash +BASE_URL=https://your-cal-instance.com k6 run tests/performance/smoke/booking.js +``` + +### Running Smoke Tests + +```bash +# Run booking flow smoke test +k6 run tests/performance/smoke/booking.js +``` + +### Running Load Tests + +```bash +# Run booking flow load test (thousands of requests per minute) +k6 run tests/performance/load/booking.js +``` + +### Running Stress Tests + +```bash +# Run booking flow stress test (tens of thousands of requests per minute) +k6 run tests/performance/stress/booking.js +``` + +### Running Spike Tests + +```bash +# Run booking flow spike test (rapid spike to tens of thousands of requests per minute) +k6 run tests/performance/spike/booking.js +``` + +### Running Tests with Docker Script + +You can also run the performance tests using the provided `run-k6-local.sh` script. +This script automatically detects your operating system (Linux or macOS) and sets up the appropriate Docker flags for running k6 tests. + +#### Prerequisites + +- Docker installed and running +- Cal.com running locally (default: `http://localhost:3000`) + +#### Usage + +From the root of the project, run: + +```bash +./tests/scripts/run-k6-local.sh +``` + +You’ll see an interactive menu: + +``` +Select a test: + 1) smoke + 2) load + 3) stress + 4) spike + 5) all + 0) exit +``` + +Alternatively, you can run directly from the CLI: + +```bash +# Run smoke tests +./tests/scripts/run-k6-local.sh smoke + +# Run all tests +./tests/scripts/run-k6-local.sh all +``` + +#### Customizing + +You can override the following environment variables: + +```bash +BASE_URL=http://localhost:3000 \ +TOKEN=your_token_here \ +./tests/scripts/run-k6-local.sh load +``` + +Supported env vars: +- `BASE_URL` +- `TOKEN` +- `TEST_USER_FREE` +- `TEST_PASSWORD_FREE` +- `TEST_USER_PRO` +- `TEST_PASSWORD_PRO` + + +## Test Scenarios + +The test suite focuses specifically on the booking flow, which is the most critical user journey in Cal.com: + +1. **Booking Page View**: Tests the performance of loading and viewing booking pages + +## Load Parameters + +The tests are configured to handle tens of thousands of requests per minute during peak times: + +- **Load Tests**: Up to 2,000 concurrent virtual users +- **Stress Tests**: Up to 4,000 concurrent virtual users +- **Spike Tests**: Rapid spike to 5,000 concurrent virtual users + +## Thresholds + +Performance thresholds are defined in `utils/config.js` and vary by test type: + +- **Smoke Tests**: Strictest thresholds to catch any performance regressions +- **Load Tests**: Moderate thresholds for normal operating conditions +- **Stress Tests**: More lenient thresholds for heavy load conditions +- **Spike Tests**: Most lenient thresholds for sudden traffic spikes + +## Environment Variables + +The following environment variables can be used to configure the tests: + +- `BASE_URL`: Base URL for the application (default: `http://localhost:3000`) +- `TEST_USER_FREE`: Username for free tier testing (default: `free`) +- `TEST_PASSWORD_FREE`: Password for free tier testing (default: `free`) +- `TEST_USER_PRO`: Username for pro tier testing (default: `pro`) +- `TEST_PASSWORD_PRO`: Password for pro tier testing (default: `pro`) + +## Adding New Tests + +To add a new test: + +1. Create a new JS file in the appropriate directory (smoke, load, stress, or spike) +2. Import necessary helpers from `utils/helpers.js` +3. Define test options including VUs, duration, and thresholds +4. Implement the test scenario using k6 and helper functions + diff --git a/tests/performance/load/booking.js b/tests/performance/load/booking.js new file mode 100644 index 0000000000..cfec940852 --- /dev/null +++ b/tests/performance/load/booking.js @@ -0,0 +1,24 @@ +import { sleep } from "k6"; + +import { THRESHOLDS } from "../utils/config.js"; +import { viewBookingPage } from "../utils/helpers.js"; + +export const options = { + stages: [ + { duration: "2m", target: 500 }, // Ramp up to 500 VUs over 2 minutes + { duration: "3m", target: 1000 }, // Ramp up to 1000 VUs over 3 minutes + { duration: "5m", target: 2000 }, // Ramp up to 2000 VUs over 5 minutes + { duration: "10m", target: 2000 }, // Stay at 2000 VUs for 10 minutes + { duration: "3m", target: 0 }, // Ramp down to 0 VUs over 3 minutes + ], + thresholds: { + http_req_failed: THRESHOLDS.HTTP_ERRORS, + http_req_duration: THRESHOLDS.RESPONSE_TIME.LOAD.p95, + }, +}; + +export default function () { + viewBookingPage("pro", "30min"); + + sleep(0.1); +} diff --git a/tests/performance/smoke/booking.js b/tests/performance/smoke/booking.js new file mode 100644 index 0000000000..0f5e6782e4 --- /dev/null +++ b/tests/performance/smoke/booking.js @@ -0,0 +1,19 @@ +import { sleep } from "k6"; + +import { THRESHOLDS } from "../utils/config.js"; +import { viewBookingPage } from "../utils/helpers.js"; + +export const options = { + vus: 10, + duration: "2m", + thresholds: { + http_req_failed: THRESHOLDS.HTTP_ERRORS, + http_req_duration: THRESHOLDS.RESPONSE_TIME.SMOKE.p95, + }, +}; + +export default function () { + viewBookingPage("pro", "30min"); + + sleep(0.5); +} diff --git a/tests/performance/spike/booking.js b/tests/performance/spike/booking.js new file mode 100644 index 0000000000..1cd57648f1 --- /dev/null +++ b/tests/performance/spike/booking.js @@ -0,0 +1,24 @@ +import { sleep } from "k6"; + +import { THRESHOLDS } from "../utils/config.js"; +import { viewBookingPage } from "../utils/helpers.js"; + +export const options = { + stages: [ + { duration: "1m", target: 500 }, // Baseline load + { duration: "2m", target: 5000 }, // Rapid spike to 5000 VUs + { duration: "5m", target: 5000 }, // Stay at 5000 VUs + { duration: "2m", target: 500 }, // Scale back to baseline + { duration: "1m", target: 0 }, // Scale down to 0 + ], + thresholds: { + http_req_failed: THRESHOLDS.HTTP_ERRORS, + http_req_duration: THRESHOLDS.RESPONSE_TIME.SPIKE.p95, + }, +}; + +export default function () { + viewBookingPage("pro", "30min"); + + sleep(0.1); +} diff --git a/tests/performance/stress/booking.js b/tests/performance/stress/booking.js new file mode 100644 index 0000000000..234f67a92f --- /dev/null +++ b/tests/performance/stress/booking.js @@ -0,0 +1,26 @@ +import { sleep } from "k6"; + +import { THRESHOLDS } from "../utils/config.js"; +import { viewBookingPage } from "../utils/helpers.js"; + +export const options = { + stages: [ + { duration: "2m", target: 1000 }, // Ramp up to 1000 VUs over 2 minutes + { duration: "3m", target: 2000 }, // Ramp up to 2000 VUs over 3 minutes + { duration: "5m", target: 3000 }, // Ramp up to 3000 VUs over 5 minutes + { duration: "10m", target: 3000 }, // Stay at 3000 VUs for 10 minutes + { duration: "5m", target: 4000 }, // Ramp up to 4000 VUs over 5 minutes + { duration: "10m", target: 4000 }, // Stay at 4000 VUs for 10 minutes + { duration: "3m", target: 0 }, // Ramp down to 0 VUs over 3 minutes + ], + thresholds: { + http_req_failed: THRESHOLDS.HTTP_ERRORS, + http_req_duration: THRESHOLDS.RESPONSE_TIME.STRESS.p95, + }, +}; + +export default function () { + viewBookingPage("pro", "30min"); + + sleep(0.05); +} diff --git a/tests/performance/utils/config.js b/tests/performance/utils/config.js new file mode 100644 index 0000000000..8a9ac0c1f5 --- /dev/null +++ b/tests/performance/utils/config.js @@ -0,0 +1,59 @@ +import { sleep } from "k6"; +import { SharedArray } from "k6/data"; + +export const BASE_URL = __ENV.BASE_URL || "http://localhost:3000"; +if (BASE_URL.includes("cal.com")) { + throw new Error("BLOCKED: Do not run against production domain"); +} + +export const SLEEP_DURATION = { + SHORT: 1, + MEDIUM: 3, + LONG: 5, +}; + +export const THRESHOLDS = { + HTTP_ERRORS: ["rate<0.01"], + + RESPONSE_TIME: { + SMOKE: { + p95: ["p(95)<500"], // 95% of requests should be below 500ms + p99: ["p(99)<1000"], // 99% of requests should be below 1000ms + }, + LOAD: { + p95: ["p(95)<1000"], // 95% of requests should be below 1000ms + p99: ["p(99)<2000"], // 99% of requests should be below 2000ms + }, + STRESS: { + p95: ["p(95)<2000"], // 95% of requests should be below 2000ms + p99: ["p(99)<4000"], // 99% of requests should be below 4000ms + }, + SPIKE: { + p95: ["p(95)<3000"], // 95% of requests should be below 3000ms + p99: ["p(99)<6000"], // 99% of requests should be below 6000ms + }, + }, +}; + +export const TEST_USERS = new SharedArray("users", function () { + return [ + { username: __ENV.TEST_USER_FREE || "free", password: __ENV.TEST_PASSWORD_FREE || "free" }, + { username: __ENV.TEST_USER_PRO || "pro", password: __ENV.TEST_PASSWORD_PRO || "pro" }, + ]; +}); + +export function getRandomUser() { + return TEST_USERS[Math.floor(Math.random() * TEST_USERS.length)]; +} + +export function randomQueryParam() { + return `nocache=${Date.now()}`; +} + +export function randomSleep(min = SLEEP_DURATION.SHORT, max = SLEEP_DURATION.MEDIUM) { + if (min > max) { + throw new Error("min cannot be greater than max"); + } + const sleepTime = Math.random() * (max - min) + min; + sleep(sleepTime); +} diff --git a/tests/performance/utils/helpers.js b/tests/performance/utils/helpers.js new file mode 100644 index 0000000000..ba47fa4aad --- /dev/null +++ b/tests/performance/utils/helpers.js @@ -0,0 +1,24 @@ +import { check, group } from "k6"; +import http from "k6/http"; + +import { randomSleep, BASE_URL, randomQueryParam } from "./config.js"; + +export function viewBookingPage(username, eventSlug) { + return group("View Booking Page", () => { + const url = `${BASE_URL}/${username}/${eventSlug}?${randomQueryParam()}`; + + const response = http.get(url, { + tags: { name: "Booking Page" }, + timeout: "30s", + }); + + check(response, { + "Booking page loaded": (r) => r.status === 200, + "Has booking form": (r) => r.body.includes('data-testid="day"') || r.body.includes("booking"), + "Response time acceptable": (r) => r.timings.duration < 5000, + }); + + randomSleep(); + return response; + }); +} diff --git a/tests/scripts/run-k6-local.sh b/tests/scripts/run-k6-local.sh new file mode 100755 index 0000000000..e8aebe338a --- /dev/null +++ b/tests/scripts/run-k6-local.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +set -euo pipefail + +# --- OS detection (macOS vs Linux) for host access from Docker --- +OS="$(uname -s || echo unknown)" +DOCKER_EXTRA=() +case "$OS" in + Linux) DOCKER_EXTRA+=(--add-host=host.docker.internal:host-gateway) ;; # needed on Linux + Darwin) : ;; # nothing extra on macOS + *) echo "Unsupported OS: $OS"; exit 1 ;; +esac + +# --- k6 image (override with K6_IMAGE if needed) --- +K6_IMAGE="${K6_IMAGE:-grafana/k6}" + +# --- Local tests folder to mount into the container --- + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +MOUNT_LOCAL="${MOUNT_LOCAL:-$SCRIPT_DIR/../../tests}" +WORKDIR_IN="/scripts" + +# --- Base URL for the system under test --- +BASE_URL_DEFAULT="${BASE_URL:-http://host.docker.internal:3000}" + +# --- Pass environment variables to k6 (only if present) --- +ENV_ARGS=() +ENV_ARGS+=(-e "BASE_URL=${BASE_URL_DEFAULT}") +for v in TOKEN TEST_USER_FREE TEST_PASSWORD_FREE TEST_USER_PRO TEST_PASSWORD_PRO; do + if [ -n "${!v:-}" ]; then ENV_ARGS+=(-e "$v=${!v}"); fi +done + +# --- Basic sanity checks --- +if [ ! -d "$MOUNT_LOCAL" ]; then + echo "Tests folder not found: $MOUNT_LOCAL" + exit 1 +fi + +# --- Run a single tier (smoke|load|stress|spike) --- +run_test () { + local tier="$1" + local file="$WORKDIR_IN/performance/$tier/booking.js" + echo "==> Running $tier ($file) against BASE_URL=${BASE_URL_DEFAULT}" + docker run -i --rm \ + ${DOCKER_EXTRA+"${DOCKER_EXTRA[@]}"} \ + -v "$MOUNT_LOCAL":"$WORKDIR_IN" -w "$WORKDIR_IN" \ + "${ENV_ARGS[@]}" \ + "$K6_IMAGE" run "$file" +} + +# --- Simple interactive menu --- +menu () { + echo "Select a test:" + echo " 1) smoke" + echo " 2) load" + echo " 3) stress" + echo " 4) spike" + echo " 5) all" + echo " 0) exit" + read -rp "> " opt + case "$opt" in + 1) run_test smoke ;; + 2) run_test load ;; + 3) run_test stress ;; + 4) run_test spike ;; + 5) run_test smoke; run_test load; run_test stress; run_test spike ;; + 0) exit 0 ;; + *) echo "invalid option"; exit 1 ;; + esac +} + +# --- CLI: ./run-k6-local.sh [smoke|load|stress|spike|all] --- +cmd="${1:-menu}" +case "$cmd" in + smoke|load|stress|spike) run_test "$cmd" ;; + all) run_test smoke; run_test load; run_test stress; run_test spike ;; + menu) menu ;; + *) echo "Usage: $0 [smoke|load|stress|spike|all]"; exit 1 ;; +esac