fix: support underscores in workflow template variables with backward compatibility (#27571)

## What does this PR do?

Fixes an issue where underscores in form field identifiers were being stripped when converting to workflow template variables. According to the documentation, users should be able to use variables like `{COMPANY_NAME}` or `{MONTHLY_INFRASTRUCTURE_SPEND}`, but the regex in `formatIdentifierToVariable` was removing underscores, causing these variables to not match.

**The problem:**
- Form field identifier: `Company_Name` or `Monthly_Infrastructure_Spend`
- Expected variable: `{COMPANY_NAME}` or `{MONTHLY_INFRASTRUCTURE_SPEND}`
- Actual variable (before fix): `{COMPANYNAME}` or `{MONTHLYINFRASTRUCTURESPEND}`

**The fix:**
- Updated the regex from `[^a-zA-Z0-9 ]` to `[^a-zA-Z0-9_ ]` to preserve underscores in `formatIdentifierToVariable`
- Added a private `formatIdentifierToVariableLegacy` function that maintains the old behavior (strips underscores)
- Added `getVariableFormats` helper that returns both current and legacy formats for backward compatibility
- Added `convertResponsesToVariableFormats` helper in `executeAIPhoneCall.ts` to convert form responses to both variable formats
- Updated `customTemplate.ts` matching logic to support both variable formats
- Updated `executeAIPhoneCall.ts` to include both variable formats when building dynamic variables

This ensures existing templates using `{COMPANYNAME}` continue to work while new templates can use the documented `{COMPANY_NAME}` format.

## Mandatory Tasks (DO NOT REMOVE)

- [x] I have self-reviewed the code (A decent size PR without self-review might be rejected).
- [x] I have updated the developer docs in /docs if this PR makes changes that would require a documentation change. N/A - no docs changes needed.
- [x] I confirm automated tests are in place that prove my fix is effective or that my feature works.

## How should this be tested?

1. Create a routing form with a field identifier containing underscores (e.g., `Company_Name`)
2. Create a workflow with a template using `{COMPANY_NAME}` (with underscore)
3. Submit the form and verify the variable is replaced correctly
4. Also test with `{COMPANYNAME}` (without underscore) to verify backward compatibility

**Automated tests added:**
- `customTemplate.test.ts` - 15 tests covering `formatIdentifierToVariable`, `getVariableFormats`, and template variable replacement
- `executeAIPhoneCall.test.ts` - 6 tests covering `convertResponsesToVariableFormats` function for form responses

## Human Review Checklist

- [ ] Verify the regex change `[^a-zA-Z0-9_ ]` correctly preserves underscores
- [ ] Verify backward compatibility: both `{COMPANY_NAME}` and `{COMPANYNAME}` should work for the same form field
- [ ] Verify `convertResponsesToVariableFormats` correctly generates both variable formats
- [ ] Review test coverage for edge cases (empty responses, undefined values)

---

Link to Devin run: https://app.devin.ai/sessions/9d5d90178714493086d94691865c3e07
Requested by: @hariombalhara
<!-- devin-review-badge-begin -->

---

<a href="https://app.devin.ai/review/calcom/cal.com/pull/27571">
  <picture>
    <source media="(prefers-color-scheme: dark)" srcset="https://static.devin.ai/assets/gh-open-in-devin-review-dark.svg?v=1">
    <img src="https://static.devin.ai/assets/gh-open-in-devin-review-light.svg?v=1" alt="Open with Devin">
  </picture>
</a>
<!-- devin-review-badge-end -->

packages/features/ee/workflows/lib/reminders/templates/customTemplate.test.ts

@@ -0,0 +1,151 @@
+import { describe, expect, it } from "vitest";
+import customTemplate, {
+  formatIdentifierToVariable,
+  getVariableFormats,
+  transformBookingResponsesToVariableFormat,
+} from "./customTemplate";
+
+describe("formatIdentifierToVariable", () => {
+  it("should convert spaces to underscores and uppercase", () => {
+    expect(formatIdentifierToVariable("Company Name")).toBe("COMPANY_NAME");
+  });
+
+  it("should preserve existing underscores", () => {
+    expect(formatIdentifierToVariable("Company_Name")).toBe("COMPANY_NAME");
+  });
+
+  it("should handle multiple underscores", () => {
+    expect(formatIdentifierToVariable("Monthly_Infrastructure_Spend")).toBe("MONTHLY_INFRASTRUCTURE_SPEND");
+  });
+
+  it("should remove special characters except underscores", () => {
+    expect(formatIdentifierToVariable("Company-Name!")).toBe("COMPANYNAME");
+  });
+
+  it("should handle mixed spaces and underscores", () => {
+    expect(formatIdentifierToVariable("Company_Name Here")).toBe("COMPANY_NAME_HERE");
+  });
+
+  it("should trim whitespace", () => {
+    expect(formatIdentifierToVariable("  Company Name  ")).toBe("COMPANY_NAME");
+  });
+  it("should preserve numbers", () => {
+    expect(formatIdentifierToVariable("Company123_Name")).toBe("COMPANY123_NAME");
+  });
+});
+
+describe("getVariableFormats", () => {
+  it("should return both formats when identifier has underscores", () => {
+    const formats = getVariableFormats("Company_Name");
+    expect(formats).toContain("COMPANY_NAME");
+    expect(formats).toContain("COMPANYNAME");
+    expect(formats).toHaveLength(2);
+  });
+
+  it("should return single format when no underscores in identifier", () => {
+    const formats = getVariableFormats("Company Name");
+    expect(formats).toEqual(["COMPANY_NAME"]);
+  });
+
+  it("should return single format when identifier has no special chars", () => {
+    const formats = getVariableFormats("CompanyName");
+    expect(formats).toEqual(["COMPANYNAME"]);
+  });
+
+  it("should handle multiple underscores", () => {
+    const formats = getVariableFormats("Monthly_Infrastructure_Spend");
+    expect(formats).toContain("MONTHLY_INFRASTRUCTURE_SPEND");
+    expect(formats).toContain("MONTHLYINFRASTRUCTURESPEND");
+    expect(formats).toHaveLength(2);
+  });
+});
+
+describe("customTemplate - variable replacement", () => {
+  const baseVariables = {
+    eventName: "Test Event",
+    organizerName: "Test Organizer",
+    attendeeName: "Test Attendee",
+    timeZone: "UTC",
+  };
+
+  it("should replace variables with underscores in template", () => {
+    const responses = transformBookingResponsesToVariableFormat({
+      Company_Name: { value: "Acme Corp", label: "Company Name" },
+    });
+
+    const result = customTemplate(
+      "Company: {COMPANY_NAME}",
+      { ...baseVariables, responses },
+      "en",
+      undefined,
+      true
+    );
+
+    expect(result.text).toBe("Company: Acme Corp");
+  });
+
+  it("should replace legacy variables without underscores in template (backward compatibility)", () => {
+    const responses = transformBookingResponsesToVariableFormat({
+      Company_Name: { value: "Acme Corp", label: "Company Name" },
+    });
+
+    const result = customTemplate(
+      "Company: {COMPANYNAME}",
+      { ...baseVariables, responses },
+      "en",
+      undefined,
+      true
+    );
+
+    expect(result.text).toBe("Company: Acme Corp");
+  });
+
+  it("should handle both variable formats in the same template", () => {
+    const responses = transformBookingResponsesToVariableFormat({
+      Company_Name: { value: "Acme Corp", label: "Company Name" },
+      Monthly_Spend: { value: "$5000", label: "Monthly Spend" },
+    });
+
+    const result = customTemplate(
+      "Company: {COMPANY_NAME}, Spend: {MONTHLYSPEND}",
+      { ...baseVariables, responses },
+      "en",
+      undefined,
+      true
+    );
+
+    expect(result.text).toBe("Company: Acme Corp, Spend: $5000");
+  });
+
+  it("should handle identifiers with spaces (converted to underscores)", () => {
+    const responses = transformBookingResponsesToVariableFormat({
+      "Company Name": { value: "Acme Corp", label: "Company Name" },
+    });
+
+    const result = customTemplate(
+      "Company: {COMPANY_NAME}",
+      { ...baseVariables, responses },
+      "en",
+      undefined,
+      true
+    );
+
+    expect(result.text).toBe("Company: Acme Corp");
+  });
+
+  it("should handle array values", () => {
+    const responses = transformBookingResponsesToVariableFormat({
+      Selected_Options: { value: ["Option A", "Option B", "Option C"], label: "Selected Options" },
+    });
+
+    const result = customTemplate(
+      "Options: {SELECTED_OPTIONS}",
+      { ...baseVariables, responses },
+      "en",
+      undefined,
+      true
+    );
+
+    expect(result.text).toBe("Options: Option A, Option B, Option C");
+  });
+});

packages/features/ee/workflows/lib/reminders/templates/customTemplate.ts

@@ -56,6 +56,19 @@ export function transformRoutingFormResponsesToVariableFormat(
 }
 
 export function formatIdentifierToVariable(key: string): string {
+  return key
+    .replace(/[^a-zA-Z0-9_ ]/g, "")
+    .trim()
+    .replaceAll(" ", "_")
+    .toUpperCase();
+}
+
+/**
+ * Legacy version of formatIdentifierToVariable that strips underscores.
+ * Used for backward compatibility with templates that were created when
+ * underscores were being stripped from identifiers.
+ */
+function formatIdentifierToVariableLegacy(key: string): string {
   return key
     .replace(/[^a-zA-Z0-9 ]/g, "")
     .trim()
@@ -63,6 +76,22 @@ export function formatIdentifierToVariable(key: string): string {
     .toUpperCase();
 }
 
+/**
+ * Returns all variable formats for a given key.
+ * Includes both the current format (with underscores preserved) and the legacy format
+ * (without underscores) for backward compatibility with existing templates.
+ * @returns Array of unique variable formats
+ */
+export function getVariableFormats(key: string): string[] {
+  const current = formatIdentifierToVariable(key);
+  const legacy = formatIdentifierToVariableLegacy(key);
+
+  if (current === legacy) {
+    return [current];
+  }
+  return [current, legacy];
+}
+
 export type VariablesType = {
   eventName?: string;
   organizerName?: string;
@@ -196,9 +225,13 @@ const customTemplate = (
     // handle custom variables from form/booking responses
     if (variables.responses) {
       Object.keys(variables.responses).forEach((customInput) => {
-        const formatedToVariable = formatIdentifierToVariable(customInput);
+        const foundVariableInTemplate = variable;
+        const availableVariable = formatIdentifierToVariable(customInput);
+        // Legacy format for backward compatibility with templates created before underscore support
+        const availableVariableLegacyFormat = formatIdentifierToVariableLegacy(customInput);
+        const isFoundTemplateVariableValid = foundVariableInTemplate === availableVariable || foundVariableInTemplate === availableVariableLegacyFormat;
 
-        if (variable === formatedToVariable && variables.responses) {
+        if (isFoundTemplateVariableValid && variables.responses) {
           const response = variables.responses[customInput];
           if (response?.value !== undefined) {
             const responseValue = response.value;
@@ -206,7 +239,7 @@ const customTemplate = (
               ? responseValue.join(", ")
               : String(responseValue);
 
-            dynamicText = dynamicText.replace(`{${variable}}`, valueString);
+            dynamicText = dynamicText.replace(`{${foundVariableInTemplate}}`, valueString);
           }
         }
       });

packages/features/tasker/tasks/executeAIPhoneCall.test.ts

@@ -0,0 +1,85 @@
+import { convertResponsesToVariableFormats } from "@calcom/features/tasker/tasks/executeAIPhoneCall";
+import { describe, expect, it } from "vitest";
+
+describe("executeAIPhoneCall - convertResponsesToVariableFormats", () => {
+  it("should generate both variable formats for identifiers with underscores", () => {
+    const responses = {
+      Company_Name: { value: "Acme Corp" },
+      Monthly_Infrastructure_Spend: { value: "$5000" },
+    };
+
+    const variables = convertResponsesToVariableFormats(responses);
+
+    // Should have both underscore and non-underscore versions
+    expect(variables.COMPANY_NAME).toBe("Acme Corp");
+    expect(variables.COMPANYNAME).toBe("Acme Corp");
+    expect(variables.MONTHLY_INFRASTRUCTURE_SPEND).toBe("$5000");
+    expect(variables.MONTHLYINFRASTRUCTURESPEND).toBe("$5000");
+  });
+
+  it("should generate single format for identifiers without underscores", () => {
+    const responses = {
+      "Company Name": { value: "Acme Corp" },
+      Email: { value: "test@example.com" },
+    };
+
+    const variables = convertResponsesToVariableFormats(responses);
+
+    // Space-separated identifiers convert to underscore format
+    expect(variables.COMPANY_NAME).toBe("Acme Corp");
+    // Simple identifiers stay as-is
+    expect(variables.EMAIL).toBe("test@example.com");
+  });
+
+  it("should handle mixed identifiers correctly", () => {
+    const responses = {
+      Company_Name: { value: "Acme Corp" },
+      "Contact Email": { value: "contact@acme.com" },
+      Notes: { value: "Some notes" },
+    };
+
+    const variables = convertResponsesToVariableFormats(responses);
+
+    // Underscore identifier: both formats
+    expect(variables.COMPANY_NAME).toBe("Acme Corp");
+    expect(variables.COMPANYNAME).toBe("Acme Corp");
+
+    // Space identifier: single format (space becomes underscore)
+    expect(variables.CONTACT_EMAIL).toBe("contact@acme.com");
+
+    // Simple identifier: single format
+    expect(variables.NOTES).toBe("Some notes");
+  });
+
+  it("should preserve the same value for both variable formats", () => {
+    const testValue = "Test Value 123";
+    const responses = {
+      Test_Field: { value: testValue },
+    };
+
+    const variables = convertResponsesToVariableFormats(responses);
+
+    expect(variables.TEST_FIELD).toBe(testValue);
+    expect(variables.TESTFIELD).toBe(testValue);
+    expect(variables.TEST_FIELD).toBe(variables.TESTFIELD);
+  });
+
+  it("should handle empty responses", () => {
+    const responses = {};
+
+    const variables = convertResponsesToVariableFormats(responses);
+
+    expect(Object.keys(variables)).toHaveLength(0);
+  });
+
+  it("should handle undefined values", () => {
+    const responses = {
+      Field_Name: { value: undefined },
+    };
+
+    const variables = convertResponsesToVariableFormats(responses);
+
+    expect(variables.FIELD_NAME).toBe("");
+    expect(variables.FIELDNAME).toBe("");
+  });
+});

packages/features/tasker/tasks/executeAIPhoneCall.ts

@@ -3,7 +3,7 @@ import dayjs from "@calcom/dayjs";
 import { getCalEventResponses } from "@calcom/features/bookings/lib/getCalEventResponses";
 import { createDefaultAIPhoneServiceProvider } from "@calcom/features/calAIPhone";
 import { handleInsufficientCredits } from "@calcom/features/ee/billing/helpers/handleInsufficientCredits";
-import { formatIdentifierToVariable } from "@calcom/features/ee/workflows/lib/reminders/templates/customTemplate";
+import { getVariableFormats } from "@calcom/features/ee/workflows/lib/reminders/templates/customTemplate";
 import { WorkflowReminderRepository } from "@calcom/features/ee/workflows/lib/repository/workflowReminder";
 import { FeaturesRepository } from "@calcom/features/flags/features.repository";
 import {
@@ -14,7 +14,6 @@ import { checkRateLimitAndThrowError } from "@calcom/lib/checkRateLimitAndThrowE
 import logger from "@calcom/lib/logger";
 import prisma from "@calcom/prisma";
 import { CreditUsageType } from "@calcom/prisma/enums";
-
 interface ExecuteAIPhoneCallPayload {
   workflowReminderId: number;
   agentId: string;
@@ -35,6 +34,23 @@ type BookingWithRelations = NonNullable<
   >["booking"]
 >;
 
+/**
+ * Converts responses to variable format entries with both current and legacy formats
+ * for backward compatibility.
+ * Accepts both FORM_SUBMITTED_WEBHOOK_RESPONSES and CalEventResponses types.
+ */
+export function convertResponsesToVariableFormats(
+  responses: Record<string, { value?: unknown }>
+) {
+  return Object.fromEntries(
+    Object.entries(responses).flatMap(([key, value]) => {
+      const formats = getVariableFormats(key);
+      const valueStr = value.value?.toString() || "";
+      return formats.map((format) => [format, valueStr]);
+    })
+  );
+}
+
 function getVariablesFromFormResponse({
   responses,
   eventTypeId,
@@ -52,13 +68,8 @@ function getVariablesFromFormResponse({
     ATTENDEE_EMAIL: submittedEmail || "",
     NUMBER_TO_CALL: numberToCall,
     eventTypeId: eventTypeId?.toString() || "",
-    // Include any custom form responses
-    ...Object.fromEntries(
-      Object.entries(responses || {}).map(([key, value]) => [
-        formatIdentifierToVariable(key),
-        value.value?.toString() || "",
-      ])
-    ),
+    // Include custom form responses with both current and legacy variable formats for backward compatibility
+    ...convertResponsesToVariableFormats(responses || {}),
   };
 }
 
@@ -102,13 +113,8 @@ function getVariablesFromBooking(booking: BookingWithRelations, numberToCall: st
       .format("h:mm A"),
     // DO NOT REMOVE THIS FIELD. It is used for conditional tool routing in prompts
     eventTypeId: booking.eventTypeId?.toString() || "",
-    // Include any custom form responses
-    ...Object.fromEntries(
-      Object.entries(responses || {}).map(([key, value]) => [
-        formatIdentifierToVariable(key),
-        value.value?.toString() || "",
-      ])
-    ),
+    // Include custom form responses with both current and legacy variable formats for backward compatibility
+    ...convertResponsesToVariableFormats(responses || {}),
   };
 }