feat: generate transcription from recording and API endpoint (#15358)

* feat: generate transcription from recording

* fix: api

* chore: undo

* tests: add unit tests

* chore: refactor move zod type

* fix: add more security check

* chore: update test

* chore: type err

* chore: test

* chore: fix tyoe

* fix: build err

* chore: add try .. catch

* chore: update desc

---------

Co-authored-by: Joe Au-Yeung <65426560+joeauyeung@users.noreply.github.com>

apps/api/v1/lib/validations/shared/queryIdTransformParseInt.ts

@@ -14,3 +14,7 @@ export const withValidQueryIdTransformParseInt = withValidation({
   type: "Zod",
   mode: "query",
 });
+
+export const getTranscriptFromRecordingId = schemaQueryIdParseInt.extend({
+  recordingId: z.string(),
+});

apps/api/v1/pages/api/bookings/[id]/transcripts/[recordingId]/_get.ts

@@ -0,0 +1,94 @@
+import type { NextApiRequest } from "next";
+
+import {
+  getTranscriptsAccessLinkFromRecordingId,
+  checkIfRoomNameMatchesInRecording,
+} from "@calcom/core/videoClient";
+import { HttpError } from "@calcom/lib/http-error";
+import { defaultResponder } from "@calcom/lib/server";
+import prisma from "@calcom/prisma";
+import type { PartialReference } from "@calcom/types/EventManager";
+
+import { getTranscriptFromRecordingId } from "~/lib/validations/shared/queryIdTransformParseInt";
+
+/**
+ * @swagger
+ * /bookings/{id}/transcripts/{recordingId}:
+ *   get:
+ *     summary: Find all Cal video transcripts of that recording
+ *     operationId: getTranscriptsByRecordingId
+ *     parameters:
+ *       - in: path
+ *         name: id
+ *         schema:
+ *           type: integer
+ *         required: true
+ *         description: ID of the booking for which transcripts need to be fetched.
+ *       - in: path
+ *         name: recordingId
+ *         schema:
+ *           type: string
+ *         required: true
+ *         description: ID of the recording(daily.co recording id) for which transcripts need to be fetched.
+ *       - in: query
+ *         name: apiKey
+ *         required: true
+ *         schema:
+ *           type: string
+ *         description: Your API key
+ *     tags:
+ *     - bookings
+ *     responses:
+ *       200:
+ *         description: OK
+ *         content:
+ *           application/json:
+ *       401:
+ *         description: Authorization information is missing or invalid.
+ *       404:
+ *         description: Booking was not found
+ */
+
+export async function getHandler(req: NextApiRequest) {
+  const { query } = req;
+  const { id, recordingId } = getTranscriptFromRecordingId.parse(query);
+
+  await checkIfRecordingBelongsToBooking(id, recordingId);
+
+  const transcriptsAccessLinks = await getTranscriptsAccessLinkFromRecordingId(recordingId);
+
+  return transcriptsAccessLinks;
+}
+
+const checkIfRecordingBelongsToBooking = async (bookingId: number, recordingId: string) => {
+  const booking = await prisma.booking.findUnique({
+    where: { id: bookingId },
+    include: { references: true },
+  });
+
+  if (!booking)
+    throw new HttpError({
+      statusCode: 404,
+      message: `No Booking found with booking id ${bookingId}`,
+    });
+
+  const roomName =
+    booking?.references?.find((reference: PartialReference) => reference.type === "daily_video")?.meetingId ??
+    undefined;
+
+  if (!roomName)
+    throw new HttpError({
+      statusCode: 404,
+      message: `No Booking Reference with Daily Video found with booking id ${bookingId}`,
+    });
+
+  const canUserAccessRecordingId = await checkIfRoomNameMatchesInRecording(roomName, recordingId);
+  if (!canUserAccessRecordingId) {
+    throw new HttpError({
+      statusCode: 403,
+      message: `This Recording Id ${recordingId} does not belong to booking ${bookingId}`,
+    });
+  }
+};
+
+export default defaultResponder(getHandler);

apps/api/v1/pages/api/bookings/[id]/transcripts/[recordingId]/index.ts

@@ -0,0 +1,16 @@
+import type { NextApiRequest, NextApiResponse } from "next";
+
+import { defaultHandler, defaultResponder } from "@calcom/lib/server";
+
+import { withMiddleware } from "~/lib/helpers/withMiddleware";
+
+import authMiddleware from "../../_auth-middleware";
+
+export default withMiddleware()(
+  defaultResponder(async (req: NextApiRequest, res: NextApiResponse) => {
+    await authMiddleware(req);
+    return defaultHandler({
+      GET: import("./_get"),
+    })(req, res);
+  })
+);

apps/api/v1/test/lib/bookings/[id]/transcripts/[recordingId]/_get.test.ts

@@ -0,0 +1,129 @@
+import prismaMock from "../../../../../../../../../tests/libs/__mocks__/prismaMock";
+
+import type { Request, Response } from "express";
+import type { NextApiRequest, NextApiResponse } from "next";
+import { createMocks } from "node-mocks-http";
+import { describe, expect, test, vi, afterEach } from "vitest";
+
+import {
+  getTranscriptsAccessLinkFromRecordingId,
+  checkIfRoomNameMatchesInRecording,
+} from "@calcom/core/videoClient";
+import { buildBooking } from "@calcom/lib/test/builder";
+
+import { getAccessibleUsers } from "~/lib/utils/retrieveScopedAccessibleUsers";
+
+import authMiddleware from "../../../../../../pages/api/bookings/[id]/_auth-middleware";
+import handler from "../../../../../../pages/api/bookings/[id]/transcripts/[recordingId]/_get";
+
+type CustomNextApiRequest = NextApiRequest & Request;
+type CustomNextApiResponse = NextApiResponse & Response;
+
+vi.mock("@calcom/core/videoClient", () => {
+  return {
+    getTranscriptsAccessLinkFromRecordingId: vi.fn(),
+    checkIfRoomNameMatchesInRecording: vi.fn(),
+  };
+});
+
+vi.mock("~/lib/utils/retrieveScopedAccessibleUsers", () => {
+  return {
+    getAccessibleUsers: vi.fn(),
+  };
+});
+
+afterEach(() => {
+  vi.resetAllMocks();
+});
+
+const mockGetTranscripts = () => {
+  const downloadLinks = [{ format: "json", link: "https://URL1" }];
+
+  vi.mocked(getTranscriptsAccessLinkFromRecordingId).mockResolvedValue(downloadLinks);
+  vi.mocked(checkIfRoomNameMatchesInRecording).mockResolvedValue(true);
+
+  return downloadLinks;
+};
+
+const recordingId = "abc-xyz";
+
+describe("GET /api/bookings/[id]/transcripts/[recordingId]", () => {
+  test("Returns transcripts if user is system-wide admin", async () => {
+    const adminUserId = 1;
+    const userId = 2;
+
+    const bookingId = 1111;
+
+    prismaMock.booking.findUnique.mockResolvedValue(
+      buildBooking({
+        id: bookingId,
+        userId,
+        references: [
+          {
+            id: 1,
+            type: "daily_video",
+            uid: "17OHkCH53pBa03FhxMbw",
+            meetingId: "17OHkCH53pBa03FhxMbw",
+            meetingPassword: "password",
+            meetingUrl: "https://URL",
+          },
+        ],
+      })
+    );
+
+    const mockedTranscripts = mockGetTranscripts();
+    const { req, res } = createMocks<CustomNextApiRequest, CustomNextApiResponse>({
+      method: "GET",
+      body: {},
+      query: {
+        id: bookingId,
+        recordingId,
+      },
+    });
+
+    req.isSystemWideAdmin = true;
+    req.userId = adminUserId;
+
+    await authMiddleware(req);
+    await handler(req, res);
+
+    expect(res.statusCode).toBe(200);
+    expect(JSON.parse(res._getData())).toEqual(mockedTranscripts);
+  });
+
+  test("Allows GET transcripts when user is org-wide admin", async () => {
+    const adminUserId = 1;
+    const memberUserId = 10;
+    const bookingId = 3333;
+
+    prismaMock.booking.findUnique.mockResolvedValue(
+      buildBooking({
+        id: bookingId,
+        userId: memberUserId,
+        references: [
+          { id: 1, type: "daily_video", uid: "17OHkCH53pBa03FhxMbw", meetingId: "17OHkCH53pBa03FhxMbw" },
+        ],
+      })
+    );
+
+    const { req, res } = createMocks<CustomNextApiRequest, CustomNextApiResponse>({
+      method: "GET",
+      body: {},
+      query: {
+        id: bookingId,
+        recordingId,
+      },
+    });
+
+    req.userId = adminUserId;
+    req.isOrganizationOwnerOrAdmin = true;
+    mockGetTranscripts();
+
+    vi.mocked(getAccessibleUsers).mockResolvedValue([memberUserId]);
+
+    await authMiddleware(req);
+    await handler(req, res);
+
+    expect(res.statusCode).toBe(200);
+  });
+});

apps/web/pages/api/recorded-daily-video.ts

@@ -3,7 +3,10 @@ import { createHmac } from "crypto";
 import type { NextApiRequest, NextApiResponse } from "next";
 import { z } from "zod";
 
-import { getDownloadLinkOfCalVideoByRecordingId } from "@calcom/core/videoClient";
+import {
+  getDownloadLinkOfCalVideoByRecordingId,
+  submitBatchProcessorTranscriptionJob,
+} from "@calcom/core/videoClient";
 import { sendDailyVideoRecordingEmails } from "@calcom/emails";
 import getWebhooks from "@calcom/features/webhooks/lib/getWebhooks";
 import sendPayload from "@calcom/features/webhooks/lib/sendOrSchedulePayload";
@@ -268,6 +271,13 @@ async function handler(req: NextApiRequest, res: NextApiResponse) {
       },
     });
 
+    try {
+      // Submit Transcription Batch Processor Job
+      await submitBatchProcessorTranscriptionJob(recording_id);
+    } catch (err) {
+      log.error("Failed to  Submit Transcription Batch Processor Job:", safeStringify(err));
+    }
+
     // send emails to all attendees only when user has team plan
     await sendDailyVideoRecordingEmails(evt, downloadLink);
     return res.status(200).json({ message: "Success" });

packages/app-store/dailyvideo/lib/VideoApiAdapter.ts

@@ -3,12 +3,18 @@ import { z } from "zod";
 import { handleErrorsJson } from "@calcom/lib/errors";
 import { prisma } from "@calcom/prisma";
 import type { GetRecordingsResponseSchema, GetAccessLinkResponseSchema } from "@calcom/prisma/zod-utils";
-import { getRecordingsResponseSchema, getAccessLinkResponseSchema } from "@calcom/prisma/zod-utils";
+import {
+  getRecordingsResponseSchema,
+  getAccessLinkResponseSchema,
+  recordingItemSchema,
+} from "@calcom/prisma/zod-utils";
 import type { CalendarEvent } from "@calcom/types/Calendar";
 import type { CredentialPayload } from "@calcom/types/Credential";
 import type { PartialReference } from "@calcom/types/EventManager";
 import type { VideoApiAdapter, VideoCallData } from "@calcom/types/VideoApiAdapter";
 
+import { ZSubmitBatchProcessorJobRes, ZGetTranscriptAccessLink } from "../zod";
+import type { TSubmitBatchProcessorJobRes, TGetTranscriptAccessLink, batchProcessorBody } from "../zod";
 import { getDailyAppKeys } from "./getDailyAppKeys";
 
 /** @link https://docs.daily.co/reference/rest-api/rooms/create-room */
@@ -48,6 +54,17 @@ const getTranscripts = z.object({
   ),
 });
 
+const getBatchProcessJobs = z.object({
+  total_count: z.number(),
+  data: z.array(
+    z.object({
+      id: z.string(),
+      preset: z.string(),
+      status: z.string(),
+    })
+  ),
+});
+
 const getRooms = z
   .object({
     id: z.string(),
@@ -272,6 +289,53 @@ const DailyVideoApiAdapter = (): VideoApiAdapter => {
         throw new Error("Something went wrong! Unable to get transcription access link");
       }
     },
+    submitBatchProcessorJob: async (body: batchProcessorBody): Promise<TSubmitBatchProcessorJobRes> => {
+      try {
+        const batchProcessorJob = await postToDailyAPI("/batch-processor", body).then(
+          ZSubmitBatchProcessorJobRes.parse
+        );
+        return batchProcessorJob;
+      } catch (err) {
+        console.log("err", err);
+        throw new Error("Something went wrong! Unable to submit batch processor job");
+      }
+    },
+    getTranscriptsAccessLinkFromRecordingId: async (
+      recordingId: string
+    ): Promise<TGetTranscriptAccessLink["transcription"] | { message: string }> => {
+      try {
+        const batchProcessorJobs = await fetcher(`/batch-processor?recordingId=${recordingId}`).then(
+          getBatchProcessJobs.parse
+        );
+        if (!batchProcessorJobs.data.length) {
+          return { message: `No Batch processor jobs found for recording id ${recordingId}` };
+        }
+
+        const transcriptJobId = batchProcessorJobs.data.filter(
+          (job) => job.preset === "transcript" && job.status === "finished"
+        )?.[0]?.id;
+
+        if (!transcriptJobId) return [];
+
+        const accessLinkRes = await fetcher(`/batch-processor/${transcriptJobId}/access-link`).then(
+          ZGetTranscriptAccessLink.parse
+        );
+
+        return accessLinkRes.transcription;
+      } catch (err) {
+        console.log("err", err);
+        throw new Error("Something went wrong! can't get transcripts");
+      }
+    },
+    checkIfRoomNameMatchesInRecording: async (roomName: string, recordingId: string): Promise<boolean> => {
+      try {
+        const recording = await fetcher(`/recordings/${recordingId}`).then(recordingItemSchema.parse);
+        return recording.room_name === roomName;
+      } catch (err) {
+        console.error("err", err);
+        throw new Error(`Something went wrong! Unable to checkIfRoomNameMatchesInRecording. ${err}`);
+      }
+    },
   };
 };
 

packages/app-store/dailyvideo/zod.ts

@@ -6,3 +6,36 @@ export const appKeysSchema = z.object({
 });
 
 export const appDataSchema = z.object({});
+
+export const ZSubmitBatchProcessorJobRes = z.object({
+  id: z.string(),
+});
+
+export type TSubmitBatchProcessorJobRes = z.infer<typeof ZSubmitBatchProcessorJobRes>;
+
+export type batchProcessorBody = {
+  preset: "transcript";
+  inParams: {
+    sourceType: "recordingId";
+    recordingId: string;
+  };
+  outParams: {
+    s3Config: {
+      s3KeyTemplate: "transcript";
+    };
+  };
+};
+
+export const ZGetTranscriptAccessLink = z.object({
+  id: z.string(),
+  preset: z.string(),
+  status: z.string(),
+  transcription: z.array(
+    z.object({
+      format: z.string(),
+      link: z.string().url(),
+    })
+  ),
+});
+
+export type TGetTranscriptAccessLink = z.infer<typeof ZGetTranscriptAccessLink>;

packages/core/videoClient.ts

@@ -301,6 +301,89 @@ const getAllTranscriptsAccessLinkFromRoomName = async (roomName: string) => {
   return videoAdapter?.getAllTranscriptsAccessLinkFromRoomName?.(roomName);
 };
 
+const submitBatchProcessorTranscriptionJob = async (recordingId: string) => {
+  let dailyAppKeys: Awaited<ReturnType<typeof getDailyAppKeys>>;
+  try {
+    dailyAppKeys = await getDailyAppKeys();
+  } catch (e) {
+    console.error("Error: Cal video provider is not installed.");
+    return;
+  }
+  const [videoAdapter] = await getVideoAdapters([
+    {
+      id: 0,
+      appId: "daily-video",
+      type: "daily_video",
+      userId: null,
+      user: { email: "" },
+      teamId: null,
+      key: dailyAppKeys,
+      invalid: false,
+    },
+  ]);
+
+  return videoAdapter?.submitBatchProcessorJob?.({
+    preset: "transcript",
+    inParams: {
+      sourceType: "recordingId",
+      recordingId: recordingId,
+    },
+    outParams: {
+      s3Config: {
+        s3KeyTemplate: "transcript",
+      },
+    },
+  });
+};
+
+const getTranscriptsAccessLinkFromRecordingId = async (recordingId: string) => {
+  let dailyAppKeys: Awaited<ReturnType<typeof getDailyAppKeys>>;
+  try {
+    dailyAppKeys = await getDailyAppKeys();
+  } catch (e) {
+    console.error("Error: Cal video provider is not installed.");
+    return;
+  }
+  const [videoAdapter] = await getVideoAdapters([
+    {
+      id: 0,
+      appId: "daily-video",
+      type: "daily_video",
+      userId: null,
+      user: { email: "" },
+      teamId: null,
+      key: dailyAppKeys,
+      invalid: false,
+    },
+  ]);
+
+  return videoAdapter?.getTranscriptsAccessLinkFromRecordingId?.(recordingId);
+};
+
+const checkIfRoomNameMatchesInRecording = async (roomName: string, recordingId: string) => {
+  let dailyAppKeys: Awaited<ReturnType<typeof getDailyAppKeys>>;
+  try {
+    dailyAppKeys = await getDailyAppKeys();
+  } catch (e) {
+    console.error("Error: Cal video provider is not installed.");
+    return;
+  }
+  const [videoAdapter] = await getVideoAdapters([
+    {
+      id: 0,
+      appId: "daily-video",
+      type: "daily_video",
+      userId: null,
+      user: { email: "" },
+      teamId: null,
+      key: dailyAppKeys,
+      invalid: false,
+    },
+  ]);
+
+  return videoAdapter?.checkIfRoomNameMatchesInRecording?.(roomName, recordingId);
+};
+
 export {
   getBusyVideoTimes,
   createMeeting,
@@ -309,4 +392,7 @@ export {
   getRecordingsOfCalVideoByRoomName,
   getDownloadLinkOfCalVideoByRecordingId,
   getAllTranscriptsAccessLinkFromRoomName,
+  submitBatchProcessorTranscriptionJob,
+  getTranscriptsAccessLinkFromRecordingId,
+  checkIfRoomNameMatchesInRecording,
 };

packages/types/VideoApiAdapter.d.ts

@@ -1,3 +1,8 @@
+import type {
+  TSubmitBatchProcessorJobRes,
+  batchProcessorBody,
+  TGetTranscriptAccessLink,
+} from "@calcom/app-store/dailyvideo/zod";
 import type { GetRecordingsResponseSchema, GetAccessLinkResponseSchema } from "@calcom/prisma/zod-utils";
 
 import type { EventBusyDate } from "./Calendar";
@@ -28,6 +33,14 @@ export type VideoApiAdapter =
       createInstantCalVideoRoom?(endTime: string): Promise<VideoCallData>;
 
       getAllTranscriptsAccessLinkFromRoomName?(roomName: string): Promise<Array<string>>;
+
+      submitBatchProcessorJob?(body: batchProcessorBody): Promise<TSubmitBatchProcessorJobRes>;
+
+      getTranscriptsAccessLinkFromRecordingId?(
+        recordingId: string
+      ): Promise<TGetTranscriptAccessLink["transcription"] | { message: string }>;
+
+      checkIfRoomNameMatchesInRecording?(roomName: string, recordingId: string): Promise<boolean>;
     }
   | undefined;