From b66d36a98e05032e783314b5fd7e9089b457fb45 Mon Sep 17 00:00:00 2001 From: Udit Takkar <53316345+Udit-takkar@users.noreply.github.com> Date: Tue, 11 Jun 2024 21:11:56 +0530 Subject: [PATCH] feat: generate transcription from recording and API endpoint (#15358) * feat: generate transcription from recording * fix: api * chore: undo * tests: add unit tests * chore: refactor move zod type * fix: add more security check * chore: update test * chore: type err * chore: test * chore: fix tyoe * fix: build err * chore: add try .. catch * chore: update desc --------- Co-authored-by: Joe Au-Yeung <65426560+joeauyeung@users.noreply.github.com> --- .../shared/queryIdTransformParseInt.ts | 4 + .../[id]/transcripts/[recordingId]/_get.ts | 94 +++++++++++++ .../[id]/transcripts/[recordingId]/index.ts | 16 +++ .../transcripts/[recordingId]/_get.test.ts | 129 ++++++++++++++++++ apps/web/pages/api/recorded-daily-video.ts | 12 +- .../dailyvideo/lib/VideoApiAdapter.ts | 66 ++++++++- packages/app-store/dailyvideo/zod.ts | 33 +++++ packages/core/videoClient.ts | 86 ++++++++++++ packages/types/VideoApiAdapter.d.ts | 13 ++ 9 files changed, 451 insertions(+), 2 deletions(-) create mode 100644 apps/api/v1/pages/api/bookings/[id]/transcripts/[recordingId]/_get.ts create mode 100644 apps/api/v1/pages/api/bookings/[id]/transcripts/[recordingId]/index.ts create mode 100644 apps/api/v1/test/lib/bookings/[id]/transcripts/[recordingId]/_get.test.ts diff --git a/apps/api/v1/lib/validations/shared/queryIdTransformParseInt.ts b/apps/api/v1/lib/validations/shared/queryIdTransformParseInt.ts index b9ec495f47..ef6d811ea9 100644 --- a/apps/api/v1/lib/validations/shared/queryIdTransformParseInt.ts +++ b/apps/api/v1/lib/validations/shared/queryIdTransformParseInt.ts @@ -14,3 +14,7 @@ export const withValidQueryIdTransformParseInt = withValidation({ type: "Zod", mode: "query", }); + +export const getTranscriptFromRecordingId = schemaQueryIdParseInt.extend({ + recordingId: z.string(), +}); diff --git a/apps/api/v1/pages/api/bookings/[id]/transcripts/[recordingId]/_get.ts b/apps/api/v1/pages/api/bookings/[id]/transcripts/[recordingId]/_get.ts new file mode 100644 index 0000000000..67cac58bdd --- /dev/null +++ b/apps/api/v1/pages/api/bookings/[id]/transcripts/[recordingId]/_get.ts @@ -0,0 +1,94 @@ +import type { NextApiRequest } from "next"; + +import { + getTranscriptsAccessLinkFromRecordingId, + checkIfRoomNameMatchesInRecording, +} from "@calcom/core/videoClient"; +import { HttpError } from "@calcom/lib/http-error"; +import { defaultResponder } from "@calcom/lib/server"; +import prisma from "@calcom/prisma"; +import type { PartialReference } from "@calcom/types/EventManager"; + +import { getTranscriptFromRecordingId } from "~/lib/validations/shared/queryIdTransformParseInt"; + +/** + * @swagger + * /bookings/{id}/transcripts/{recordingId}: + * get: + * summary: Find all Cal video transcripts of that recording + * operationId: getTranscriptsByRecordingId + * parameters: + * - in: path + * name: id + * schema: + * type: integer + * required: true + * description: ID of the booking for which transcripts need to be fetched. + * - in: path + * name: recordingId + * schema: + * type: string + * required: true + * description: ID of the recording(daily.co recording id) for which transcripts need to be fetched. + * - in: query + * name: apiKey + * required: true + * schema: + * type: string + * description: Your API key + * tags: + * - bookings + * responses: + * 200: + * description: OK + * content: + * application/json: + * 401: + * description: Authorization information is missing or invalid. + * 404: + * description: Booking was not found + */ + +export async function getHandler(req: NextApiRequest) { + const { query } = req; + const { id, recordingId } = getTranscriptFromRecordingId.parse(query); + + await checkIfRecordingBelongsToBooking(id, recordingId); + + const transcriptsAccessLinks = await getTranscriptsAccessLinkFromRecordingId(recordingId); + + return transcriptsAccessLinks; +} + +const checkIfRecordingBelongsToBooking = async (bookingId: number, recordingId: string) => { + const booking = await prisma.booking.findUnique({ + where: { id: bookingId }, + include: { references: true }, + }); + + if (!booking) + throw new HttpError({ + statusCode: 404, + message: `No Booking found with booking id ${bookingId}`, + }); + + const roomName = + booking?.references?.find((reference: PartialReference) => reference.type === "daily_video")?.meetingId ?? + undefined; + + if (!roomName) + throw new HttpError({ + statusCode: 404, + message: `No Booking Reference with Daily Video found with booking id ${bookingId}`, + }); + + const canUserAccessRecordingId = await checkIfRoomNameMatchesInRecording(roomName, recordingId); + if (!canUserAccessRecordingId) { + throw new HttpError({ + statusCode: 403, + message: `This Recording Id ${recordingId} does not belong to booking ${bookingId}`, + }); + } +}; + +export default defaultResponder(getHandler); diff --git a/apps/api/v1/pages/api/bookings/[id]/transcripts/[recordingId]/index.ts b/apps/api/v1/pages/api/bookings/[id]/transcripts/[recordingId]/index.ts new file mode 100644 index 0000000000..3085d27a86 --- /dev/null +++ b/apps/api/v1/pages/api/bookings/[id]/transcripts/[recordingId]/index.ts @@ -0,0 +1,16 @@ +import type { NextApiRequest, NextApiResponse } from "next"; + +import { defaultHandler, defaultResponder } from "@calcom/lib/server"; + +import { withMiddleware } from "~/lib/helpers/withMiddleware"; + +import authMiddleware from "../../_auth-middleware"; + +export default withMiddleware()( + defaultResponder(async (req: NextApiRequest, res: NextApiResponse) => { + await authMiddleware(req); + return defaultHandler({ + GET: import("./_get"), + })(req, res); + }) +); diff --git a/apps/api/v1/test/lib/bookings/[id]/transcripts/[recordingId]/_get.test.ts b/apps/api/v1/test/lib/bookings/[id]/transcripts/[recordingId]/_get.test.ts new file mode 100644 index 0000000000..c6320fc9e7 --- /dev/null +++ b/apps/api/v1/test/lib/bookings/[id]/transcripts/[recordingId]/_get.test.ts @@ -0,0 +1,129 @@ +import prismaMock from "../../../../../../../../../tests/libs/__mocks__/prismaMock"; + +import type { Request, Response } from "express"; +import type { NextApiRequest, NextApiResponse } from "next"; +import { createMocks } from "node-mocks-http"; +import { describe, expect, test, vi, afterEach } from "vitest"; + +import { + getTranscriptsAccessLinkFromRecordingId, + checkIfRoomNameMatchesInRecording, +} from "@calcom/core/videoClient"; +import { buildBooking } from "@calcom/lib/test/builder"; + +import { getAccessibleUsers } from "~/lib/utils/retrieveScopedAccessibleUsers"; + +import authMiddleware from "../../../../../../pages/api/bookings/[id]/_auth-middleware"; +import handler from "../../../../../../pages/api/bookings/[id]/transcripts/[recordingId]/_get"; + +type CustomNextApiRequest = NextApiRequest & Request; +type CustomNextApiResponse = NextApiResponse & Response; + +vi.mock("@calcom/core/videoClient", () => { + return { + getTranscriptsAccessLinkFromRecordingId: vi.fn(), + checkIfRoomNameMatchesInRecording: vi.fn(), + }; +}); + +vi.mock("~/lib/utils/retrieveScopedAccessibleUsers", () => { + return { + getAccessibleUsers: vi.fn(), + }; +}); + +afterEach(() => { + vi.resetAllMocks(); +}); + +const mockGetTranscripts = () => { + const downloadLinks = [{ format: "json", link: "https://URL1" }]; + + vi.mocked(getTranscriptsAccessLinkFromRecordingId).mockResolvedValue(downloadLinks); + vi.mocked(checkIfRoomNameMatchesInRecording).mockResolvedValue(true); + + return downloadLinks; +}; + +const recordingId = "abc-xyz"; + +describe("GET /api/bookings/[id]/transcripts/[recordingId]", () => { + test("Returns transcripts if user is system-wide admin", async () => { + const adminUserId = 1; + const userId = 2; + + const bookingId = 1111; + + prismaMock.booking.findUnique.mockResolvedValue( + buildBooking({ + id: bookingId, + userId, + references: [ + { + id: 1, + type: "daily_video", + uid: "17OHkCH53pBa03FhxMbw", + meetingId: "17OHkCH53pBa03FhxMbw", + meetingPassword: "password", + meetingUrl: "https://URL", + }, + ], + }) + ); + + const mockedTranscripts = mockGetTranscripts(); + const { req, res } = createMocks({ + method: "GET", + body: {}, + query: { + id: bookingId, + recordingId, + }, + }); + + req.isSystemWideAdmin = true; + req.userId = adminUserId; + + await authMiddleware(req); + await handler(req, res); + + expect(res.statusCode).toBe(200); + expect(JSON.parse(res._getData())).toEqual(mockedTranscripts); + }); + + test("Allows GET transcripts when user is org-wide admin", async () => { + const adminUserId = 1; + const memberUserId = 10; + const bookingId = 3333; + + prismaMock.booking.findUnique.mockResolvedValue( + buildBooking({ + id: bookingId, + userId: memberUserId, + references: [ + { id: 1, type: "daily_video", uid: "17OHkCH53pBa03FhxMbw", meetingId: "17OHkCH53pBa03FhxMbw" }, + ], + }) + ); + + const { req, res } = createMocks({ + method: "GET", + body: {}, + query: { + id: bookingId, + recordingId, + }, + }); + + req.userId = adminUserId; + req.isOrganizationOwnerOrAdmin = true; + mockGetTranscripts(); + + vi.mocked(getAccessibleUsers).mockResolvedValue([memberUserId]); + + await authMiddleware(req); + await handler(req, res); + + expect(res.statusCode).toBe(200); + }); +}); diff --git a/apps/web/pages/api/recorded-daily-video.ts b/apps/web/pages/api/recorded-daily-video.ts index 3fcb734960..9d6fc4d5d4 100644 --- a/apps/web/pages/api/recorded-daily-video.ts +++ b/apps/web/pages/api/recorded-daily-video.ts @@ -3,7 +3,10 @@ import { createHmac } from "crypto"; import type { NextApiRequest, NextApiResponse } from "next"; import { z } from "zod"; -import { getDownloadLinkOfCalVideoByRecordingId } from "@calcom/core/videoClient"; +import { + getDownloadLinkOfCalVideoByRecordingId, + submitBatchProcessorTranscriptionJob, +} from "@calcom/core/videoClient"; import { sendDailyVideoRecordingEmails } from "@calcom/emails"; import getWebhooks from "@calcom/features/webhooks/lib/getWebhooks"; import sendPayload from "@calcom/features/webhooks/lib/sendOrSchedulePayload"; @@ -268,6 +271,13 @@ async function handler(req: NextApiRequest, res: NextApiResponse) { }, }); + try { + // Submit Transcription Batch Processor Job + await submitBatchProcessorTranscriptionJob(recording_id); + } catch (err) { + log.error("Failed to Submit Transcription Batch Processor Job:", safeStringify(err)); + } + // send emails to all attendees only when user has team plan await sendDailyVideoRecordingEmails(evt, downloadLink); return res.status(200).json({ message: "Success" }); diff --git a/packages/app-store/dailyvideo/lib/VideoApiAdapter.ts b/packages/app-store/dailyvideo/lib/VideoApiAdapter.ts index 8f981ce72b..3910706a72 100644 --- a/packages/app-store/dailyvideo/lib/VideoApiAdapter.ts +++ b/packages/app-store/dailyvideo/lib/VideoApiAdapter.ts @@ -3,12 +3,18 @@ import { z } from "zod"; import { handleErrorsJson } from "@calcom/lib/errors"; import { prisma } from "@calcom/prisma"; import type { GetRecordingsResponseSchema, GetAccessLinkResponseSchema } from "@calcom/prisma/zod-utils"; -import { getRecordingsResponseSchema, getAccessLinkResponseSchema } from "@calcom/prisma/zod-utils"; +import { + getRecordingsResponseSchema, + getAccessLinkResponseSchema, + recordingItemSchema, +} from "@calcom/prisma/zod-utils"; import type { CalendarEvent } from "@calcom/types/Calendar"; import type { CredentialPayload } from "@calcom/types/Credential"; import type { PartialReference } from "@calcom/types/EventManager"; import type { VideoApiAdapter, VideoCallData } from "@calcom/types/VideoApiAdapter"; +import { ZSubmitBatchProcessorJobRes, ZGetTranscriptAccessLink } from "../zod"; +import type { TSubmitBatchProcessorJobRes, TGetTranscriptAccessLink, batchProcessorBody } from "../zod"; import { getDailyAppKeys } from "./getDailyAppKeys"; /** @link https://docs.daily.co/reference/rest-api/rooms/create-room */ @@ -48,6 +54,17 @@ const getTranscripts = z.object({ ), }); +const getBatchProcessJobs = z.object({ + total_count: z.number(), + data: z.array( + z.object({ + id: z.string(), + preset: z.string(), + status: z.string(), + }) + ), +}); + const getRooms = z .object({ id: z.string(), @@ -272,6 +289,53 @@ const DailyVideoApiAdapter = (): VideoApiAdapter => { throw new Error("Something went wrong! Unable to get transcription access link"); } }, + submitBatchProcessorJob: async (body: batchProcessorBody): Promise => { + try { + const batchProcessorJob = await postToDailyAPI("/batch-processor", body).then( + ZSubmitBatchProcessorJobRes.parse + ); + return batchProcessorJob; + } catch (err) { + console.log("err", err); + throw new Error("Something went wrong! Unable to submit batch processor job"); + } + }, + getTranscriptsAccessLinkFromRecordingId: async ( + recordingId: string + ): Promise => { + try { + const batchProcessorJobs = await fetcher(`/batch-processor?recordingId=${recordingId}`).then( + getBatchProcessJobs.parse + ); + if (!batchProcessorJobs.data.length) { + return { message: `No Batch processor jobs found for recording id ${recordingId}` }; + } + + const transcriptJobId = batchProcessorJobs.data.filter( + (job) => job.preset === "transcript" && job.status === "finished" + )?.[0]?.id; + + if (!transcriptJobId) return []; + + const accessLinkRes = await fetcher(`/batch-processor/${transcriptJobId}/access-link`).then( + ZGetTranscriptAccessLink.parse + ); + + return accessLinkRes.transcription; + } catch (err) { + console.log("err", err); + throw new Error("Something went wrong! can't get transcripts"); + } + }, + checkIfRoomNameMatchesInRecording: async (roomName: string, recordingId: string): Promise => { + try { + const recording = await fetcher(`/recordings/${recordingId}`).then(recordingItemSchema.parse); + return recording.room_name === roomName; + } catch (err) { + console.error("err", err); + throw new Error(`Something went wrong! Unable to checkIfRoomNameMatchesInRecording. ${err}`); + } + }, }; }; diff --git a/packages/app-store/dailyvideo/zod.ts b/packages/app-store/dailyvideo/zod.ts index 2774e0c0ec..fbfdf4af64 100644 --- a/packages/app-store/dailyvideo/zod.ts +++ b/packages/app-store/dailyvideo/zod.ts @@ -6,3 +6,36 @@ export const appKeysSchema = z.object({ }); export const appDataSchema = z.object({}); + +export const ZSubmitBatchProcessorJobRes = z.object({ + id: z.string(), +}); + +export type TSubmitBatchProcessorJobRes = z.infer; + +export type batchProcessorBody = { + preset: "transcript"; + inParams: { + sourceType: "recordingId"; + recordingId: string; + }; + outParams: { + s3Config: { + s3KeyTemplate: "transcript"; + }; + }; +}; + +export const ZGetTranscriptAccessLink = z.object({ + id: z.string(), + preset: z.string(), + status: z.string(), + transcription: z.array( + z.object({ + format: z.string(), + link: z.string().url(), + }) + ), +}); + +export type TGetTranscriptAccessLink = z.infer; diff --git a/packages/core/videoClient.ts b/packages/core/videoClient.ts index 7ebe925d62..aa0cab0af5 100644 --- a/packages/core/videoClient.ts +++ b/packages/core/videoClient.ts @@ -301,6 +301,89 @@ const getAllTranscriptsAccessLinkFromRoomName = async (roomName: string) => { return videoAdapter?.getAllTranscriptsAccessLinkFromRoomName?.(roomName); }; +const submitBatchProcessorTranscriptionJob = async (recordingId: string) => { + let dailyAppKeys: Awaited>; + try { + dailyAppKeys = await getDailyAppKeys(); + } catch (e) { + console.error("Error: Cal video provider is not installed."); + return; + } + const [videoAdapter] = await getVideoAdapters([ + { + id: 0, + appId: "daily-video", + type: "daily_video", + userId: null, + user: { email: "" }, + teamId: null, + key: dailyAppKeys, + invalid: false, + }, + ]); + + return videoAdapter?.submitBatchProcessorJob?.({ + preset: "transcript", + inParams: { + sourceType: "recordingId", + recordingId: recordingId, + }, + outParams: { + s3Config: { + s3KeyTemplate: "transcript", + }, + }, + }); +}; + +const getTranscriptsAccessLinkFromRecordingId = async (recordingId: string) => { + let dailyAppKeys: Awaited>; + try { + dailyAppKeys = await getDailyAppKeys(); + } catch (e) { + console.error("Error: Cal video provider is not installed."); + return; + } + const [videoAdapter] = await getVideoAdapters([ + { + id: 0, + appId: "daily-video", + type: "daily_video", + userId: null, + user: { email: "" }, + teamId: null, + key: dailyAppKeys, + invalid: false, + }, + ]); + + return videoAdapter?.getTranscriptsAccessLinkFromRecordingId?.(recordingId); +}; + +const checkIfRoomNameMatchesInRecording = async (roomName: string, recordingId: string) => { + let dailyAppKeys: Awaited>; + try { + dailyAppKeys = await getDailyAppKeys(); + } catch (e) { + console.error("Error: Cal video provider is not installed."); + return; + } + const [videoAdapter] = await getVideoAdapters([ + { + id: 0, + appId: "daily-video", + type: "daily_video", + userId: null, + user: { email: "" }, + teamId: null, + key: dailyAppKeys, + invalid: false, + }, + ]); + + return videoAdapter?.checkIfRoomNameMatchesInRecording?.(roomName, recordingId); +}; + export { getBusyVideoTimes, createMeeting, @@ -309,4 +392,7 @@ export { getRecordingsOfCalVideoByRoomName, getDownloadLinkOfCalVideoByRecordingId, getAllTranscriptsAccessLinkFromRoomName, + submitBatchProcessorTranscriptionJob, + getTranscriptsAccessLinkFromRecordingId, + checkIfRoomNameMatchesInRecording, }; diff --git a/packages/types/VideoApiAdapter.d.ts b/packages/types/VideoApiAdapter.d.ts index 03cab0a658..136a4fdf9d 100644 --- a/packages/types/VideoApiAdapter.d.ts +++ b/packages/types/VideoApiAdapter.d.ts @@ -1,3 +1,8 @@ +import type { + TSubmitBatchProcessorJobRes, + batchProcessorBody, + TGetTranscriptAccessLink, +} from "@calcom/app-store/dailyvideo/zod"; import type { GetRecordingsResponseSchema, GetAccessLinkResponseSchema } from "@calcom/prisma/zod-utils"; import type { EventBusyDate } from "./Calendar"; @@ -28,6 +33,14 @@ export type VideoApiAdapter = createInstantCalVideoRoom?(endTime: string): Promise; getAllTranscriptsAccessLinkFromRoomName?(roomName: string): Promise>; + + submitBatchProcessorJob?(body: batchProcessorBody): Promise; + + getTranscriptsAccessLinkFromRecordingId?( + recordingId: string + ): Promise; + + checkIfRoomNameMatchesInRecording?(roomName: string, recordingId: string): Promise; } | undefined;