From 006aecb066799daa9c2e8c87c3f8e0822f4dd3c9 Mon Sep 17 00:00:00 2001 From: Eunjae Lee Date: Tue, 5 Aug 2025 02:40:22 +0200 Subject: [PATCH] fix: add backfill migration for routing response denormalized tables (#21474) * fix: add backfill migration for routing response denormalized tables * update script * implement select type * improve number handling * clean up queries * rename * handle zero record case * revert * rename * fix division problem * log clean up * force adding new rows after removing existing rows * backfill only missing or wrong data * remove sleep --- .../migration.sql | 197 ++++++++++++++++++ .../migration.sql | 125 +++++++++++ 2 files changed, 322 insertions(+) create mode 100644 packages/prisma/migrations/20250618093846_routing_form_response_denormalize_backfill/migration.sql create mode 100644 packages/prisma/migrations/20250618093923_routing_form_response_field_backfill/migration.sql diff --git a/packages/prisma/migrations/20250618093846_routing_form_response_denormalize_backfill/migration.sql b/packages/prisma/migrations/20250618093846_routing_form_response_denormalize_backfill/migration.sql new file mode 100644 index 0000000000..8b34508fa7 --- /dev/null +++ b/packages/prisma/migrations/20250618093846_routing_form_response_denormalize_backfill/migration.sql @@ -0,0 +1,197 @@ +DO $$ +DECLARE + chunk_size INTEGER := 1000; + start_id INTEGER := 1; -- Starting ID + end_id INTEGER; -- Will be set dynamically + current_id INTEGER; + + processed_count INTEGER := 0; + chunk_updated_count INTEGER := 0; + total_count INTEGER; +BEGIN + -- Get the maximum ID and total count from the App_RoutingForms_FormResponse table + SELECT COALESCE(MAX(id), 0) INTO end_id FROM "App_RoutingForms_FormResponse"; + SELECT COUNT(*) INTO total_count FROM "App_RoutingForms_FormResponse"; + + -- Handle case where there are no records to process + IF total_count = 0 THEN + RAISE NOTICE 'No records found in App_RoutingForms_FormResponse table. Migration completed.'; + RETURN; + END IF; + + RAISE NOTICE 'Starting migration: processing up to ID % (total responses: %)', end_id, total_count; + + FOR current_id IN SELECT * FROM generate_series(start_id, end_id, chunk_size) + LOOP + -- Use UPSERT to only update records that are missing or have incorrect data + WITH expected_data AS ( + SELECT + r.id, + r."formId" as expected_form_id, + f.name as expected_form_name, + f."teamId" as expected_form_team_id, + f."userId" as expected_form_user_id, + b.uid as expected_booking_uid, + b.id as expected_booking_id, + b.status as expected_booking_status, + calculate_booking_status_order(b.status::text) as expected_booking_status_order, + b."createdAt" as expected_booking_created_at, + b."startTime" as expected_booking_start_time, + b."endTime" as expected_booking_end_time, + b."userId" as expected_booking_user_id, + u.name as expected_booking_user_name, + u.email as expected_booking_user_email, + u."avatarUrl" as expected_booking_user_avatar_url, + COALESCE( + ( + SELECT ar."reasonString" + FROM "AssignmentReason" ar + WHERE ar."bookingId" = b.id + LIMIT 1 + ), + '' + ) as expected_booking_assignment_reason, + et.id as expected_event_type_id, + et."parentId" as expected_event_type_parent_id, + et."schedulingType"::text as expected_event_type_scheduling_type, + r."createdAt" as expected_created_at, + t.utm_source as expected_utm_source, + t.utm_medium as expected_utm_medium, + t.utm_campaign as expected_utm_campaign, + t.utm_term as expected_utm_term, + t.utm_content as expected_utm_content + FROM "App_RoutingForms_FormResponse" r + INNER JOIN "App_RoutingForms_Form" f ON r."formId" = f.id + LEFT JOIN "users" u ON f."userId" = u.id + LEFT JOIN "Booking" b ON b.uid = r."routedToBookingUid" + LEFT JOIN "EventType" et ON b."eventTypeId" = et.id + LEFT JOIN "Tracking" t ON t."bookingId" = b.id + WHERE r.id BETWEEN current_id AND current_id + chunk_size - 1 + ), + records_to_update AS ( + SELECT e.* + FROM expected_data e + LEFT JOIN "RoutingFormResponseDenormalized" d ON e.id = d.id + WHERE CASE + WHEN d.id IS NULL THEN true -- Include missing records + WHEN d."formId" IS DISTINCT FROM e.expected_form_id THEN true + WHEN d."formName" IS DISTINCT FROM e.expected_form_name THEN true + WHEN d."formTeamId" IS DISTINCT FROM e.expected_form_team_id THEN true + WHEN d."formUserId" IS DISTINCT FROM e.expected_form_user_id THEN true + WHEN d."bookingUid" IS DISTINCT FROM e.expected_booking_uid THEN true + WHEN d."bookingId" IS DISTINCT FROM e.expected_booking_id THEN true + WHEN d."bookingStatus" IS DISTINCT FROM e.expected_booking_status THEN true + WHEN d."bookingStatusOrder" IS DISTINCT FROM e.expected_booking_status_order THEN true + WHEN d."bookingCreatedAt" IS DISTINCT FROM e.expected_booking_created_at THEN true + WHEN d."bookingStartTime" IS DISTINCT FROM e.expected_booking_start_time THEN true + WHEN d."bookingEndTime" IS DISTINCT FROM e.expected_booking_end_time THEN true + WHEN d."bookingUserId" IS DISTINCT FROM e.expected_booking_user_id THEN true + WHEN d."bookingUserName" IS DISTINCT FROM e.expected_booking_user_name THEN true + WHEN d."bookingUserEmail" IS DISTINCT FROM e.expected_booking_user_email THEN true + WHEN d."bookingUserAvatarUrl" IS DISTINCT FROM e.expected_booking_user_avatar_url THEN true + WHEN d."bookingAssignmentReason" IS DISTINCT FROM e.expected_booking_assignment_reason THEN true + WHEN d."eventTypeId" IS DISTINCT FROM e.expected_event_type_id THEN true + WHEN d."eventTypeParentId" IS DISTINCT FROM e.expected_event_type_parent_id THEN true + WHEN d."eventTypeSchedulingType"::text IS DISTINCT FROM e.expected_event_type_scheduling_type THEN true + WHEN d."createdAt" IS DISTINCT FROM e.expected_created_at THEN true + WHEN d."utm_source" IS DISTINCT FROM e.expected_utm_source THEN true + WHEN d."utm_medium" IS DISTINCT FROM e.expected_utm_medium THEN true + WHEN d."utm_campaign" IS DISTINCT FROM e.expected_utm_campaign THEN true + WHEN d."utm_term" IS DISTINCT FROM e.expected_utm_term THEN true + WHEN d."utm_content" IS DISTINCT FROM e.expected_utm_content THEN true + ELSE false -- Don't include valid records + END + ) + INSERT INTO "RoutingFormResponseDenormalized" ( + id, + "formId", + "formName", + "formTeamId", + "formUserId", + "bookingUid", + "bookingId", + "bookingStatus", + "bookingStatusOrder", + "bookingCreatedAt", + "bookingStartTime", + "bookingEndTime", + "bookingUserId", + "bookingUserName", + "bookingUserEmail", + "bookingUserAvatarUrl", + "bookingAssignmentReason", + "eventTypeId", + "eventTypeParentId", + "eventTypeSchedulingType", + "createdAt", + "utm_source", + "utm_medium", + "utm_campaign", + "utm_term", + "utm_content" + ) + SELECT + r.id, + r.expected_form_id, + r.expected_form_name, + r.expected_form_team_id, + r.expected_form_user_id, + r.expected_booking_uid, + r.expected_booking_id, + r.expected_booking_status, + r.expected_booking_status_order, + r.expected_booking_created_at, + r.expected_booking_start_time, + r.expected_booking_end_time, + r.expected_booking_user_id, + r.expected_booking_user_name, + r.expected_booking_user_email, + r.expected_booking_user_avatar_url, + r.expected_booking_assignment_reason, + r.expected_event_type_id, + r.expected_event_type_parent_id, + r.expected_event_type_scheduling_type::text, + r.expected_created_at, + r.expected_utm_source, + r.expected_utm_medium, + r.expected_utm_campaign, + r.expected_utm_term, + r.expected_utm_content + FROM records_to_update r + ON CONFLICT (id) DO UPDATE SET + "formId" = EXCLUDED."formId", + "formName" = EXCLUDED."formName", + "formTeamId" = EXCLUDED."formTeamId", + "formUserId" = EXCLUDED."formUserId", + "bookingUid" = EXCLUDED."bookingUid", + "bookingId" = EXCLUDED."bookingId", + "bookingStatus" = EXCLUDED."bookingStatus", + "bookingStatusOrder" = EXCLUDED."bookingStatusOrder", + "bookingCreatedAt" = EXCLUDED."bookingCreatedAt", + "bookingStartTime" = EXCLUDED."bookingStartTime", + "bookingEndTime" = EXCLUDED."bookingEndTime", + "bookingUserId" = EXCLUDED."bookingUserId", + "bookingUserName" = EXCLUDED."bookingUserName", + "bookingUserEmail" = EXCLUDED."bookingUserEmail", + "bookingUserAvatarUrl" = EXCLUDED."bookingUserAvatarUrl", + "bookingAssignmentReason" = EXCLUDED."bookingAssignmentReason", + "eventTypeId" = EXCLUDED."eventTypeId", + "eventTypeParentId" = EXCLUDED."eventTypeParentId", + "eventTypeSchedulingType" = EXCLUDED."eventTypeSchedulingType", + "createdAt" = EXCLUDED."createdAt", + "utm_source" = EXCLUDED."utm_source", + "utm_medium" = EXCLUDED."utm_medium", + "utm_campaign" = EXCLUDED."utm_campaign", + "utm_term" = EXCLUDED."utm_term", + "utm_content" = EXCLUDED."utm_content"; + + GET DIAGNOSTICS chunk_updated_count = ROW_COUNT; + processed_count := processed_count + chunk_updated_count; + + RAISE NOTICE 'Chunk processed: IDs %-% (updated/inserted: % records, total updated: %)', + current_id, current_id + chunk_size - 1, chunk_updated_count, processed_count; + END LOOP; + + RAISE NOTICE 'Migration completed: processed up to ID % (total updated/inserted: % records out of % total records)', + end_id, processed_count, total_count; +END $$; \ No newline at end of file diff --git a/packages/prisma/migrations/20250618093923_routing_form_response_field_backfill/migration.sql b/packages/prisma/migrations/20250618093923_routing_form_response_field_backfill/migration.sql new file mode 100644 index 0000000000..6c17303214 --- /dev/null +++ b/packages/prisma/migrations/20250618093923_routing_form_response_field_backfill/migration.sql @@ -0,0 +1,125 @@ +DO $$ +DECLARE + chunk_size INTEGER := 1000; + sleep_interval FLOAT := 1; + start_id INTEGER := 1; + end_id INTEGER; + current_id INTEGER; + response_record RECORD; + processed_count INTEGER := 0; + chunk_updated_count INTEGER := 0; + total_count INTEGER; +BEGIN + -- Get the maximum ID and total count from the App_RoutingForms_FormResponse table + SELECT COALESCE(MAX(id), 0) INTO end_id FROM "App_RoutingForms_FormResponse"; + SELECT COUNT(*) INTO total_count FROM "App_RoutingForms_FormResponse"; + + -- Handle case where there are no records to process + IF total_count = 0 THEN + RAISE NOTICE 'No records found in App_RoutingForms_FormResponse table. Migration completed.'; + RETURN; + END IF; + + RAISE NOTICE 'Starting migration: processing up to ID % (total responses: %)', end_id, total_count; + + FOR current_id IN SELECT * FROM generate_series(start_id, end_id, chunk_size) + LOOP + -- Process only responses that have missing or incorrect field data + FOR response_record IN + WITH form_response_fields AS ( + -- Extract expected fields from the response data + SELECT + r.id as "responseId", + r."formId", + field->>'id' as "fieldId", + field->>'type' as "fieldType", + response_value->>'value' as raw_value, + jsonb_typeof(response_value->'value') as value_type, + -- Extract array values for multiselect + CASE + WHEN field->>'type' = 'multiselect' AND jsonb_typeof(response_value->'value') = 'array' + THEN ARRAY(SELECT jsonb_array_elements_text(response_value->'value')) + END as expected_array, + -- Extract first element for select type when it's an array + CASE + WHEN field->>'type' = 'select' AND jsonb_typeof(response_value->'value') = 'array' + THEN (response_value->'value'->0)::text + WHEN field->>'type' = 'select' AND jsonb_typeof(response_value->'value') = 'string' + THEN response_value->>'value' + END as expected_select_value + FROM "App_RoutingForms_FormResponse" r + CROSS JOIN LATERAL jsonb_array_elements( + ( + SELECT fields::jsonb + FROM "App_RoutingForms_Form" f + WHERE f.id = r."formId" + ) + ) as field + CROSS JOIN LATERAL ( + SELECT r.response::jsonb->(field->>'id') as response_value + ) as rv + WHERE r.id BETWEEN current_id AND current_id + chunk_size - 1 + AND r.response::jsonb ? (field->>'id') -- Only include fields that exist in response + ), + responses_needing_update AS ( + SELECT DISTINCT f."responseId" + FROM form_response_fields f + LEFT JOIN "RoutingFormResponseField" rf ON + rf."responseId" = f."responseId" AND + rf."fieldId" = f."fieldId" + WHERE + -- Case 1: Field doesn't exist in denormalized table + rf.id IS NULL OR + -- Case 2: Multiselect field validation - compare actual array contents + (f."fieldType" = 'multiselect' AND f.value_type = 'array' AND ( + rf."valueStringArray" IS NULL OR + rf."valueStringArray" != f.expected_array OR + array_length(rf."valueStringArray", 1) != array_length(f.expected_array, 1) + )) OR + -- Case 3: Number field validation - compare as numbers + (f."fieldType" = 'number' AND f.value_type = 'number' AND ( + rf."valueNumber" IS NULL OR + rf."valueNumber" != (f.raw_value)::decimal + )) OR + -- Case 4: Select field validation - compare expected select value + (f."fieldType" = 'select' AND ( + rf."valueString" IS NULL OR + rf."valueString" != f.expected_select_value + )) OR + -- Case 5: Other string field validation + (f."fieldType" NOT IN ('multiselect', 'number', 'select') AND ( + rf."valueString" IS NULL OR + rf."valueString" != f.raw_value + )) + ) + SELECT r.id + FROM responses_needing_update rnu + INNER JOIN "App_RoutingForms_FormResponse" r ON r.id = rnu."responseId" + ORDER BY r.id + LOOP + BEGIN + -- Use the reprocess_routing_form_response_fields function + PERFORM reprocess_routing_form_response_fields(response_record.id); + chunk_updated_count := chunk_updated_count + 1; + + EXCEPTION WHEN OTHERS THEN + RAISE WARNING 'Failed to process responseId %: %', response_record.id, SQLERRM; + CONTINUE; + END; + END LOOP; + + processed_count := processed_count + chunk_updated_count; + + RAISE NOTICE 'Chunk processed: IDs %-% (updated: % records, total updated: %)', + current_id, current_id + chunk_size - 1, chunk_updated_count, processed_count; + + -- Reset chunk counter for next iteration + chunk_updated_count := 0; + + -- Sleep after each chunk (outside transaction to avoid holding locks) + PERFORM pg_sleep(sleep_interval); + END LOOP; + + RAISE NOTICE 'Migration completed: processed up to ID % (total updated: % records out of % total records)', + end_id, processed_count, total_count; +END $$; \ No newline at end of file