refactor: Insert by chunks

src/app/[lang]/dashboard/bulk/Csv.tsx

@@ -100,6 +100,7 @@ export function Csv({ ...props }: { d: Dictionary }): React.ReactElement {
 			return;
 		}
 		setLoading(true);
+		setUpload(<Uploading d={d} />);
 		postData<CheckEmailOutput>({
 			url: `/api/v1/bulk`,
 			data: {
@@ -227,6 +228,16 @@ function Analyzed({
 	);
 }
 
+function Uploading({ d }: { d: Dictionary["dashboard"]["get_started_bulk"] }) {
+	return (
+		<>
+			<Upload />
+			<h4>{d.step_loading.title}</h4>
+			<p>{d.step_loading.description}</p>
+		</>
+	);
+}
+
 function Uploaded({
 	emailsNum,
 	d,

src/app/api/v1/bulk/route.ts

@@ -5,7 +5,7 @@ import { sentryException } from "@/util/sentry";
 import { getWebappURL } from "@/util/helpers";
 import { isEarlyResponse } from "@/app/api/v0/check_email/checkUserInDb";
 import { SAAS_100K_PRODUCT_ID, getApiUsage } from "@/util/subs";
-import { Json } from "@/supabase/database.types";
+import { Json, Tables } from "@/supabase/database.types";
 import { cookies } from "next/headers";
 import { createClient } from "@/supabase/server";
 import { getSubscription } from "@/supabase/supabaseServer";
@@ -72,17 +72,34 @@ export const POST = async (req: NextRequest): Promise<Response> => {
 			throw res1.error;
 		}
 		const bulkJob = res1.data[0];
-		const res2 = await supabaseAdmin
-			.from("bulk_emails")
-			.insert(
-				payload.input.map((email) => ({
-					bulk_job_id: bulkJob.id,
-					email,
-				}))
-			)
-			.select("*");
-		if (res2.error) {
-			throw res2.error;
+		console.log(`[💪] Created job ${bulkJob.id}`);
+
+		// Split Array in chunks of 5000 so that Supabase can handle it.
+		const chunkSize = 5000;
+		const chunks = [];
+		for (let i = 0; i < payload.input.length; i += chunkSize) {
+			chunks.push(payload.input.slice(i, i + chunkSize));
+		}
+		let i = 1;
+		let bulkEmails: Tables<"bulk_emails">[] = [];
+		for (const chunk of chunks) {
+			const res2 = await supabaseAdmin
+				.from("bulk_emails")
+				.insert(
+					chunk.map((email) => ({
+						bulk_job_id: bulkJob.id,
+						email,
+					}))
+				)
+				.select("*");
+			if (res2.error) {
+				throw res2.error;
+			}
+			bulkEmails = bulkEmails.concat(res2.data);
+
+			console.log(
+				`[💪] Inserted job ${bulkJob.id} chunk ${i++}/${chunks.length}`
+			);
 		}
 
 		const conn = await amqplib
@@ -107,7 +124,7 @@ export const POST = async (req: NextRequest): Promise<Response> => {
 			maxPriority: 5,
 		});
 
-		res2.data.forEach(({ email, id }) => {
+		bulkEmails.forEach(({ email, id }) => {
 			ch1.sendToQueue(
 				queueName,
 				Buffer.from(
@@ -137,7 +154,7 @@ export const POST = async (req: NextRequest): Promise<Response> => {
 
 		return Response.json({
 			bulk_job_id: bulkJob.id,
-			emails: res2.data.length,
+			emails: bulkEmails.length,
 		});
 	} catch (err) {
 		if (isEarlyResponse(err)) {

src/dictionaries/en.json

@@ -42,6 +42,10 @@
 				"title": "Analyzed %s",
 				"label": "Found %s emails"
 			},
+			"step_loading": {
+				"title": "Loading...",
+				"description": "This might take up to 1 minute. Please don't close this page."
+			},
 			"step_complete": {
 				"title": "Congratulations!",
 				"description": "Your %s emails are currently being verified. You'll get notified by email once the job is done. You can also check the progress in the table below.",

src/dictionaries/fr.json

@@ -42,6 +42,10 @@
 				"title": "Analyse de %s terminée",
 				"label": "%s emails trouvés"
 			},
+			"step_loading": {
+				"title": "Upload en cours...",
+				"description": "Cette opération peut prendre jusqu'à une minute, veuillez ne pas fermer cette fenêtre."
+			},
 			"step_complete": {
 				"title": "Félicitations !",
 				"description": "Vos %s emails sont actuellement en cours de vérification. Vous serez notifié par email une fois le travail terminé. Vous pouvez également vérifier la progression dans le tableau ci-dessous.",

src/util/helpers.ts

@@ -106,4 +106,4 @@ export function formatDate(d: string | Date, locale?: string): string {
 	});
 }
 
-export const ENABLE_BULK: boolean = false;
+export const ENABLE_BULK: boolean = true;