feat: support bucketting for large whitelist csv

frontend/src/features/admin-form/settings/components/AuthSettingsSection/FormWhitelistAttachmentField.tsx

@@ -25,7 +25,7 @@ interface FormWhitelistAttachmentFieldProps {
   isDisabled: boolean
 }
 
-const MAX_SIZE_IN_BYTES = 3 * MB
+const MAX_SIZE_IN_BYTES = 12 * MB
 const FormWhitelistAttachmentFieldContainerName =
   'whitelist-csv-attachment-field-container'
 const FormWhitelistAttachmentFieldName = 'whitelist-csv-attachment-field'

shared/types/form/form.ts

@@ -215,7 +215,7 @@ export interface WhitelistedSubmitterIds {
 
 export interface WhitelistedSubmitterIdsWithReferenceOid
   extends WhitelistedSubmitterIds {
-  encryptedWhitelistedSubmitterIds: string // Object id of the encrypted whitelist
+  encryptedWhitelistedSubmitterIds: string[] // Object ids of the encrypted whitelist
 }
 
 export interface StorageFormBase extends FormBase {

src/app/models/form.server.model.ts

@@ -201,12 +201,14 @@ const whitelistedSubmitterIdNestedPath = {
     required: true,
     default: false,
   },
-  encryptedWhitelistedSubmitterIds: {
-    type: Schema.Types.ObjectId,
-    ref: FORM_WHITELISTED_SUBMITTER_IDS_ID,
-    required: false,
-    default: undefined,
-  },
+  encryptedWhitelistedSubmitterIds: [
+    {
+      type: Schema.Types.ObjectId,
+      ref: FORM_WHITELISTED_SUBMITTER_IDS_ID,
+      required: false,
+      default: undefined,
+    },
+  ],
   _id: { id: false },
 }
 
@@ -282,9 +284,26 @@ const EncryptedFormDocumentSchema =
   EncryptedFormSchema as unknown as Schema<IEncryptedFormDocument>
 
 EncryptedFormDocumentSchema.methods.getWhitelistedSubmitterIds = function () {
-  return this.get('whitelistedSubmitterIds', null, {
+  const whitelistedSubmitterIds = this.get('whitelistedSubmitterIds', null, {
     getters: false,
   })
+
+  // NOTE: this is to support legacy format for encryptedWhitelistedSubmitterIds which only stores 1 document.
+  const whitelistedSubmitterIdsObject = whitelistedSubmitterIds
+  if (
+    !Array.isArray(
+      whitelistedSubmitterIdsObject.encryptedWhitelistedSubmitterIds,
+    )
+  ) {
+    return {
+      ...whitelistedSubmitterIdsObject,
+      encryptedWhitelistedSubmitterIds: [
+        whitelistedSubmitterIdsObject.encryptedWhitelistedSubmitterIds,
+      ],
+    }
+  }
+
+  return whitelistedSubmitterIdsObject
 }
 
 EncryptedFormDocumentSchema.methods.addPaymentAccountId = function ({

src/app/models/form_whitelist.server.model.ts

@@ -44,9 +44,9 @@ const formWhitelistedSubmitterIdsSchema = new Schema<
 })
 
 formWhitelistedSubmitterIdsSchema.statics.checkIfSubmitterIdIsWhitelisted =
-  async function (whitelistId: string, submitterId: string) {
+  async function (whitelistId: string[], submitterId: string) {
     return this.exists({
-      _id: whitelistId,
+      _id: { $in: whitelistId },
       cipherTexts: submitterId,
     }).exec()
   }

src/app/modules/form/admin-form/admin-form.controller.ts

@@ -1656,7 +1656,7 @@ export const handleDeleteWorkflowStep: ControllerHandler<
   )
 }
 
-const LIMIT_IN_MB = 3
+const LIMIT_IN_MB = 12
 const STRING_MAX_LENGTH = LIMIT_IN_MB * MB
 const _handleUpdateWhitelistSettingValidator = celebrate({
   [Segments.PARAMS]: Joi.object({

src/app/modules/form/admin-form/admin-form.service.ts

@@ -1388,6 +1388,22 @@ export const checkIsWhitelistSettingValid = (
   }
 }
 
+const combineFormWhitelistedSubmitterIdsBuckets = (
+  whitelistSettingBuckets: EncryptedStringsMessageContent[],
+): EncryptedStringsMessageContent => {
+  const whitelistedSubmitterIds = whitelistSettingBuckets.reduce(
+    (acc, bucket) => {
+      return acc.concat(bucket.cipherTexts)
+    },
+    [] as string[],
+  )
+  return {
+    myPublicKey: whitelistSettingBuckets[0].myPublicKey,
+    nonce: whitelistSettingBuckets[0].nonce,
+    cipherTexts: whitelistedSubmitterIds,
+  }
+}
+
 /**
  * Fetches the whitelist setting document without myPrivateKey for the client to use for decryption.
  */
@@ -1403,15 +1419,17 @@ export const getFormWhitelistSetting = (
   if (!isWhitelistEnabled) {
     return okAsync(null)
   }
-
   if (isWhitelistEnabled && !encryptedWhitelistedSubmitterIds) {
     return errAsync(new FormWhitelistSettingNotFoundError())
   }
 
   return ResultAsync.fromPromise(
-    FormWhitelistedSubmitterIdsModel.findById(encryptedWhitelistedSubmitterIds)
+    FormWhitelistedSubmitterIdsModel.where({
+      _id: { $in: encryptedWhitelistedSubmitterIds },
+    })
       .lean()
       .exec()
+      .then(combineFormWhitelistedSubmitterIdsBuckets)
       .then((whitelistSetting) =>
         pick(whitelistSetting, WHITELISTED_SUBMITTER_ID_DECRYPTION_FIELDS),
       ) as Promise<EncryptedStringsMessageContent>,
@@ -1434,6 +1452,26 @@ export const getFormWhitelistSetting = (
   })
 }
 
+const NUM_SUBMITTER_IDS_PER_BUCKET = 200_000
+const getFormWhitelistedSubmitterIdsBuckets = (
+  encryptedWhitelistedSubmitterIdsContent: EncryptedStringsMessageContentWithMyPrivateKey,
+) => {
+  const buckets: string[][] = []
+  for (
+    let i = 0;
+    i < encryptedWhitelistedSubmitterIdsContent.cipherTexts.length;
+    i += NUM_SUBMITTER_IDS_PER_BUCKET
+  ) {
+    buckets.push(
+      encryptedWhitelistedSubmitterIdsContent.cipherTexts.slice(
+        i,
+        i + NUM_SUBMITTER_IDS_PER_BUCKET,
+      ),
+    )
+  }
+  return buckets
+}
+
 export const updateFormWhitelistSetting = (
   originalForm: IPopulatedForm,
   encryptedWhitelistedSubmitterIdsContent: EncryptedStringsMessageContentWithMyPrivateKey | null,
@@ -1453,19 +1491,30 @@ export const updateFormWhitelistSetting = (
     session.startTransaction()
 
     if (encryptedWhitelistedSubmitterIdsContent) {
-      // create whitelisted submitter id collection document and update reference to it
-      const createdWhitelistedSubmitterIdsDocument =
-        await FormWhitelistedSubmitterIdsModel.create({
-          formId: originalForm._id,
-          ...encryptedWhitelistedSubmitterIdsContent,
-        })
+      const whitelistedSubmitterIdsBuckets =
+        getFormWhitelistedSubmitterIdsBuckets(
+          encryptedWhitelistedSubmitterIdsContent,
+        )
+
+      const documentIds = await FormWhitelistedSubmitterIdsModel.insertMany(
+        whitelistedSubmitterIdsBuckets.map((bucket) => {
+          const bucketContent = {
+            ...encryptedWhitelistedSubmitterIdsContent,
+            cipherTexts: bucket,
+          }
+          return {
+            formId: originalForm._id,
+            ...bucketContent,
+          }
+        }),
+      )
+
       const updatedForm = await FormModelToUse.findByIdAndUpdate(
         originalForm._id,
         {
           whitelistedSubmitterIds: {
             isWhitelistEnabled: true,
-            encryptedWhitelistedSubmitterIds:
-              createdWhitelistedSubmitterIdsDocument._id,
+            encryptedWhitelistedSubmitterIds: documentIds,
           },
         },
         {

src/app/modules/form/form.service.ts

@@ -292,13 +292,13 @@ export const checkHasRespondentNotWhitelistedFailure = (
     return okAsync(false)
   }
 
-  const { isWhitelistEnabled, encryptedWhitelistedSubmitterIds: whitelistId } =
+  const { isWhitelistEnabled, encryptedWhitelistedSubmitterIds: whitelistIds } =
     form.getWhitelistedSubmitterIds()
 
   if (!isWhitelistEnabled) {
     return okAsync(false)
   }
-  if (isWhitelistEnabled && !whitelistId) {
+  if (isWhitelistEnabled && (!whitelistIds || whitelistIds.length <= 0)) {
     return errAsync(new FormWhitelistSettingNotFoundError())
   }
 
@@ -317,7 +317,7 @@ export const checkHasRespondentNotWhitelistedFailure = (
   }
   return ResultAsync.fromPromise(
     FormWhitelistSubmitterIdsModel.findEncryptionPropertiesById(
-      whitelistId,
+      whitelistIds[0],
     ).then(({ myPublicKey, myPrivateKey, nonce }) => {
       const myKeyPair = {
         publicKey: myPublicKey,
@@ -335,7 +335,7 @@ export const checkHasRespondentNotWhitelistedFailure = (
       ).cipherText
 
       return FormWhitelistSubmitterIdsModel.checkIfSubmitterIdIsWhitelisted(
-        whitelistId,
+        whitelistIds,
         submitterIdForLookup,
       ).then((isWhitelisted) => !isWhitelisted)
     }),

src/types/form_whitelisted_submitter_ids.ts

@@ -22,7 +22,7 @@ export interface IFormWhitelistedSubmitterIdsModel
   >
 
   checkIfSubmitterIdIsWhitelisted(
-    whitelistId: string,
+    whitelistId: string[],
     submitterId: string,
   ): Promise<boolean>
 }