From 10f9e52c63bcf066babf69a71ebc5a40069a2555 Mon Sep 17 00:00:00 2001 From: Kevin Foong <55353265+kevin9foong@users.noreply.github.com> Date: Wed, 15 Jan 2025 17:50:19 +0800 Subject: [PATCH] feat: support bucketting for large whitelist csv --- .../FormWhitelistAttachmentField.tsx | 2 +- shared/types/form/form.ts | 2 +- src/app/models/form.server.model.ts | 33 +++++++-- src/app/models/form_whitelist.server.model.ts | 4 +- .../form/admin-form/admin-form.controller.ts | 2 +- .../form/admin-form/admin-form.service.ts | 69 ++++++++++++++++--- src/app/modules/form/form.service.ts | 8 +-- src/types/form_whitelisted_submitter_ids.ts | 2 +- 8 files changed, 95 insertions(+), 27 deletions(-) diff --git a/frontend/src/features/admin-form/settings/components/AuthSettingsSection/FormWhitelistAttachmentField.tsx b/frontend/src/features/admin-form/settings/components/AuthSettingsSection/FormWhitelistAttachmentField.tsx index 120125e977..372534e336 100644 --- a/frontend/src/features/admin-form/settings/components/AuthSettingsSection/FormWhitelistAttachmentField.tsx +++ b/frontend/src/features/admin-form/settings/components/AuthSettingsSection/FormWhitelistAttachmentField.tsx @@ -25,7 +25,7 @@ interface FormWhitelistAttachmentFieldProps { isDisabled: boolean } -const MAX_SIZE_IN_BYTES = 3 * MB +const MAX_SIZE_IN_BYTES = 12 * MB const FormWhitelistAttachmentFieldContainerName = 'whitelist-csv-attachment-field-container' const FormWhitelistAttachmentFieldName = 'whitelist-csv-attachment-field' diff --git a/shared/types/form/form.ts b/shared/types/form/form.ts index b88f8dea98..f34475ad76 100644 --- a/shared/types/form/form.ts +++ b/shared/types/form/form.ts @@ -215,7 +215,7 @@ export interface WhitelistedSubmitterIds { export interface WhitelistedSubmitterIdsWithReferenceOid extends WhitelistedSubmitterIds { - encryptedWhitelistedSubmitterIds: string // Object id of the encrypted whitelist + encryptedWhitelistedSubmitterIds: string[] // Object ids of the encrypted whitelist } export interface StorageFormBase extends FormBase { diff --git a/src/app/models/form.server.model.ts b/src/app/models/form.server.model.ts index bbe1360902..dd9df405ca 100644 --- a/src/app/models/form.server.model.ts +++ b/src/app/models/form.server.model.ts @@ -201,12 +201,14 @@ const whitelistedSubmitterIdNestedPath = { required: true, default: false, }, - encryptedWhitelistedSubmitterIds: { - type: Schema.Types.ObjectId, - ref: FORM_WHITELISTED_SUBMITTER_IDS_ID, - required: false, - default: undefined, - }, + encryptedWhitelistedSubmitterIds: [ + { + type: Schema.Types.ObjectId, + ref: FORM_WHITELISTED_SUBMITTER_IDS_ID, + required: false, + default: undefined, + }, + ], _id: { id: false }, } @@ -282,9 +284,26 @@ const EncryptedFormDocumentSchema = EncryptedFormSchema as unknown as Schema EncryptedFormDocumentSchema.methods.getWhitelistedSubmitterIds = function () { - return this.get('whitelistedSubmitterIds', null, { + const whitelistedSubmitterIds = this.get('whitelistedSubmitterIds', null, { getters: false, }) + + // NOTE: this is to support legacy format for encryptedWhitelistedSubmitterIds which only stores 1 document. + const whitelistedSubmitterIdsObject = whitelistedSubmitterIds + if ( + !Array.isArray( + whitelistedSubmitterIdsObject.encryptedWhitelistedSubmitterIds, + ) + ) { + return { + ...whitelistedSubmitterIdsObject, + encryptedWhitelistedSubmitterIds: [ + whitelistedSubmitterIdsObject.encryptedWhitelistedSubmitterIds, + ], + } + } + + return whitelistedSubmitterIdsObject } EncryptedFormDocumentSchema.methods.addPaymentAccountId = function ({ diff --git a/src/app/models/form_whitelist.server.model.ts b/src/app/models/form_whitelist.server.model.ts index 799b7f5b2c..b129192f97 100644 --- a/src/app/models/form_whitelist.server.model.ts +++ b/src/app/models/form_whitelist.server.model.ts @@ -44,9 +44,9 @@ const formWhitelistedSubmitterIdsSchema = new Schema< }) formWhitelistedSubmitterIdsSchema.statics.checkIfSubmitterIdIsWhitelisted = - async function (whitelistId: string, submitterId: string) { + async function (whitelistId: string[], submitterId: string) { return this.exists({ - _id: whitelistId, + _id: { $in: whitelistId }, cipherTexts: submitterId, }).exec() } diff --git a/src/app/modules/form/admin-form/admin-form.controller.ts b/src/app/modules/form/admin-form/admin-form.controller.ts index 5524540ed4..bd7a551409 100644 --- a/src/app/modules/form/admin-form/admin-form.controller.ts +++ b/src/app/modules/form/admin-form/admin-form.controller.ts @@ -1656,7 +1656,7 @@ export const handleDeleteWorkflowStep: ControllerHandler< ) } -const LIMIT_IN_MB = 3 +const LIMIT_IN_MB = 12 const STRING_MAX_LENGTH = LIMIT_IN_MB * MB const _handleUpdateWhitelistSettingValidator = celebrate({ [Segments.PARAMS]: Joi.object({ diff --git a/src/app/modules/form/admin-form/admin-form.service.ts b/src/app/modules/form/admin-form/admin-form.service.ts index 74545217c0..c6c3a541bf 100644 --- a/src/app/modules/form/admin-form/admin-form.service.ts +++ b/src/app/modules/form/admin-form/admin-form.service.ts @@ -1388,6 +1388,22 @@ export const checkIsWhitelistSettingValid = ( } } +const combineFormWhitelistedSubmitterIdsBuckets = ( + whitelistSettingBuckets: EncryptedStringsMessageContent[], +): EncryptedStringsMessageContent => { + const whitelistedSubmitterIds = whitelistSettingBuckets.reduce( + (acc, bucket) => { + return acc.concat(bucket.cipherTexts) + }, + [] as string[], + ) + return { + myPublicKey: whitelistSettingBuckets[0].myPublicKey, + nonce: whitelistSettingBuckets[0].nonce, + cipherTexts: whitelistedSubmitterIds, + } +} + /** * Fetches the whitelist setting document without myPrivateKey for the client to use for decryption. */ @@ -1403,15 +1419,17 @@ export const getFormWhitelistSetting = ( if (!isWhitelistEnabled) { return okAsync(null) } - if (isWhitelistEnabled && !encryptedWhitelistedSubmitterIds) { return errAsync(new FormWhitelistSettingNotFoundError()) } return ResultAsync.fromPromise( - FormWhitelistedSubmitterIdsModel.findById(encryptedWhitelistedSubmitterIds) + FormWhitelistedSubmitterIdsModel.where({ + _id: { $in: encryptedWhitelistedSubmitterIds }, + }) .lean() .exec() + .then(combineFormWhitelistedSubmitterIdsBuckets) .then((whitelistSetting) => pick(whitelistSetting, WHITELISTED_SUBMITTER_ID_DECRYPTION_FIELDS), ) as Promise, @@ -1434,6 +1452,26 @@ export const getFormWhitelistSetting = ( }) } +const NUM_SUBMITTER_IDS_PER_BUCKET = 200_000 +const getFormWhitelistedSubmitterIdsBuckets = ( + encryptedWhitelistedSubmitterIdsContent: EncryptedStringsMessageContentWithMyPrivateKey, +) => { + const buckets: string[][] = [] + for ( + let i = 0; + i < encryptedWhitelistedSubmitterIdsContent.cipherTexts.length; + i += NUM_SUBMITTER_IDS_PER_BUCKET + ) { + buckets.push( + encryptedWhitelistedSubmitterIdsContent.cipherTexts.slice( + i, + i + NUM_SUBMITTER_IDS_PER_BUCKET, + ), + ) + } + return buckets +} + export const updateFormWhitelistSetting = ( originalForm: IPopulatedForm, encryptedWhitelistedSubmitterIdsContent: EncryptedStringsMessageContentWithMyPrivateKey | null, @@ -1453,19 +1491,30 @@ export const updateFormWhitelistSetting = ( session.startTransaction() if (encryptedWhitelistedSubmitterIdsContent) { - // create whitelisted submitter id collection document and update reference to it - const createdWhitelistedSubmitterIdsDocument = - await FormWhitelistedSubmitterIdsModel.create({ - formId: originalForm._id, - ...encryptedWhitelistedSubmitterIdsContent, - }) + const whitelistedSubmitterIdsBuckets = + getFormWhitelistedSubmitterIdsBuckets( + encryptedWhitelistedSubmitterIdsContent, + ) + + const documentIds = await FormWhitelistedSubmitterIdsModel.insertMany( + whitelistedSubmitterIdsBuckets.map((bucket) => { + const bucketContent = { + ...encryptedWhitelistedSubmitterIdsContent, + cipherTexts: bucket, + } + return { + formId: originalForm._id, + ...bucketContent, + } + }), + ) + const updatedForm = await FormModelToUse.findByIdAndUpdate( originalForm._id, { whitelistedSubmitterIds: { isWhitelistEnabled: true, - encryptedWhitelistedSubmitterIds: - createdWhitelistedSubmitterIdsDocument._id, + encryptedWhitelistedSubmitterIds: documentIds, }, }, { diff --git a/src/app/modules/form/form.service.ts b/src/app/modules/form/form.service.ts index 67461ca653..3556eec4d2 100644 --- a/src/app/modules/form/form.service.ts +++ b/src/app/modules/form/form.service.ts @@ -292,13 +292,13 @@ export const checkHasRespondentNotWhitelistedFailure = ( return okAsync(false) } - const { isWhitelistEnabled, encryptedWhitelistedSubmitterIds: whitelistId } = + const { isWhitelistEnabled, encryptedWhitelistedSubmitterIds: whitelistIds } = form.getWhitelistedSubmitterIds() if (!isWhitelistEnabled) { return okAsync(false) } - if (isWhitelistEnabled && !whitelistId) { + if (isWhitelistEnabled && (!whitelistIds || whitelistIds.length <= 0)) { return errAsync(new FormWhitelistSettingNotFoundError()) } @@ -317,7 +317,7 @@ export const checkHasRespondentNotWhitelistedFailure = ( } return ResultAsync.fromPromise( FormWhitelistSubmitterIdsModel.findEncryptionPropertiesById( - whitelistId, + whitelistIds[0], ).then(({ myPublicKey, myPrivateKey, nonce }) => { const myKeyPair = { publicKey: myPublicKey, @@ -335,7 +335,7 @@ export const checkHasRespondentNotWhitelistedFailure = ( ).cipherText return FormWhitelistSubmitterIdsModel.checkIfSubmitterIdIsWhitelisted( - whitelistId, + whitelistIds, submitterIdForLookup, ).then((isWhitelisted) => !isWhitelisted) }), diff --git a/src/types/form_whitelisted_submitter_ids.ts b/src/types/form_whitelisted_submitter_ids.ts index 8cea89409b..c44560f9ca 100644 --- a/src/types/form_whitelisted_submitter_ids.ts +++ b/src/types/form_whitelisted_submitter_ids.ts @@ -22,7 +22,7 @@ export interface IFormWhitelistedSubmitterIdsModel > checkIfSubmitterIdIsWhitelisted( - whitelistId: string, + whitelistId: string[], submitterId: string, ): Promise }