Change error checks in K6 tests

k6/run-all-tests.sh

@@ -1,7 +1,5 @@
-
 # Array to collect failed tests
 failed_tests=()
-
 for file in tests/*.js; do
   echo "Test: $file"
   echo "Starting services"
@@ -12,7 +10,6 @@ for file in tests/*.js; do
   # Log the contents of summary.json for debugging
   echo "Contents of summary.json:"
   cat summary.json
-
   # Check if there are any failed checks
   if [ $(jq '.metrics.checks.fails' summary.json) -gt 0 ]; then
       echo "Test failed: $file"
@@ -21,7 +18,6 @@ for file in tests/*.js; do
   echo "Stopping services"
   (cd ../services ; docker compose -f docker-compose.yml down)
 done
-
 # Check if there were any failed tests
 if [ ${#failed_tests[@]} -ne 0 ]; then
   echo "The following tests failed:"

k6/tests/getProgramWithManyAttributes.js

@@ -1,4 +1,5 @@
-import { check, sleep } from 'k6';
+import { check, fail, sleep } from 'k6';
+import { Counter } from 'k6/metrics';
 
 import { registrationVisa } from '../helpers/registration-default.data.js';
 import loginModel from '../models/login.js';
@@ -18,22 +19,33 @@ const programId = 2;
 export const options = {
   thresholds: {
     http_req_failed: ['rate<0.01'], // http errors should be less than 1%
+    failed_checks: ['count<1'], // fail the test if any check fails
   },
   vus: 1,
-  duration: '20m',
+  duration: '30s',
   iterations: 1,
 };
 
+const failedChecks = new Counter('failed_checks');
+
+function checkAndFail(response, checks) {
+  const result = check(response, checks);
+  if (!result) {
+    failedChecks.add(1);
+    fail('One or more checks failed');
+  }
+}
+
 export default function () {
   // reset db
   const reset = resetPage.resetDB(resetScript);
-  check(reset, {
+  checkAndFail(reset, {
     'Reset succesfull status was 202': (r) => r.status == 202,
   });
 
   // login
   const login = loginPage.login();
-  check(login, {
+  checkAndFail(login, {
     'Login succesfull status was 200': (r) => r.status == 201,
     'Login time is less than 200ms': (r) => {
       if (r.timings.duration >= 200) {
@@ -49,7 +61,7 @@ export default function () {
       programsPage.createProgramRegistrationAttribute(programId, attributeName);
     registrationVisa[attributeName] = 'bla';
 
-    check(programRegistrationAttributes, {
+    checkAndFail(programRegistrationAttributes, {
       'Program registration attributes added successfully status was 201': (
         r,
       ) => {
@@ -66,20 +78,20 @@ export default function () {
     programId,
     registrationVisa,
   );
-  check(registrationImport, {
+  checkAndFail(registrationImport, {
     'Import of registration successful status was 201': (r) => r.status == 201,
   });
 
   // Duplicate registration
   const duplicateRegistration =
     resetPage.duplicateRegistrations(duplicateNumber);
-  check(duplicateRegistration, {
+  checkAndFail(duplicateRegistration, {
     'Duplication successful status was 201': (r) => r.status == 201,
   });
 
   // get program by id and validte load time is less than 200ms
   const program = programsPage.getProgramById(2);
-  check(program, {
+  checkAndFail(program, {
     'Programme loaded succesfully status was 200': (r) => r.status == 200,
     'Programme load time is less than 200ms': (r) => {
       if (r.timings.duration >= 200) {

k6/tests/import1000Registrations.js

@@ -1,4 +1,5 @@
-import { check, sleep } from 'k6';
+import { check, fail, sleep } from 'k6';
+import { Counter } from 'k6/metrics';
 
 import LoginModel from '../models/login.js';
 import RegistrationsModel from '../models/registrations.js';
@@ -24,23 +25,34 @@ if (!csvFile) {
 export const options = {
   thresholds: {
     http_req_failed: ['rate<0.01'], // http errors should be less than 1%
+    failed_checks: ['count<1'], // fail the test if any check fails
   },
   vus: 1,
   iterations: 1,
   // REFACTOR: should we investigate if the duration can be reduced?
   duration: '9m', // At the time of writing this test, this test took ~7m both locally and on GH actions. Setting the limit to 9m, so it's below the API timeout limit of10m. Change this value only deliberatedly. If the tests takes longer because of regression effects, it should fail.
 };
 
+const failedChecks = new Counter('failed_checks');
+
+function checkAndFail(response, checks) {
+  const result = check(response, checks);
+  if (!result) {
+    failedChecks.add(1);
+    fail('One or more checks failed');
+  }
+}
+
 export default function () {
   // reset db
   const reset = resetPage.resetDB(resetScript);
-  check(reset, {
+  checkAndFail(reset, {
     'Reset successful status was 202': (r) => r.status == 202,
   });
 
   // login
   const login = loginPage.login();
-  check(login, {
+  checkAndFail(login, {
     'Login successful status was 200': (r) => r.status == 201,
     'Login time is less than 200ms': (r) => {
       if (r.timings.duration >= 200) {
@@ -55,7 +67,7 @@ export default function () {
     programId,
     csvFile,
   );
-  check(registrationImport, {
+  checkAndFail(registrationImport, {
     'Import of registration successful status was 201': (r) => r.status == 201,
   });
 

k6/tests/payment100kRegistrationOCW.js

@@ -1,4 +1,5 @@
-import { check, sleep } from 'k6';
+import { check, fail, sleep } from 'k6';
+import { Counter } from 'k6/metrics';
 
 import { registrationVisa } from '../helpers/registration-default.data.js';
 import InitializePaymentModel from '../models/initalize-payment.js';
@@ -9,19 +10,30 @@ const duplicateNumber = 17; // '17' leads to 131k registrations
 const resetScript = 'nlrc-multiple';
 const programId = 3;
 const paymentId = 3;
-const maxTimeoutAttempts = 200;
+const maxTimeoutAttempts = 800;
 const minPassRatePercentage = 10;
-const amount = 11.11; // Using an amount with cents. To ensure we handle javascript floating point precision issues
+const amount = 11.11;
 
 export const options = {
   thresholds: {
     http_req_failed: ['rate<0.01'], // http errors should be less than 1%
+    failed_checks: ['count<1'], // fail the test if any check fails
   },
   vus: 1,
-  duration: '20m',
+  duration: '80m',
   iterations: 1,
 };
 
+const failedChecks = new Counter('failed_checks');
+
+function checkAndFail(response, checks) {
+  const result = check(response, checks);
+  if (!result) {
+    failedChecks.add(1);
+    fail('One or more checks failed');
+  }
+}
+
 export default function () {
   const monitorPayment = initializePayment.initializePayment(
     resetScript,
@@ -33,12 +45,13 @@ export default function () {
     minPassRatePercentage,
     amount,
   );
-  check(monitorPayment, {
+  checkAndFail(monitorPayment, {
     'Payment progressed successfully status 200': (r) => {
       if (r.status != 200) {
         const responseBody = JSON.parse(r.body);
         console.log(responseBody.error || r.status);
       }
+      console.log('status: ', r.status);
       return r.status == 200;
     },
   });

k6/tests/payment100kRegistrationSafaricom.js

@@ -1,27 +1,39 @@
-import { check, sleep } from 'k6';
+import { check, fail, sleep } from 'k6';
+import { Counter } from 'k6/metrics';
 
 import { registrationSafaricom } from '../helpers/registration-default.data.js';
 import InitializePaymentModel from '../models/initalize-payment.js';
 
 const initializePayment = new InitializePaymentModel();
 
-const duplicateNumber = 17; // '17' leads to 131k registrations
+const duplicateNumber = 7; // '17' leads to 131k registrations
 const resetScript = 'safari-program';
 const programId = 1;
 const paymentId = 3;
-const maxTimeoutAttempts = 200;
+const maxTimeoutAttempts = 800;
 const minPassRatePercentage = 10;
 const amount = 10;
 
 export const options = {
   thresholds: {
     http_req_failed: ['rate<0.01'], // http errors should be less than 1%
+    failed_checks: ['count<1'], // fail the test if any check fails
   },
   vus: 1,
-  duration: '20m',
+  duration: '80m',
   iterations: 1,
 };
 
+const failedChecks = new Counter('failed_checks');
+
+function checkAndFail(response, checks) {
+  const result = check(response, checks);
+  if (!result) {
+    failedChecks.add(1);
+    fail('One or more checks failed');
+  }
+}
+
 export default function () {
   const monitorPayment = initializePayment.initializePayment(
     resetScript,
@@ -33,7 +45,7 @@ export default function () {
     minPassRatePercentage,
     amount,
   );
-  check(monitorPayment, {
+  checkAndFail(monitorPayment, {
     'Payment progressed successfully status 200': (r) => {
       if (r.status != 200) {
         const responseBody = JSON.parse(r.body);

k6/tests/performanceDuringPayment.js

@@ -1,4 +1,5 @@
-import { check, sleep } from 'k6';
+import { check, fail, sleep } from 'k6';
+import { Counter } from 'k6/metrics';
 
 import loginModel from '../models/login.js';
 import metricstsModel from '../models/metrics.js';
@@ -12,32 +13,43 @@ const loginPage = new loginModel();
 const programsPage = new programsModel();
 const metricsPage = new metricstsModel();
 
-const duplicateNumber = 15;
+const duplicateNumber = 5;
 const programId = 3;
 const paymentId = 3;
-const maxTimeoutAttempts = 400;
+const maxTimeoutAttempts = 600;
 const minPassRatePercentage = 50;
-const amount = 10;
+const amount = 11.11;
 
 export const options = {
   thresholds: {
     http_req_failed: ['rate<0.01'], // http errors should be less than 1%
+    failed_checks: ['count<1'], // fail the test if any check fails
   },
   vus: 1,
-  duration: '40m',
+  duration: '60m',
   iterations: 1,
 };
 
+const failedChecks = new Counter('failed_checks');
+
+function checkAndFail(response, checks) {
+  const result = check(response, checks);
+  if (!result) {
+    failedChecks.add(1);
+    fail('One or more checks failed');
+  }
+}
+
 export default function () {
   // reset db
   const reset = resetPage.resetDBMockRegistrations(duplicateNumber);
-  check(reset, {
+  checkAndFail(reset, {
     'Reset succesfull status was 202': (r) => r.status == 202,
   });
 
   // login
   const login = loginPage.login();
-  check(login, {
+  checkAndFail(login, {
     'Login succesfull status was 200': (r) => r.status == 201,
     'Login time is less than 200ms': (r) => {
       if (r.timings.duration >= 200) {
@@ -49,7 +61,7 @@ export default function () {
 
   // Do the payment
   const doPayment = paymentsPage.createPayment(programId, amount);
-  check(doPayment, {
+  checkAndFail(doPayment, {
     'Payment successfully done status 202': (r) => {
       if (r.status != 202) {
         console.log(r.body);
@@ -65,8 +77,9 @@ export default function () {
     paymentId,
     duplicateNumber,
     minPassRatePercentage,
+    amount,
   );
-  check(monitorPayment, {
+  checkAndFail(monitorPayment, {
     'Payment progressed successfully status 200': (r) => {
       if (r.status != 200) {
         const responseBody = JSON.parse(r.body);
@@ -78,13 +91,13 @@ export default function () {
 
   // get export list
   const exportList = metricsPage.getExportList(3);
-  check(exportList, {
+  checkAndFail(exportList, {
     'Export list loaded succesfully status was 200': (r) => r.status == 200,
   });
 
   // send bulk message
   const message = programsPage.sendBulkMessage(3);
-  check(message, {
+  checkAndFail(message, {
     'Message sent succesfully status was 202': (r) => r.status == 202,
   });