From a4071ee6e5ef9aad8f0aaccae85cc4eae69ab298 Mon Sep 17 00:00:00 2001 From: TOUATI Osema Date: Mon, 20 Jan 2025 14:41:58 +0100 Subject: [PATCH 1/5] fix: Fixing logger for custom S3 stream --- packages/server/src/utils/logger.ts | 34 +++++++++++++++++------------ 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/packages/server/src/utils/logger.ts b/packages/server/src/utils/logger.ts index 66078e93ca8..1040b32addf 100644 --- a/packages/server/src/utils/logger.ts +++ b/packages/server/src/utils/logger.ts @@ -17,32 +17,38 @@ if (process.env.STORAGE_TYPE === 's3') { const secretAccessKey = process.env.S3_STORAGE_SECRET_ACCESS_KEY const region = process.env.S3_STORAGE_REGION const s3Bucket = process.env.S3_STORAGE_BUCKET_NAME + const customURL = process.env.S3_ENDPOINT_URL + const forcePathStyle = process.env.S3_FORCE_PATH_STYLE === 'true' + + const s3Config = { + region: region, + endpoint: customURL, + s3ForcePathStyle: forcePathStyle, + credentials: { + accessKeyId: accessKeyId, + secretAccessKey: secretAccessKey + } + } s3ServerStream = new S3StreamLogger({ bucket: s3Bucket, folder: 'logs/server', - region, - access_key_id: accessKeyId, - secret_access_key: secretAccessKey, - name_format: `server-%Y-%m-%d-%H-%M-%S-%L-${hostname()}.log` + name_format: `server-%Y-%m-%d-%H-%M-%S-%L-${hostname()}.log`, + config: s3Config }) s3ErrorStream = new S3StreamLogger({ bucket: s3Bucket, - folder: 'logs/error', - region, - access_key_id: accessKeyId, - secret_access_key: secretAccessKey, - name_format: `server-error-%Y-%m-%d-%H-%M-%S-%L-${hostname()}.log` + folder: 'logs/server', + name_format: `server-error-%Y-%m-%d-%H-%M-%S-%L-${hostname()}.log`, + config: s3Config }) s3ServerReqStream = new S3StreamLogger({ bucket: s3Bucket, - folder: 'logs/requests', - region, - access_key_id: accessKeyId, - secret_access_key: secretAccessKey, - name_format: `server-requests-%Y-%m-%d-%H-%M-%S-%L-${hostname()}.log.jsonl` + folder: 'logs/server', + name_format: `server-requests-%Y-%m-%d-%H-%M-%S-%L-${hostname()}.log`, + config: s3Config }) } From 080ef127b012ab855feda8c1684a8502f30f6655 Mon Sep 17 00:00:00 2001 From: TOUATI Osema Date: Mon, 20 Jan 2025 14:53:30 +0100 Subject: [PATCH 2/5] fix: adjusting s3 stream folder --- packages/server/src/utils/logger.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/utils/logger.ts b/packages/server/src/utils/logger.ts index 1040b32addf..a80f78020c1 100644 --- a/packages/server/src/utils/logger.ts +++ b/packages/server/src/utils/logger.ts @@ -39,14 +39,14 @@ if (process.env.STORAGE_TYPE === 's3') { s3ErrorStream = new S3StreamLogger({ bucket: s3Bucket, - folder: 'logs/server', + folder: 'logs/error', name_format: `server-error-%Y-%m-%d-%H-%M-%S-%L-${hostname()}.log`, config: s3Config }) s3ServerReqStream = new S3StreamLogger({ bucket: s3Bucket, - folder: 'logs/server', + folder: 'logs/requests', name_format: `server-requests-%Y-%m-%d-%H-%M-%S-%L-${hostname()}.log`, config: s3Config }) From b099b6ad597f9302f5409c8a2e417444bfef6178 Mon Sep 17 00:00:00 2001 From: TOUATI Osema Date: Mon, 20 Jan 2025 16:50:11 +0100 Subject: [PATCH 3/5] fix: fixing forcedPathStyle variable name. --- packages/server/src/utils/logger.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/utils/logger.ts b/packages/server/src/utils/logger.ts index a80f78020c1..f3017d49d7c 100644 --- a/packages/server/src/utils/logger.ts +++ b/packages/server/src/utils/logger.ts @@ -23,7 +23,7 @@ if (process.env.STORAGE_TYPE === 's3') { const s3Config = { region: region, endpoint: customURL, - s3ForcePathStyle: forcePathStyle, + forcePathStyle: forcePathStyle, credentials: { accessKeyId: accessKeyId, secretAccessKey: secretAccessKey From 75f58d2d4351014c8bf9f9e37788c4f50cdb0fb4 Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Mon, 20 Jan 2025 18:30:34 +0000 Subject: [PATCH 4/5] Update logger.ts --- packages/server/src/utils/logger.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/utils/logger.ts b/packages/server/src/utils/logger.ts index f3017d49d7c..9924c8c108b 100644 --- a/packages/server/src/utils/logger.ts +++ b/packages/server/src/utils/logger.ts @@ -47,7 +47,7 @@ if (process.env.STORAGE_TYPE === 's3') { s3ServerReqStream = new S3StreamLogger({ bucket: s3Bucket, folder: 'logs/requests', - name_format: `server-requests-%Y-%m-%d-%H-%M-%S-%L-${hostname()}.log`, + name_format: `server-requests-%Y-%m-%d-%H-%M-%S-%L-${hostname()}.log.jsonl`, config: s3Config }) } From 61f02af49c9672c62857537815597cca717b940b Mon Sep 17 00:00:00 2001 From: Henry Heng Date: Mon, 20 Jan 2025 18:36:26 +0000 Subject: [PATCH 5/5] add S3ClientConfig types --- packages/server/src/utils/logger.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/server/src/utils/logger.ts b/packages/server/src/utils/logger.ts index 9924c8c108b..207f7ce34e9 100644 --- a/packages/server/src/utils/logger.ts +++ b/packages/server/src/utils/logger.ts @@ -4,6 +4,7 @@ import { hostname } from 'node:os' import config from './config' // should be replaced by node-config or similar import { createLogger, transports, format } from 'winston' import { NextFunction, Request, Response } from 'express' +import { S3ClientConfig } from '@aws-sdk/client-s3' const { S3StreamLogger } = require('s3-streamlogger') @@ -20,13 +21,13 @@ if (process.env.STORAGE_TYPE === 's3') { const customURL = process.env.S3_ENDPOINT_URL const forcePathStyle = process.env.S3_FORCE_PATH_STYLE === 'true' - const s3Config = { + const s3Config: S3ClientConfig = { region: region, endpoint: customURL, forcePathStyle: forcePathStyle, credentials: { - accessKeyId: accessKeyId, - secretAccessKey: secretAccessKey + accessKeyId: accessKeyId as string, + secretAccessKey: secretAccessKey as string } }