saving json now in chunks

esbuild.js

@@ -64,7 +64,9 @@ esbuild.build({
 		"misc/perMinuteMerger.ts",
 	],
 	plugins: [
-		nodeExternalsPlugin()
+		nodeExternalsPlugin({
+			packagePath: "package.json",
+		})
 	],
 	...makeGeneralConfig("Node TS", true, "node")
 })

src/redditApi.ts

@@ -68,6 +68,8 @@ export class RedditAuth {
 	private rateLimitCheck(headers: Headers) {
 		const rlReqRemaining = parseInt(headers.get("x-ratelimit-remaining"));
 		const rlTimeRemaining = parseInt(headers.get("x-ratelimit-reset"));
+		if (isNaN(rlReqRemaining) || isNaN(rlTimeRemaining))
+			return;
 		if (rlReqRemaining < 30)
 			console.log(`Rate limit: ${rlReqRemaining} requests remaining, ${rlTimeRemaining} seconds until reset`);
 	}

src/utils.ts

@@ -1,4 +1,6 @@
 import fs, {promises as fsp} from "fs";
+import stream, {Readable} from 'stream';
+import {promisify} from 'util';
 
 
 export async function fileExists(path: string): Promise<boolean> {
@@ -10,7 +12,55 @@ export async function fileExists(path: string): Promise<boolean> {
 	}
 }
 
-export async function saveJsonSafely(obj: any, file: string): Promise<void> {
+function objectOrArrayToStream(data: Record<string, unknown> | unknown[]): Readable {
+	let entries: [string, unknown][] | unknown[];
+	let startChar: string, endChar: string, isObject: boolean;
+
+	if (Array.isArray(data)) {
+		entries = data;
+		startChar = '[';
+		endChar = ']';
+		isObject = false;
+	} else {
+		entries = Object.entries(data);
+		startChar = '{';
+		endChar = '}';
+		isObject = true;
+	}
+
+	let index = 0;
+	return new Readable({
+		objectMode: true,
+		read() {
+			if (index === 0)
+				this.push(startChar);
+			else if (index < entries.length)
+				this.push(',');
+
+			if (index < entries.length) {
+				if (isObject) {
+					let [key, value] = entries[index] as [string, unknown];
+					this.push(JSON.stringify({[key]: value}).replace(/^\{|}$/g, ''));
+				} else {
+					this.push(JSON.stringify(entries[index]));
+				}
+				index += 1;
+			} else {
+				this.push(endChar);
+				this.push(null);
+			}
+		},
+	});
+}
+
+function saveBigJson(file: string, obj: Record<string, unknown> | unknown[]): Promise<void> {
+	const writeStream = fs.createWriteStream(file);
+	const readStream = objectOrArrayToStream(obj);
+
+	return promisify(stream.pipeline)(readStream, writeStream);
+}
+
+export async function saveJsonSafely(obj: Record<string, unknown> | unknown[], file: string): Promise<void> {
 	try {
 		// backup old file
 		if (await fileExists(file)) {
@@ -19,14 +69,16 @@ export async function saveJsonSafely(obj: any, file: string): Promise<void> {
 				await fsp.copyFile(file, backupFile);
 			}
 		}
-		const json = JSON.stringify(obj);
 		// save new file to temp file
 		const tempFile = file + ".tmp";
-		await fsp.writeFile(tempFile, json);
+		const t1 = Date.now();
+		await saveBigJson(tempFile, obj);
+		const t2 = Date.now();
+		console.log(`Saved ${file} in ${t2 - t1}ms`);
 		// rename temp file to actual file
 		await fsp.rename(tempFile, file);
 	} catch (e) {
-		console.error("Error saving json file", e);
+		console.error("Error saving json file");
 		console.error(e);
 	}
 }