From a0fbb70e0d21d78f8c395ac19f37d10ab964b91a Mon Sep 17 00:00:00 2001
From: Ahmed Farghal <me@asoli.dev>
Date: Tue, 4 Jun 2024 08:09:17 +0100
Subject: [PATCH] Allow bifrost wal fsync to be disabled

---
 crates/bifrost/src/loglets/local_loglet/log_store.rs      | 1 +
 .../bifrost/src/loglets/local_loglet/log_store_writer.rs  | 3 +--
 crates/rocksdb/src/db_manager.rs                          | 3 ++-
 crates/types/src/config/bifrost.rs                        | 8 ++++++++
 4 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/crates/bifrost/src/loglets/local_loglet/log_store.rs b/crates/bifrost/src/loglets/local_loglet/log_store.rs
index 141d69fca..b955cc1d5 100644
--- a/crates/bifrost/src/loglets/local_loglet/log_store.rs
+++ b/crates/bifrost/src/loglets/local_loglet/log_store.rs
@@ -129,6 +129,7 @@ fn db_options(options: &LocalLogletOptions) -> rocksdb::Options {
     if options.rocksdb.rocksdb_disable_wal() {
         opts.set_atomic_flush(true);
     }
+    opts.set_wal_recovery_mode(rocksdb::DBRecoveryMode::AbsoluteConsistency);
 
     opts
 }
diff --git a/crates/bifrost/src/loglets/local_loglet/log_store_writer.rs b/crates/bifrost/src/loglets/local_loglet/log_store_writer.rs
index 5bdbb03a6..777735463 100644
--- a/crates/bifrost/src/loglets/local_loglet/log_store_writer.rs
+++ b/crates/bifrost/src/loglets/local_loglet/log_store_writer.rs
@@ -243,8 +243,7 @@ impl LogStoreWriter {
     async fn commit(&mut self, opts: &LocalLogletOptions, write_batch: WriteBatch) {
         let mut write_opts = rocksdb::WriteOptions::new();
         write_opts.disable_wal(opts.rocksdb.rocksdb_disable_wal());
-        // if WAL is enabled, we sync after every write.
-        write_opts.set_sync(!opts.rocksdb.rocksdb_disable_wal());
+        write_opts.set_sync(!opts.rocksdb_disable_wal_fsync());
 
         trace!(
             "Committing local loglet current write batch: {} items",
diff --git a/crates/rocksdb/src/db_manager.rs b/crates/rocksdb/src/db_manager.rs
index 00401ba2b..7c5a6d70c 100644
--- a/crates/rocksdb/src/db_manager.rs
+++ b/crates/rocksdb/src/db_manager.rs
@@ -286,7 +286,8 @@ impl RocksDbManager {
 
         // no need to retain 1000 log files by default.
         //
-        db_options.set_keep_log_file_num(1);
+        db_options.set_keep_log_file_num(2);
+        db_options.set_recycle_log_file_num(4);
         // Disable WAL archiving.
         // the following two options has to be both 0 to disable WAL log archive.
         db_options.set_wal_size_limit_mb(0);
diff --git a/crates/types/src/config/bifrost.rs b/crates/types/src/config/bifrost.rs
index cc5fabec0..d3520e159 100644
--- a/crates/types/src/config/bifrost.rs
+++ b/crates/types/src/config/bifrost.rs
@@ -70,6 +70,9 @@ pub struct LocalLogletOptions {
     /// (See `rocksdb-total-memtables-ratio` in common).
     rocksdb_memory_ratio: f32,
 
+    /// Disable fsync of WAL on every batch
+    rocksdb_disable_wal_fsync: bool,
+
     /// Trigger a commit when the batch size exceeds this threshold.
     ///
     /// Set to 0 or 1 to commit the write batch on every command.
@@ -97,6 +100,10 @@ impl LocalLogletOptions {
         }
     }
 
+    pub fn rocksdb_disable_wal_fsync(&self) -> bool {
+        self.rocksdb_disable_wal_fsync
+    }
+
     pub fn rocksdb_memory_budget(&self) -> usize {
         self.rocksdb_memory_budget
             .unwrap_or_else(|| {
@@ -125,6 +132,7 @@ impl Default for LocalLogletOptions {
             rocksdb_memory_ratio: 0.5,
             writer_batch_commit_count: 2000,
             writer_batch_commit_duration: Duration::ZERO.into(),
+            rocksdb_disable_wal_fsync: false,
         }
     }
 }