Merge pull request #472 from gyuho/initial_mmap

Introduce InitialMmapSize to prevent deadlock

db.go

@@ -196,7 +196,7 @@ func Open(path string, mode os.FileMode, options *Options) (*DB, error) {
 	}
 
 	// Memory map the data file.
-	if err := db.mmap(0); err != nil {
+	if err := db.mmap(options.InitialMmapSize); err != nil {
 		_ = db.close()
 		return nil, err
 	}
@@ -411,6 +411,10 @@ func (db *DB) close() error {
 // writer to deadlock because the database periodically needs to re-mmap itself
 // as it grows and it cannot do that while a read transaction is open.
 //
+// If a long running read transaction (for example, a snapshot transaction) is
+// needed, you might want to set DB.InitialMmapSize to a large enough value
+// to avoid potential blocking of write transaction.
+//
 // IMPORTANT: You must close read-only transactions after you are finished or
 // else the database will not reclaim old pages.
 func (db *DB) Begin(writable bool) (*Tx, error) {
@@ -680,6 +684,16 @@ type Options struct {
 
 	// Sets the DB.MmapFlags flag before memory mapping the file.
 	MmapFlags int
+
+	// InitialMmapSize is the initial mmap size of the database
+	// in bytes. Read transactions won't block write transaction
+	// if the InitialMmapSize is large enough to hold database mmap
+	// size. (See DB.Begin for more information)
+	//
+	// If <=0, the initial map size is 0.
+	// If initialMmapSize is smaller than the previous database size,
+	// it takes no effect.
+	InitialMmapSize int
 }
 
 // DefaultOptions represent the options used if nil options are passed into Open().

db_test.go

@@ -368,6 +368,51 @@ func TestOpen_ReadOnly(t *testing.T) {
 	}
 }
 
+// TestDB_Open_InitialMmapSize tests if having InitialMmapSize large enough
+// to hold data from concurrent write transaction resolves the issue that
+// read transaction blocks the write transaction and causes deadlock.
+// This is a very hacky test since the mmap size is not exposed.
+func TestDB_Open_InitialMmapSize(t *testing.T) {
+	path := tempfile()
+	defer os.Remove(path)
+
+	initMmapSize := 1 << 31  // 2GB
+	testWriteSize := 1 << 27 // 134MB
+
+	db, err := bolt.Open(path, 0666, &bolt.Options{InitialMmapSize: initMmapSize})
+	assert(t, err == nil, "")
+
+	// create a long-running read transaction
+	// that never gets closed while writing
+	rtx, err := db.Begin(false)
+	assert(t, err == nil, "")
+	defer rtx.Rollback()
+
+	// create a write transaction
+	wtx, err := db.Begin(true)
+	assert(t, err == nil, "")
+
+	b, err := wtx.CreateBucket([]byte("test"))
+	assert(t, err == nil, "")
+
+	// and commit a large write
+	err = b.Put([]byte("foo"), make([]byte, testWriteSize))
+	assert(t, err == nil, "")
+
+	done := make(chan struct{})
+
+	go func() {
+		wtx.Commit()
+		done <- struct{}{}
+	}()
+
+	select {
+	case <-time.After(5 * time.Second):
+		t.Errorf("unexpected that the reader blocks writer")
+	case <-done:
+	}
+}
+
 // TODO(benbjohnson): Test corruption at every byte of the first two pages.
 
 // Ensure that a database cannot open a transaction when it's not open.