checkpoint: WAL copying can omit a committed batch while retaining a later ingest

Open
#6,255 1 comment 0 reactions 0 assignees View on GitHub

Maintainers usually reply within 1 day

@akasakariko is already working on this.

Since Nov 14, 2026.

Assessment

Difficulty
4/5
Estimated time
3-5 days
Newbie friendliness
22/100
Issue type
Bug
Clarity
Clearly specified
Activity status
Active
Tech stack
go
Domain
databases

Research direction

The failure sits in Checkpoint in checkpoint.go, where the version is captured and then the WAL is copied with wal.Copy; the diagnostic patch touches the code just after the version is captured. Start with the repro test in the issue, which uses errorfs to hold WAL writes and MANIFEST syncs, and check whether the copy can miss records still buffered in the log writer. Done means the checkpoint contains every batch with sequence number at or below the captured visibleSeqNum, and the repro passes reliably.

Written by the indexing model from the issue text.

Description

Checkpoint can include an ingest but omit a NoSync batch committed before it. The captured visibleSeqNum includes the batch, but its WAL record can still be buffered when wal.Copy reaches the file's end.

The repro holds WAL and MANIFEST operations to put that batch before the ingest while keeping its record out of the WAL file. Checkpoint returns nil, but the checkpoint has the ingested z and no k. I'd expect the same contents as an iterator at the captured sequence number. WithFlushedWAL doesn't help because the batch is committed during the call.

This fails 10/10 on v2.1.7 and public master a995e890. Flushing the WAL after capturing the version makes it pass 10/10 on both: patched Checkpoint waits for the held write until the test's one-second fallback releases it. That's a diagnostic change, not a finished fix.

Standalone reproduction and run instructions

Save as repro_test.go in an empty directory. The test uses an in-memory file system and does not modify an existing database.

go mod init repro
go get github.com/cockroachdb/pebble/[email protected]
go mod tidy
go test -v .

For master, remove /v2 from the Pebble imports and get github.com/cockroachdb/pebble@a995e890c06e instead. Runs used Go 1.27.3 on darwin/arm64.

Also fails 3/3 on crl-release-26.2 ad3f0555, which carries #6224, built with master's cockroachdb/swiss because the pinned version doesn't compile under Go 1.27.3.

package ckptwalrepro

import (
	"errors"
	"strings"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/cockroachdb/pebble/v2"
	"github.com/cockroachdb/pebble/v2/objstorage/objstorageprovider"
	"github.com/cockroachdb/pebble/v2/sstable"
	"github.com/cockroachdb/pebble/v2/vfs"
	"github.com/cockroachdb/pebble/v2/vfs/errorfs"
)

// gate blocks the first file operation that matches it after it is armed,
// until it is released.
type gate struct {
	armed   atomic.Bool
	fired   atomic.Bool
	reached chan struct{}
	release chan struct{}
	once    sync.Once
}

func newGate() *gate {
	return &gate{reached: make(chan struct{}), release: make(chan struct{})}
}

func (g *gate) hold() {
	if g.armed.Load() && g.fired.CompareAndSwap(false, true) {
		close(g.reached)
		<-g.release
	}
}

func (g *gate) open() { g.once.Do(func() { close(g.release) }) }

func (g *gate) wait(t *testing.T, what string) {
	t.Helper()
	select {
	case <-g.reached:
	case <-time.After(10 * time.Second):
		t.Fatalf("%s was not reached", what)
	}
}

func recv(t *testing.T, ch <-chan error, what string) error {
	t.Helper()
	select {
	case err := <-ch:
		return err
	case <-time.After(10 * time.Second):
		t.Fatalf("%s did not return", what)
		return nil
	}
}

// TestCheckpointOmitsBatchBeforeIngest: a batch is committed without sync
// while Checkpoint flushes the WAL, and an ingest is committed after it.
// Checkpoint captures a version that holds the ingest and a visible sequence
// number above both, but copies the WAL before the batch's record reaches
// the file. The checkpoint then holds the ingest and not the batch committed
// before it, which no iterator at that sequence number would see.
func TestCheckpointOmitsBatchBeforeIngest(t *testing.T) {
	logSync, logWrite, manifestSync := newGate(), newGate(), newGate()
	fs := errorfs.Wrap(vfs.NewMem(), errorfs.InjectorFunc(func(op errorfs.Op) error {
		if !strings.HasPrefix(op.Path, "db/") {
			return nil
		}
		isSync := op.Kind == errorfs.OpFileSync || op.Kind == errorfs.OpFileSyncData || op.Kind == errorfs.OpFileSyncTo
		switch {
		case isSync && strings.HasSuffix(op.Path, ".log"):
			logSync.hold()
		case op.Kind == errorfs.OpFileWrite && strings.HasSuffix(op.Path, ".log"):
			logWrite.hold()
		case isSync && strings.Contains(op.Path, "MANIFEST"):
			manifestSync.hold()
		}
		return nil
	}))
	opts := &pebble.Options{
		FS:                          fs,
		FormatMajorVersion:          pebble.FormatNewest,
		DisableAutomaticCompactions: true,
	}
	db, err := pebble.Open("db", opts)
	if err != nil {
		t.Fatal(err)
	}
	defer db.Close()

	// An sstable to ingest whose key does not overlap the memtable, so the
	// ingest goes straight to the LSM through a MANIFEST write.
	f, err := fs.Create("ext.sst", vfs.WriteCategoryUnspecified)
	if err != nil {
		t.Fatal(err)
	}
	o := opts.Clone()
	o.EnsureDefaults()
	w := sstable.NewWriter(objstorageprovider.NewFileWritable(f), o.MakeWriterOptions(0, db.TableFormat()))
	if err := w.Set([]byte("z"), []byte("ingested")); err != nil {
		t.Fatal(err)
	}
	if err := w.Close(); err != nil {
		t.Fatal(err)
	}
	if err := db.Set([]byte("a"), []byte("synced"), pebble.Sync); err != nil {
		t.Fatal(err)
	}

	var wg sync.WaitGroup
	defer wg.Wait()
	defer logWrite.open()
	defer manifestSync.open()
	defer logSync.open()

	// 1. Checkpoint flushes the WAL; hold its sync.
	logSync.armed.Store(true)
	ckptErr := make(chan error, 1)
	wg.Go(func() { ckptErr <- db.Checkpoint("ckpt", pebble.WithFlushedWAL()) })
	logSync.wait(t, "the WAL sync of Checkpoint's flush")

	// 2. A batch without sync: its record is queued behind the held sync.
	setErr := make(chan error, 1)
	wg.Go(func() { setErr <- db.Set([]byte("k"), []byte("v"), pebble.NoSync) })
	time.Sleep(100 * time.Millisecond)

	// 3. An ingest, sequenced after the batch; hold its MANIFEST sync, during
	// which it holds the manifest lock that Checkpoint needs next.
	manifestSync.armed.Store(true)
	ingestErr := make(chan error, 1)
	wg.Go(func() { ingestErr <- db.Ingest(t.Context(), []string{"ext.sst"}) })
	manifestSync.wait(t, "the ingest's MANIFEST sync")

	// 4. Release the WAL sync but hold the next WAL write, which carries the
	// batch's record. The batch is published and Set returns; Checkpoint
	// moves on and waits for the manifest lock.
	logWrite.armed.Store(true)
	logSync.open()
	if err := recv(t, setErr, "Set(k)"); err != nil {
		t.Fatalf("Set(k): %v", err)
	}
	logWrite.wait(t, "the WAL write of k's record")
	time.Sleep(100 * time.Millisecond)

	// 5. Let the ingest finish. Checkpoint captures a version holding it and
	// copies the WAL while k's record is not yet in the file.
	manifestSync.open()
	if err := recv(t, ingestErr, "Ingest"); err != nil {
		t.Fatalf("Ingest: %v", err)
	}
	select {
	case err = <-ckptErr:
	case <-time.After(time.Second):
		t.Logf("Checkpoint waits for the WAL write of k's record; releasing it")
		logWrite.open()
		err = recv(t, ckptErr, "Checkpoint")
	}
	if err != nil {
		t.Fatalf("Checkpoint: %v", err)
	}
	logWrite.open()
	if err := db.Flush(); err != nil {
		t.Fatal(err)
	}
	t.Logf("db LSM after a flush:\n%s", db.DebugString())

	for _, key := range []string{"a", "k", "z"} {
		v, closer, err := db.Get([]byte(key))
		if err != nil {
			t.Fatalf("db Get(%s): %v", key, err)
		}
		t.Logf("db: %s = %q", key, v)
		closer.Close()
	}

	ck, err := pebble.Open("ckpt", &pebble.Options{FS: fs, ReadOnly: true})
	if err != nil {
		t.Fatalf("opening the checkpoint: %v", err)
	}
	defer ck.Close()
	found := map[string]bool{}
	for _, key := range []string{"a", "k", "z"} {
		v, closer, err := ck.Get([]byte(key))
		switch {
		case errors.Is(err, pebble.ErrNotFound):
			t.Logf("checkpoint: %s not found", key)
		case err != nil:
			t.Fatalf("checkpoint Get(%s): %v", key, err)
		default:
			t.Logf("checkpoint: %s = %q", key, v)
			found[key] = true
			closer.Close()
		}
	}
	if found["z"] && !found["k"] {
		t.Errorf("the checkpoint holds the ingested z but not k, which was committed before the ingest")
	}
}
Diagnostic WAL-flush patch
--- a/checkpoint.go
+++ b/checkpoint.go
@@ -244,6 +244,14 @@
 	d.mu.versions.logUnlock()
 	d.mu.Unlock()
 	defer current.Unref()
+
+	// Batches below visibleSeqNum may still have their WAL records queued in
+	// the log writer. Flush them, so that the WAL copy below reads every one.
+	if !d.opts.DisableWAL {
+		if err := d.LogData(nil /* data */, Sync); err != nil {
+			return err
+		}
+	}
 
 	// Wrap the normal filesystem with one which wraps newly created files with
 	// vfs.NewSyncingFile.

Jira issue: PEBBLE-1711

Dominant language
Go
Stars
6.1k
Forks
608
Avg merge
16h 40m
Merged PRs (30d)
9

Getting set up

This project ships no dev container, Dockerfile or contributing guide, so setting up is up to you: start from its README, and see our first-contribution guide for the general steps.

First steps

  1. Read the whole issue, then the project's contributing guide.
  2. Comment on the issue to say you are picking it up — it saves two people doing the same work.
  3. Fork the repository and make your change on a branch.
  4. Open a pull request that references the issue number.

More from cockroachdb/pebble

All issues in cockroachdb/pebble

Similar issues

More Go issues

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.