etcdserver/*, wal/*: add Sync method

etcdserver/raft.go

@@ -245,6 +245,13 @@ func (r *raftNode) start(rh *raftReadyHandler) {
 				// gofail: var raftAfterSave struct{}
 
 				if !raft.IsEmptySnap(rd.Snapshot) {
+					// Force WAL to fsync its hard state before Release() releases
+					// old data from the WAL. Otherwise could get an error like:
+					// panic: tocommit(107) is out of range [lastIndex(84)]. Was the raft log corrupted, truncated, or lost?
+					if err := r.storage.Sync(); err != nil {
+						log.Fatal(err)
+					}
+
 					// etcdserver now claim the snapshot has been persisted onto the disk
 					notifyc <- struct{}{}
 

etcdserver/server_test.go

@@ -995,7 +995,7 @@ func TestSnapshot(t *testing.T) {
 	ch := make(chan struct{}, 2)
 
 	go func() {
-		gaction, _ := p.Wait(1)
+		gaction, _ := p.Wait(2)
 		defer func() { ch <- struct{}{} }()
 
 		if len(gaction) != 2 {
@@ -1111,6 +1111,10 @@ func TestSnapshotOrdering(t *testing.T) {
 	}
 
 	// unblock SaveSnapshot, etcdserver now permitted to move snapshot file
+	if ac := <-p.Chan(); ac.Name != "Sync" {
+		t.Fatalf("expected Sync, got %+v", ac)
+	}
+
 	if ac := <-p.Chan(); ac.Name != "Release" {
 		t.Fatalf("expected Release, got %+v", ac)
 	}

etcdserver/storage.go

@@ -38,6 +38,8 @@ type Storage interface {
 	Close() error
 	// Release release release the locked wal files since they will not be used.
 	Release(snap raftpb.Snapshot) error
+	// Sync WAL
+	Sync() error
 }
 
 type storage struct {

pkg/mock/mockstorage/storage_recorder.go

@@ -52,4 +52,9 @@ func (p *storageRecorder) Release(st raftpb.Snapshot) error {
 	return nil
 }
 
+func (p *storageRecorder) Sync() error {
+	p.Record(testutil.Action{Name: "Sync"})
+	return nil
+}
+
 func (p *storageRecorder) Close() error { return nil }

wal/wal.go

@@ -723,6 +723,10 @@ func (w *WAL) sync() error {
 	return err
 }
 
+func (w *WAL) Sync() error {
+	return w.sync()
+}
+
 // ReleaseLockTo releases the locks, which has smaller index than the given index
 // except the largest one among them.
 // For example, if WAL is holding lock 1,2,3,4,5,6, ReleaseLockTo(4) will release