Fix Validator Backups (#9446)

* fix val backups

* fix

* debug

Co-authored-by: prylabs-bulldozer[bot] <58059840+prylabs-bulldozer[bot]@users.noreply.github.com>

validator/db/kv/backup.go

@@ -53,14 +53,30 @@ func (s *Store) Backup(ctx context.Context, outputDir string, permissionOverride
 
 	return s.db.View(func(tx *bolt.Tx) error {
 		return tx.ForEach(func(name []byte, b *bolt.Bucket) error {
-			log.Debugf("Copying bucket %s\n", name)
+			log.Debugf("Copying bucket %s\n with %d keys", name, b.Stats().KeyN)
 			return copyDB.Update(func(tx2 *bolt.Tx) error {
 				b2, err := tx2.CreateBucketIfNotExists(name)
 				if err != nil {
 					return err
 				}
-				return b.ForEach(b2.Put)
+				return b.ForEach(createNestedBuckets(b, b2, b2.Put))
 			})
 		})
 	})
 }
+
+// Walks through each buckets and looks out for nested buckets so that
+// the backup db also includes them.
+func createNestedBuckets(srcBucket *bolt.Bucket, dstBucket *bolt.Bucket, fn func(k, v []byte) error) func(k, v []byte) error {
+	return func(k, v []byte) error {
+		bkt := srcBucket.Bucket(k)
+		if bkt != nil {
+			b2, err := dstBucket.CreateBucketIfNotExists(k)
+			if err != nil {
+				return err
+			}
+			return bkt.ForEach(createNestedBuckets(bkt, b2, b2.Put))
+		}
+		return fn(k, v)
+	}
+}

validator/db/kv/backup_test.go

@@ -7,6 +7,7 @@ import (
 	"path/filepath"
 	"testing"
 
+	ethpb "github.com/prysmaticlabs/prysm/proto/prysm/v1alpha1"
 	"github.com/prysmaticlabs/prysm/shared/testutil/require"
 )
 
@@ -38,3 +39,80 @@ func TestStore_Backup(t *testing.T) {
 	require.NoError(t, err)
 	require.DeepEqual(t, root[:], genesisRoot)
 }
+
+func TestStore_NestedBackup(t *testing.T) {
+	keys := [][48]byte{{'A'}, {'B'}}
+	db := setupDB(t, keys)
+	ctx := context.Background()
+	root := [32]byte{1}
+	idxAtt := &ethpb.IndexedAttestation{
+		AttestingIndices: nil,
+		Data: &ethpb.AttestationData{
+			Slot:            0,
+			CommitteeIndex:  0,
+			BeaconBlockRoot: root[:],
+			Source: &ethpb.Checkpoint{
+				Epoch: 10,
+				Root:  root[:],
+			},
+			Target: &ethpb.Checkpoint{
+				Epoch: 0,
+				Root:  root[:],
+			},
+		},
+		Signature: make([]byte, 96),
+	}
+	require.NoError(t, db.SaveGenesisValidatorsRoot(ctx, root[:]))
+	require.NoError(t, db.SaveAttestationForPubKey(context.Background(), keys[0], [32]byte{'C'}, idxAtt))
+	require.NoError(t, db.SaveAttestationForPubKey(context.Background(), keys[1], [32]byte{'C'}, idxAtt))
+	require.NoError(t, db.Backup(ctx, "", true))
+
+	backupsPath := filepath.Join(db.databasePath, backupsDirectoryName)
+	files, err := ioutil.ReadDir(backupsPath)
+	require.NoError(t, err)
+	require.NotEqual(t, 0, len(files), "No backups created")
+	require.NoError(t, db.Close(), "Failed to close database")
+
+	oldFilePath := filepath.Join(backupsPath, files[0].Name())
+	newFilePath := filepath.Join(backupsPath, ProtectionDbFileName)
+	// We rename the file to match the database file name
+	// our NewKVStore function expects when opening a database.
+	require.NoError(t, os.Rename(oldFilePath, newFilePath))
+
+	backedDB, err := NewKVStore(ctx, backupsPath, &Config{})
+	require.NoError(t, err, "Failed to instantiate DB")
+	t.Cleanup(func() {
+		require.NoError(t, backedDB.Close(), "Failed to close database")
+	})
+	genesisRoot, err := backedDB.GenesisValidatorsRoot(ctx)
+	require.NoError(t, err)
+	require.DeepEqual(t, root[:], genesisRoot)
+
+	hist, err := backedDB.AttestationHistoryForPubKey(context.Background(), keys[0])
+	require.NoError(t, err)
+	require.DeepEqual(t, &AttestationRecord{
+		PubKey:      keys[0],
+		Source:      10,
+		Target:      0,
+		SigningRoot: [32]byte{'C'},
+	}, hist[0])
+
+	hist, err = backedDB.AttestationHistoryForPubKey(context.Background(), keys[1])
+	require.NoError(t, err)
+	require.DeepEqual(t, &AttestationRecord{
+		PubKey:      keys[1],
+		Source:      10,
+		Target:      0,
+		SigningRoot: [32]byte{'C'},
+	}, hist[0])
+
+	ep, exists, err := backedDB.LowestSignedSourceEpoch(context.Background(), keys[0])
+	require.NoError(t, err)
+	require.Equal(t, true, exists)
+	require.Equal(t, 10, int(ep))
+
+	ep, exists, err = backedDB.LowestSignedSourceEpoch(context.Background(), keys[1])
+	require.NoError(t, err)
+	require.Equal(t, true, exists)
+	require.Equal(t, 10, int(ep))
+}