Organize blobs on disk by epoch (#14023)

* organize blob directories by period and epoch

* changelog

* remove Indices and replace with Summary

* old PR feedback

* log to advise about the speed of blob migration

* rename level->layer (hoping term is more clear)

* assert path in tests for increased legibility

* lint

* lint

* remove test covering a newly impossible error

* improve feedback from flag validation failure

* Try to clean dangling dirs epoch->flat migration

* lint

* Preston feedback

* try all layouts and short-circuit if base not found

---------

Co-authored-by: Kasey Kirkham <kasey@users.noreply.github.com>
This commit is contained in:
kasey
2025-02-05 14:09:38 -06:00
committed by GitHub
parent 2a7fc84044
commit 41daac1b04
42 changed files with 2137 additions and 1137 deletions

View File

@@ -11,7 +11,6 @@ import (
"github.com/pkg/errors"
"github.com/prysmaticlabs/prysm/v5/beacon-chain/core/transition"
"github.com/prysmaticlabs/prysm/v5/beacon-chain/das"
"github.com/prysmaticlabs/prysm/v5/beacon-chain/db/filesystem"
"github.com/prysmaticlabs/prysm/v5/beacon-chain/sync"
"github.com/prysmaticlabs/prysm/v5/beacon-chain/verification"
"github.com/prysmaticlabs/prysm/v5/consensus-types/blocks"
@@ -70,14 +69,6 @@ func (s *Service) startBlocksQueue(ctx context.Context, highestSlot primitives.S
return nil, errors.Wrapf(err, "unable to initialize context version map using genesis validator root = %#x", vr)
}
summarizer, err := s.cfg.BlobStorage.WaitForSummarizer(ctx)
if err != nil {
// The summarizer is an optional optimization, we can continue without, only stop if there is a different error.
if !errors.Is(err, filesystem.ErrBlobStorageSummarizerUnavailable) {
return nil, err
}
summarizer = nil // This should already be nil, but we'll set it just to be safe.
}
cfg := &blocksQueueConfig{
p2p: s.cfg.P2P,
db: s.cfg.DB,
@@ -86,7 +77,7 @@ func (s *Service) startBlocksQueue(ctx context.Context, highestSlot primitives.S
ctxMap: ctxMap,
highestExpectedSlot: highestSlot,
mode: mode,
bs: summarizer,
bs: s.cfg.BlobStorage,
}
queue := newBlocksQueue(ctx, cfg)
if err := queue.start(); err != nil {