From 8c58ffc333bbe5d6a3a2664c64f688e7245e5a4b Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Tue, 22 May 2018 14:36:55 -0400 Subject: [PATCH 01/11] sharding: begin shard local storage Former-commit-id: 92fbba3d64ae6543595dd3270d5c3a75a8f54175 [formerly 5782e4c437022bb6827617ab05d7417afb9ceec3] Former-commit-id: 030ef0240481a764c549d7cc323d30a958447413 --- sharding/database/database.go | 21 +++++++++++++++++++++ sharding/interfaces.go | 10 ++++++++++ sharding/notary/service.go | 1 + sharding/shard.go | 9 --------- 4 files changed, 32 insertions(+), 9 deletions(-) create mode 100644 sharding/database/database.go diff --git a/sharding/database/database.go b/sharding/database/database.go new file mode 100644 index 0000000000..fe2bb74380 --- /dev/null +++ b/sharding/database/database.go @@ -0,0 +1,21 @@ +package database + +import ( + "path/filepath" + + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/sharding" + "github.com/micro/cli" +) + +// CreateShardDB initializes a shardDB that writes to local disk. +func CreateShardDB(ctx *cli.Context, name string) (sharding.ShardBackend, error) { + + dataDir := ctx.GlobalString(utils.DataDir.Name) + path := filepath.Join(dataDir, name) + + // Uses default cache and handles values. + // TODO: fix interface. + return ethdb.NewLDBDatabase(path, 16, 16) +} diff --git a/sharding/interfaces.go b/sharding/interfaces.go index a75a04ca62..8795e8eb6b 100644 --- a/sharding/interfaces.go +++ b/sharding/interfaces.go @@ -1,6 +1,7 @@ package sharding import ( + "github.com/ethereum/go-ethereum/common" cli "gopkg.in/urfave/cli.v1" ) @@ -22,3 +23,12 @@ type Service interface { // ServiceConstructor defines the callback passed in when registering a service // to a sharding node. type ServiceConstructor func(ctx *cli.Context) (Service, error) + +// ShardBackend defines an interface for a shardDB's necessary method +// signatures. +type ShardBackend interface { + Get(k common.Hash) (*[]byte, error) + Has(k common.Hash) bool + Put(k common.Hash, val []byte) error + Delete(k common.Hash) error +} diff --git a/sharding/notary/service.go b/sharding/notary/service.go index f1123f2985..7281be0cf2 100644 --- a/sharding/notary/service.go +++ b/sharding/notary/service.go @@ -3,6 +3,7 @@ package notary import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/sharding/node" + cli "gopkg.in/urfave/cli.v1" ) diff --git a/sharding/shard.go b/sharding/shard.go index 3408fbe75c..9e84ca0f10 100644 --- a/sharding/shard.go +++ b/sharding/shard.go @@ -9,15 +9,6 @@ import ( "github.com/ethereum/go-ethereum/rlp" ) -// ShardBackend defines an interface for a shardDB's necessary method -// signatures. -type ShardBackend interface { - Get(k common.Hash) (*[]byte, error) - Has(k common.Hash) bool - Put(k common.Hash, val []byte) error - Delete(k common.Hash) error -} - // Shard base struct. type Shard struct { shardDB ShardBackend From 0f27660b1836e931e0facc5e362b77827339394e Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Tue, 22 May 2018 14:49:59 -0400 Subject: [PATCH 02/11] sharding: using eth leveldb, interface mismatch Former-commit-id: 127630fadb68deff3418e499f303f7eab16e775f [formerly 20bf39d9d62f72857512b505cfac7e122002c4ab] Former-commit-id: d3baf1deac782fa9ee4bcceae658723b65d1b08d --- sharding/database/database.go | 9 +++++---- sharding/notary/service.go | 14 ++++++++++++-- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/sharding/database/database.go b/sharding/database/database.go index fe2bb74380..ee1aebc23c 100644 --- a/sharding/database/database.go +++ b/sharding/database/database.go @@ -9,13 +9,14 @@ import ( "github.com/micro/cli" ) -// CreateShardDB initializes a shardDB that writes to local disk. -func CreateShardDB(ctx *cli.Context, name string) (sharding.ShardBackend, error) { +// NewShardDB initializes a shardDB that writes to local disk. +func NewShardDB(ctx *cli.Context, name string) (sharding.ShardBackend, error) { - dataDir := ctx.GlobalString(utils.DataDir.Name) + dataDir := ctx.GlobalString(utils.DataDirFlag.Name) path := filepath.Join(dataDir, name) // Uses default cache and handles values. - // TODO: fix interface. + // TODO: allow these to be set based on cli context. + // TODO: fix interface - lots of methods do not match. return ethdb.NewLDBDatabase(path, 16, 16) } diff --git a/sharding/notary/service.go b/sharding/notary/service.go index 7281be0cf2..2f52e3e8cf 100644 --- a/sharding/notary/service.go +++ b/sharding/notary/service.go @@ -2,6 +2,8 @@ package notary import ( "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/sharding" + "github.com/ethereum/go-ethereum/sharding/database" "github.com/ethereum/go-ethereum/sharding/node" cli "gopkg.in/urfave/cli.v1" @@ -11,12 +13,20 @@ import ( // in a sharded system. Must satisfy the Service interface defined in // sharding/service.go. type Notary struct { - node node.Node + node node.Node + shardDB sharding.ShardBackend } // NewNotary creates a new notary instance. func NewNotary(ctx *cli.Context, node node.Node) (*Notary, error) { - return &Notary{node}, nil + // Initializes a shardDB that writes to disk at /path/to/datadir/shardchaindata. + // This DB can be used by the Notary service to create Shard struct + // instances. + shardDB, err := database.NewShardDB(node.Context(), "shardchaindata") + if err != nil { + return nil, err + } + return &Notary{node, shardDB}, nil } // Start the main routine for a notary. From 407a112ec015ca517c79a8c5328ef3d3cc449041 Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Thu, 24 May 2018 17:36:20 -0600 Subject: [PATCH 03/11] sharding: fix datadir and import cycle Former-commit-id: 787b53e195a8ffde14f18139516ff510e03969a7 [formerly e7d0d7c28711060a8f2545de3218dd5a06e53e4e] Former-commit-id: 0eb1bbe8fa5c9ad0b7a42d6c1043641ca3542016 --- sharding/database/database.go | 9 ++------- sharding/node/node.go | 6 ++++++ sharding/notary/service.go | 8 ++++---- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/sharding/database/database.go b/sharding/database/database.go index d61a7b0152..a97d04cfab 100644 --- a/sharding/database/database.go +++ b/sharding/database/database.go @@ -5,7 +5,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" - "github.com/micro/cli" ) // ShardBackend defines an interface for a shardDB's necessary method @@ -19,13 +18,9 @@ type ShardBackend interface { // NewShardDB initializes a shardDB that writes to local disk. // TODO: make it return ShardBackend but modify interface methods. -func NewShardDB(ctx *cli.Context, name string) (*ethdb.LDBDatabase, error) { - - dataDir := "" - path := filepath.Join(dataDir, name) - +func NewShardDB(dataDir string, name string) (*ethdb.LDBDatabase, error) { // Uses default cache and handles values. // TODO: allow these to be set based on cli context. // TODO: fix interface - lots of methods do not match. - return ethdb.NewLDBDatabase(path, 16, 16) + return ethdb.NewLDBDatabase(filepath.Join(dataDir, name), 16, 16) } diff --git a/sharding/node/node.go b/sharding/node/node.go index a5eec20fe4..4a370e955e 100644 --- a/sharding/node/node.go +++ b/sharding/node/node.go @@ -44,6 +44,7 @@ type Node interface { SMCCaller() *contracts.SMCCaller SMCTransactor() *contracts.SMCTransactor DepositFlagSet() bool + DataDirFlag() string } // General node for a sharding-enabled system. @@ -213,6 +214,11 @@ func (n *shardingNode) DepositFlagSet() bool { return n.ctx.GlobalBool(utils.DepositFlag.Name) } +// DataDirFlag returns the datadir flag as a string. +func (n *shardingNode) DataDirFlag() string { + return n.ctx.GlobalString(utils.DataDirFlag.Name) +} + // Client to interact with a geth node via JSON-RPC. func (n *shardingNode) ethereumClient() *ethclient.Client { return n.client diff --git a/sharding/notary/service.go b/sharding/notary/service.go index e08c827134..bbd5d00c43 100644 --- a/sharding/notary/service.go +++ b/sharding/notary/service.go @@ -2,7 +2,6 @@ package notary import ( "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/sharding" "github.com/ethereum/go-ethereum/sharding/database" "github.com/ethereum/go-ethereum/sharding/node" ) @@ -12,7 +11,7 @@ import ( // sharding/service.go. type Notary struct { node node.Node - shardDB sharding.ShardBackend + shardDB database.ShardBackend } // NewNotary creates a new notary instance. @@ -20,11 +19,12 @@ func NewNotary(node node.Node) (*Notary, error) { // Initializes a shardDB that writes to disk at /path/to/datadir/shardchaindata. // This DB can be used by the Notary service to create Shard struct // instances. - shardDB, err := database.NewShardDB(node.Context(), "shardchaindata") + shardDB, err := database.NewShardDB(node.DataDirFlag(), "shardchaindata") if err != nil { return nil, err } - return &Notary{node, shardDB}, nil + // return &Notary{node, shardDB}, nil + return &Notary{node: node}, nil } // Start the main routine for a notary. From 52cc968c57f4140a3daeed04008d54ef9b2b9186 Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Thu, 24 May 2018 18:03:24 -0600 Subject: [PATCH 04/11] sharding: generalized shardbackend, refactor database package and tests Former-commit-id: 806e44466e6484a0028fa39f364c2ee63e828983 [formerly 89d7697ddc25156265f8f49622a775c879e3bd88] Former-commit-id: 8cf56c2760f2b9fcf1121a94967276c2a34cae58 --- sharding/database/database.go | 11 +++--- sharding/database/inmemory.go | 24 ++++++------- sharding/database/inmemory_test.go | 54 ++++++++++++++++-------------- 3 files changed, 46 insertions(+), 43 deletions(-) diff --git a/sharding/database/database.go b/sharding/database/database.go index a97d04cfab..9e30b5fa92 100644 --- a/sharding/database/database.go +++ b/sharding/database/database.go @@ -3,22 +3,21 @@ package database import ( "path/filepath" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" ) // ShardBackend defines an interface for a shardDB's necessary method // signatures. type ShardBackend interface { - Get(k common.Hash) (*[]byte, error) - Has(k common.Hash) bool - Put(k common.Hash, val []byte) error - Delete(k common.Hash) error + Get(k []byte) ([]byte, error) + Has(k []byte) (bool, error) + Put(k []byte, val []byte) error + Delete(k []byte) error } // NewShardDB initializes a shardDB that writes to local disk. // TODO: make it return ShardBackend but modify interface methods. -func NewShardDB(dataDir string, name string) (*ethdb.LDBDatabase, error) { +func NewShardDB(dataDir string, name string) (ShardBackend, error) { // Uses default cache and handles values. // TODO: allow these to be set based on cli context. // TODO: fix interface - lots of methods do not match. diff --git a/sharding/database/inmemory.go b/sharding/database/inmemory.go index 6d7ca35fe0..515bcf9761 100644 --- a/sharding/database/inmemory.go +++ b/sharding/database/inmemory.go @@ -12,48 +12,48 @@ import ( // ShardKV is an in-memory mapping of hashes to RLP encoded values. type ShardKV struct { - kv map[common.Hash]*[]byte + kv map[common.Hash][]byte lock sync.RWMutex } // NewShardKV initializes a keyval store in memory. func NewShardKV() *ShardKV { - return &ShardKV{kv: make(map[common.Hash]*[]byte)} + return &ShardKV{kv: make(map[common.Hash][]byte)} } // Get fetches a val from the mappping by key. -func (sb *ShardKV) Get(k common.Hash) (*[]byte, error) { +func (sb *ShardKV) Get(k []byte) ([]byte, error) { sb.lock.RLock() defer sb.lock.RUnlock() - v, ok := sb.kv[k] + v, ok := sb.kv[common.BytesToHash(k)] if !ok { - return nil, fmt.Errorf("key not found: %v", k) + return []byte{}, fmt.Errorf("key not found: %v", k) } return v, nil } // Has checks if the key exists in the mapping. -func (sb *ShardKV) Has(k common.Hash) bool { +func (sb *ShardKV) Has(k []byte) (bool, error) { sb.lock.RLock() defer sb.lock.RUnlock() - v := sb.kv[k] - return v != nil + v := sb.kv[common.BytesToHash(k)] + return v != nil, nil } // Put updates a key's value in the mapping. -func (sb *ShardKV) Put(k common.Hash, v []byte) error { +func (sb *ShardKV) Put(k []byte, v []byte) error { sb.lock.Lock() defer sb.lock.Unlock() // there is no error in a simple setting of a value in a go map. - sb.kv[k] = &v + sb.kv[common.BytesToHash(k)] = v return nil } // Delete removes the key and value from the mapping. -func (sb *ShardKV) Delete(k common.Hash) error { +func (sb *ShardKV) Delete(k []byte) error { sb.lock.Lock() defer sb.lock.Unlock() // There is no return value for deleting a simple key in a go map. - delete(sb.kv, k) + delete(sb.kv, common.BytesToHash(k)) return nil } diff --git a/sharding/database/inmemory_test.go b/sharding/database/inmemory_test.go index 2f9a54dae9..4043c20d44 100644 --- a/sharding/database/inmemory_test.go +++ b/sharding/database/inmemory_test.go @@ -2,73 +2,77 @@ package database import ( "testing" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/sharding" ) // Verifies that ShardKV implements the ShardBackend interface. -var _ = sharding.ShardBackend(&ShardKV{}) +var _ = ShardBackend(&ShardKV{}) func Test_ShardKVPut(t *testing.T) { kv := NewShardKV() - hash := common.BytesToHash([]byte("ralph merkle")) - if err := kv.Put(hash, []byte{1, 2, 3}); err != nil { + if err := kv.Put([]byte("ralph merkle"), []byte{1, 2, 3}); err != nil { t.Errorf("could not save value in kv store: %v", err) } } func Test_ShardKVHas(t *testing.T) { kv := NewShardKV() - hash := common.BytesToHash([]byte("ralph merkle")) + key := []byte("ralph merkle") - if err := kv.Put(hash, []byte{1, 2, 3}); err != nil { + if err := kv.Put(key, []byte{1, 2, 3}); err != nil { t.Fatalf("could not save value in kv store: %v", err) } - if !kv.Has(hash) { - t.Errorf("kv store does not have hash: %v", hash) + has, err := kv.Has(key) + if err != nil { + t.Errorf("could not check if kv store has key: %v", err) + } + if !has { + t.Errorf("kv store should have key: %v", key) } - hash2 := common.BytesToHash([]byte{}) - if kv.Has(hash2) { - t.Errorf("kv store should not contain unset key: %v", hash2) + key2 := []byte{} + has2, err := kv.Has(key2) + if err != nil { + t.Errorf("could not check if kv store has key: %v", err) + } + if has2 { + t.Errorf("kv store should not have non-existent key: %v", key2) } } func Test_ShardKVGet(t *testing.T) { kv := NewShardKV() - hash := common.BytesToHash([]byte("ralph merkle")) + key := []byte("ralph merkle") - if err := kv.Put(hash, []byte{1, 2, 3}); err != nil { + if err := kv.Put(key, []byte{1, 2, 3}); err != nil { t.Fatalf("could not save value in kv store: %v", err) } - val, err := kv.Get(hash) + val, err := kv.Get(key) if err != nil { t.Errorf("get failed: %v", err) } - if val == nil { + if len(val) == 0 { t.Errorf("no value stored for key") } - hash2 := common.BytesToHash([]byte{}) - val2, err := kv.Get(hash2) - if val2 != nil { - t.Errorf("non-existent key should not have a value. key=%v, value=%v", hash2, val2) + key2 := []byte{} + val2, err := kv.Get(key2) + if len(val2) != 0 { + t.Errorf("non-existent key should not have a value. key=%v, value=%v", key2, val2) } } func Test_ShardKVDelete(t *testing.T) { kv := NewShardKV() - hash := common.BytesToHash([]byte("ralph merkle")) + key := []byte("ralph merkle") - if err := kv.Put(hash, []byte{1, 2, 3}); err != nil { + if err := kv.Put(key, []byte{1, 2, 3}); err != nil { t.Fatalf("could not save value in kv store: %v", err) } - if err := kv.Delete(hash); err != nil { - t.Errorf("could not delete key: %v", hash) + if err := kv.Delete(key); err != nil { + t.Errorf("could not delete key: %v", key) } } From b813c4b33e97b04e4b49ffbb66469dd364d742c7 Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Thu, 24 May 2018 18:09:42 -0600 Subject: [PATCH 05/11] sharding: refactor shard.go for new interface changes Former-commit-id: 7b507d71c2e1dd4b19b5d9895458fb090b6ca6f6 [formerly af227af2ce595eda9cfb0a7b930c66654ce9cc46] Former-commit-id: 5fcc1fad7f3a85e05d78818e1abe37408b78de9a --- sharding/shard.go | 31 +++++++++++++++---------------- sharding/shard_test.go | 5 +++-- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/sharding/shard.go b/sharding/shard.go index 66a90d043f..ca2905ee8b 100644 --- a/sharding/shard.go +++ b/sharding/shard.go @@ -39,17 +39,17 @@ func (s *Shard) ValidateShardID(h *CollationHeader) error { // HeaderByHash looks up a collation header from the shardDB using the header's hash. func (s *Shard) HeaderByHash(hash *common.Hash) (*CollationHeader, error) { - encoded, err := s.shardDB.Get(*hash) + encoded, err := s.shardDB.Get(hash.Bytes()) if err != nil { return nil, fmt.Errorf("get failed: %v", err) } - if encoded == nil { + if len(encoded) == 0 { return nil, fmt.Errorf("no value set for header hash: %vs", hash.Hex()) } var header CollationHeader - stream := rlp.NewStream(bytes.NewReader(*encoded), uint64(len(*encoded))) + stream := rlp.NewStream(bytes.NewReader(encoded), uint64(len(encoded))) if err := header.DecodeRLP(stream); err != nil { return nil, fmt.Errorf("could not decode RLP header: %v", err) } @@ -80,18 +80,18 @@ func (s *Shard) CanonicalHeaderHash(shardID *big.Int, period *big.Int) (*common. key := canonicalCollationLookupKey(shardID, period) // fetches the RLP encoded collation header corresponding to the key. - encoded, err := s.shardDB.Get(key) + encoded, err := s.shardDB.Get(key.Bytes()) if err != nil { return nil, err } - if encoded == nil { + if len(encoded) == 0 { return nil, fmt.Errorf("no canonical collation header set for period=%v, shardID=%v pair: %v", shardID, period, err) } // RLP decodes the header, computes its hash. var header CollationHeader - stream := rlp.NewStream(bytes.NewReader(*encoded), uint64(len(*encoded))) + stream := rlp.NewStream(bytes.NewReader(encoded), uint64(len(encoded))) if err := header.DecodeRLP(stream); err != nil { return nil, fmt.Errorf("could not decode RLP header: %v", err) } @@ -112,27 +112,26 @@ func (s *Shard) CanonicalCollation(shardID *big.Int, period *big.Int) (*Collatio // BodyByChunkRoot fetches a collation body. func (s *Shard) BodyByChunkRoot(chunkRoot *common.Hash) ([]byte, error) { - body, err := s.shardDB.Get(*chunkRoot) + body, err := s.shardDB.Get(chunkRoot.Bytes()) if err != nil { return nil, err } - if body == nil { + if len(body) == 0 { return nil, fmt.Errorf("no corresponding body with chunk root found: %s", chunkRoot) } - return *body, nil + return body, nil } // CheckAvailability is used by notaries to confirm a header's data availability. func (s *Shard) CheckAvailability(header *CollationHeader) (bool, error) { key := dataAvailabilityLookupKey(header.ChunkRoot()) - val, err := s.shardDB.Get(key) + availability, err := s.shardDB.Get(key.Bytes()) if err != nil { return false, err } - if val == nil { + if len(availability) == 0 { return false, fmt.Errorf("availability not set for header") } - availability := *val // availability is a byte array of length 1. return availability[0] != 0, nil } @@ -146,7 +145,7 @@ func (s *Shard) SetAvailability(chunkRoot *common.Hash, availability bool) error } else { encoded = []byte{0} } - return s.shardDB.Put(key, encoded) + return s.shardDB.Put(key.Bytes(), encoded) } // SaveHeader adds the collation header to shardDB. @@ -162,7 +161,7 @@ func (s *Shard) SaveHeader(header *CollationHeader) error { } // uses the hash of the header as the key. - return s.shardDB.Put(header.Hash(), encoded) + return s.shardDB.Put(header.Hash().Bytes(), encoded) } // SaveBody adds the collation body to the shardDB and sets availability. @@ -173,7 +172,7 @@ func (s *Shard) SaveBody(body []byte) error { // right now we will just take the raw keccak256 of the body until #92 is merged. chunkRoot := common.BytesToHash(body) s.SetAvailability(&chunkRoot, true) - return s.shardDB.Put(chunkRoot, body) + return s.shardDB.Put(chunkRoot.Bytes(), body) } // SaveCollation adds the collation's header and body to shardDB. @@ -214,7 +213,7 @@ func (s *Shard) SetCanonical(header *CollationHeader) error { } // sets the key to be the canonical collation lookup key and val as RLP encoded // collation header. - return s.shardDB.Put(key, encoded) + return s.shardDB.Put(key.Bytes(), encoded) } // dataAvailabilityLookupKey formats a string that will become a lookup diff --git a/sharding/shard_test.go b/sharding/shard_test.go index e4258c38d2..32ccc462a3 100644 --- a/sharding/shard_test.go +++ b/sharding/shard_test.go @@ -16,8 +16,9 @@ type mockShardDB struct { kv map[common.Hash]*[]byte } -func (m *mockShardDB) Get(k common.Hash) (*[]byte, error) { - return nil, nil +// TOOD: FINISH MOCK CLIENT +func (m *mockShardDB) Get(k common.Hash) ([]byte, error) { + return []byte{, nil } func (m *mockShardDB) Has(k common.Hash) bool { From aa8734ea5607b62c5012d6c62d0d2394f0345d5e Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Fri, 25 May 2018 09:06:39 -0600 Subject: [PATCH 06/11] sharding: refactor all tests, travis passes Former-commit-id: 979b3f3f7424ed9c2855f37acfbb36e1cb05bb3a [formerly 638fb7ddd3e39f136d5d1eb32ddc01d23bac5729] Former-commit-id: 47be397ed0173b73eaeb2ffb7b194689e76be7a5 --- sharding/database/database.go | 4 +--- sharding/shard.go | 2 +- sharding/shard_test.go | 26 +++++++++++++------------- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/sharding/database/database.go b/sharding/database/database.go index 9e30b5fa92..a06b34f18f 100644 --- a/sharding/database/database.go +++ b/sharding/database/database.go @@ -16,10 +16,8 @@ type ShardBackend interface { } // NewShardDB initializes a shardDB that writes to local disk. -// TODO: make it return ShardBackend but modify interface methods. func NewShardDB(dataDir string, name string) (ShardBackend, error) { // Uses default cache and handles values. - // TODO: allow these to be set based on cli context. - // TODO: fix interface - lots of methods do not match. + // TODO: allow these arguments to be set based on cli context. return ethdb.NewLDBDatabase(filepath.Join(dataDir, name), 16, 16) } diff --git a/sharding/shard.go b/sharding/shard.go index ca2905ee8b..218cbfbcc5 100644 --- a/sharding/shard.go +++ b/sharding/shard.go @@ -114,7 +114,7 @@ func (s *Shard) CanonicalCollation(shardID *big.Int, period *big.Int) (*Collatio func (s *Shard) BodyByChunkRoot(chunkRoot *common.Hash) ([]byte, error) { body, err := s.shardDB.Get(chunkRoot.Bytes()) if err != nil { - return nil, err + return []byte{}, err } if len(body) == 0 { return nil, fmt.Errorf("no corresponding body with chunk root found: %s", chunkRoot) diff --git a/sharding/shard_test.go b/sharding/shard_test.go index 32ccc462a3..58b6823e23 100644 --- a/sharding/shard_test.go +++ b/sharding/shard_test.go @@ -13,23 +13,23 @@ import ( ) type mockShardDB struct { - kv map[common.Hash]*[]byte + kv map[common.Hash][]byte } // TOOD: FINISH MOCK CLIENT -func (m *mockShardDB) Get(k common.Hash) ([]byte, error) { - return []byte{, nil +func (m *mockShardDB) Get(k []byte) ([]byte, error) { + return []byte{}, nil } -func (m *mockShardDB) Has(k common.Hash) bool { - return false +func (m *mockShardDB) Has(k []byte) (bool, error) { + return false, nil } -func (m *mockShardDB) Put(k common.Hash, v []byte) error { +func (m *mockShardDB) Put(k []byte, v []byte) error { return fmt.Errorf("error updating db") } -func (m *mockShardDB) Delete(k common.Hash) error { +func (m *mockShardDB) Delete(k []byte) error { return fmt.Errorf("error deleting value in db") } @@ -65,7 +65,7 @@ func TestShard_HeaderByHash(t *testing.T) { header := NewCollationHeader(big.NewInt(1), &emptyHash, big.NewInt(1), &emptyAddr, []byte{}) // creates a mockDB that always returns nil values from .Get and errors in every other method. - mockDB := &mockShardDB{kv: make(map[common.Hash]*[]byte)} + mockDB := &mockShardDB{kv: make(map[common.Hash][]byte)} // creates a well-functioning shardDB. shardDB := database.NewShardKV() @@ -296,10 +296,10 @@ func TestShard_BodyByChunkRoot(t *testing.T) { } // setting the val of the key to nil. - if err := shard.shardDB.Put(emptyHash, nil); err != nil { + if err := shard.shardDB.Put([]byte{}, nil); err != nil { t.Fatalf("could not update shardDB: %v", err) } - if _, err := shard.BodyByChunkRoot(&emptyHash); err != nil { + if _, err := shard.BodyByChunkRoot(&emptyHash); err == nil { t.Errorf("value set as nil in shardDB should return error from BodyByChunkRoot") } @@ -347,7 +347,7 @@ func TestShard_SetAvailability(t *testing.T) { header := NewCollationHeader(big.NewInt(1), &chunkRoot, big.NewInt(1), nil, []byte{}) // creates a mockDB that always returns nil values from .Get and errors in every other method. - mockDB := &mockShardDB{kv: make(map[common.Hash]*[]byte)} + mockDB := &mockShardDB{kv: make(map[common.Hash][]byte)} // creates a well-functioning shardDB. shardDB := database.NewShardKV() @@ -402,7 +402,7 @@ func TestShard_SaveCollation(t *testing.T) { func TestShard_SaveHeader(t *testing.T) { // creates a mockDB that always returns nil values from .Get and errors in every other method. - mockDB := &mockShardDB{kv: make(map[common.Hash]*[]byte)} + mockDB := &mockShardDB{kv: make(map[common.Hash][]byte)} emptyHash := common.BytesToHash([]byte{}) errorShard := NewShard(big.NewInt(1), mockDB) @@ -414,7 +414,7 @@ func TestShard_SaveHeader(t *testing.T) { func TestShard_SaveBody(t *testing.T) { // creates a mockDB that always returns nil values from .Get and errors in every other method. - mockDB := &mockShardDB{kv: make(map[common.Hash]*[]byte)} + mockDB := &mockShardDB{kv: make(map[common.Hash][]byte)} errorShard := NewShard(big.NewInt(1), mockDB) if err := errorShard.SaveBody([]byte{1, 2, 3}); err == nil { From 1c0d4e848e00e783ea13166ab4a4cbded6ee6cb2 Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Fri, 25 May 2018 09:52:18 -0600 Subject: [PATCH 07/11] sharding: tests pass, services updated to include shardDB Former-commit-id: 8b8b695ddc0a39bfe6a536fd7b7f40d74c4105da [formerly 829d481af0eebd1647a34006018761e3e64c5e3e] Former-commit-id: 1d607fcb8fae1327c346bdf3557ec40a8816a09b --- sharding/database/database_test.go | 92 ++++++++++++++++++++++++++++++ sharding/notary/service.go | 3 +- sharding/proposer/service.go | 10 +++- 3 files changed, 101 insertions(+), 4 deletions(-) create mode 100644 sharding/database/database_test.go diff --git a/sharding/database/database_test.go b/sharding/database/database_test.go new file mode 100644 index 0000000000..80e4409f6a --- /dev/null +++ b/sharding/database/database_test.go @@ -0,0 +1,92 @@ +package database + +import ( + "strconv" + "testing" +) + +var db ShardBackend + +func init() { + shardDB, err := NewShardDB("/tmp/datadir", "shardchaindata") + if err != nil { + panic(err) + } + db = shardDB +} + +// Testing the concurrency of the shardDB with multiple processes attempting to write. +func Test_DBConcurrent(t *testing.T) { + for i := 0; i < 100; i++ { + go func(val string) { + if err := db.Put([]byte("ralph merkle"), []byte(val)); err != nil { + t.Errorf("could not save value in db: %v", err) + } + }(strconv.Itoa(i)) + } +} + +func Test_DBPut(t *testing.T) { + if err := db.Put([]byte("ralph merkle"), []byte{1, 2, 3}); err != nil { + t.Errorf("could not save value in db: %v", err) + } +} + +func Test_DBHas(t *testing.T) { + key := []byte("ralph merkle") + + if err := db.Put(key, []byte{1, 2, 3}); err != nil { + t.Fatalf("could not save value in db: %v", err) + } + + has, err := db.Has(key) + if err != nil { + t.Errorf("could not check if db has key: %v", err) + } + if !has { + t.Errorf("db should have key: %v", key) + } + + key2 := []byte{} + has2, err := db.Has(key2) + if err != nil { + t.Errorf("could not check if db has key: %v", err) + } + if has2 { + t.Errorf("db should not have non-existent key: %v", key2) + } +} + +func Test_DBGet(t *testing.T) { + key := []byte("ralph merkle") + + if err := db.Put(key, []byte{1, 2, 3}); err != nil { + t.Fatalf("could not save value in db: %v", err) + } + + val, err := db.Get(key) + if err != nil { + t.Errorf("get failed: %v", err) + } + if len(val) == 0 { + t.Errorf("no value stored for key") + } + + key2 := []byte{} + val2, err := db.Get(key2) + if len(val2) != 0 { + t.Errorf("non-existent key should not have a value. key=%v, value=%v", key2, val2) + } +} + +func Test_DBDelete(t *testing.T) { + key := []byte("ralph merkle") + + if err := db.Put(key, []byte{1, 2, 3}); err != nil { + t.Fatalf("could not save value in db: %v", err) + } + + if err := db.Delete(key); err != nil { + t.Errorf("could not delete key: %v", key) + } +} diff --git a/sharding/notary/service.go b/sharding/notary/service.go index bbd5d00c43..bfd0e0f822 100644 --- a/sharding/notary/service.go +++ b/sharding/notary/service.go @@ -23,8 +23,7 @@ func NewNotary(node node.Node) (*Notary, error) { if err != nil { return nil, err } - // return &Notary{node, shardDB}, nil - return &Notary{node: node}, nil + return &Notary{node, shardDB}, nil } // Start the main routine for a notary. diff --git a/sharding/proposer/service.go b/sharding/proposer/service.go index afa490bbae..f1dd818283 100644 --- a/sharding/proposer/service.go +++ b/sharding/proposer/service.go @@ -2,6 +2,7 @@ package proposer import ( "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/sharding/database" "github.com/ethereum/go-ethereum/sharding/node" ) @@ -9,14 +10,19 @@ import ( // in a sharded system. Must satisfy the Service interface defined in // sharding/service.go. type Proposer struct { - node node.Node + node node.Node + shardDB database.ShardBackend } // NewProposer creates a struct instance. It is initialized and // registered as a service upon start of a sharding node. // Has access to the public methods of this node. func NewProposer(node node.Node) (*Proposer, error) { - return &Proposer{node}, nil + shardDB, err := database.NewShardDB(node.DataDirFlag(), "shardchaindata") + if err != nil { + return nil, err + } + return &Proposer{node, shardDB}, nil } // Start the main loop for proposing collations. From 84c8c58c3dd6cb68f7d9c9aaaf646def403c81a6 Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Fri, 25 May 2018 20:16:29 -0600 Subject: [PATCH 08/11] sharding: revert master changes - travis was broken Former-commit-id: 726c06eb4248536e23b143f8e0af51cc91a8f19f [formerly 305291115def673a9b862d4701da1f1d6b4439d4] Former-commit-id: 643294b138f2ff848b32319f8e60fad93e81095b --- sharding/proposer/service.go | 1 + 1 file changed, 1 insertion(+) diff --git a/sharding/proposer/service.go b/sharding/proposer/service.go index f1dd818283..da672f34b2 100644 --- a/sharding/proposer/service.go +++ b/sharding/proposer/service.go @@ -18,6 +18,7 @@ type Proposer struct { // registered as a service upon start of a sharding node. // Has access to the public methods of this node. func NewProposer(node node.Node) (*Proposer, error) { + // Initializes a shardchaindata directory persistent db. shardDB, err := database.NewShardDB(node.DataDirFlag(), "shardchaindata") if err != nil { return nil, err From 54cf3c9a4f853d09fb45c3f239e1840280758c06 Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Fri, 25 May 2018 21:19:04 -0600 Subject: [PATCH 09/11] sharding: fix comment Former-commit-id: 58eb9982deb7122f6f1890f2e25aeb8259cb8d76 [formerly d28feb35046373f36c3be3543156e1b12c7c595f] Former-commit-id: 842b7609b844df8e282cce43f13a1c94515933af --- sharding/shard_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/sharding/shard_test.go b/sharding/shard_test.go index 58b6823e23..49247fcda2 100644 --- a/sharding/shard_test.go +++ b/sharding/shard_test.go @@ -16,7 +16,6 @@ type mockShardDB struct { kv map[common.Hash][]byte } -// TOOD: FINISH MOCK CLIENT func (m *mockShardDB) Get(k []byte) ([]byte, error) { return []byte{}, nil } From 50811bcd8f0b8297398d7542d379c80a7191f802 Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Sat, 26 May 2018 12:25:43 -0600 Subject: [PATCH 10/11] sharding: fix lint Former-commit-id: f820d6e22ebe00f68cb7b6a210d9a2a4bb6ee027 [formerly 3d492416a550b77a7cf5565e54588310b758bb40] Former-commit-id: 6be1b4da840c8c77a227043fef7fe655f7ab2ad7 --- sharding/notary/service_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sharding/notary/service_test.go b/sharding/notary/service_test.go index 52fb00e378..683ef9c637 100644 --- a/sharding/notary/service_test.go +++ b/sharding/notary/service_test.go @@ -63,6 +63,10 @@ func (m *mockNode) DepositFlagSet() bool { return m.DepositFlag } +func (m *mockNode) DataDirFlag() string { + return "/tmp/datadir" +} + // Unused mockClient methods. func (m *mockNode) Start() error { m.t.Fatal("Start called") From 689800d0652c7450596206c8016c77c87bd76ff3 Mon Sep 17 00:00:00 2001 From: Preston Van Loon Date: Mon, 28 May 2018 20:32:10 -0400 Subject: [PATCH 11/11] Quick fixes (#141) Sharding: * Sharding: add actor flag to sharding group for app help * Sharding: ensure rpcClient is not nil before calling Close() * Sharding: revert shardingcmd.go comments * Sharding: more address Str() fixes Former-commit-id: 0b3d8b73ac1b20eb5c018a71be1d935a1fb860ef [formerly 6cd1df472713e95f6b9f8a0a7b6bbd88a5dd2ca2] Former-commit-id: 90c7e1a70d46c1f889576a7f2f253a02414c3654 --- cmd/geth/usage.go | 1 + sharding/node/node.go | 5 ++++- sharding/node/utils.go | 6 +++--- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 34937948f1..bafc3d48c6 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -86,6 +86,7 @@ var AppHelpFlagGroups = []flagGroup{ { Name: "SHARDING", Flags: []cli.Flag{ + utils.ActorFlag, utils.DepositFlag, }, }, diff --git a/sharding/node/node.go b/sharding/node/node.go index 4a370e955e..73c818cce8 100644 --- a/sharding/node/node.go +++ b/sharding/node/node.go @@ -162,7 +162,10 @@ func (n *shardingNode) Register(constructor sharding.ServiceConstructor) error { // Close the RPC client connection. func (n *shardingNode) Close() { - n.rpcClient.Close() + // rpcClient could be nil if the connection failed. + if n.rpcClient != nil { + n.rpcClient.Close() + } } // CreateTXOpts creates a *TransactOpts with a signer using the default account on the keystore. diff --git a/sharding/node/utils.go b/sharding/node/utils.go index 1838d99739..1a701f5b2e 100644 --- a/sharding/node/utils.go +++ b/sharding/node/utils.go @@ -26,14 +26,14 @@ func dialRPC(endpoint string) (*rpc.Client, error) { func initSMC(n *shardingNode) (*contracts.SMC, error) { b, err := n.client.CodeAt(context.Background(), sharding.ShardingManagerAddress, nil) if err != nil { - return nil, fmt.Errorf("unable to get contract code at %s: %v", sharding.ShardingManagerAddress, err) + return nil, fmt.Errorf("unable to get contract code at %s: %v", sharding.ShardingManagerAddress.Str(), err) } // Deploy SMC for development only. // TODO: Separate contract deployment from the sharding node. It would only need to be deployed // once on the mainnet, so this code would not need to ship with the node. if len(b) == 0 { - log.Info(fmt.Sprintf("No sharding manager contract found at %s. Deploying new contract.", sharding.ShardingManagerAddress)) + log.Info(fmt.Sprintf("No sharding manager contract found at %s. Deploying new contract.", sharding.ShardingManagerAddress.Str())) txOps, err := n.CreateTXOpts(big.NewInt(0)) if err != nil { @@ -52,7 +52,7 @@ func initSMC(n *shardingNode) (*contracts.SMC, error) { time.Sleep(1 * time.Second) } - log.Info(fmt.Sprintf("New contract deployed at %s", addr)) + log.Info(fmt.Sprintf("New contract deployed at %s", addr.Str())) return contract, nil }