beacon: Define a Core Blockchain Package and Persisted Structure for Beacon (#278)

Former-commit-id: bbd5b46e7f64f762350d6fb496492207e70d7130 [formerly 43a37f7139b7d1d90f0c27a7406b63bdf390ad96]
Former-commit-id: bb7a2ff0a7619f8de0bd38cd2c9eb0de7c189edb

beacon-chain/BUILD.bazel

@@ -7,7 +7,8 @@ go_library(
     visibility = ["//beacon-chain:__subpackages__"],
     deps = [
         "//beacon-chain/node:go_default_library",
-        "//beacon-chain/types:go_default_library",
+        "//beacon-chain/utils:go_default_library",
+        "//shared/cmd:go_default_library",
         "//shared/debug:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
         "@com_github_urfave_cli//:go_default_library",

beacon-chain/VALIDATOR_REGISTER.md

@@ -0,0 +1,34 @@
+# Validator Registration Workflow
+
+This doc summarizes the work flow of registering to become a validator in the beacon chain. The scope is within Ruby Release.
+
+### Step 1: Deploy validator registration contract if it hasn't been done
+To deploy VRC, we can use [deployVRC](https://github.com/terenc3t/geth-sharding/tree/contract-util/contracts/deployVRC) utility.  
+Once we get the VRC contract address, we can launch our beacon chain node
+```
+# Deploy VRC with keystore UTCJSON and password
+go run deployVRC.go --UTCPath /path/to/your/keystore/UTCJSON --passwordFile /path/to/your/password.txt
+# Deploy VRC with private key
+go run deployVRC.go --privKey 8a6db3b30934439c9f71f1fa777019810fd538c9c1e396809bcf9fd5535e20ca
+
+INFO[0039] New contract deployed at 0x559eDab2b5896C2Bc37951325666ed08CD41099d
+```
+
+### Step 2: Launch beacon chain node
+Launch beacon chain node with account holder's public key and the VRC address we just deployed
+```
+./bazel-bin/path/to/your/beacon-chain/binary --vrcaddr 0x527580dd995c0ab81d01f9993eb39166796877a1 --pubkey aaace816cdab194b4bc6c0de3575ccf917a9b9ecfead263720968e0e1b45739c
+
+```
+
+### Step 3: Send a transaction to the deposit function in VRC with 32 ETH and beacon chain node account holder's public key as argument
+
+
+### Step 4: Wait for deposit transaction to mine.
+After the deposit transaction gets mined, beacon chain node will report account holder has been registered. Congrats! Now, you are contributing to the security of Ethereum 2.0 : )
+```
+INFO[0000] Starting beacon node
+INFO[0000] Starting web3 PoW chain service at ws://127.0.0.1:8546
+INFO[0152] Validator registered in VRC with public key: aaace816cdab194b4bc6c0de3575ccf917a9b9ecfead263720968e0e1b45739c
+```
+

beacon-chain/blockchain/BUILD.bazel

@@ -0,0 +1,34 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "core.go",
+        "service.go",
+    ],
+    importpath = "github.com/prysmaticlabs/geth-sharding/beacon-chain/blockchain",
+    visibility = ["//beacon-chain:__subpackages__"],
+    deps = [
+        "//beacon-chain/database:go_default_library",
+        "//beacon-chain/types:go_default_library",
+        "@com_github_ethereum_go_ethereum//ethdb:go_default_library",
+        "@com_github_ethereum_go_ethereum//rlp:go_default_library",
+        "@com_github_sirupsen_logrus//:go_default_library",
+        "@com_github_syndtr_goleveldb//leveldb/errors:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "core_test.go",
+        "service_test.go",
+    ],
+    embed = [":go_default_library"],
+    deps = [
+        "//beacon-chain/database:go_default_library",
+        "//beacon-chain/types:go_default_library",
+        "@com_github_ethereum_go_ethereum//common:go_default_library",
+        "@com_github_sirupsen_logrus//hooks/test:go_default_library",
+    ],
+)

beacon-chain/blockchain/core.go

@@ -0,0 +1,87 @@
+package blockchain
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/prysmaticlabs/geth-sharding/beacon-chain/types"
+	log "github.com/sirupsen/logrus"
+	leveldberrors "github.com/syndtr/goleveldb/leveldb/errors"
+)
+
+var stateLookupKey = "beaconchainstate"
+
+// BeaconChain represents the core PoS blockchain object containing
+// both a crystallized and active state.
+type BeaconChain struct {
+	state *beaconState
+	lock  sync.Mutex
+	db    ethdb.Database
+}
+
+type beaconState struct {
+	ActiveState       *types.ActiveState
+	CrystallizedState *types.CrystallizedState
+}
+
+// NewBeaconChain initializes an instance using genesis state parameters if
+// none provided.
+func NewBeaconChain(db ethdb.Database) (*BeaconChain, error) {
+	beaconChain := &BeaconChain{
+		db:    db,
+		state: &beaconState{},
+	}
+	enc, err := db.Get([]byte(stateLookupKey))
+	if err != nil && err.Error() == leveldberrors.ErrNotFound.Error() {
+		log.Info("No chainstate found on disk, initializing beacon from genesis")
+		active, crystallized := types.NewGenesisStates()
+		beaconChain.state.ActiveState = active
+		beaconChain.state.CrystallizedState = crystallized
+		return beaconChain, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	// Deserializes the encoded object into a beacon chain.
+	if err := rlp.DecodeBytes(enc, &beaconChain.state); err != nil {
+		return nil, fmt.Errorf("could not deserialize chainstate from disk: %v", err)
+	}
+	return beaconChain, nil
+}
+
+// ActiveState exposes a getter to external services.
+func (b *BeaconChain) ActiveState() *types.ActiveState {
+	return b.state.ActiveState
+}
+
+// CrystallizedState exposes a getter to external services.
+func (b *BeaconChain) CrystallizedState() *types.CrystallizedState {
+	return b.state.CrystallizedState
+}
+
+// MutateActiveState allows external services to modify the active state.
+func (b *BeaconChain) MutateActiveState(activeState *types.ActiveState) error {
+	defer b.lock.Unlock()
+	b.lock.Lock()
+	b.state.ActiveState = activeState
+	return b.persist()
+}
+
+// MutateCrystallizedState allows external services to modify the crystallized state.
+func (b *BeaconChain) MutateCrystallizedState(crystallizedState *types.CrystallizedState) error {
+	defer b.lock.Unlock()
+	b.lock.Lock()
+	b.state.CrystallizedState = crystallizedState
+	return b.persist()
+}
+
+// persist stores the RLP encoding of the latest beacon chain state into the db.
+func (b *BeaconChain) persist() error {
+	encodedState, err := rlp.EncodeToBytes(b.state)
+	if err != nil {
+		return err
+	}
+	return b.db.Put([]byte(stateLookupKey), encodedState)
+}

beacon-chain/blockchain/core_test.go

@@ -0,0 +1,121 @@
+package blockchain
+
+import (
+	"fmt"
+	"os"
+	"reflect"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/prysmaticlabs/geth-sharding/beacon-chain/database"
+	"github.com/prysmaticlabs/geth-sharding/beacon-chain/types"
+	logTest "github.com/sirupsen/logrus/hooks/test"
+)
+
+func TestNewBeaconChain(t *testing.T) {
+	hook := logTest.NewGlobal()
+	tmp := fmt.Sprintf("%s/beacontest", os.TempDir())
+	config := &database.BeaconDBConfig{DataDir: tmp, Name: "beacontest", InMemory: false}
+	db, err := database.NewBeaconDB(config)
+	if err != nil {
+		t.Fatalf("unable to setup db: %v", err)
+	}
+	db.Start()
+	beaconChain, err := NewBeaconChain(db.DB())
+	if err != nil {
+		t.Fatalf("unable to setup beacon chain: %v", err)
+	}
+
+	msg := hook.LastEntry().Message
+	want := "No chainstate found on disk, initializing beacon from genesis"
+	if msg != want {
+		t.Errorf("incorrect log, expected %s, got %s", want, msg)
+	}
+
+	hook.Reset()
+	active, crystallized := types.NewGenesisStates()
+	if !reflect.DeepEqual(beaconChain.ActiveState(), active) {
+		t.Errorf("active states not equal. received: %v, wanted: %v", beaconChain.ActiveState(), active)
+	}
+	if !reflect.DeepEqual(beaconChain.CrystallizedState(), crystallized) {
+		t.Errorf("crystallized states not equal. received: %v, wanted: %v", beaconChain.CrystallizedState(), crystallized)
+	}
+}
+
+func TestMutateActiveState(t *testing.T) {
+	tmp := fmt.Sprintf("%s/beacontest", os.TempDir())
+	config := &database.BeaconDBConfig{DataDir: tmp, Name: "beacontest2", InMemory: false}
+	db, err := database.NewBeaconDB(config)
+	if err != nil {
+		t.Fatalf("unable to setup db: %v", err)
+	}
+	db.Start()
+	beaconChain, err := NewBeaconChain(db.DB())
+	if err != nil {
+		t.Fatalf("unable to setup beacon chain: %v", err)
+	}
+
+	randao := common.BytesToHash([]byte("hello"))
+	active := &types.ActiveState{
+		Height: 100,
+		Randao: randao,
+	}
+	if err := beaconChain.MutateActiveState(active); err != nil {
+		t.Fatalf("unable to mutate active state: %v", err)
+	}
+	if !reflect.DeepEqual(beaconChain.state.ActiveState, active) {
+		t.Errorf("active state was not updated. wanted %v, got %v", active, beaconChain.state.ActiveState)
+	}
+
+	// Initializing a new beacon chain should deserialize persisted state from disk.
+	newBeaconChain, err := NewBeaconChain(db.DB())
+	if err != nil {
+		t.Fatalf("unable to setup beacon chain: %v", err)
+	}
+	// The active state should still be the one we mutated and persited earlier.
+	if active.Height != newBeaconChain.state.ActiveState.Height {
+		t.Errorf("active state height incorrect. wanted %v, got %v", active.Height, newBeaconChain.state.ActiveState.Height)
+	}
+	if active.Randao.Hex() != newBeaconChain.state.ActiveState.Randao.Hex() {
+		t.Errorf("active state randao incorrect. wanted %v, got %v", active.Randao.Hex(), newBeaconChain.state.ActiveState.Randao.Hex())
+	}
+}
+
+func TestMutateCrystallizedState(t *testing.T) {
+	tmp := fmt.Sprintf("%s/beacontest", os.TempDir())
+	config := &database.BeaconDBConfig{DataDir: tmp, Name: "beacontest3", InMemory: false}
+	db, err := database.NewBeaconDB(config)
+	if err != nil {
+		t.Fatalf("unable to setup db: %v", err)
+	}
+	db.Start()
+	beaconChain, err := NewBeaconChain(db.DB())
+	if err != nil {
+		t.Fatalf("unable to setup beacon chain: %v", err)
+	}
+
+	currentCheckpoint := common.BytesToHash([]byte("checkpoint"))
+	crystallized := &types.CrystallizedState{
+		Dynasty:           3,
+		CurrentCheckpoint: currentCheckpoint,
+	}
+	if err := beaconChain.MutateCrystallizedState(crystallized); err != nil {
+		t.Fatalf("unable to mutate crystallized state: %v", err)
+	}
+	if !reflect.DeepEqual(beaconChain.state.CrystallizedState, crystallized) {
+		t.Errorf("crystallized state was not updated. wanted %v, got %v", crystallized, beaconChain.state.CrystallizedState)
+	}
+
+	// Initializing a new beacon chain should deserialize persisted state from disk.
+	newBeaconChain, err := NewBeaconChain(db.DB())
+	if err != nil {
+		t.Fatalf("unable to setup beacon chain: %v", err)
+	}
+	// The crystallized state should still be the one we mutated and persited earlier.
+	if crystallized.Dynasty != newBeaconChain.state.CrystallizedState.Dynasty {
+		t.Errorf("crystallized state dynasty incorrect. wanted %v, got %v", crystallized.Dynasty, newBeaconChain.state.CrystallizedState.Dynasty)
+	}
+	if crystallized.CurrentCheckpoint.Hex() != newBeaconChain.state.CrystallizedState.CurrentCheckpoint.Hex() {
+		t.Errorf("crystallized state current checkpoint incorrect. wanted %v, got %v", crystallized.CurrentCheckpoint.Hex(), newBeaconChain.state.CrystallizedState.CurrentCheckpoint.Hex())
+	}
+}

beacon-chain/blockchain/service.go

@@ -0,0 +1,39 @@
+package blockchain
+
+import (
+	"context"
+
+	"github.com/prysmaticlabs/geth-sharding/beacon-chain/database"
+	log "github.com/sirupsen/logrus"
+)
+
+// ChainService represents a service that handles the internal
+// logic of managing the full PoS beacon chain.
+type ChainService struct {
+	ctx      context.Context
+	cancel   context.CancelFunc
+	beaconDB *database.BeaconDB
+	chain    *BeaconChain
+}
+
+// NewChainService instantiates a new service instance that will
+// be registered into a running beacon node.
+func NewChainService(ctx context.Context, beaconDB *database.BeaconDB) (*ChainService, error) {
+	ctx, cancel := context.WithCancel(ctx)
+	return &ChainService{ctx, cancel, beaconDB, nil}, nil
+}
+
+// Start a blockchain service's main event loop.
+func (c *ChainService) Start() {
+	log.Infof("Starting blockchain service")
+	if _, err := NewBeaconChain(c.beaconDB.DB()); err != nil {
+		log.Errorf("Unable to setup blockchain: %v", err)
+	}
+}
+
+// Stop the blockchain service's main event loop and associated goroutines.
+func (c *ChainService) Stop() error {
+	defer c.cancel()
+	log.Info("Stopping blockchain service")
+	return nil
+}

beacon-chain/blockchain/service_test.go

@@ -0,0 +1,63 @@
+package blockchain
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/prysmaticlabs/geth-sharding/beacon-chain/database"
+	logTest "github.com/sirupsen/logrus/hooks/test"
+)
+
+func TestStartStop(t *testing.T) {
+	hook := logTest.NewGlobal()
+	ctx := context.Background()
+	tmp := fmt.Sprintf("%s/beacontest", os.TempDir())
+	config := &database.BeaconDBConfig{DataDir: tmp, Name: "beacontestdata", InMemory: false}
+	db, err := database.NewBeaconDB(config)
+	if err != nil {
+		t.Fatalf("could not setup beaconDB: %v", err)
+	}
+	db.Start()
+	chainService, err := NewChainService(ctx, db)
+	if err != nil {
+		t.Fatalf("unable to setup chain service: %v", err)
+	}
+
+	chainService.Start()
+
+	if err := chainService.Stop(); err != nil {
+		t.Fatalf("unable to stop chain service: %v", err)
+	}
+
+	msg := hook.AllEntries()[0].Message
+	want := "Starting beaconDB service"
+	if msg != want {
+		t.Errorf("incorrect log, expected %s, got %s", want, msg)
+	}
+
+	msg = hook.AllEntries()[1].Message
+	want = "Starting blockchain service"
+	if msg != want {
+		t.Errorf("incorrect log, expected %s, got %s", want, msg)
+	}
+
+	msg = hook.AllEntries()[2].Message
+	want = "No chainstate found on disk, initializing beacon from genesis"
+	if msg != want {
+		t.Errorf("incorrect log, expected %s, got %s", want, msg)
+	}
+
+	msg = hook.AllEntries()[3].Message
+	want = "Stopping blockchain service"
+	if msg != want {
+		t.Errorf("incorrect log, expected %s, got %s", want, msg)
+	}
+
+	// The context should have been canceled.
+	if chainService.ctx.Err() == nil {
+		t.Error("context was not canceled")
+	}
+	hook.Reset()
+}

beacon-chain/database/BUILD.bazel

@@ -0,0 +1,24 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["database.go"],
+    importpath = "github.com/prysmaticlabs/geth-sharding/beacon-chain/database",
+    visibility = ["//beacon-chain:__subpackages__"],
+    deps = [
+        "//shared/database:go_default_library",
+        "@com_github_ethereum_go_ethereum//ethdb:go_default_library",
+        "@com_github_sirupsen_logrus//:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["database_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "//shared:go_default_library",
+        "@com_github_sirupsen_logrus//hooks/test:go_default_library",
+        "@com_github_syndtr_goleveldb//leveldb/errors:go_default_library",
+    ],
+)

beacon-chain/database/database.go

@@ -0,0 +1,73 @@
+// Package database defines a beacon chain DB service that can be
+// initialized with either a persistent db, or an in-memory kv-store.
+package database
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/ethereum/go-ethereum/ethdb"
+	sharedDB "github.com/prysmaticlabs/geth-sharding/shared/database"
+	log "github.com/sirupsen/logrus"
+)
+
+// BeaconDB defines a service for the beacon chain system's persistent storage.
+type BeaconDB struct {
+	inmemory bool
+	dataDir  string
+	name     string
+	cache    int
+	handles  int
+	db       ethdb.Database
+}
+
+// BeaconDBConfig specifies configuration options for the db service.
+type BeaconDBConfig struct {
+	DataDir  string
+	Name     string
+	InMemory bool
+}
+
+// NewBeaconDB initializes a beaconDB instance.
+func NewBeaconDB(config *BeaconDBConfig) (*BeaconDB, error) {
+	// Uses default cache and handles values.
+	// TODO: allow these arguments to be set based on cli context.
+	beaconDB := &BeaconDB{
+		name:    config.Name,
+		dataDir: config.DataDir,
+	}
+	if config.InMemory {
+		beaconDB.inmemory = true
+		beaconDB.db = sharedDB.NewKVStore()
+	} else {
+		beaconDB.inmemory = false
+		beaconDB.cache = 16
+		beaconDB.handles = 16
+	}
+	return beaconDB, nil
+}
+
+// Start the beacon DB service.
+func (b *BeaconDB) Start() {
+	log.Info("Starting beaconDB service")
+	if !b.inmemory {
+		db, err := ethdb.NewLDBDatabase(filepath.Join(b.dataDir, b.name), b.cache, b.handles)
+		if err != nil {
+			log.Error(fmt.Sprintf("Could not start beaconDB: %v", err))
+			return
+		}
+		b.db = db
+	}
+}
+
+// Stop the beaconDB service gracefully.
+func (b *BeaconDB) Stop() error {
+	log.Info("Stopping beaconDB service")
+	b.db.Close()
+	return nil
+}
+
+// DB returns the attached ethdb instance.
+func (b *BeaconDB) DB() ethdb.Database {
+	return b.db
+}

beacon-chain/database/database_test.go

@@ -0,0 +1,139 @@
+package database
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+	"sync"
+	"testing"
+
+	"github.com/prysmaticlabs/geth-sharding/shared"
+	logTest "github.com/sirupsen/logrus/hooks/test"
+	leveldberrors "github.com/syndtr/goleveldb/leveldb/errors"
+)
+
+// Verifies that BeaconDB implements the sharding Service inteface.
+var _ = shared.Service(&BeaconDB{})
+
+var testDB *BeaconDB
+
+func init() {
+	tmp := fmt.Sprintf("%s/datadir", os.TempDir())
+	config := &BeaconDBConfig{DataDir: tmp, Name: "beaconchaindata", InMemory: false}
+	beaconDB, _ := NewBeaconDB(config)
+	testDB = beaconDB
+	testDB.Start()
+}
+
+func TestLifecycle(t *testing.T) {
+	hook := logTest.NewGlobal()
+
+	tmp := fmt.Sprintf("%s/lifecycledir", os.TempDir())
+	config := &BeaconDBConfig{DataDir: tmp, Name: "beaconchaindata", InMemory: false}
+	b, err := NewBeaconDB(config)
+	if err != nil {
+		t.Fatalf("could not initialize a new DB: %v", err)
+	}
+
+	b.Start()
+	msg := hook.LastEntry().Message
+	if msg != "Starting beaconDB service" {
+		t.Errorf("incorrect log, expected %s, got %s", "Starting beaconDB service", msg)
+	}
+
+	b.Stop()
+	msg = hook.LastEntry().Message
+	if msg != "Stopping beaconDB service" {
+		t.Errorf("incorrect log, expected %s, got %s", "Stopping beaconDB service", msg)
+	}
+
+	// Access DB after it's stopped, this should fail.
+	_, err = b.db.Get([]byte("ralph merkle"))
+
+	if err.Error() != "leveldb: closed" {
+		t.Fatalf("beaconDB close function did not work")
+	}
+}
+
+// Testing the concurrency with multiple processes attempting to write.
+func Test_DBConcurrent(t *testing.T) {
+	var wg sync.WaitGroup
+	wg.Add(100)
+	for i := 0; i < 100; i++ {
+		go func(val string) {
+			defer wg.Done()
+			if err := testDB.db.Put([]byte("ralph merkle"), []byte(val)); err != nil {
+				t.Errorf("could not save value in db: %v", err)
+			}
+		}(strconv.Itoa(i))
+	}
+	wg.Wait()
+}
+
+func Test_DBPut(t *testing.T) {
+	if err := testDB.db.Put([]byte("ralph merkle"), []byte{1, 2, 3}); err != nil {
+		t.Errorf("could not save value in db: %v", err)
+	}
+}
+
+func Test_DBHas(t *testing.T) {
+	key := []byte("ralph merkle")
+
+	if err := testDB.db.Put(key, []byte{1, 2, 3}); err != nil {
+		t.Fatalf("could not save value in db: %v", err)
+	}
+
+	has, err := testDB.db.Has(key)
+	if err != nil {
+		t.Errorf("could not check if db has key: %v", err)
+	}
+	if !has {
+		t.Errorf("db should have key: %v", key)
+	}
+
+	key2 := []byte{}
+	has2, err := testDB.db.Has(key2)
+	if err != nil {
+		t.Errorf("could not check if db has key: %v", err)
+	}
+	if has2 {
+		t.Errorf("db should not have non-existent key: %v", key2)
+	}
+}
+
+func Test_DBGet(t *testing.T) {
+	key := []byte("ralph merkle")
+
+	if err := testDB.db.Put(key, []byte{1, 2, 3}); err != nil {
+		t.Fatalf("could not save value in db: %v", err)
+	}
+
+	val, err := testDB.db.Get(key)
+	if err != nil {
+		t.Errorf("get failed: %v", err)
+	}
+	if len(val) == 0 {
+		t.Errorf("no value stored for key")
+	}
+
+	key2 := []byte{}
+	val2, err := testDB.db.Get(key2)
+	if err == nil || err.Error() != leveldberrors.ErrNotFound.Error() {
+		t.Errorf("Expected error %v but got %v", leveldberrors.ErrNotFound, err)
+	}
+	if len(val2) != 0 {
+		t.Errorf("non-existent key should not have a value. key=%v, value=%v", key2, val2)
+	}
+}
+
+func Test_DBDelete(t *testing.T) {
+	key := []byte("ralph merkle")
+
+	if err := testDB.db.Put(key, []byte{1, 2, 3}); err != nil {
+		t.Fatalf("could not save value in db: %v", err)
+	}
+
+	if err := testDB.db.Delete(key); err != nil {
+		t.Errorf("could not delete key: %v", key)
+	}
+}

beacon-chain/main.go

@@ -5,7 +5,8 @@ import (
 	"runtime"
 
 	"github.com/prysmaticlabs/geth-sharding/beacon-chain/node"
-	"github.com/prysmaticlabs/geth-sharding/beacon-chain/types"
+	"github.com/prysmaticlabs/geth-sharding/beacon-chain/utils"
+	"github.com/prysmaticlabs/geth-sharding/shared/cmd"
 	"github.com/prysmaticlabs/geth-sharding/shared/debug"
 	log "github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -44,7 +45,7 @@ VERSION:
 	app.Usage = "this is a beacon chain implementation for Ethereum 2.0"
 	app.Action = startNode
 
-	app.Flags = []cli.Flag{types.Web3ProviderFlag, debug.PProfFlag, debug.PProfAddrFlag, debug.PProfPortFlag, debug.MemProfileRateFlag, debug.CPUProfileFlag, debug.TraceFlag}
+	app.Flags = []cli.Flag{cmd.DataDirFlag, utils.VrcContractFlag, utils.PubKeyFlag, utils.Web3ProviderFlag, debug.PProfFlag, debug.PProfAddrFlag, debug.PProfPortFlag, debug.MemProfileRateFlag, debug.CPUProfileFlag, debug.TraceFlag}
 
 	app.Before = func(ctx *cli.Context) error {
 		runtime.GOMAXPROCS(runtime.NumCPU())

beacon-chain/node/BUILD.bazel

@@ -6,10 +6,14 @@ go_library(
     importpath = "github.com/prysmaticlabs/geth-sharding/beacon-chain/node",
     visibility = ["//beacon-chain:__subpackages__"],
     deps = [
+        "//beacon-chain/blockchain:go_default_library",
+        "//beacon-chain/database:go_default_library",
         "//beacon-chain/powchain:go_default_library",
-        "//beacon-chain/types:go_default_library",
+        "//beacon-chain/utils:go_default_library",
         "//shared:go_default_library",
+        "//shared/cmd:go_default_library",
         "//shared/debug:go_default_library",
+        "@com_github_ethereum_go_ethereum//common:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
         "@com_github_urfave_cli//:go_default_library",
     ],

beacon-chain/node/node.go

@@ -8,14 +8,20 @@ import (
 	"sync"
 	"syscall"
 
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/prysmaticlabs/geth-sharding/beacon-chain/blockchain"
+	"github.com/prysmaticlabs/geth-sharding/beacon-chain/database"
 	"github.com/prysmaticlabs/geth-sharding/beacon-chain/powchain"
-	"github.com/prysmaticlabs/geth-sharding/beacon-chain/types"
+	"github.com/prysmaticlabs/geth-sharding/beacon-chain/utils"
 	"github.com/prysmaticlabs/geth-sharding/shared"
+	"github.com/prysmaticlabs/geth-sharding/shared/cmd"
 	"github.com/prysmaticlabs/geth-sharding/shared/debug"
 	log "github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )
 
+var beaconChainDBName = "beaconchaindata"
+
 // BeaconNode defines a struct that handles the services running a random beacon chain
 // full PoS node. It handles the lifecycle of the entire system and registers
 // services to a service registry.
@@ -30,13 +36,23 @@ type BeaconNode struct {
 // every required service to the node.
 func New(ctx *cli.Context) (*BeaconNode, error) {
 	registry := shared.NewServiceRegistry()
+
 	beacon := &BeaconNode{
 		ctx:      ctx,
 		services: registry,
 		stop:     make(chan struct{}),
 	}
 
-	if err := beacon.registerWeb3Service(); err != nil {
+	path := ctx.GlobalString(cmd.DataDirFlag.Name)
+	if err := beacon.registerBeaconDB(path); err != nil {
+		return nil, err
+	}
+
+	if err := beacon.registerBlockchainService(); err != nil {
+		return nil, err
+	}
+
+	if err := beacon.registerPOWChainService(); err != nil {
 		return nil, err
 	}
 
@@ -85,11 +101,36 @@ func (b *BeaconNode) Close() {
 	close(b.stop)
 }
 
-func (b *BeaconNode) registerWeb3Service() error {
+func (b *BeaconNode) registerBeaconDB(path string) error {
-	endpoint := b.ctx.GlobalString(types.Web3ProviderFlag.Name)
+	config := &database.BeaconDBConfig{DataDir: path, Name: beaconChainDBName, InMemory: false}
-	web3Service, err := powchain.NewWeb3Service(context.TODO(), endpoint)
+	beaconDB, err := database.NewBeaconDB(config)
 	if err != nil {
-		return fmt.Errorf("could not register web3Service: %v", err)
+		return fmt.Errorf("could not register beaconDB service: %v", err)
+	}
+	return b.services.RegisterService(beaconDB)
+}
+
+func (b *BeaconNode) registerBlockchainService() error {
+	var beaconDB *database.BeaconDB
+	if err := b.services.FetchService(&beaconDB); err != nil {
+		return err
+	}
+
+	blockchainService, err := blockchain.NewChainService(context.TODO(), beaconDB)
+	if err != nil {
+		return fmt.Errorf("could not register blockchain service: %v", err)
+	}
+	return b.services.RegisterService(blockchainService)
+}
+
+func (b *BeaconNode) registerPOWChainService() error {
+	web3Service, err := powchain.NewWeb3Service(context.TODO(), &powchain.Web3ServiceConfig{
+		Endpoint: b.ctx.GlobalString(utils.Web3ProviderFlag.Name),
+		Pubkey:   b.ctx.GlobalString(utils.PubKeyFlag.Name),
+		VrcAddr:  common.HexToAddress(b.ctx.GlobalString(utils.VrcContractFlag.Name)),
+	})
+	if err != nil {
+		return fmt.Errorf("could not register proof-of-work chain web3Service: %v", err)
 	}
 	return b.services.RegisterService(web3Service)
 }

beacon-chain/powchain/BUILD.bazel

@@ -21,6 +21,7 @@ go_test(
     embed = [":go_default_library"],
     deps = [
         "@com_github_ethereum_go_ethereum//:go_default_library",
+        "@com_github_ethereum_go_ethereum//common:go_default_library",
         "@com_github_ethereum_go_ethereum//core/types:go_default_library",
         "@com_github_sirupsen_logrus//hooks/test:go_default_library",
     ],

beacon-chain/powchain/service.go

@@ -6,7 +6,7 @@ import (
 	"math/big"
 	"strings"
 
-	ethereum "github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/common"
 	gethTypes "github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethclient"
@@ -19,6 +19,11 @@ type Reader interface {
 	SubscribeNewHead(ctx context.Context, ch chan<- *gethTypes.Header) (ethereum.Subscription, error)
 }
 
+// Logger subscribe filtered log on the PoW chain
+type Logger interface {
+	SubscribeFilterLogs(ctx context.Context, q ethereum.FilterQuery, ch chan<- gethTypes.Log) (ethereum.Subscription, error)
+}
+
 // Web3Service fetches important information about the canonical
 // Ethereum PoW chain via a web3 endpoint using an ethclient. The Random
 // Beacon Chain requires synchronization with the PoW chain's current
@@ -26,34 +31,49 @@ type Reader interface {
 // Validator Registration Contract on the PoW chain to kick off the beacon
 // chain's validator registration process.
 type Web3Service struct {
-	ctx         context.Context
+	ctx                 context.Context
-	cancel      context.CancelFunc
+	cancel              context.CancelFunc
-	headerChan  chan *gethTypes.Header
+	headerChan          chan *gethTypes.Header
-	endpoint    string
+	logChan             chan gethTypes.Log
-	blockNumber *big.Int    // the latest PoW chain blocknumber.
+	pubKey              string
-	blockHash   common.Hash // the latest PoW chain blockhash.
+	endpoint            string
+	validatorRegistered bool
+	vrcAddress          common.Address
+	blockNumber         *big.Int    // the latest PoW chain blocknumber.
+	blockHash           common.Hash // the latest PoW chain blockhash.
+}
+
+// Web3ServiceConfig defines a config struct for web3 service to use through its life cycle.
+type Web3ServiceConfig struct {
+	Endpoint string
+	Pubkey   string
+	VrcAddr  common.Address
 }
 
 // NewWeb3Service sets up a new instance with an ethclient when
 // given a web3 endpoint as a string.
-func NewWeb3Service(ctx context.Context, endpoint string) (*Web3Service, error) {
+func NewWeb3Service(ctx context.Context, config *Web3ServiceConfig) (*Web3Service, error) {
-	if !strings.HasPrefix(endpoint, "ws") && !strings.HasPrefix(endpoint, "ipc") {
+	if !strings.HasPrefix(config.Endpoint, "ws") && !strings.HasPrefix(config.Endpoint, "ipc") {
-		return nil, fmt.Errorf("web3service requires either an IPC or WebSocket endpoint, provided %s", endpoint)
+		return nil, fmt.Errorf("web3service requires either an IPC or WebSocket endpoint, provided %s", config.Endpoint)
 	}
 	web3ctx, cancel := context.WithCancel(ctx)
 	return &Web3Service{
-		ctx:         web3ctx,
+		ctx:                 web3ctx,
-		cancel:      cancel,
+		cancel:              cancel,
-		headerChan:  make(chan *gethTypes.Header),
+		headerChan:          make(chan *gethTypes.Header),
-		endpoint:    endpoint,
+		logChan:             make(chan gethTypes.Log),
-		blockNumber: nil,
+		pubKey:              config.Pubkey,
-		blockHash:   common.BytesToHash([]byte{}),
+		endpoint:            config.Endpoint,
+		validatorRegistered: false,
+		blockNumber:         nil,
+		blockHash:           common.BytesToHash([]byte{}),
+		vrcAddress:          config.VrcAddr,
 	}, nil
 }
 
 // Start a web3 service's main event loop.
 func (w *Web3Service) Start() {
-	log.Infof("Starting web3 PoW chain service at %s", w.endpoint)
+	log.Infof("Starting web3 proof-of-work chain service at %s", w.endpoint)
 	rpcClient, err := rpc.Dial(w.endpoint)
 	if err != nil {
 		log.Errorf("Cannot connect to PoW chain RPC client: %v", err)
@@ -61,13 +81,14 @@ func (w *Web3Service) Start() {
 	}
 	client := ethclient.NewClient(rpcClient)
 	go w.latestPOWChainInfo(client, w.ctx.Done())
+	go w.queryValidatorStatus(client, w.ctx.Done())
 }
 
 // Stop the web3 service's main event loop and associated goroutines.
 func (w *Web3Service) Stop() error {
 	defer w.cancel()
 	defer close(w.headerChan)
-	log.Info("Stopping web3 PoW chain service")
+	log.Info("Stopping web3 proof-of-work chain service")
 	return nil
 }
 
@@ -89,6 +110,33 @@ func (w *Web3Service) latestPOWChainInfo(reader Reader, done <-chan struct{}) {
 	}
 }
 
+func (w *Web3Service) queryValidatorStatus(logger Logger, done <-chan struct{}) {
+	query := ethereum.FilterQuery{
+		Addresses: []common.Address{
+			w.vrcAddress,
+		},
+	}
+	_, err := logger.SubscribeFilterLogs(context.Background(), query, w.logChan)
+	if err != nil {
+		log.Errorf("Unable to query logs from VRC: %v", err)
+		return
+	}
+	for {
+		select {
+		case <-done:
+			return
+		case VRClog := <-w.logChan:
+			// public key is the second topic from validatorRegistered log and strip off 0x
+			pubKeyLog := VRClog.Topics[1].Hex()[2:]
+			if pubKeyLog == w.pubKey {
+				log.Infof("Validator registered in VRC with public key: %v", pubKeyLog)
+				w.validatorRegistered = true
+				return
+			}
+		}
+	}
+}
+
 // LatestBlockNumber is a getter for blockNumber to make it read-only.
 func (w *Web3Service) LatestBlockNumber() *big.Int {
 	return w.blockNumber
@@ -98,3 +146,8 @@ func (w *Web3Service) LatestBlockNumber() *big.Int {
 func (w *Web3Service) LatestBlockHash() common.Hash {
 	return w.blockHash
 }
+
+// ValidatorRegistered is a getter for validatorRegistered to make it read-only.
+func (w *Web3Service) ValidatorRegistered() bool {
+	return w.validatorRegistered
+}

beacon-chain/powchain/service_test.go

@@ -7,7 +7,8 @@ import (
 	"strings"
 	"testing"
 
-	ethereum "github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/common"
 	gethTypes "github.com/ethereum/go-ethereum/core/types"
 	logTest "github.com/sirupsen/logrus/hooks/test"
 )
@@ -24,22 +25,34 @@ func (g *goodReader) SubscribeNewHead(ctx context.Context, ch chan<- *gethTypes.
 	return nil, nil
 }
 
+type badLogger struct{}
+
+func (b *badLogger) SubscribeFilterLogs(ctx context.Context, q ethereum.FilterQuery, ch chan<- gethTypes.Log) (ethereum.Subscription, error) {
+	return nil, errors.New("subscription has failed")
+}
+
+type goodLogger struct{}
+
+func (g *goodLogger) SubscribeFilterLogs(ctx context.Context, q ethereum.FilterQuery, ch chan<- gethTypes.Log) (ethereum.Subscription, error) {
+	return nil, nil
+}
+
 func TestNewWeb3Service(t *testing.T) {
 	endpoint := "http://127.0.0.1"
 	ctx := context.Background()
-	if _, err := NewWeb3Service(ctx, endpoint); err == nil {
+	if _, err := NewWeb3Service(ctx, &Web3ServiceConfig{endpoint, "", common.Address{}}); err == nil {
 		t.Errorf("passing in an HTTP endpoint should throw an error, received nil")
 	}
 	endpoint = "ftp://127.0.0.1"
-	if _, err := NewWeb3Service(ctx, endpoint); err == nil {
+	if _, err := NewWeb3Service(ctx, &Web3ServiceConfig{endpoint, "", common.Address{}}); err == nil {
 		t.Errorf("passing in a non-ws, wss, or ipc endpoint should throw an error, received nil")
 	}
 	endpoint = "ws://127.0.0.1"
-	if _, err := NewWeb3Service(ctx, endpoint); err != nil {
+	if _, err := NewWeb3Service(ctx, &Web3ServiceConfig{endpoint, "", common.Address{}}); err != nil {
 		t.Errorf("passing in as ws endpoint should not throw error, received %v", err)
 	}
 	endpoint = "ipc://geth.ipc"
-	if _, err := NewWeb3Service(ctx, endpoint); err != nil {
+	if _, err := NewWeb3Service(ctx, &Web3ServiceConfig{endpoint, "", common.Address{}}); err != nil {
 		t.Errorf("passing in an ipc endpoint should not throw error, received %v", err)
 	}
 }
@@ -48,7 +61,7 @@ func TestStart(t *testing.T) {
 	hook := logTest.NewGlobal()
 
 	endpoint := "ws://127.0.0.1"
-	web3Service, err := NewWeb3Service(context.Background(), endpoint)
+	web3Service, err := NewWeb3Service(context.Background(), &Web3ServiceConfig{endpoint, "", common.Address{}})
 	if err != nil {
 		t.Fatalf("unable to setup web3 PoW chain service: %v", err)
 	}
@@ -67,7 +80,7 @@ func TestStop(t *testing.T) {
 	hook := logTest.NewGlobal()
 
 	endpoint := "ws://127.0.0.1"
-	web3Service, err := NewWeb3Service(context.Background(), endpoint)
+	web3Service, err := NewWeb3Service(context.Background(), &Web3ServiceConfig{endpoint, "", common.Address{}})
 	if err != nil {
 		t.Fatalf("unable to setup web3 PoW chain service: %v", err)
 	}
@@ -77,7 +90,7 @@ func TestStop(t *testing.T) {
 	}
 
 	msg := hook.LastEntry().Message
-	want := "Stopping web3 PoW chain service"
+	want := "Stopping web3 proof-of-work chain service"
 	if msg != want {
 		t.Errorf("incorrect log, expected %s, got %s", want, msg)
 	}
@@ -92,7 +105,7 @@ func TestStop(t *testing.T) {
 func TestBadReader(t *testing.T) {
 	hook := logTest.NewGlobal()
 	endpoint := "ws://127.0.0.1"
-	web3Service, err := NewWeb3Service(context.Background(), endpoint)
+	web3Service, err := NewWeb3Service(context.Background(), &Web3ServiceConfig{endpoint, "", common.Address{}})
 	if err != nil {
 		t.Fatalf("unable to setup web3 PoW chain service: %v", err)
 	}
@@ -107,7 +120,7 @@ func TestBadReader(t *testing.T) {
 
 func TestLatestMainchainInfo(t *testing.T) {
 	endpoint := "ws://127.0.0.1"
-	web3Service, err := NewWeb3Service(context.Background(), endpoint)
+	web3Service, err := NewWeb3Service(context.Background(), &Web3ServiceConfig{endpoint, "", common.Address{}})
 	if err != nil {
 		t.Fatalf("unable to setup web3 PoW chain service: %v", err)
 	}
@@ -134,3 +147,55 @@ func TestLatestMainchainInfo(t *testing.T) {
 		t.Errorf("block hash not set, expected %v, got %v", header.Hash().Hex(), web3Service.blockHash.Hex())
 	}
 }
+
+func TestBadLogger(t *testing.T) {
+	hook := logTest.NewGlobal()
+	endpoint := "ws://127.0.0.1"
+	web3Service, err := NewWeb3Service(context.Background(), &Web3ServiceConfig{endpoint, "", common.Address{}})
+	if err != nil {
+		t.Fatalf("unable to setup web3 PoW chain service: %v", err)
+	}
+	web3Service.queryValidatorStatus(&badLogger{}, web3Service.ctx.Done())
+	msg := hook.LastEntry().Message
+	want := "Unable to query logs from VRC: subscription has failed"
+	if msg != want {
+		t.Errorf("incorrect log, expected %s, got %s", want, msg)
+	}
+	hook.Reset()
+}
+
+func TestGoodLogger(t *testing.T) {
+	hook := logTest.NewGlobal()
+	endpoint := "ws://127.0.0.1"
+	web3Service, err := NewWeb3Service(context.Background(), &Web3ServiceConfig{endpoint, "", common.Address{}})
+	if err != nil {
+		t.Fatalf("unable to setup web3 PoW chain service: %v", err)
+	}
+
+	web3Service.pubKey = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
+	pubkey := common.HexToHash(web3Service.pubKey)
+
+	doneChan := make(chan struct{})
+	exitRoutine := make(chan bool)
+
+	go func() {
+		web3Service.queryValidatorStatus(&goodLogger{}, doneChan)
+		<-exitRoutine
+	}()
+
+	log := gethTypes.Log{Topics: []common.Hash{[32]byte{}, pubkey}}
+	web3Service.logChan <- log
+	exitRoutine <- true
+
+	msg := hook.LastEntry().Message
+	want := "Validator registered in VRC with public key: 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
+	if msg != want {
+		t.Errorf("incorrect log, expected %s, got %s", want, msg)
+	}
+
+	if !web3Service.validatorRegistered {
+		t.Error("validatorRegistered status expected true, got %v", web3Service.validatorRegistered)
+	}
+
+	hook.Reset()
+}

beacon-chain/types/BUILD.bazel

@@ -4,13 +4,9 @@ go_library(
     name = "go_default_library",
     srcs = [
         "block.go",
-        "flags.go",
         "state.go",
     ],
     importpath = "github.com/prysmaticlabs/geth-sharding/beacon-chain/types",
     visibility = ["//beacon-chain:__subpackages__"],
-    deps = [
+    deps = ["@com_github_ethereum_go_ethereum//common:go_default_library"],
-        "@com_github_ethereum_go_ethereum//common:go_default_library",
-        "@com_github_urfave_cli//:go_default_library",
-    ],
 )

beacon-chain/types/flags.go

@@ -1,12 +0,0 @@
-package types
-
-import "github.com/urfave/cli"
-
-var (
-	// Web3ProviderFlag defines a flag for a mainchain RPC endpoint.
-	Web3ProviderFlag = cli.StringFlag{
-		Name:  "web3provider",
-		Usage: "A mainchain web3 provider string endpoint. Can either be an IPC file string or a WebSocket endpoint. Uses WebSockets by default at ws://127.0.0.1:8546. Cannot be an HTTP endpoint.",
-		Value: "ws://127.0.0.1:8546",
-	}
-)

beacon-chain/types/state.go

@@ -60,3 +60,28 @@ type CrosslinkRecord struct {
 	Epoch uint64      // Epoch records the epoch the crosslink was submitted in.
 	Hash  common.Hash // Hash is the block hash.
 }
+
+// NewGenesisStates initializes a beacon chain with starting parameters.
+func NewGenesisStates() (*ActiveState, *CrystallizedState) {
+	active := &ActiveState{
+		Height:            0,
+		Randao:            common.BytesToHash([]byte{}),
+		FfgVoterBitmask:   []byte{},
+		BalanceDeltas:     []uint{},
+		PartialCrosslinks: []PartialCrosslinkRecord{},
+		TotalSkipCount:    0,
+	}
+	crystallized := &CrystallizedState{
+		ActiveValidators:   []ValidatorRecord{},
+		QueuedValidators:   []ValidatorRecord{},
+		ExitedValidators:   []ValidatorRecord{},
+		CurrentShuffling:   []uint16{},
+		CurrentEpoch:       0,
+		LastJustifiedEpoch: 0,
+		LastFinalizedEpoch: 0,
+		Dynasty:            0,
+		TotalDeposits:      0,
+		CrosslinkSeed:      common.BytesToHash([]byte{}),
+	}
+	return active, crystallized
+}

beacon-chain/utils/BUILD.bazel

@@ -0,0 +1,9 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["flags.go"],
+    importpath = "github.com/prysmaticlabs/geth-sharding/beacon-chain/utils",
+    visibility = ["//beacon-chain:__subpackages__"],
+    deps = ["@com_github_urfave_cli//:go_default_library"],
+)

beacon-chain/utils/flags.go

@@ -0,0 +1,24 @@
+package utils
+
+import (
+	"github.com/urfave/cli"
+)
+
+var (
+	// Web3ProviderFlag defines a flag for a mainchain RPC endpoint.
+	Web3ProviderFlag = cli.StringFlag{
+		Name:  "web3provider",
+		Usage: "A mainchain web3 provider string endpoint. Can either be an IPC file string or a WebSocket endpoint. Uses WebSockets by default at ws://127.0.0.1:8546. Cannot be an HTTP endpoint.",
+		Value: "ws://127.0.0.1:8546",
+	}
+	// VrcContractFlag defines a flag for VRC contract address.
+	VrcContractFlag = cli.StringFlag{
+		Name:  "vrcaddr",
+		Usage: "Validator registration contract address. Beacon chain node will listen logs coming from VRC to determine when validator is eligible to participate.",
+	}
+	// PubKeyFlag defines a flag for validator's public key on the mainchain
+	PubKeyFlag = cli.StringFlag{
+		Name:  "pubkey",
+		Usage: "Validator's public key. Beacon chain node will listen to VRC log to determine when registration has completed based on this public key address.",
+	}
+)

contracts/validator_registration.go

@@ -16,10 +16,10 @@ import (
 )
 
 // ValidatorRegistrationABI is the input ABI used to generate the binding from.
-const ValidatorRegistrationABI = "[{\"constant\":true,\"inputs\":[{\"name\":\"\",\"type\":\"bytes32\"}],\"name\":\"usedPubkey\",\"outputs\":[{\"name\":\"\",\"type\":\"bool\"}],\"payable\":false,\"stateMutability\":\"view\",\"type\":\"function\"},{\"constant\":true,\"inputs\":[],\"name\":\"VALIDATOR_DEPOSIT\",\"outputs\":[{\"name\":\"\",\"type\":\"uint256\"}],\"payable\":false,\"stateMutability\":\"view\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"_pubkey\",\"type\":\"bytes32\"},{\"name\":\"_withdrawalShardID\",\"type\":\"uint256\"},{\"name\":\"_withdrawalAddressbytes32\",\"type\":\"address\"},{\"name\":\"_randaoCommitment\",\"type\":\"bytes32\"}],\"name\":\"deposit\",\"outputs\":[],\"payable\":true,\"stateMutability\":\"payable\",\"type\":\"function\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":false,\"name\":\"pubKey\",\"type\":\"bytes32\"},{\"indexed\":false,\"name\":\"withdrawalShardID\",\"type\":\"uint256\"},{\"indexed\":false,\"name\":\"withdrawalAddressbytes32\",\"type\":\"address\"},{\"indexed\":false,\"name\":\"randaoCommitment\",\"type\":\"bytes32\"}],\"name\":\"ValidatorRegistered\",\"type\":\"event\"}]"
+const ValidatorRegistrationABI = "[{\"constant\":true,\"inputs\":[{\"name\":\"\",\"type\":\"bytes32\"}],\"name\":\"usedPubkey\",\"outputs\":[{\"name\":\"\",\"type\":\"bool\"}],\"payable\":false,\"stateMutability\":\"view\",\"type\":\"function\"},{\"constant\":true,\"inputs\":[],\"name\":\"VALIDATOR_DEPOSIT\",\"outputs\":[{\"name\":\"\",\"type\":\"uint256\"}],\"payable\":false,\"stateMutability\":\"view\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"name\":\"_pubkey\",\"type\":\"bytes32\"},{\"name\":\"_withdrawalShardID\",\"type\":\"uint256\"},{\"name\":\"_withdrawalAddressbytes32\",\"type\":\"address\"},{\"name\":\"_randaoCommitment\",\"type\":\"bytes32\"}],\"name\":\"deposit\",\"outputs\":[],\"payable\":true,\"stateMutability\":\"payable\",\"type\":\"function\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"name\":\"pubKey\",\"type\":\"bytes32\"},{\"indexed\":false,\"name\":\"withdrawalShardID\",\"type\":\"uint256\"},{\"indexed\":true,\"name\":\"withdrawalAddressbytes32\",\"type\":\"address\"},{\"indexed\":true,\"name\":\"randaoCommitment\",\"type\":\"bytes32\"}],\"name\":\"ValidatorRegistered\",\"type\":\"event\"}]"
 
 // ValidatorRegistrationBin is the compiled bytecode used for deploying new contracts.
-const ValidatorRegistrationBin = `0x608060405234801561001057600080fd5b506101d3806100206000396000f3006080604052600436106100565763ffffffff7c010000000000000000000000000000000000000000000000000000000060003504166301110845811461005b578063441d92cc14610087578063881d2135146100ae575b600080fd5b34801561006757600080fd5b506100736004356100da565b604080519115158252519081900360200190f35b34801561009357600080fd5b5061009c6100ef565b60408051918252519081900360200190f35b6100d860043560243573ffffffffffffffffffffffffffffffffffffffff604435166064356100fc565b005b60006020819052908152604090205460ff1681565b6801bc16d674ec80000081565b346801bc16d674ec8000001461011157600080fd5b60008481526020819052604090205460ff161561012d57600080fd5b60008481526020818152604091829020805460ff19166001179055815186815290810185905273ffffffffffffffffffffffffffffffffffffffff8416818301526060810183905290517f7b0678aab009b61a805f5004869728b53a444f9a3e6bb9e22b8537c89af512749181900360800190a1505050505600a165627a7a7230582030b51cf5829c9fac611cd2060acc062996b9cc9cf3d57d32b2b54c509a32b85d0029`
+const ValidatorRegistrationBin = `0x608060405234801561001057600080fd5b506101c7806100206000396000f3006080604052600436106100565763ffffffff7c010000000000000000000000000000000000000000000000000000000060003504166301110845811461005b578063441d92cc14610087578063881d2135146100ae575b600080fd5b34801561006757600080fd5b506100736004356100da565b604080519115158252519081900360200190f35b34801561009357600080fd5b5061009c6100ef565b60408051918252519081900360200190f35b6100d860043560243573ffffffffffffffffffffffffffffffffffffffff604435166064356100fc565b005b60006020819052908152604090205460ff1681565b6801bc16d674ec80000081565b346801bc16d674ec8000001461011157600080fd5b60008481526020819052604090205460ff161561012d57600080fd5b60008481526020818152604091829020805460ff1916600117905581518581529151839273ffffffffffffffffffffffffffffffffffffffff86169288927f7b0678aab009b61a805f5004869728b53a444f9a3e6bb9e22b8537c89af512749281900390910190a4505050505600a165627a7a7230582010cee12b801046464a3d4ffaad0081c2c3cf734f886a556d1e3b4f3565b649380029`
 
 // DeployValidatorRegistration deploys a new Ethereum contract, binding an instance of ValidatorRegistration to it.
 func DeployValidatorRegistration(auth *bind.TransactOpts, backend bind.ContractBackend) (common.Address, *types.Transaction, *ValidatorRegistration, error) {
@@ -327,10 +327,24 @@ type ValidatorRegistrationValidatorRegistered struct {
 
 // FilterValidatorRegistered is a free log retrieval operation binding the contract event 0x7b0678aab009b61a805f5004869728b53a444f9a3e6bb9e22b8537c89af51274.
 //
-// Solidity: event ValidatorRegistered(pubKey bytes32, withdrawalShardID uint256, withdrawalAddressbytes32 address, randaoCommitment bytes32)
+// Solidity: event ValidatorRegistered(pubKey indexed bytes32, withdrawalShardID uint256, withdrawalAddressbytes32 indexed address, randaoCommitment indexed bytes32)
-func (_ValidatorRegistration *ValidatorRegistrationFilterer) FilterValidatorRegistered(opts *bind.FilterOpts) (*ValidatorRegistrationValidatorRegisteredIterator, error) {
+func (_ValidatorRegistration *ValidatorRegistrationFilterer) FilterValidatorRegistered(opts *bind.FilterOpts, pubKey [][32]byte, withdrawalAddressbytes32 []common.Address, randaoCommitment [][32]byte) (*ValidatorRegistrationValidatorRegisteredIterator, error) {
 
-	logs, sub, err := _ValidatorRegistration.contract.FilterLogs(opts, "ValidatorRegistered")
+	var pubKeyRule []interface{}
+	for _, pubKeyItem := range pubKey {
+		pubKeyRule = append(pubKeyRule, pubKeyItem)
+	}
+
+	var withdrawalAddressbytes32Rule []interface{}
+	for _, withdrawalAddressbytes32Item := range withdrawalAddressbytes32 {
+		withdrawalAddressbytes32Rule = append(withdrawalAddressbytes32Rule, withdrawalAddressbytes32Item)
+	}
+	var randaoCommitmentRule []interface{}
+	for _, randaoCommitmentItem := range randaoCommitment {
+		randaoCommitmentRule = append(randaoCommitmentRule, randaoCommitmentItem)
+	}
+
+	logs, sub, err := _ValidatorRegistration.contract.FilterLogs(opts, "ValidatorRegistered", pubKeyRule, withdrawalAddressbytes32Rule, randaoCommitmentRule)
 	if err != nil {
 		return nil, err
 	}
@@ -339,10 +353,24 @@ func (_ValidatorRegistration *ValidatorRegistrationFilterer) FilterValidatorRegi
 
 // WatchValidatorRegistered is a free log subscription operation binding the contract event 0x7b0678aab009b61a805f5004869728b53a444f9a3e6bb9e22b8537c89af51274.
 //
-// Solidity: event ValidatorRegistered(pubKey bytes32, withdrawalShardID uint256, withdrawalAddressbytes32 address, randaoCommitment bytes32)
+// Solidity: event ValidatorRegistered(pubKey indexed bytes32, withdrawalShardID uint256, withdrawalAddressbytes32 indexed address, randaoCommitment indexed bytes32)
-func (_ValidatorRegistration *ValidatorRegistrationFilterer) WatchValidatorRegistered(opts *bind.WatchOpts, sink chan<- *ValidatorRegistrationValidatorRegistered) (event.Subscription, error) {
+func (_ValidatorRegistration *ValidatorRegistrationFilterer) WatchValidatorRegistered(opts *bind.WatchOpts, sink chan<- *ValidatorRegistrationValidatorRegistered, pubKey [][32]byte, withdrawalAddressbytes32 []common.Address, randaoCommitment [][32]byte) (event.Subscription, error) {
 
-	logs, sub, err := _ValidatorRegistration.contract.WatchLogs(opts, "ValidatorRegistered")
+	var pubKeyRule []interface{}
+	for _, pubKeyItem := range pubKey {
+		pubKeyRule = append(pubKeyRule, pubKeyItem)
+	}
+
+	var withdrawalAddressbytes32Rule []interface{}
+	for _, withdrawalAddressbytes32Item := range withdrawalAddressbytes32 {
+		withdrawalAddressbytes32Rule = append(withdrawalAddressbytes32Rule, withdrawalAddressbytes32Item)
+	}
+	var randaoCommitmentRule []interface{}
+	for _, randaoCommitmentItem := range randaoCommitment {
+		randaoCommitmentRule = append(randaoCommitmentRule, randaoCommitmentItem)
+	}
+
+	logs, sub, err := _ValidatorRegistration.contract.WatchLogs(opts, "ValidatorRegistered", pubKeyRule, withdrawalAddressbytes32Rule, randaoCommitmentRule)
 	if err != nil {
 		return nil, err
 	}

contracts/validator_registration.sol

@@ -2,10 +2,10 @@ pragma solidity 0.4.23;
 
 contract ValidatorRegistration {
     event ValidatorRegistered(
-        bytes32 pubKey,
+        bytes32 indexed pubKey,
         uint256 withdrawalShardID,
-        address withdrawalAddressbytes32,
+        address indexed withdrawalAddressbytes32,
-        bytes32 randaoCommitment
+        bytes32 indexed randaoCommitment
     );
 
     mapping (bytes32 => bool) public usedPubkey;

contracts/validator_registration_test.go

@@ -136,7 +136,7 @@ func TestRegister(t *testing.T) {
 	if err != nil {
 		t.Errorf("Validator registration failed: %v", err)
 	}
-	log, err := testAccount.contract.FilterValidatorRegistered(&bind.FilterOpts{})
+	log, err := testAccount.contract.FilterValidatorRegistered(&bind.FilterOpts{}, [][32]byte{}, []common.Address{}, [][32]byte{})
 	if err != nil {
 		t.Fatal(err)
 	}

sharding/BUILD.bazel

@@ -8,6 +8,7 @@ go_library(
     deps = [
         "//sharding/node:go_default_library",
         "//sharding/utils:go_default_library",
+        "//shared/cmd:go_default_library",
         "//shared/debug:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
         "@com_github_urfave_cli//:go_default_library",

sharding/database/BUILD.bazel

@@ -2,14 +2,11 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "go_default_library",
-    srcs = [
+    srcs = ["database.go"],
-        "database.go",
-        "inmemory.go",
-    ],
     importpath = "github.com/prysmaticlabs/geth-sharding/sharding/database",
     visibility = ["//sharding:__subpackages__"],
     deps = [
-        "@com_github_ethereum_go_ethereum//common:go_default_library",
+        "//shared/database:go_default_library",
         "@com_github_ethereum_go_ethereum//ethdb:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
     ],
@@ -17,14 +14,10 @@ go_library(
 
 go_test(
     name = "go_default_test",
-    srcs = [
+    srcs = ["database_test.go"],
-        "database_test.go",
-        "inmemory_test.go",
-    ],
     embed = [":go_default_library"],
     deps = [
         "//sharding/types:go_default_library",
-        "@com_github_ethereum_go_ethereum//ethdb:go_default_library",
         "@com_github_sirupsen_logrus//hooks/test:go_default_library",
         "@com_github_syndtr_goleveldb//leveldb/errors:go_default_library",
     ],

sharding/database/database.go

@@ -8,9 +8,11 @@ import (
 	"path/filepath"
 
 	"github.com/ethereum/go-ethereum/ethdb"
+	sharedDB "github.com/prysmaticlabs/geth-sharding/shared/database"
 	log "github.com/sirupsen/logrus"
 )
 
+// ShardDB defines a service for the sharding system's persistent storage.
 type ShardDB struct {
 	inmemory bool
 	dataDir  string
@@ -20,27 +22,30 @@ type ShardDB struct {
 	db       ethdb.Database
 }
 
+// ShardDBConfig specifies configuration options for the db service.
+type ShardDBConfig struct {
+	DataDir  string
+	Name     string
+	InMemory bool
+}
+
 // NewShardDB initializes a shardDB.
-func NewShardDB(dataDir string, name string, inmemory bool) (*ShardDB, error) {
+func NewShardDB(config *ShardDBConfig) (*ShardDB, error) {
 	// Uses default cache and handles values.
 	// TODO: allow these arguments to be set based on cli context.
-	if inmemory {
+	shardDB := &ShardDB{
-		return &ShardDB{
+		name:    config.Name,
-			inmemory: inmemory,
+		dataDir: config.DataDir,
-			dataDir:  dataDir,
-			name:     name,
-			cache:    16,
-			handles:  16,
-			db:       NewShardKV(),
-		}, nil
 	}
-	return &ShardDB{
+	if config.InMemory {
-		dataDir: dataDir,
+		shardDB.inmemory = true
-		name:    name,
+		shardDB.db = sharedDB.NewKVStore()
-		cache:   16,
+	} else {
-		handles: 16,
+		shardDB.inmemory = false
-		db:      nil,
+		shardDB.cache = 16
-	}, nil
+		shardDB.handles = 16
+	}
+	return shardDB, nil
 }
 
 // Start the shard DB service.

sharding/database/database_test.go

@@ -1,7 +1,10 @@
 package database
 
 import (
+	"fmt"
+	"os"
 	"strconv"
+	"sync"
 	"testing"
 
 	"github.com/prysmaticlabs/geth-sharding/sharding/types"
@@ -15,15 +18,18 @@ var _ = types.Service(&ShardDB{})
 var testDB *ShardDB
 
 func init() {
-	shardDB, _ := NewShardDB("/tmp/datadir", "shardchaindata", false)
+	tmp := fmt.Sprintf("%s/datadir", os.TempDir())
+	config := &ShardDBConfig{DataDir: tmp, Name: "shardchaindata", InMemory: false}
+	shardDB, _ := NewShardDB(config)
 	testDB = shardDB
 	testDB.Start()
 }
 
 func TestLifecycle(t *testing.T) {
 	hook := logTest.NewGlobal()
-
+	tmp := fmt.Sprintf("%s/lifecycledir", os.TempDir())
-	s, err := NewShardDB("/tmp/datadir", "shardchaindb", false)
+	config := &ShardDBConfig{DataDir: tmp, Name: "shardchaindata", InMemory: false}
+	s, err := NewShardDB(config)
 	if err != nil {
 		t.Fatalf("could not initialize a new sb: %v", err)
 	}
@@ -50,13 +56,17 @@ func TestLifecycle(t *testing.T) {
 
 // Testing the concurrency of the shardDB with multiple processes attempting to write.
 func Test_DBConcurrent(t *testing.T) {
+	var wg sync.WaitGroup
+	wg.Add(100)
 	for i := 0; i < 100; i++ {
 		go func(val string) {
+			defer wg.Done()
 			if err := testDB.db.Put([]byte("ralph merkle"), []byte(val)); err != nil {
 				t.Errorf("could not save value in db: %v", err)
 			}
 		}(strconv.Itoa(i))
 	}
+	wg.Wait()
 }
 
 func Test_DBPut(t *testing.T) {

sharding/database/inmemory.go

@@ -1,67 +0,0 @@
-package database
-
-import (
-	"fmt"
-	"sync"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/ethdb"
-)
-
-// ShardKV is an in-memory mapping of hashes to RLP encoded values.
-type ShardKV struct {
-	kv   map[common.Hash][]byte
-	lock sync.RWMutex
-}
-
-// NewShardKV creates an in-memory, key-value store.
-func NewShardKV() *ShardKV {
-	return &ShardKV{kv: make(map[common.Hash][]byte)}
-}
-
-// Get fetches a val from the mappping by key.
-func (sb *ShardKV) Get(k []byte) ([]byte, error) {
-	sb.lock.RLock()
-	defer sb.lock.RUnlock()
-	v, ok := sb.kv[common.BytesToHash(k)]
-	if !ok {
-		return []byte{}, fmt.Errorf("key not found: %v", k)
-	}
-	return v, nil
-}
-
-// Has checks if the key exists in the mapping.
-func (sb *ShardKV) Has(k []byte) (bool, error) {
-	sb.lock.RLock()
-	defer sb.lock.RUnlock()
-	v := sb.kv[common.BytesToHash(k)]
-	return v != nil, nil
-}
-
-// Put updates a key's value in the mapping.
-func (sb *ShardKV) Put(k []byte, v []byte) error {
-	sb.lock.Lock()
-	defer sb.lock.Unlock()
-	// there is no error in a simple setting of a value in a go map.
-	sb.kv[common.BytesToHash(k)] = v
-	return nil
-}
-
-// Delete removes the key and value from the mapping.
-func (sb *ShardKV) Delete(k []byte) error {
-	sb.lock.Lock()
-	defer sb.lock.Unlock()
-	// There is no return value for deleting a simple key in a go map.
-	delete(sb.kv, common.BytesToHash(k))
-	return nil
-}
-
-func (sb *ShardKV) Close() {
-	//TODO: Implement Close for ShardKV
-	panic("ShardKV Close() isnt implemented yet")
-}
-
-func (sb *ShardKV) NewBatch() ethdb.Batch {
-	//TODO: Implement NewBatch for ShardKV
-	panic("ShardKV NewBatch() isnt implemented yet")
-}

sharding/main.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/prysmaticlabs/geth-sharding/sharding/node"
 	"github.com/prysmaticlabs/geth-sharding/sharding/utils"
+	"github.com/prysmaticlabs/geth-sharding/shared/cmd"
 	"github.com/prysmaticlabs/geth-sharding/shared/debug"
 	log "github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -46,7 +47,7 @@ VERSION:
 	app.Usage = `launches a sharding client that interacts with a beacon chain, starts proposer services, shardp2p connections, and more
 `
 	app.Action = startNode
-	app.Flags = []cli.Flag{utils.ActorFlag, utils.DataDirFlag, utils.PasswordFileFlag, utils.NetworkIdFlag, utils.IPCPathFlag, utils.DepositFlag, utils.ShardIDFlag, debug.PProfFlag, debug.PProfAddrFlag, debug.PProfPortFlag, debug.MemProfileRateFlag, debug.CPUProfileFlag, debug.TraceFlag}
+	app.Flags = []cli.Flag{utils.ActorFlag, cmd.DataDirFlag, cmd.PasswordFileFlag, cmd.NetworkIdFlag, cmd.IPCPathFlag, utils.DepositFlag, utils.ShardIDFlag, debug.PProfFlag, debug.PProfAddrFlag, debug.PProfPortFlag, debug.MemProfileRateFlag, debug.CPUProfileFlag, debug.TraceFlag}
 
 	app.Before = func(ctx *cli.Context) error {
 		runtime.GOMAXPROCS(runtime.NumCPU())

sharding/node/BUILD.bazel

@@ -18,6 +18,7 @@ go_library(
         "//sharding/txpool:go_default_library",
         "//sharding/utils:go_default_library",
         "//shared:go_default_library",
+        "//shared/cmd:go_default_library",
         "//shared/debug:go_default_library",
         "@com_github_ethereum_go_ethereum//node:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",

sharding/node/backend.go

@@ -25,6 +25,7 @@ import (
 	"github.com/prysmaticlabs/geth-sharding/sharding/txpool"
 	"github.com/prysmaticlabs/geth-sharding/sharding/utils"
 	"github.com/prysmaticlabs/geth-sharding/shared"
+	"github.com/prysmaticlabs/geth-sharding/shared/cmd"
 	"github.com/prysmaticlabs/geth-sharding/shared/debug"
 	log "github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
@@ -131,10 +132,11 @@ func (s *ShardEthereum) Close() {
 // registerShardChainDB attaches a LevelDB wrapped object to the shardEthereum instance.
 func (s *ShardEthereum) registerShardChainDB(ctx *cli.Context) error {
 	path := node.DefaultDataDir()
-	if ctx.GlobalIsSet(utils.DataDirFlag.Name) {
+	if ctx.GlobalIsSet(cmd.DataDirFlag.Name) {
-		path = ctx.GlobalString(utils.DataDirFlag.Name)
+		path = ctx.GlobalString(cmd.DataDirFlag.Name)
 	}
-	shardDB, err := database.NewShardDB(path, shardChainDBName, false)
+	config := &database.ShardDBConfig{DataDir: path, Name: shardChainDBName, InMemory: false}
+	shardDB, err := database.NewShardDB(config)
 	if err != nil {
 		return fmt.Errorf("could not register shardDB service: %v", err)
 	}
@@ -152,18 +154,18 @@ func (s *ShardEthereum) registerP2P() error {
 
 func (s *ShardEthereum) registerMainchainClient(ctx *cli.Context) error {
 	path := node.DefaultDataDir()
-	if ctx.GlobalIsSet(utils.DataDirFlag.Name) {
+	if ctx.GlobalIsSet(cmd.DataDirFlag.Name) {
-		path = ctx.GlobalString(utils.DataDirFlag.Name)
+		path = ctx.GlobalString(cmd.DataDirFlag.Name)
 	}
 
 	endpoint := ctx.Args().First()
 	if endpoint == "" {
 		endpoint = fmt.Sprintf("%s/%s.ipc", path, mainchain.ClientIdentifier)
 	}
-	if ctx.GlobalIsSet(utils.IPCPathFlag.Name) {
+	if ctx.GlobalIsSet(cmd.IPCPathFlag.Name) {
-		endpoint = ctx.GlobalString(utils.IPCPathFlag.Name)
+		endpoint = ctx.GlobalString(cmd.IPCPathFlag.Name)
 	}
-	passwordFile := ctx.GlobalString(utils.PasswordFileFlag.Name)
+	passwordFile := ctx.GlobalString(cmd.PasswordFileFlag.Name)
 	depositFlag := ctx.GlobalBool(utils.DepositFlag.Name)
 
 	client, err := mainchain.NewSMCClient(endpoint, path, depositFlag, passwordFile)

sharding/observer/service_test.go

@@ -23,7 +23,8 @@ func TestStartStop(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Unable to setup p2p server: %v", err)
 	}
-	shardChainDB, err := database.NewShardDB("", "", true)
+	config := &database.ShardDBConfig{Name: "", DataDir: "", InMemory: true}
+	shardChainDB, err := database.NewShardDB(config)
 	if err != nil {
 		t.Fatalf("Unable to setup db: %v", err)
 	}

sharding/syncer/service_test.go

@@ -28,7 +28,8 @@ func init() {
 func TestStop(t *testing.T) {
 	hook := logTest.NewGlobal()
 
-	shardChainDB, err := database.NewShardDB("", "", true)
+	config := &database.ShardDBConfig{Name: "", DataDir: "", InMemory: true}
+	shardChainDB, err := database.NewShardDB(config)
 	if err != nil {
 		t.Fatalf("unable to setup db: %v", err)
 	}
@@ -70,7 +71,8 @@ func TestStop(t *testing.T) {
 func TestHandleCollationBodyRequests(t *testing.T) {
 	hook := logTest.NewGlobal()
 
-	shardChainDB, err := database.NewShardDB("", "", true)
+	config := &database.ShardDBConfig{Name: "", DataDir: "", InMemory: true}
+	shardChainDB, err := database.NewShardDB(config)
 	if err != nil {
 		t.Fatalf("unable to setup db: %v", err)
 	}

sharding/types/BUILD.bazel

@@ -29,8 +29,8 @@ go_test(
     ],
     embed = [":go_default_library"],
     deps = [
-        "//sharding/database:go_default_library",
         "//shared:go_default_library",
+        "//shared/database:go_default_library",
         "@com_github_ethereum_go_ethereum//common:go_default_library",
         "@com_github_ethereum_go_ethereum//core/types:go_default_library",
         "@com_github_ethereum_go_ethereum//crypto/sha3:go_default_library",

sharding/types/shard_test.go

@@ -11,7 +11,7 @@ import (
 	"github.com/ethereum/go-ethereum/crypto/sha3"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/prysmaticlabs/geth-sharding/sharding/database"
+	sharedDB "github.com/prysmaticlabs/geth-sharding/shared/database"
 )
 
 type mockShardDB struct {
@@ -52,7 +52,7 @@ func TestShard_ValidateShardID(t *testing.T) {
 	emptyHash := common.BytesToHash([]byte{})
 	emptyAddr := common.BytesToAddress([]byte{})
 	header := NewCollationHeader(big.NewInt(1), &emptyHash, big.NewInt(1), &emptyAddr, [32]byte{})
-	shardDB := database.NewShardKV()
+	shardDB := sharedDB.NewKVStore()
 	shard := NewShard(big.NewInt(3), shardDB)
 
 	if err := shard.ValidateShardID(header); err == nil {
@@ -76,7 +76,7 @@ func TestShard_HeaderByHash(t *testing.T) {
 	mockDB := &mockShardDB{kv: make(map[common.Hash][]byte)}
 
 	// creates a well-functioning shardDB.
-	shardDB := database.NewShardKV()
+	shardDB := sharedDB.NewKVStore()
 
 	// creates a shard with a functioning DB and another one with a faulty DB.
 	shard := NewShard(big.NewInt(1), shardDB)
@@ -115,7 +115,7 @@ func TestShard_CollationByHeaderHash(t *testing.T) {
 		body:   []byte{1, 2, 3},
 	}
 
-	shardDB := database.NewShardKV()
+	shardDB := sharedDB.NewKVStore()
 	shard := NewShard(big.NewInt(1), shardDB)
 
 	// should throw error if saving the collation before setting the chunk root
@@ -171,7 +171,7 @@ func TestShard_ChunkRootfromHeaderHash(t *testing.T) {
 
 	collation := NewCollation(header, []byte{1, 2, 3}, nil)
 	collation.CalculateChunkRoot()
-	shardDB := database.NewShardKV()
+	shardDB := sharedDB.NewKVStore()
 	shard := NewShard(shardID, shardDB)
 
 	if err := shard.SaveCollation(collation); err != nil {
@@ -208,7 +208,7 @@ func TestShard_CanonicalHeaderHash(t *testing.T) {
 
 	collation.CalculateChunkRoot()
 
-	shardDB := database.NewShardKV()
+	shardDB := sharedDB.NewKVStore()
 	shard := NewShard(shardID, shardDB)
 
 	// should not be able to set as canonical before saving the header and body first.
@@ -248,7 +248,7 @@ func TestShard_CanonicalCollation(t *testing.T) {
 	emptyHash := common.BytesToHash([]byte{})
 	header := NewCollationHeader(shardID, &emptyHash, period, &proposerAddress, proposerSignature)
 
-	shardDB := database.NewShardKV()
+	shardDB := sharedDB.NewKVStore()
 	shard := NewShard(shardID, shardDB)
 
 	collation := &Collation{
@@ -288,7 +288,7 @@ func TestShard_SetCanonical(t *testing.T) {
 	chunkRoot := common.BytesToHash([]byte{})
 	header := NewCollationHeader(big.NewInt(1), &chunkRoot, big.NewInt(1), nil, [32]byte{})
 
-	shardDB := database.NewShardKV()
+	shardDB := sharedDB.NewKVStore()
 	shard := NewShard(big.NewInt(1), shardDB)
 	otherShard := NewShard(big.NewInt(2), shardDB)
 
@@ -311,7 +311,7 @@ func TestShard_SetCanonical(t *testing.T) {
 func TestShard_BodyByChunkRoot(t *testing.T) {
 	body := []byte{1, 2, 3, 4, 5}
 	shardID := big.NewInt(1)
-	shardDB := database.NewShardKV()
+	shardDB := sharedDB.NewKVStore()
 	shard := NewShard(shardID, shardDB)
 
 	if err := shard.SaveBody(body); err != nil {
@@ -354,7 +354,7 @@ func TestShard_CheckAvailability(t *testing.T) {
 	emptyHash := common.BytesToHash([]byte{})
 	header := NewCollationHeader(shardID, &emptyHash, period, &proposerAddress, proposerSignature)
 
-	shardDB := database.NewShardKV()
+	shardDB := sharedDB.NewKVStore()
 	shard := NewShard(shardID, shardDB)
 
 	collation := &Collation{
@@ -391,7 +391,7 @@ func TestShard_SetAvailability(t *testing.T) {
 	mockDB := &mockShardDB{kv: make(map[common.Hash][]byte)}
 
 	// creates a well-functioning shardDB.
-	shardDB := database.NewShardKV()
+	shardDB := sharedDB.NewKVStore()
 
 	// creates a shard with a functioning DB and another one with a faulty DB.
 	shard := NewShard(big.NewInt(1), shardDB)
@@ -425,7 +425,7 @@ func TestShard_SaveCollation(t *testing.T) {
 	emptyHash := common.BytesToHash([]byte{})
 	header := NewCollationHeader(headerShardID, &emptyHash, period, &proposerAddress, proposerSignature)
 
-	shardDB := database.NewShardKV()
+	shardDB := sharedDB.NewKVStore()
 	shard := NewShard(big.NewInt(2), shardDB)
 
 	collation := &Collation{

sharding/utils/BUILD.bazel

@@ -1,22 +1,12 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
 
 go_library(
     name = "go_default_library",
-    srcs = [
+    srcs = ["flags.go"],
-        "customflags.go",
-        "flags.go",
-    ],
     importpath = "github.com/prysmaticlabs/geth-sharding/sharding/utils",
     visibility = ["//sharding:__subpackages__"],
     deps = [
         "//sharding/params:go_default_library",
-        "@com_github_ethereum_go_ethereum//node:go_default_library",
         "@com_github_urfave_cli//:go_default_library",
     ],
 )
-
-go_test(
-    name = "go_default_test",
-    srcs = ["customflags_test.go"],
-    embed = [":go_default_library"],
-)

sharding/utils/flags.go

@@ -1,60 +1,24 @@
-// Copyright 2015 The go-ethereum Authors
-// This file is part of go-ethereum.
-//
-// go-ethereum is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// go-ethereum is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
-
-// Package utils contains internal helper functions for go-ethereum commands.
 package utils
 
 import (
 	"math/big"
 
-	"github.com/ethereum/go-ethereum/node"
 	shardparams "github.com/prysmaticlabs/geth-sharding/sharding/params"
 	"github.com/urfave/cli"
 )
 
 var (
-	// General settings
+	// DepositFlag defines whether a node will withdraw ETH from the user's account.
-	IPCPathFlag = DirectoryFlag{
-		Name:  "ipcpath",
-		Usage: "Filename for IPC socket/pipe within the datadir (explicit paths escape it)",
-	}
-	DataDirFlag = DirectoryFlag{
-		Name:  "datadir",
-		Usage: "Data directory for the databases and keystore",
-		Value: DirectoryString{node.DefaultDataDir()},
-	}
-	NetworkIdFlag = cli.Uint64Flag{
-		Name:  "networkid",
-		Usage: "Network identifier (integer, 1=Frontier, 2=Morden (disused), 3=Ropsten, 4=Rinkeby)",
-		Value: 1,
-	}
-	PasswordFileFlag = cli.StringFlag{
-		Name:  "password",
-		Usage: "Password file to use for non-interactive password input",
-		Value: "",
-	}
-	// Sharding Settings
 	DepositFlag = cli.BoolFlag{
 		Name:  "deposit",
 		Usage: "To become a notary in a sharding node, " + new(big.Int).Div(shardparams.DefaultConfig.NotaryDeposit, new(big.Int).Exp(big.NewInt(10), big.NewInt(18), nil)).String() + " ETH will be deposited into SMC",
 	}
+	// ActorFlag defines the role of the sharding client. Either proposer, notary, or simulator.
 	ActorFlag = cli.StringFlag{
 		Name:  "actor",
 		Usage: `use the --actor notary or --actor proposer to start a notary or proposer service in the sharding node. If omitted, the sharding node registers an Observer service that simply observes the activity in the sharded network`,
 	}
+	// ShardIDFlag specifies which shard to listen to.
 	ShardIDFlag = cli.IntFlag{
 		Name:  "shardid",
 		Usage: `use the --shardid to determine which shard to start p2p server, listen for incoming transactions and perform proposer/observer duties`,

shared/cmd/BUILD.bazel

@@ -0,0 +1,21 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "customflags.go",
+        "flags.go",
+    ],
+    importpath = "github.com/prysmaticlabs/geth-sharding/shared/cmd",
+    visibility = ["//visibility:public"],
+    deps = [
+        "@com_github_ethereum_go_ethereum//node:go_default_library",
+        "@com_github_urfave_cli//:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["customflags_test.go"],
+    embed = [":go_default_library"],
+)

shared/cmd/customflags.go

@@ -1,4 +1,4 @@
-package utils
+package cmd
 
 import (
 	"flag"

shared/cmd/customflags_test.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
 
-package utils
+package cmd
 
 import (
 	"os"

shared/cmd/flags.go

@@ -0,0 +1,32 @@
+package cmd
+
+import (
+	"github.com/ethereum/go-ethereum/node"
+	"github.com/urfave/cli"
+)
+
+var (
+	// IPCPathFlag defines the filename of a pipe within the datadir.
+	IPCPathFlag = DirectoryFlag{
+		Name:  "ipcpath",
+		Usage: "Filename for IPC socket/pipe within the datadir (explicit paths escape it)",
+	}
+	// DataDirFlag defines a path on disk.
+	DataDirFlag = DirectoryFlag{
+		Name:  "datadir",
+		Usage: "Data directory for the databases and keystore",
+		Value: DirectoryString{node.DefaultDataDir()},
+	}
+	// NetworkIdFlag defines the specific network identifier.
+	NetworkIdFlag = cli.Uint64Flag{
+		Name:  "networkid",
+		Usage: "Network identifier (integer, 1=Frontier, 2=Morden (disused), 3=Ropsten, 4=Rinkeby)",
+		Value: 1,
+	}
+	// PasswordFileFlag defines the path to the user's account password file.
+	PasswordFileFlag = cli.StringFlag{
+		Name:  "password",
+		Usage: "Password file to use for non-interactive password input",
+		Value: "",
+	}
+)

shared/database/BUILD.bazel

@@ -0,0 +1,20 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["inmemory.go"],
+    importpath = "github.com/prysmaticlabs/geth-sharding/shared/database",
+    visibility = ["//visibility:public"],
+    deps = [
+        "@com_github_ethereum_go_ethereum//common:go_default_library",
+        "@com_github_ethereum_go_ethereum//ethdb:go_default_library",
+        "@com_github_sirupsen_logrus//:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["inmemory_test.go"],
+    embed = [":go_default_library"],
+    deps = ["@com_github_ethereum_go_ethereum//ethdb:go_default_library"],
+)

shared/database/inmemory.go

@@ -0,0 +1,71 @@
+package database
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/ethdb"
+	log "github.com/sirupsen/logrus"
+)
+
+// KVStore is an in-memory mapping of hashes to RLP encoded values.
+type KVStore struct {
+	kv   map[common.Hash][]byte
+	lock sync.RWMutex
+}
+
+// NewKVStore creates an in-memory, key-value store.
+func NewKVStore() *KVStore {
+	return &KVStore{kv: make(map[common.Hash][]byte)}
+}
+
+// Get fetches a val from the mappping by key.
+func (s *KVStore) Get(k []byte) ([]byte, error) {
+	s.lock.RLock()
+	defer s.lock.RUnlock()
+	v, ok := s.kv[common.BytesToHash(k)]
+	if !ok {
+		return []byte{}, fmt.Errorf("key not found: %v", k)
+	}
+	return v, nil
+}
+
+// Has checks if the key exists in the mapping.
+func (s *KVStore) Has(k []byte) (bool, error) {
+	s.lock.RLock()
+	defer s.lock.RUnlock()
+	v := s.kv[common.BytesToHash(k)]
+	return v != nil, nil
+}
+
+// Put updates a key's value in the mapping.
+func (s *KVStore) Put(k []byte, v []byte) error {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+	// there is no error in a simple setting of a value in a go map.
+	s.kv[common.BytesToHash(k)] = v
+	return nil
+}
+
+// Delete removes the key and value from the mapping.
+func (s *KVStore) Delete(k []byte) error {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+	// There is no return value for deleting a simple key in a go map.
+	delete(s.kv, common.BytesToHash(k))
+	return nil
+}
+
+// Close satisfies ethdb.Database.
+func (s *KVStore) Close() {
+	//TODO: Implement Close for KVStore
+	log.Debug("ShardKV Close() isnt implemented yet")
+}
+
+// NewBatch satisfies ethdb.Database.
+func (s *KVStore) NewBatch() ethdb.Batch {
+	//TODO: Implement NewBatch for KVStore
+	log.Debug("ShardKV NewBatch() isnt implemented yet")
+	return nil
+}

shared/database/inmemory_test.go

@@ -6,19 +6,19 @@ import (
 	"github.com/ethereum/go-ethereum/ethdb"
 )
 
-// Verifies that ShardKV implements the ethdb interface.
+// Verifies that KVStore implements the ethdb interface.
-var _ = ethdb.Database(&ShardKV{})
+var _ = ethdb.Database(&KVStore{})
 
-func Test_ShardKVPut(t *testing.T) {
+func Test_KVStorePut(t *testing.T) {
-	kv := NewShardKV()
+	kv := NewKVStore()
 
 	if err := kv.Put([]byte("ralph merkle"), []byte{1, 2, 3}); err != nil {
 		t.Errorf("could not save value in kv store: %v", err)
 	}
 }
 
-func Test_ShardKVHas(t *testing.T) {
+func Test_KVStoreHas(t *testing.T) {
-	kv := NewShardKV()
+	kv := NewKVStore()
 	key := []byte("ralph merkle")
 
 	if err := kv.Put(key, []byte{1, 2, 3}); err != nil {
@@ -43,8 +43,8 @@ func Test_ShardKVHas(t *testing.T) {
 	}
 }
 
-func Test_ShardKVGet(t *testing.T) {
+func Test_KVStoreGet(t *testing.T) {
-	kv := NewShardKV()
+	kv := NewKVStore()
 	key := []byte("ralph merkle")
 
 	if err := kv.Put(key, []byte{1, 2, 3}); err != nil {
@@ -69,8 +69,8 @@ func Test_ShardKVGet(t *testing.T) {
 	}
 }
 
-func Test_ShardKVDelete(t *testing.T) {
+func Test_KVStoreDelete(t *testing.T) {
-	kv := NewShardKV()
+	kv := NewKVStore()
 	key := []byte("ralph merkle")
 
 	if err := kv.Put(key, []byte{1, 2, 3}); err != nil {