Merge branch 'master' into construct-collation

Former-commit-id: d4fc305467544cd1a3d95ee5a338aa211b1e73f2 [formerly 87b957424793737b6324bdf63af936bfdc36f099]
Former-commit-id: b3bab328bb947ddac5e0dcf48616479456b73d83

README.md

@@ -119,16 +119,16 @@ Concurrently, you will need to run another service that is tasked with processin
 ## Running a Collation Proposal Node
 
 ```
-geth sharding --actor "proposer" --datadir /path/to/your/datadir --password /path/to/your/password.txt --networkid 12345
+geth sharding --actor "proposer" --datadir /path/to/your/datadir --password /path/to/your/password.txt --shardid 0 --networkid 12345
 ```
 
-This node is tasked with processing pending transactions into blobs within collations by serializing data into collation bodies. It is responsible for submitting proposals (collation headers) to the SMC via the `addHeader` function.
+This node is tasked with processing pending transactions into blobs within collations by serializing data into collation bodies. It is responsible for submitting proposals on shard 0 (collation headers) to the SMC via the `addHeader` function.
 
 ## Running an Observer Node
 
-    geth sharding --datadir /path/to/your/datadir --password /path/to/your/password.txt --networkid 12345
+    geth sharding --datadir /path/to/your/datadir --password /path/to/your/password.txt --shardid 0 --networkid 12345
 
-Omitting the `--actor` flag will launch a simple observer service attached to the sharding client that is able to listen to changes happening throughout the sharded Ethereum network.
+Omitting the `--actor` flag will launch a simple observer service attached to the sharding client that is able to listen to changes happening throughout the sharded Ethereum network on shard 0.
 
 # Making Changes
 

cmd/geth/main.go

@@ -120,6 +120,7 @@ var (
 		utils.ExtraDataFlag,
 		utils.DepositFlag,
 		utils.ActorFlag,
+		utils.ShardIDFlag,
 		configFileFlag,
 	}
 

cmd/geth/shardingcmd.go

@@ -14,7 +14,7 @@ var (
 		Name:      "sharding",
 		Usage:     "Start a sharding-enabled node",
 		ArgsUsage: "[endpoint]",
-		Flags:     []cli.Flag{utils.ActorFlag, utils.DataDirFlag, utils.PasswordFileFlag, utils.NetworkIdFlag, utils.IPCPathFlag, utils.DepositFlag},
+		Flags:     []cli.Flag{utils.ActorFlag, utils.DataDirFlag, utils.PasswordFileFlag, utils.NetworkIdFlag, utils.IPCPathFlag, utils.DepositFlag, utils.ShardIDFlag},
 		Category:  "SHARDING COMMANDS",
 		Description: `
 Launches a sharding node that manages services related to submitting collations to a Sharding Manager Contract, notary and proposer services, and shardp2p connections. This feature is a work in progress.

cmd/utils/flags.go

@@ -542,6 +542,10 @@ var (
 		Name:  "actor",
 		Usage: `use the --actor notary or --actor proposer to start a notary or proposer service in the sharding node. If omitted, the sharding node registers an Observer service that simply observes the activity in the sharded network`,
 	}
+	ShardIDFlag = cli.IntFlag{
+		Name:  "shardid",
+		Usage: `use the --shardid to determine which shard to start p2p server, listen for incoming transactions and perform proposer/observer duties`,
+	}
 )
 
 // MakeDataDir retrieves the currently requested data directory, terminating

sharding/collation.go

@@ -70,6 +70,14 @@ func (h *CollationHeader) Hash() (hash common.Hash) {
 	return hash
 }
 
+// AddSig adds the signature of proposer after collationHeader gets signed.
+func (h *CollationHeader) AddSig(sig []byte) {
+	h.data.ProposerSignature = sig
+}
+
+// Signature of the collation corresponds to.
+func (h *CollationHeader) Sig() []byte { return h.data.ProposerSignature }
+
 // ShardID the collation corresponds to.
 func (h *CollationHeader) ShardID() *big.Int { return h.data.ShardID }
 

sharding/contracts/sharding_manager.sol

@@ -51,6 +51,9 @@ contract SMC {
     uint sampleSizeLastUpdatedPeriod;
 
     // Constant values
+    // Length of challenge period for notary's proof of custody
+    uint public constant CHALLENGE_PERIOD = 25;
+    // Number of blocks per period
     uint constant PERIOD_LENGTH = 5;
     // Number of shards
     uint constant SHARD_COUNT = 100;

sharding/database/database.go

@@ -9,17 +9,8 @@ import (
 	"github.com/ethereum/go-ethereum/ethdb"
 )
 
-// ShardBackend defines an interface for a shardDB's necessary method
-// signatures.
-type ShardBackend interface {
-	Get(k []byte) ([]byte, error)
-	Has(k []byte) (bool, error)
-	Put(k []byte, val []byte) error
-	Delete(k []byte) error
-}
-
 // NewShardDB initializes a shardDB that writes to local disk.
-func NewShardDB(dataDir string, name string) (ShardBackend, error) {
+func NewShardDB(dataDir string, name string) (ethdb.Database, error) {
 	// Uses default cache and handles values.
 	// TODO: allow these arguments to be set based on cli context.
 	return ethdb.NewLDBDatabase(filepath.Join(dataDir, name), 16, 16)

sharding/database/database_test.go

@@ -3,9 +3,11 @@ package database
 import (
 	"strconv"
 	"testing"
+
+	"github.com/ethereum/go-ethereum/ethdb"
 )
 
-var db ShardBackend
+var db ethdb.Database
 
 func init() {
 	shardDB, err := NewShardDB("/tmp/datadir", "shardchaindata")

sharding/database/inmemory.go

@@ -5,6 +5,7 @@ import (
 	"sync"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/ethdb"
 )
 
 // ShardKV is an in-memory mapping of hashes to RLP encoded values.
@@ -54,3 +55,13 @@ func (sb *ShardKV) Delete(k []byte) error {
 	delete(sb.kv, common.BytesToHash(k))
 	return nil
 }
+
+func (sb *ShardKV) Close() {
+	//TODO: Implement Close for ShardKV
+	panic("ShardKV Close() isnt implemented yet")
+}
+
+func (sb *ShardKV) NewBatch() ethdb.Batch {
+	//TODO: Implement NewBatch for ShardKV
+	panic("ShardKV NewBatch() isnt implemented yet")
+}

sharding/database/inmemory_test.go

@@ -2,10 +2,12 @@ package database
 
 import (
 	"testing"
+
+	"github.com/ethereum/go-ethereum/ethdb"
 )
 
-// Verifies that ShardKV implements the ShardBackend interface.
-var _ = ShardBackend(&ShardKV{})
+// Verifies that ShardKV implements the ethdb interface.
+var _ = ethdb.Database(&ShardKV{})
 
 func Test_ShardKVPut(t *testing.T) {
 	kv := NewShardKV()

sharding/mainchain/smc_client.go

@@ -42,6 +42,7 @@ type Client interface {
 	DepositFlag() bool
 	SetDepositFlag(deposit bool)
 	DataDirPath() string
+	Sign(hash common.Hash) ([]byte, error)
 }
 
 // SMCClient defines a struct that interacts with a
@@ -217,3 +218,10 @@ func (s *SMCClient) unlockAccount(account accounts.Account) error {
 
 	return s.keystore.Unlock(account, pass)
 }
+
+// Sign signs the hash of collationHeader contents by
+// using default account on keystore and returns signed signature.
+func (s *SMCClient) Sign(hash common.Hash) ([]byte, error) {
+	account := s.Account()
+	return s.keystore.SignHash(*account, hash.Bytes())
+}

sharding/node/backend.go

@@ -22,11 +22,14 @@ import (
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/sharding/database"
 	"github.com/ethereum/go-ethereum/sharding/params"
 	"github.com/ethereum/go-ethereum/sharding/txpool"
 	cli "gopkg.in/urfave/cli.v1"
 )
 
+const shardChainDbName = "shardchaindata"
+
 // ShardEthereum is a service that is registered and started when geth is launched.
 // it contains APIs and fields that handle the different components of the sharded
 // Ethereum network.
@@ -67,15 +70,24 @@ func New(ctx *cli.Context) (*ShardEthereum, error) {
 	passwordFile := ctx.GlobalString(utils.PasswordFileFlag.Name)
 	depositFlag := ctx.GlobalBool(utils.DepositFlag.Name)
 	actorFlag := ctx.GlobalString(utils.ActorFlag.Name)
+	shardIDFlag := ctx.GlobalInt(utils.ShardIDFlag.Name)
 
 	smcClient, err := mainchain.NewSMCClient(endpoint, path, depositFlag, passwordFile)
 	if err != nil {
 		return nil, err
 	}
 
+	shardChainDb, err := database.NewShardDB(path, shardChainDbName)
+	if err != nil {
+		return nil, err
+	}
+
 	// Adds the initialized SMCClient to the ShardEthereum instance.
 	shardEthereum.smcClient = smcClient
 
+	// Adds the initialized shardChainDb to the ShardEthereum instance.
+	shardEthereum.shardChainDb = shardChainDb
+
 	if err := shardEthereum.registerP2P(); err != nil {
 		return nil, err
 	}
@@ -84,7 +96,7 @@ func New(ctx *cli.Context) (*ShardEthereum, error) {
 		return nil, err
 	}
 
-	if err := shardEthereum.registerActorService(actorFlag); err != nil {
+	if err := shardEthereum.registerActorService(actorFlag, shardIDFlag); err != nil {
 		return nil, err
 	}
 
@@ -174,20 +186,19 @@ func (s *ShardEthereum) registerTXPool(actor string) error {
 }
 
 // Registers the actor according to CLI flags. Either notary/proposer/observer.
-func (s *ShardEthereum) registerActorService(actor string) error {
+func (s *ShardEthereum) registerActorService(actor string, shardID int) error {
 	return s.Register(func(ctx *sharding.ServiceContext) (sharding.Service, error) {
 
 		var p2p *shardp2p.Server
 		ctx.RetrieveService(&p2p)
 
 		if actor == "notary" {
-			return notary.NewNotary(s.smcClient, p2p)
+			return notary.NewNotary(s.smcClient, p2p, s.shardChainDb)
 		} else if actor == "proposer" {
 			var txPool *txpool.ShardTXPool
 			ctx.RetrieveService(&txPool)
-			return proposer.NewProposer(s.smcClient, p2p, txPool)
+			return proposer.NewProposer(s.smcClient, p2p, txPool, s.shardChainDb, shardID)
 		}
-
-		return observer.NewObserver(p2p)
+		return observer.NewObserver(p2p, s.shardChainDb, shardID)
 	})
 }

sharding/notary/notary.go

@@ -144,7 +144,7 @@ func joinNotaryPool(client mainchain.Client) error {
 	log.Info("Joining notary pool")
 	txOps, err := client.CreateTXOpts(sharding.NotaryDeposit)
 	if err != nil {
-		return fmt.Errorf("unable to intiate the deposit transaction: %v", err)
+		return fmt.Errorf("unable to initiate the deposit transaction: %v", err)
 	}
 
 	tx, err := client.SMCTransactor().RegisterNotary(txOps)

sharding/notary/service.go

@@ -3,6 +3,7 @@
 package notary
 
 import (
+	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/sharding"
 	"github.com/ethereum/go-ethereum/sharding/mainchain"
@@ -12,13 +13,14 @@ import (
 // in a sharded system. Must satisfy the Service interface defined in
 // sharding/service.go.
 type Notary struct {
-	smcClient *mainchain.SMCClient
-	shardp2p  sharding.ShardP2P
+	smcClient    *mainchain.SMCClient
+	shardp2p     sharding.ShardP2P
+	shardChainDb ethdb.Database
 }
 
 // NewNotary creates a new notary instance.
-func NewNotary(smcClient *mainchain.SMCClient, shardp2p sharding.ShardP2P) (*Notary, error) {
-	return &Notary{smcClient, shardp2p}, nil
+func NewNotary(smcClient *mainchain.SMCClient, shardp2p sharding.ShardP2P, shardChainDb ethdb.Database) (*Notary, error) {
+	return &Notary{smcClient, shardp2p, shardChainDb}, nil
 }
 
 // Start the main routine for a notary.

sharding/notary/service_test.go

@@ -75,6 +75,20 @@ func (s *smcClient) SetDepositFlag(deposit bool) {
 	s.depositFlag = deposit
 }
 
+func (m *smcClient) Sign(hash common.Hash) ([]byte, error) {
+	return nil, nil
+}
+
+// Unused mockClient methods.
+func (m *smcClient) Start() error {
+	m.t.Fatal("Start called")
+	return nil
+}
+
+func (m *smcClient) Close() {
+	m.t.Fatal("Close called")
+}
+
 func (s *smcClient) DataDirPath() string {
 	return "/tmp/datadir"
 }

sharding/observer/service.go

@@ -3,6 +3,9 @@
 package observer
 
 import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/sharding"
 )
@@ -11,22 +14,24 @@ import (
 // in a sharded system. Must satisfy the Service interface defined in
 // sharding/service.go.
 type Observer struct {
-	shardp2p sharding.ShardP2P
+	shardp2p     sharding.ShardP2P
+	shardChainDb ethdb.Database
+	shardID      int
 }
 
 // NewObserver creates a new observer instance.
-func NewObserver(shardp2p sharding.ShardP2P) (*Observer, error) {
-	return &Observer{shardp2p}, nil
+func NewObserver(shardp2p sharding.ShardP2P, shardChainDb ethdb.Database, shardID int) (*Observer, error) {
+	return &Observer{shardp2p, shardChainDb, shardID}, nil
 }
 
 // Start the main routine for an observer.
 func (o *Observer) Start() error {
-	log.Info("Starting shard observer service")
+	log.Info(fmt.Sprintf("Starting observer service in shard %d", o.shardID))
 	return nil
 }
 
 // Stop the main loop for observing the shard network.
 func (o *Observer) Stop() error {
-	log.Info("Stopping shard observer service")
+	log.Info(fmt.Sprintf("Starting observer service in shard %d", o.shardID))
 	return nil
 }

sharding/proposer/proposer.go

@@ -0,0 +1,89 @@
+package proposer
+
+import (
+	"fmt"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/accounts/abi/bind"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/sharding"
+	"github.com/ethereum/go-ethereum/sharding/mainchain"
+)
+
+// createCollation creates collation base struct with header
+// and body. Header consists of shardID, ChunkRoot, period,
+// proposer addr and signatures. Body contains serialized blob
+// of a collations transactions.
+func createCollation(client mainchain.Client, shardId *big.Int, period *big.Int, txs []*types.Transaction) (*sharding.Collation, error) {
+	// shardId has to be within range
+	if shardId.Cmp(big.NewInt(0)) < 0 || shardId.Cmp(big.NewInt(sharding.ShardCount)) > 0 {
+		return nil, fmt.Errorf("can't create collation for shard %v. Must be between 0 and %v", shardId, sharding.ShardCount)
+	}
+
+	// check with SMC to see if we can add the header.
+	if a, _ := checkHeaderAdded(client, shardId, period); !a {
+		return nil, fmt.Errorf("can't create collation, collation with same period has already been added")
+	}
+
+	// serialized tx to blob for collation body.
+	blobs, err := sharding.SerializeTxToBlob(txs)
+	if err != nil {
+		return nil, fmt.Errorf("can't create collation, serialization to blob failed: %v", err)
+	}
+
+	// construct the header, leave chunkRoot and signature fields empty, to be filled later.
+	addr := client.Account().Address
+	header := sharding.NewCollationHeader(shardId, nil, period, &addr, nil)
+
+	// construct the body with header, blobs(serialized txs) and txs.
+	collation := sharding.NewCollation(header, blobs, txs)
+	collation.CalculateChunkRoot()
+	sig, err := client.Sign(collation.Header().Hash())
+	if err != nil {
+		return nil, fmt.Errorf("can't create collation, sign collationHeader failed: %v", err)
+	}
+
+	// add proposer signature to collation header.
+	collation.Header().AddSig(sig)
+	log.Info(fmt.Sprintf("Collation %v created for shardID %v period %v", collation.Header().Hash().Hex(), collation.Header().ShardID(), collation.Header().Period()))
+	return collation, nil
+}
+
+// addHeader adds the collation header to the main chain by sending
+// an addHeader transaction to the sharding manager contract.
+// There can only exist one header per period per shard, it's proposer's
+// responsibility to check if a header has been added.
+func addHeader(client mainchain.Client, collation *sharding.Collation) error {
+	log.Info("Adding header to SMC")
+
+	txOps, err := client.CreateTXOpts(big.NewInt(0))
+	if err != nil {
+		return fmt.Errorf("unable to initiate add header transaction: %v", err)
+	}
+
+	// TODO: Copy is inefficient here. Let's research how to best convert hash to [32]byte.
+	var chunkRoot [32]byte
+	copy(chunkRoot[:], collation.Header().ChunkRoot().Bytes())
+
+	tx, err := client.SMCTransactor().AddHeader(txOps, collation.Header().ShardID(), collation.Header().Period(), chunkRoot)
+	if err != nil {
+		return fmt.Errorf("unable to add header to SMC: %v", err)
+	}
+	log.Info(fmt.Sprintf("Add header transaction hash: %v", tx.Hash().Hex()))
+	return nil
+}
+
+// checkHeaderAdded checks if a collation header has already
+// submitted to the main chain. There can only be one header per shard
+// per period, proposer should check if a header's already submitted,
+// checkHeaderAdded returns true if it's available, false if it's unavailable.
+func checkHeaderAdded(client mainchain.Client, shardId *big.Int, period *big.Int) (bool, error) {
+	// Get the period of the last header.
+	lastPeriod, err := client.SMCCaller().LastSubmittedCollation(&bind.CallOpts{}, shardId)
+	if err != nil {
+		return false, fmt.Errorf("unable to get the period of last submitted collation: %v", err)
+	}
+	// True if current period is greater than last added period.
+	return period.Cmp(lastPeriod) > 0, nil
+}

sharding/proposer/service.go

@@ -3,6 +3,14 @@
 package proposer
 
 import (
+	"context"
+	"crypto/rand"
+	"fmt"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/sharding"
 	"github.com/ethereum/go-ethereum/sharding/mainchain"
@@ -12,28 +20,61 @@ import (
 // in a sharded system. Must satisfy the Service interface defined in
 // sharding/service.go.
 type Proposer struct {
-	client   mainchain.Client
-	shardp2p sharding.ShardP2P
-	txpool   sharding.TXPool
+	client       *mainchain.SMCClient
+	shardp2p     sharding.ShardP2P
+	txpool       sharding.TXPool
+	shardChainDb ethdb.Database
+	shardID      int
 }
 
 // NewProposer creates a struct instance of a proposer service.
 // It will have access to a mainchain client, a shardp2p network,
 // and a shard transaction pool.
-func NewProposer(client mainchain.Client, shardp2p sharding.ShardP2P, txpool sharding.TXPool) (*Proposer, error) {
+func NewProposer(client *mainchain.SMCClient, shardp2p sharding.ShardP2P, txpool sharding.TXPool, shardChainDb ethdb.Database, shardID int) (*Proposer, error) {
 	// Initializes a  directory persistent db.
-	return &Proposer{client, shardp2p, txpool}, nil
+	return &Proposer{client, shardp2p, txpool, shardChainDb, shardID}, nil
 }
 
 // Start the main loop for proposing collations.
 func (p *Proposer) Start() error {
-	log.Info("Starting proposer service")
-	// TODO: Propose collations.
+	log.Info(fmt.Sprintf("Starting proposer service in shard %d", p.shardID))
+
+	// TODO: Receive TXs from shard TX generator or TXpool (Github Issues 153 and 161)
+	var txs []*types.Transaction
+	for i := 0; i < 10; i++ {
+		data := make([]byte, 1024)
+		rand.Read(data)
+		txs = append(txs, types.NewTransaction(0, common.HexToAddress("0x0"),
+			nil, 0, nil, data))
+	}
+
+	// Get current block number.
+	blockNumber, err := p.client.ChainReader().BlockByNumber(context.Background(), nil)
+	if err != nil {
+		return err
+	}
+	period := new(big.Int).Div(blockNumber.Number(), big.NewInt(sharding.PeriodLength))
+
+	// Create collation.
+	collation, err := createCollation(p.client, big.NewInt(int64(p.shardID)), period, txs)
+	if err != nil {
+		return err
+	}
+
+	// Check SMC if we can submit header before addHeader
+	canAdd, err := checkHeaderAdded(p.client, big.NewInt(int64(p.shardID)), period)
+	if err != nil {
+		return err
+	}
+	if canAdd {
+		addHeader(p.client, collation)
+	}
+
 	return nil
 }
 
 // Stop the main loop for proposing collations.
 func (p *Proposer) Stop() error {
-	log.Info("Stopping proposer service")
+	log.Info(fmt.Sprintf("Stopping proposer service in shard %d", p.shardID))
 	return nil
 }

sharding/proposer/service_test.go

@@ -1,6 +1,255 @@
 package proposer
 
-import "github.com/ethereum/go-ethereum/sharding"
+import (
+	"math/big"
+	"testing"
 
-// Verifies that Proposer implements the Actor interface.
-var _ = sharding.Actor(&Proposer{})
+	"crypto/rand"
+
+	"github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/accounts/abi/bind"
+	"github.com/ethereum/go-ethereum/accounts/abi/bind/backends"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/sharding"
+	"github.com/ethereum/go-ethereum/sharding/contracts"
+	"gopkg.in/urfave/cli.v1"
+)
+
+var (
+	key, _            = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
+	addr              = crypto.PubkeyToAddress(key.PublicKey)
+	accountBalance, _ = new(big.Int).SetString("1001000000000000000000", 10)
+)
+
+// Mock client for testing proposer.
+type mockNode struct {
+	smc         *contracts.SMC
+	t           *testing.T
+	depositFlag bool
+	backend     *backends.SimulatedBackend
+}
+
+func (s *mockNode) Account() *accounts.Account {
+	return &accounts.Account{Address: addr}
+}
+
+func (s *mockNode) SMCCaller() *contracts.SMCCaller {
+	return &s.smc.SMCCaller
+}
+
+func (s *mockNode) ChainReader() ethereum.ChainReader {
+	return nil
+}
+
+func (s *mockNode) Context() *cli.Context {
+	return nil
+}
+
+func (s *mockNode) SMCTransactor() *contracts.SMCTransactor {
+	return &s.smc.SMCTransactor
+}
+
+func (s *mockNode) SMCFilterer() *contracts.SMCFilterer {
+	return &s.smc.SMCFilterer
+}
+
+func (s *mockNode) TransactionReceipt(hash common.Hash) (*types.Receipt, error) {
+	return nil, nil
+}
+
+func (s *mockNode) CreateTXOpts(value *big.Int) (*bind.TransactOpts, error) {
+	txOpts := transactOpts()
+	txOpts.Value = value
+	return txOpts, nil
+}
+
+func (s *mockNode) DepositFlag() bool {
+	return false
+}
+
+func (s *mockNode) SetDepositFlag(deposit bool) {
+	s.depositFlag = deposit
+}
+
+func (m *mockNode) Sign(hash common.Hash) ([]byte, error) {
+	return nil, nil
+}
+
+// Unused mockClient methods.
+func (m *mockNode) Start() error {
+	m.t.Fatal("Start called")
+	return nil
+}
+
+func (m *mockNode) Close() {
+	m.t.Fatal("Close called")
+}
+
+func (s *mockNode) DataDirPath() string {
+	return "/tmp/datadir"
+}
+
+func transactOpts() *bind.TransactOpts {
+	return bind.NewKeyedTransactor(key)
+}
+
+func setup(t *testing.T) (*backends.SimulatedBackend, *contracts.SMC) {
+	backend := backends.NewSimulatedBackend(core.GenesisAlloc{addr: {Balance: accountBalance}})
+	_, _, smc, err := contracts.DeploySMC(transactOpts(), backend)
+	if err != nil {
+		t.Fatalf("Failed to deploy SMC contract: %v", err)
+	}
+	backend.Commit()
+	return backend, smc
+}
+
+func TestCreateCollation(t *testing.T) {
+	backend, smc := setup(t)
+	node := &mockNode{smc: smc, t: t, backend: backend}
+	var txs []*types.Transaction
+	for i := 0; i < 10; i++ {
+		data := make([]byte, 1024)
+		rand.Read(data)
+		txs = append(txs, types.NewTransaction(0, common.HexToAddress("0x0"),
+			nil, 0, nil, data))
+	}
+
+	collation, err := createCollation(node, big.NewInt(0), big.NewInt(1), txs)
+	if err != nil {
+		t.Fatalf("Create collation failed: %v", err)
+	}
+
+	// fast forward to 2nd period.
+	for i := 0; i < 2*int(sharding.PeriodLength); i++ {
+		backend.Commit()
+	}
+
+	// negative test case #1: create collation with shard > shardCount.
+	collation, err = createCollation(node, big.NewInt(101), big.NewInt(2), txs)
+	if err == nil {
+		t.Errorf("Create collation should have failed with invalid shard number")
+	}
+	// negative test case #2, create collation with blob size > collationBodySizeLimit.
+	var badTxs []*types.Transaction
+	for i := 0; i <= 1024; i++ {
+		data := make([]byte, 1024)
+		rand.Read(data)
+		badTxs = append(badTxs, types.NewTransaction(0, common.HexToAddress("0x0"),
+			nil, 0, nil, data))
+	}
+	collation, err = createCollation(node, big.NewInt(0), big.NewInt(2), badTxs)
+	if err == nil {
+		t.Errorf("Create collation should have failed with Txs longer than collation body limit")
+	}
+
+	// normal test case #1 create collation with correct parameters.
+	collation, err = createCollation(node, big.NewInt(5), big.NewInt(5), txs)
+	if err != nil {
+		t.Errorf("Create collation failed: %v", err)
+	}
+	if collation.Header().Period().Cmp(big.NewInt(5)) != 0 {
+		t.Errorf("Incorrect collation period, want 5, got %v ", collation.Header().Period())
+	}
+	if collation.Header().ShardID().Cmp(big.NewInt(5)) != 0 {
+		t.Errorf("Incorrect shard id, want 5, got %v ", collation.Header().ShardID())
+	}
+	if *collation.ProposerAddress() != node.Account().Address {
+		t.Errorf("Incorrect proposer address, got %v", *collation.ProposerAddress())
+	}
+	if collation.Header().Sig() != nil {
+		t.Errorf("Proposer signaure can not be empty")
+	}
+}
+
+func TestAddCollation(t *testing.T) {
+	backend, smc := setup(t)
+	node := &mockNode{smc: smc, t: t, backend: backend}
+	var txs []*types.Transaction
+	for i := 0; i < 10; i++ {
+		data := make([]byte, 1024)
+		rand.Read(data)
+		txs = append(txs, types.NewTransaction(0, common.HexToAddress("0x0"),
+			nil, 0, nil, data))
+	}
+
+	collation, err := createCollation(node, big.NewInt(0), big.NewInt(1), txs)
+	if err != nil {
+		t.Errorf("Create collation failed: %v", err)
+	}
+
+	// fast forward to next period.
+	for i := 0; i < int(sharding.PeriodLength); i++ {
+		backend.Commit()
+	}
+
+	// normal test case #1 create collation with normal parameters.
+	err = addHeader(node, collation)
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	backend.Commit()
+
+	// verify collation was correctly added from SMC.
+	collationFromSMC, err := smc.CollationRecords(&bind.CallOpts{}, big.NewInt(0), big.NewInt(1))
+	if err != nil {
+		t.Errorf("Failed to get collation record")
+	}
+	if collationFromSMC.Proposer != node.Account().Address {
+		t.Errorf("Incorrect proposer address, got %v", *collation.ProposerAddress())
+	}
+	if common.BytesToHash(collationFromSMC.ChunkRoot[:]) != *collation.Header().ChunkRoot() {
+		t.Errorf("Incorrect chunk root, got %v", collationFromSMC.ChunkRoot)
+	}
+
+	// negative test case #1 create the same collation that just got added to SMC.
+	collation, err = createCollation(node, big.NewInt(0), big.NewInt(1), txs)
+	if err == nil {
+		t.Errorf("Create collation should fail due to same collation in SMC")
+	}
+
+}
+
+func TestCheckCollation(t *testing.T) {
+	backend, smc := setup(t)
+	node := &mockNode{smc: smc, t: t, backend: backend}
+	var txs []*types.Transaction
+	for i := 0; i < 10; i++ {
+		data := make([]byte, 1024)
+		rand.Read(data)
+		txs = append(txs, types.NewTransaction(0, common.HexToAddress("0x0"),
+			nil, 0, nil, data))
+	}
+
+	collation, err := createCollation(node, big.NewInt(0), big.NewInt(1), txs)
+	if err != nil {
+		t.Errorf("Create collation failed: %v", err)
+	}
+
+	for i := 0; i < int(sharding.PeriodLength); i++ {
+		backend.Commit()
+	}
+
+	err = addHeader(node, collation)
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+	backend.Commit()
+
+	// normal test case 1: check if we can still add header for period 1, should return false.
+	a, err := checkHeaderAdded(node, big.NewInt(0), big.NewInt(1))
+	if err != nil {
+		t.Errorf("Can not check header submitted: %v", err)
+	}
+	if a {
+		t.Errorf("Check header submitted shouldn't return: %v", a)
+	}
+	// normal test case 2: check if we can add header for period 2, should return true.
+	a, err = checkHeaderAdded(node, big.NewInt(0), big.NewInt(2))
+	if !a {
+		t.Errorf("Check header submitted shouldn't return: %v", a)
+	}
+}

sharding/shard.go

@@ -7,8 +7,8 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/ethereum/go-ethereum/sharding/database"
 )
 
 // Shard defines a way for services attached to a sharding-enabled node to
@@ -16,12 +16,12 @@ import (
 // an abstraction that contains useful methods to fetch collations corresponding to
 // a shard from the DB, methods to check for data availability, and more.
 type Shard struct {
-	shardDB database.ShardBackend
+	shardDB ethdb.Database
 	shardID *big.Int
 }
 
 // NewShard creates an instance of a Shard struct given a shardID.
-func NewShard(shardID *big.Int, shardDB database.ShardBackend) *Shard {
+func NewShard(shardID *big.Int, shardDB ethdb.Database) *Shard {
 	return &Shard{
 		shardID: shardID,
 		shardDB: shardDB,

sharding/shard_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto/sha3"
+	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/sharding/database"
 )
@@ -33,6 +34,13 @@ func (m *mockShardDB) Delete(k []byte) error {
 	return fmt.Errorf("error deleting value in db")
 }
 
+func (m *mockShardDB) Close() {
+}
+
+func (m *mockShardDB) NewBatch() ethdb.Batch {
+	return nil
+}
+
 // Hash returns the hash of a collation's entire contents. Useful for comparison tests.
 func (c *Collation) Hash() (hash common.Hash) {
 	hw := sha3.NewKeccak256()