From 89f0a4566859d65e371fffd1d376002993c9afb4 Mon Sep 17 00:00:00 2001 From: Eli Date: Wed, 2 May 2018 01:24:34 -0700 Subject: [PATCH 01/40] trie: golint iterator fixes (#16639) Former-commit-id: fe3d7c1e0956ba6ce2dc5b2e4a8142e3753c9b7c [formerly cbd57f04bdfc7105341b7c0286a0d2f432373fc1] Former-commit-id: e2c38a0175a5f52a75e44b373a6d35de6cc3f58f --- trie/iterator.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/trie/iterator.go b/trie/iterator.go index 76146c0d64..3bae8e186b 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -303,7 +303,7 @@ func (it *nodeIterator) push(state *nodeIteratorState, parentIndex *int, path [] it.path = path it.stack = append(it.stack, state) if parentIndex != nil { - *parentIndex += 1 + *parentIndex++ } } @@ -380,7 +380,7 @@ func (it *differenceIterator) Next(bool) bool { if !it.b.Next(true) { return false } - it.count += 1 + it.count++ if it.eof { // a has reached eof, so we just return all elements from b @@ -395,7 +395,7 @@ func (it *differenceIterator) Next(bool) bool { it.eof = true return true } - it.count += 1 + it.count++ case 1: // b is before a return true @@ -405,12 +405,12 @@ func (it *differenceIterator) Next(bool) bool { if !it.b.Next(hasHash) { return false } - it.count += 1 + it.count++ if !it.a.Next(hasHash) { it.eof = true return true } - it.count += 1 + it.count++ } } } @@ -504,14 +504,14 @@ func (it *unionIterator) Next(descend bool) bool { skipped := heap.Pop(it.items).(NodeIterator) // Skip the whole subtree if the nodes have hashes; otherwise just skip this node if skipped.Next(skipped.Hash() == common.Hash{}) { - it.count += 1 + it.count++ // If there are more elements, push the iterator back on the heap heap.Push(it.items, skipped) } } if least.Next(descend) { - it.count += 1 + it.count++ heap.Push(it.items, least) } From fc1fb315bc7fed41682c63e8b3582631f4ae5eb9 Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Wed, 16 May 2018 16:51:45 -0400 Subject: [PATCH 02/40] sharding: replace StringToHash and StringToAddress deprecated methods Former-commit-id: ff398a7f1b661f52735aceed6cd2d52ede02fc49 [formerly 42fbcc22fc08bf6d14b7a783adc9aecfabc83c60] Former-commit-id: de4aa4af63aa42bced0c592b6e8afec404361022 --- sharding/collation_test.go | 2 +- sharding/shard_test.go | 32 ++++++++++++++++---------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/sharding/collation_test.go b/sharding/collation_test.go index 2d815f3360..c94ec71996 100644 --- a/sharding/collation_test.go +++ b/sharding/collation_test.go @@ -28,7 +28,7 @@ func TestCollation_Transactions(t *testing.T) { } func TestCollation_ProposerAddress(t *testing.T) { - proposerAddr := common.StringToAddress("proposer") + proposerAddr := common.BytesToAddress([]byte("proposer")) header := NewCollationHeader(big.NewInt(1), nil, big.NewInt(1), &proposerAddr, []byte{}) body := []byte{} diff --git a/sharding/shard_test.go b/sharding/shard_test.go index 710c4e4888..e4258c38d2 100644 --- a/sharding/shard_test.go +++ b/sharding/shard_test.go @@ -40,8 +40,8 @@ func (c *Collation) Hash() (hash common.Hash) { return hash } func TestShard_ValidateShardID(t *testing.T) { - emptyHash := common.StringToHash("") - emptyAddr := common.StringToAddress("") + emptyHash := common.BytesToHash([]byte{}) + emptyAddr := common.BytesToAddress([]byte{}) header := NewCollationHeader(big.NewInt(1), &emptyHash, big.NewInt(1), &emptyAddr, []byte{}) shardDB := database.NewShardKV() shard := NewShard(big.NewInt(3), shardDB) @@ -59,8 +59,8 @@ func TestShard_ValidateShardID(t *testing.T) { } func TestShard_HeaderByHash(t *testing.T) { - emptyHash := common.StringToHash("") - emptyAddr := common.StringToAddress("") + emptyHash := common.BytesToHash([]byte{}) + emptyAddr := common.BytesToAddress([]byte{}) header := NewCollationHeader(big.NewInt(1), &emptyHash, big.NewInt(1), &emptyAddr, []byte{}) // creates a mockDB that always returns nil values from .Get and errors in every other method. @@ -96,7 +96,7 @@ func TestShard_HeaderByHash(t *testing.T) { } func TestShard_CollationByHash(t *testing.T) { - emptyAddr := common.StringToAddress("") + emptyAddr := common.BytesToAddress([]byte{}) // Empty chunk root. header := NewCollationHeader(big.NewInt(1), nil, big.NewInt(1), &emptyAddr, []byte{}) @@ -156,7 +156,7 @@ func TestShard_CollationByHash(t *testing.T) { func TestShard_CanonicalHeaderHash(t *testing.T) { shardID := big.NewInt(1) period := big.NewInt(1) - proposerAddress := common.StringToAddress("") + proposerAddress := common.BytesToAddress([]byte{}) proposerSignature := []byte{} header := NewCollationHeader(shardID, nil, period, &proposerAddress, proposerSignature) @@ -199,9 +199,9 @@ func TestShard_CanonicalHeaderHash(t *testing.T) { func TestShard_CanonicalCollation(t *testing.T) { shardID := big.NewInt(1) period := big.NewInt(1) - proposerAddress := common.StringToAddress("") + proposerAddress := common.BytesToAddress([]byte{}) proposerSignature := []byte{} - emptyHash := common.StringToHash("") + emptyHash := common.BytesToHash([]byte{}) header := NewCollationHeader(shardID, &emptyHash, period, &proposerAddress, proposerSignature) shardDB := database.NewShardKV() @@ -241,7 +241,7 @@ func TestShard_CanonicalCollation(t *testing.T) { } func TestShard_SetCanonical(t *testing.T) { - chunkRoot := common.StringToHash("") + chunkRoot := common.BytesToHash([]byte{}) header := NewCollationHeader(big.NewInt(1), &chunkRoot, big.NewInt(1), nil, []byte{}) shardDB := database.NewShardKV() @@ -285,7 +285,7 @@ func TestShard_BodyByChunkRoot(t *testing.T) { } // it should throw error if fetching non-existent chunk root. - emptyHash := common.StringToHash("") + emptyHash := common.BytesToHash([]byte{}) if _, err := shard.BodyByChunkRoot(&emptyHash); err == nil { t.Errorf("non-existent chunk root should throw error: %v", err) } @@ -307,9 +307,9 @@ func TestShard_BodyByChunkRoot(t *testing.T) { func TestShard_CheckAvailability(t *testing.T) { shardID := big.NewInt(1) period := big.NewInt(1) - proposerAddress := common.StringToAddress("") + proposerAddress := common.BytesToAddress([]byte{}) proposerSignature := []byte{} - emptyHash := common.StringToHash("") + emptyHash := common.BytesToHash([]byte{}) header := NewCollationHeader(shardID, &emptyHash, period, &proposerAddress, proposerSignature) shardDB := database.NewShardKV() @@ -342,7 +342,7 @@ func TestShard_CheckAvailability(t *testing.T) { } func TestShard_SetAvailability(t *testing.T) { - chunkRoot := common.StringToHash("") + chunkRoot := common.BytesToHash([]byte{}) header := NewCollationHeader(big.NewInt(1), &chunkRoot, big.NewInt(1), nil, []byte{}) // creates a mockDB that always returns nil values from .Get and errors in every other method. @@ -378,9 +378,9 @@ func TestShard_SetAvailability(t *testing.T) { func TestShard_SaveCollation(t *testing.T) { headerShardID := big.NewInt(1) period := big.NewInt(1) - proposerAddress := common.StringToAddress("") + proposerAddress := common.BytesToAddress([]byte{}) proposerSignature := []byte{} - emptyHash := common.StringToHash("") + emptyHash := common.BytesToHash([]byte{}) header := NewCollationHeader(headerShardID, &emptyHash, period, &proposerAddress, proposerSignature) shardDB := database.NewShardKV() @@ -402,7 +402,7 @@ func TestShard_SaveCollation(t *testing.T) { func TestShard_SaveHeader(t *testing.T) { // creates a mockDB that always returns nil values from .Get and errors in every other method. mockDB := &mockShardDB{kv: make(map[common.Hash]*[]byte)} - emptyHash := common.StringToHash("") + emptyHash := common.BytesToHash([]byte{}) errorShard := NewShard(big.NewInt(1), mockDB) header := NewCollationHeader(big.NewInt(1), &emptyHash, big.NewInt(1), nil, []byte{}) From dfaf00522991f0a2d4c8bd503dbdc47a7b8f7b82 Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Wed, 16 May 2018 17:06:35 -0400 Subject: [PATCH 03/40] sharding: address linter errors Former-commit-id: 70fee9bf5816e9520fb927b84ab8b93d83014869 [formerly 523192e37e40878079d7981bd28ed25f618ab0a6] Former-commit-id: 604f33826c63ff42dd90d75bb517ed09f5d52b6a --- sharding/contracts/sharding_manager_test.go | 8 ++++---- sharding/database/inmemory_test.go | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/sharding/contracts/sharding_manager_test.go b/sharding/contracts/sharding_manager_test.go index 25267971ba..9bd02e46c9 100644 --- a/sharding/contracts/sharding_manager_test.go +++ b/sharding/contracts/sharding_manager_test.go @@ -27,10 +27,10 @@ var ( notaryDepositInsufficient, _ = new(big.Int).SetString("999000000000000000000", 10) notaryDeposit, _ = new(big.Int).SetString("1000000000000000000000", 10) FastForward100Blocks = 100 - smcConfig = SMCConfig{ - notaryLockupLenght: new(big.Int).SetInt64(1), - proposerLockupLength: new(big.Int).SetInt64(1), - } + // smcConfig = SMCConfig{ + // notaryLockupLenght: new(big.Int).SetInt64(1), + // proposerLockupLength: new(big.Int).SetInt64(1), + // } ctx = context.Background() ) diff --git a/sharding/database/inmemory_test.go b/sharding/database/inmemory_test.go index 67c1d11d1a..2f9a54dae9 100644 --- a/sharding/database/inmemory_test.go +++ b/sharding/database/inmemory_test.go @@ -12,7 +12,7 @@ var _ = sharding.ShardBackend(&ShardKV{}) func Test_ShardKVPut(t *testing.T) { kv := NewShardKV() - hash := common.StringToHash("ralph merkle") + hash := common.BytesToHash([]byte("ralph merkle")) if err := kv.Put(hash, []byte{1, 2, 3}); err != nil { t.Errorf("could not save value in kv store: %v", err) @@ -21,7 +21,7 @@ func Test_ShardKVPut(t *testing.T) { func Test_ShardKVHas(t *testing.T) { kv := NewShardKV() - hash := common.StringToHash("ralph merkle") + hash := common.BytesToHash([]byte("ralph merkle")) if err := kv.Put(hash, []byte{1, 2, 3}); err != nil { t.Fatalf("could not save value in kv store: %v", err) @@ -31,7 +31,7 @@ func Test_ShardKVHas(t *testing.T) { t.Errorf("kv store does not have hash: %v", hash) } - hash2 := common.StringToHash("") + hash2 := common.BytesToHash([]byte{}) if kv.Has(hash2) { t.Errorf("kv store should not contain unset key: %v", hash2) } @@ -39,7 +39,7 @@ func Test_ShardKVHas(t *testing.T) { func Test_ShardKVGet(t *testing.T) { kv := NewShardKV() - hash := common.StringToHash("ralph merkle") + hash := common.BytesToHash([]byte("ralph merkle")) if err := kv.Put(hash, []byte{1, 2, 3}); err != nil { t.Fatalf("could not save value in kv store: %v", err) @@ -53,7 +53,7 @@ func Test_ShardKVGet(t *testing.T) { t.Errorf("no value stored for key") } - hash2 := common.StringToHash("") + hash2 := common.BytesToHash([]byte{}) val2, err := kv.Get(hash2) if val2 != nil { t.Errorf("non-existent key should not have a value. key=%v, value=%v", hash2, val2) @@ -62,7 +62,7 @@ func Test_ShardKVGet(t *testing.T) { func Test_ShardKVDelete(t *testing.T) { kv := NewShardKV() - hash := common.StringToHash("ralph merkle") + hash := common.BytesToHash([]byte("ralph merkle")) if err := kv.Put(hash, []byte{1, 2, 3}); err != nil { t.Fatalf("could not save value in kv store: %v", err) From f2e110680d158559aaa64ecf9961e930d8b22abe Mon Sep 17 00:00:00 2001 From: nisdas Date: Thu, 19 Apr 2018 01:07:39 +0800 Subject: [PATCH 04/40] sharding/client: Add utils.go for blob serialization(#92) Former-commit-id: 553671c1f1e1e1cb50c16fce53ecf0a5acc0595c [formerly 454f56eb8103b7a2b341ec2852c849611c55a1c0] Former-commit-id: 1a8305e422b5dcc1173afc0ed9d81c49ce426eb0 --- sharding/client/utils.go | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 sharding/client/utils.go diff --git a/sharding/client/utils.go b/sharding/client/utils.go new file mode 100644 index 0000000000..a4c577ec81 --- /dev/null +++ b/sharding/client/utils.go @@ -0,0 +1,39 @@ +package client + +import ( + "fmt" +) + +type collationbody []byte + +var ( + collationsizelimit = int64(2 * *20) + chunkSize = int64(32) + indicatorSize = int64(1) + chunkDataSize = chunkSize - indicatorSize +) + +func (c *collationbody) validateBody() bool { + return len(c) < 2^20 && len(c) > 0 +} + +func createChunks(c collationbody) { + validCollation, err := c.validateBody + if err != nil { + fmt.Errorf("Error %v", err) + } + if !validCollation { + fmt.Errorf("Error %v", err) + } + index := int64(len(c)) / chunkDataSize + + /* + for i = 0; i < index; i++ { + serialisedblob[i*chunksize] = 0 + for f = 0; f Date: Sun, 22 Apr 2018 17:57:12 +0800 Subject: [PATCH 05/40] sharding/client: Changing Validate Body(#92) Former-commit-id: 155ec90649a1226fc3a3f442014bdc29dae9d62f [formerly fe02510a1af1287d54296a4051745b02fa1aca3d] Former-commit-id: c5d489d9e640a3460c32136809016c0a9fb970ef --- sharding/client/utils.go | 49 ++++++++++++++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 9 deletions(-) diff --git a/sharding/client/utils.go b/sharding/client/utils.go index a4c577ec81..cecff0b956 100644 --- a/sharding/client/utils.go +++ b/sharding/client/utils.go @@ -7,33 +7,64 @@ import ( type collationbody []byte var ( - collationsizelimit = int64(2 * *20) + collationsizelimit = int64(2 ^ 20) chunkSize = int64(32) indicatorSize = int64(1) chunkDataSize = chunkSize - indicatorSize ) -func (c *collationbody) validateBody() bool { - return len(c) < 2^20 && len(c) > 0 +/* Validate that the collation body is within its bounds and if +the size of the body is below the limit it is simply appended +till it reaches the required limit */ + +func (cb collationbody) validateBody() error { + + length := int64(len(cb)) + + if length == 0 { + return fmt.Errorf("Collation Body has to be a non-zero value") + } + + if length > collationsizelimit { + return fmt.Errorf("Collation Body is over the size limit") + } + + if length < collationsizelimit { + x := make([]byte, (collationsizelimit - length)) + cb = append(cb, x...) + fmt.Printf("%b", x) + + } + + return nil } -func createChunks(c collationbody) { - validCollation, err := c.validateBody +func main() { + + x := []byte{'h', 'e', 'g', 'g'} + t := x.validateBody() + fmt.Printf("%b %s", x, t) + +} + +/* +func createChunks(cb collationbody) { + validCollation := cb.validateBody if err != nil { fmt.Errorf("Error %v", err) } if !validCollation { fmt.Errorf("Error %v", err) } - index := int64(len(c)) / chunkDataSize + index := int64(len(cb)) / chunkDataSize + - /* for i = 0; i < index; i++ { serialisedblob[i*chunksize] = 0 for f = 0; f Date: Sun, 22 Apr 2018 23:09:22 +0800 Subject: [PATCH 06/40] sharding/client: Adding Parse Blob(#92) Former-commit-id: 2ae72f191f84b88ab28a0752130b591c01b7a2fd [formerly c9a496eb646c098a1b46b9880d8059a094b93574] Former-commit-id: a1ef1e7d5f2d880c6678ca725ecd4837349a21f6 --- sharding/client/utils.go | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/sharding/client/utils.go b/sharding/client/utils.go index cecff0b956..29f12c1879 100644 --- a/sharding/client/utils.go +++ b/sharding/client/utils.go @@ -10,27 +10,32 @@ var ( collationsizelimit = int64(2 ^ 20) chunkSize = int64(32) indicatorSize = int64(1) + numberOfChunks = collationsizelimit / chunkSize chunkDataSize = chunkSize - indicatorSize + totalDatasize = numberOfChunks * chunkDataSize ) +func (cb collationbody) length() int64 { + + return int64(len(cb)) +} + /* Validate that the collation body is within its bounds and if the size of the body is below the limit it is simply appended till it reaches the required limit */ func (cb collationbody) validateBody() error { - length := int64(len(cb)) - - if length == 0 { + if cb.length() == 0 { return fmt.Errorf("Collation Body has to be a non-zero value") } - if length > collationsizelimit { + if cb.length() > totalDatasize { return fmt.Errorf("Collation Body is over the size limit") } - if length < collationsizelimit { - x := make([]byte, (collationsizelimit - length)) + if cb.length() < totalDatasize { + x := make([]byte, (totalDatasize - cb.length())) cb = append(cb, x...) fmt.Printf("%b", x) @@ -39,12 +44,17 @@ func (cb collationbody) validateBody() error { return nil } -func main() { +/* + add +*/ - x := []byte{'h', 'e', 'g', 'g'} - t := x.validateBody() - fmt.Printf("%b %s", x, t) +func (cb collationbody) ParseBlob() { + terminalLength := cb.length() % chunkDataSize + chunksNumber := cb.length() / chunkDataSize + if terminalLength != 0 { + + } } /* From b5454d2dca9271b2fa946f8a571eb8edc8658868 Mon Sep 17 00:00:00 2001 From: nisdas Date: Mon, 23 Apr 2018 00:39:41 +0800 Subject: [PATCH 07/40] sharding/client: Add conditions for terminal chunks(#92) Former-commit-id: d74291a1c8ce13da79aacb3e91c890554ef0c704 [formerly 03bf20998003e0beb9cc3f611d5023720c946369] Former-commit-id: 49ebefec9aacaff995e178112504b5f77c71140f --- sharding/client/utils.go | 53 ++++++++++++++++------------------------ 1 file changed, 21 insertions(+), 32 deletions(-) diff --git a/sharding/client/utils.go b/sharding/client/utils.go index 29f12c1879..c7972f6197 100644 --- a/sharding/client/utils.go +++ b/sharding/client/utils.go @@ -34,47 +34,36 @@ func (cb collationbody) validateBody() error { return fmt.Errorf("Collation Body is over the size limit") } - if cb.length() < totalDatasize { - x := make([]byte, (totalDatasize - cb.length())) - cb = append(cb, x...) - fmt.Printf("%b", x) - - } - return nil } -/* - add -*/ +// Parse Collation body and modify it accordingly func (cb collationbody) ParseBlob() { + terminalLength := cb.length() % chunkDataSize chunksNumber := cb.length() / chunkDataSize + indicatorByte := make([]byte, 1) + indicatorByte[0] = 0 + var tempbody collationbody + // Appends empty indicator bytes to non terminal-chunks + for i := int64(1); i <= chunksNumber; i++ { + tempbody = append(tempbody, append(indicatorByte, cb[(i-1)*chunkDataSize:i*chunkDataSize]...)...) + + } + // Appends indicator bytes to terminal-chunks , and if the index of the chunk delimiter is non-zero adds it to the chunk if terminalLength != 0 { + indicatorByte[0] = byte(terminalLength) + tempbody = append(tempbody, append(indicatorByte, cb[chunksNumber*chunkDataSize:(chunksNumber*chunkDataSize)+(terminalLength+1)]...)...) + + } + cb = tempbody + + // Pad the collation body with empty bytes until it is equal to 1 Mib + if cb.length() < collationsizelimit { + emptyBytes := make([]byte, (collationsizelimit - cb.length())) + cb = append(cb, emptyBytes...) } } - -/* -func createChunks(cb collationbody) { - validCollation := cb.validateBody - if err != nil { - fmt.Errorf("Error %v", err) - } - if !validCollation { - fmt.Errorf("Error %v", err) - } - index := int64(len(cb)) / chunkDataSize - - - for i = 0; i < index; i++ { - serialisedblob[i*chunksize] = 0 - for f = 0; f Date: Wed, 2 May 2018 00:36:16 +0800 Subject: [PATCH 08/40] sharding/client: Change to serialize blobs(#92) Former-commit-id: c2dea5d8ba8c469a304a1855e1f262892b3ea0a9 [formerly 4d77a3e8815c3cd4803f4f785131c9aac86a7795] Former-commit-id: 8b5d6485900122b46d88418a7c7f8ddabae6dd09 --- sharding/client/utils.go | 56 ++++++++++++++++++++++++++++------------ 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/sharding/client/utils.go b/sharding/client/utils.go index c7972f6197..0950dbdb4a 100644 --- a/sharding/client/utils.go +++ b/sharding/client/utils.go @@ -2,12 +2,11 @@ package client import ( "fmt" + "math" ) -type collationbody []byte - var ( - collationsizelimit = int64(2 ^ 20) + collationsizelimit = int64(math.Pow(float64(2), float64(20))) chunkSize = int64(32) indicatorSize = int64(1) numberOfChunks = collationsizelimit / chunkSize @@ -15,6 +14,14 @@ var ( totalDatasize = numberOfChunks * chunkDataSize ) +type collationbody []byte + +type body interface { + length() int64 + validateblob() error + ParseBlob() +} + func (cb collationbody) length() int64 { return int64(len(cb)) @@ -39,13 +46,36 @@ func (cb collationbody) validateBody() error { // Parse Collation body and modify it accordingly -func (cb collationbody) ParseBlob() { +func (cb collationbody) serializeBlob() []byte { terminalLength := cb.length() % chunkDataSize chunksNumber := cb.length() / chunkDataSize indicatorByte := make([]byte, 1) indicatorByte[0] = 0 - var tempbody collationbody + var tempbody []byte + + // if blob is less than 31 bytes, it adds the indicator chunk and pads the remaining empty bytes to the right + + if chunksNumber == 0 { + paddedbytes := make([]byte, cb.length()-terminalLength) + indicatorByte[0] = byte(terminalLength) + tempbody = append(indicatorByte, append(cb, paddedbytes...)...) + return tempbody + } + + //if there is no need to pad empty bytes, then the indicator byte is added as 00011111 + + if terminalLength == 0 { + + for i := int64(1); i < chunksNumber; i++ { + tempbody = append(tempbody, append(indicatorByte, cb[(i-1)*chunkDataSize:i*chunkDataSize]...)...) + + } + indicatorByte[0] = byte(chunkDataSize) + tempbody = append(tempbody, append(indicatorByte, cb[(chunksNumber-1)*chunkDataSize:chunksNumber*chunkDataSize]...)...) + return tempbody + + } // Appends empty indicator bytes to non terminal-chunks for i := int64(1); i <= chunksNumber; i++ { @@ -53,17 +83,11 @@ func (cb collationbody) ParseBlob() { } // Appends indicator bytes to terminal-chunks , and if the index of the chunk delimiter is non-zero adds it to the chunk - if terminalLength != 0 { - indicatorByte[0] = byte(terminalLength) - tempbody = append(tempbody, append(indicatorByte, cb[chunksNumber*chunkDataSize:(chunksNumber*chunkDataSize)+(terminalLength+1)]...)...) + indicatorByte[0] = byte(terminalLength) + tempbody = append(tempbody, append(indicatorByte, cb[chunksNumber*chunkDataSize:(chunksNumber*chunkDataSize)+(terminalLength+1)]...)...) + emptyBytes := make([]byte, (chunkDataSize - cb.length())) + cb = append(cb, emptyBytes...) - } - cb = tempbody + return tempbody - // Pad the collation body with empty bytes until it is equal to 1 Mib - if cb.length() < collationsizelimit { - emptyBytes := make([]byte, (collationsizelimit - cb.length())) - cb = append(cb, emptyBytes...) - - } } From e88c1343d402f4928b155e8f297f9a9d2283c788 Mon Sep 17 00:00:00 2001 From: nisdas Date: Wed, 2 May 2018 13:00:37 +0800 Subject: [PATCH 09/40] sharding/client: Adding deserialize blobs(#92) Former-commit-id: f11427ff5b305c1f9ebb30b2436e75b5ff9a9dc9 [formerly 3def7d93a525e3bb8348de8fb24ad9d828580860] Former-commit-id: 6dc530e57dc5a35ee50564ecc082d035f6ceead1 --- sharding/client/utils.go | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/sharding/client/utils.go b/sharding/client/utils.go index 0950dbdb4a..30c2845702 100644 --- a/sharding/client/utils.go +++ b/sharding/client/utils.go @@ -3,6 +3,7 @@ package client import ( "fmt" "math" + "reflect" ) var ( @@ -18,8 +19,8 @@ type collationbody []byte type body interface { length() int64 - validateblob() error - ParseBlob() + validateBody() error + serializeBlob() []byte } func (cb collationbody) length() int64 { @@ -44,6 +45,24 @@ func (cb collationbody) validateBody() error { return nil } +func deserializeBlob(blob body) []byte { + deserializedblob := blob.(collationbody) + length := deserializedblob.length() + chunksNumber := chunkSize / length + indicatorByte := make([]byte, 1) + indicatorByte[0] = 0 + tempbody := []byte{0} + + for i := int64(1); i <= chunksNumber; i++ { + + if reflect.TypeOf(deserializedblob[:(i-1)*chunksNumber]) == reflect.TypeOf(indicatorByte) { + + } + + } + +} + // Parse Collation body and modify it accordingly func (cb collationbody) serializeBlob() []byte { @@ -52,7 +71,7 @@ func (cb collationbody) serializeBlob() []byte { chunksNumber := cb.length() / chunkDataSize indicatorByte := make([]byte, 1) indicatorByte[0] = 0 - var tempbody []byte + tempbody := []byte{} // if blob is less than 31 bytes, it adds the indicator chunk and pads the remaining empty bytes to the right From 256078b48ed620afde0f6b41320f85f6939f93e2 Mon Sep 17 00:00:00 2001 From: nisdas Date: Wed, 2 May 2018 15:46:46 +0800 Subject: [PATCH 10/40] sharding/client: Fixing deserializeblobs(#92) Former-commit-id: 956c46610107382fbf8d6447f10d47290afbeb79 [formerly 3387c9666b09f695bcde26cc236df3db03c453fa] Former-commit-id: 4cc2bc2ac8ab198eb7fb99c773fa34d09b8b9acf --- sharding/client/utils.go | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/sharding/client/utils.go b/sharding/client/utils.go index 30c2845702..554425e8cb 100644 --- a/sharding/client/utils.go +++ b/sharding/client/utils.go @@ -15,7 +15,7 @@ var ( totalDatasize = numberOfChunks * chunkDataSize ) -type collationbody []byte +type txblob []byte type body interface { length() int64 @@ -23,7 +23,7 @@ type body interface { serializeBlob() []byte } -func (cb collationbody) length() int64 { +func (cb txblob) length() int64 { return int64(len(cb)) } @@ -32,7 +32,7 @@ func (cb collationbody) length() int64 { the size of the body is below the limit it is simply appended till it reaches the required limit */ -func (cb collationbody) validateBody() error { +func (cb txblob) validateBody() error { if cb.length() == 0 { return fmt.Errorf("Collation Body has to be a non-zero value") @@ -45,27 +45,36 @@ func (cb collationbody) validateBody() error { return nil } -func deserializeBlob(blob body) []byte { - deserializedblob := blob.(collationbody) - length := deserializedblob.length() +func deserializebody(collationbody []byte) []body { + length := int64(len(collationbody)) chunksNumber := chunkSize / length indicatorByte := make([]byte, 1) indicatorByte[0] = 0 - tempbody := []byte{0} + txblobs := []body{} + var tempbody txblob for i := int64(1); i <= chunksNumber; i++ { - if reflect.TypeOf(deserializedblob[:(i-1)*chunksNumber]) == reflect.TypeOf(indicatorByte) { + if reflect.TypeOf(collationbody[(i-1)*chunkSize]) == reflect.TypeOf(indicatorByte) { + tempbody = append(tempbody, collationbody[((i-1)*chunkSize+1):(i)*chunkSize]...) + + } else { + terminalIndex := int64(collationbody[(i-1)*chunkSize]) + tempbody = append(tempbody, collationbody[((i-1)*chunkSize+1):((i-1)*chunkSize+2+terminalIndex)]...) + txblobs = append(txblobs, tempbody) + tempbody = txblob{} } } + return txblobs + } // Parse Collation body and modify it accordingly -func (cb collationbody) serializeBlob() []byte { +func (cb txblob) serializeBlob() []byte { terminalLength := cb.length() % chunkDataSize chunksNumber := cb.length() / chunkDataSize From 1129d94303224b0c6960a9d08150cb08354b6dab Mon Sep 17 00:00:00 2001 From: nisdas Date: Wed, 2 May 2018 16:16:57 +0800 Subject: [PATCH 11/40] sharding/client: Adding Main serialize function(#92) Former-commit-id: abb3b3d0213daf4b6e84c880cf202bf91d04d086 [formerly d3e6aad05cba1272ab84fe9c03140d3d3b092584] Former-commit-id: 1185e61d50a11f7fa15e986521f91856cb1df3d5 --- sharding/client/utils.go | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/sharding/client/utils.go b/sharding/client/utils.go index 554425e8cb..c0d2c13fda 100644 --- a/sharding/client/utils.go +++ b/sharding/client/utils.go @@ -17,7 +17,7 @@ var ( type txblob []byte -type body interface { +type blob interface { length() int64 validateBody() error serializeBlob() []byte @@ -45,12 +45,12 @@ func (cb txblob) validateBody() error { return nil } -func deserializebody(collationbody []byte) []body { +func deserializebody(collationbody []byte) []blob { length := int64(len(collationbody)) chunksNumber := chunkSize / length indicatorByte := make([]byte, 1) indicatorByte[0] = 0 - txblobs := []body{} + txblobs := []blob{} var tempbody txblob for i := int64(1); i <= chunksNumber; i++ { @@ -72,7 +72,21 @@ func deserializebody(collationbody []byte) []body { } -// Parse Collation body and modify it accordingly +func serialize(rawtx []blob) []byte { + length := int64(len(rawtx)) + serialisedData := []byte{} + + for i := int64(1); i < length; i++ { + data := rawtx[length].(txblob) + refinedData := data.serializeBlob() + serialisedData = append(serialisedData, refinedData...) + txblob(serialisedData).validateBody() + + } + return serialisedData +} + +// Parse blob and modify it accordingly func (cb txblob) serializeBlob() []byte { From efb8f3a80f70d0657c17442fa268c2f288b94256 Mon Sep 17 00:00:00 2001 From: nisdas Date: Wed, 2 May 2018 16:57:59 +0800 Subject: [PATCH 12/40] sharding/client: Cleaning Up and adding errors(#92) Former-commit-id: 2047d553c3226550d07852d91352ba5287b86f6f [formerly 1257adb782b252461bb9f4b176e1d7f0a09f7b92] Former-commit-id: 75ad116eb88c6d8e0e94de65ec5c137d47e6429a --- sharding/client/utils.go | 117 +++++++++++++++++++-------------------- 1 file changed, 58 insertions(+), 59 deletions(-) diff --git a/sharding/client/utils.go b/sharding/client/utils.go index c0d2c13fda..ec5efe9ee3 100644 --- a/sharding/client/utils.go +++ b/sharding/client/utils.go @@ -4,6 +4,8 @@ import ( "fmt" "math" "reflect" + + "github.com/ethereum/go-ethereum/log" ) var ( @@ -19,7 +21,6 @@ type txblob []byte type blob interface { length() int64 - validateBody() error serializeBlob() []byte } @@ -28,64 +29,6 @@ func (cb txblob) length() int64 { return int64(len(cb)) } -/* Validate that the collation body is within its bounds and if -the size of the body is below the limit it is simply appended -till it reaches the required limit */ - -func (cb txblob) validateBody() error { - - if cb.length() == 0 { - return fmt.Errorf("Collation Body has to be a non-zero value") - } - - if cb.length() > totalDatasize { - return fmt.Errorf("Collation Body is over the size limit") - } - - return nil -} - -func deserializebody(collationbody []byte) []blob { - length := int64(len(collationbody)) - chunksNumber := chunkSize / length - indicatorByte := make([]byte, 1) - indicatorByte[0] = 0 - txblobs := []blob{} - var tempbody txblob - - for i := int64(1); i <= chunksNumber; i++ { - - if reflect.TypeOf(collationbody[(i-1)*chunkSize]) == reflect.TypeOf(indicatorByte) { - tempbody = append(tempbody, collationbody[((i-1)*chunkSize+1):(i)*chunkSize]...) - - } else { - terminalIndex := int64(collationbody[(i-1)*chunkSize]) - tempbody = append(tempbody, collationbody[((i-1)*chunkSize+1):((i-1)*chunkSize+2+terminalIndex)]...) - txblobs = append(txblobs, tempbody) - tempbody = txblob{} - - } - - } - - return txblobs - -} - -func serialize(rawtx []blob) []byte { - length := int64(len(rawtx)) - serialisedData := []byte{} - - for i := int64(1); i < length; i++ { - data := rawtx[length].(txblob) - refinedData := data.serializeBlob() - serialisedData = append(serialisedData, refinedData...) - txblob(serialisedData).validateBody() - - } - return serialisedData -} - // Parse blob and modify it accordingly func (cb txblob) serializeBlob() []byte { @@ -133,3 +76,59 @@ func (cb txblob) serializeBlob() []byte { return tempbody } + +func Serialize(rawtx []blob) ([]byte, error) { + length := int64(len(rawtx)) + + if length == 0 { + return nil, fmt.Errorf("Validation failed: Collation Body has to be a non-zero value") + } + serialisedData := []byte{} + + for i := int64(0); i < length; i++ { + + blobLength := txblob(serialisedData).length() + data := rawtx[i].(txblob) + refinedData := data.serializeBlob() + serialisedData = append(serialisedData, refinedData...) + + if txblob(serialisedData).length() > collationsizelimit { + log.Info(fmt.Sprintf("The total number of interfaces added to the collation body are: %d", i)) + serialisedData = serialisedData[:blobLength] + return serialisedData, nil + + } + + } + return serialisedData, nil +} + +// Collation body deserialised and separated into its respective interfaces + +func Deserializebody(collationbody []byte) []blob { + + length := int64(len(collationbody)) + chunksNumber := chunkSize / length + indicatorByte := make([]byte, 1) + indicatorByte[0] = 0 + txblobs := []blob{} + var tempbody txblob + + for i := int64(1); i <= chunksNumber; i++ { + + if reflect.TypeOf(collationbody[(i-1)*chunkSize]) == reflect.TypeOf(indicatorByte) { + tempbody = append(tempbody, collationbody[((i-1)*chunkSize+1):(i)*chunkSize]...) + + } else { + terminalIndex := int64(collationbody[(i-1)*chunkSize]) + tempbody = append(tempbody, collationbody[((i-1)*chunkSize+1):((i-1)*chunkSize+2+terminalIndex)]...) + txblobs = append(txblobs, tempbody) + tempbody = txblob{} + + } + + } + + return txblobs + +} From 2cf1bfc2b7a689dd3ad707768a24bbf7d96a8e6e Mon Sep 17 00:00:00 2001 From: nisdas Date: Thu, 3 May 2018 20:37:22 +0800 Subject: [PATCH 13/40] sharding/client: Make serialization similar to JSON marshalling(#92) Former-commit-id: 4d236b8b6a8b9d242beaeb2c035d0fae25fb4f63 [formerly 0a9b7aa774c9e6175b6fe70859c7c861bdbe6332] Former-commit-id: c7e62ad5effc66da4603cccaf65a789712796f04 --- sharding/client/utils.go | 56 +++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/sharding/client/utils.go b/sharding/client/utils.go index ec5efe9ee3..1eeb064bfa 100644 --- a/sharding/client/utils.go +++ b/sharding/client/utils.go @@ -1,6 +1,7 @@ package client import ( + "errors" "fmt" "math" "reflect" @@ -17,24 +18,27 @@ var ( totalDatasize = numberOfChunks * chunkDataSize ) -type txblob []byte +func convertInterface(arg interface{}, kind reflect.Kind) (reflect.Value, error) { + val := reflect.ValueOf(arg) + if val.Kind() == kind { + return val, nil -type blob interface { - length() int64 - serializeBlob() []byte -} - -func (cb txblob) length() int64 { - - return int64(len(cb)) + } + err := errors.New("Interface Conversion a failure") + return val, err } // Parse blob and modify it accordingly -func (cb txblob) serializeBlob() []byte { +func serializeBlob(cb interface{}) ([]byte, error) { - terminalLength := cb.length() % chunkDataSize - chunksNumber := cb.length() / chunkDataSize + blob, err := convertInterface(cb, reflect.Slice) + if err != nil { + return nil, fmt.Errorf("Error: %v", err) + } + length := int64(blob.Len()) + terminalLength := length % chunkDataSize + chunksNumber := length / chunkDataSize indicatorByte := make([]byte, 1) indicatorByte[0] = 0 tempbody := []byte{} @@ -42,10 +46,10 @@ func (cb txblob) serializeBlob() []byte { // if blob is less than 31 bytes, it adds the indicator chunk and pads the remaining empty bytes to the right if chunksNumber == 0 { - paddedbytes := make([]byte, cb.length()-terminalLength) + paddedbytes := make([]byte, length-terminalLength) indicatorByte[0] = byte(terminalLength) - tempbody = append(indicatorByte, append(cb, paddedbytes...)...) - return tempbody + tempbody = append(indicatorByte, append(cb.([]byte), paddedbytes...)...) + return tempbody, nil } //if there is no need to pad empty bytes, then the indicator byte is added as 00011111 @@ -53,41 +57,41 @@ func (cb txblob) serializeBlob() []byte { if terminalLength == 0 { for i := int64(1); i < chunksNumber; i++ { - tempbody = append(tempbody, append(indicatorByte, cb[(i-1)*chunkDataSize:i*chunkDataSize]...)...) + tempbody = append(tempbody, append(indicatorByte, blob.Bytes()[(i-1)*chunkDataSize:i*chunkDataSize]...)...) } indicatorByte[0] = byte(chunkDataSize) - tempbody = append(tempbody, append(indicatorByte, cb[(chunksNumber-1)*chunkDataSize:chunksNumber*chunkDataSize]...)...) - return tempbody + tempbody = append(tempbody, append(indicatorByte, blob.Bytes()[(chunksNumber-1)*chunkDataSize:chunksNumber*chunkDataSize]...)...) + return tempbody, nil } // Appends empty indicator bytes to non terminal-chunks for i := int64(1); i <= chunksNumber; i++ { - tempbody = append(tempbody, append(indicatorByte, cb[(i-1)*chunkDataSize:i*chunkDataSize]...)...) + tempbody = append(tempbody, append(indicatorByte, blob.Bytes()[(i-1)*chunkDataSize:i*chunkDataSize]...)...) } // Appends indicator bytes to terminal-chunks , and if the index of the chunk delimiter is non-zero adds it to the chunk indicatorByte[0] = byte(terminalLength) - tempbody = append(tempbody, append(indicatorByte, cb[chunksNumber*chunkDataSize:(chunksNumber*chunkDataSize)+(terminalLength+1)]...)...) - emptyBytes := make([]byte, (chunkDataSize - cb.length())) - cb = append(cb, emptyBytes...) + tempbody = append(tempbody, append(indicatorByte, blob.Bytes()[chunksNumber*chunkDataSize:(chunksNumber*chunkDataSize)+(terminalLength+1)]...)...) + emptyBytes := make([]byte, (chunkDataSize - terminalLength)) + tempbody = append(tempbody, emptyBytes...) - return tempbody + return tempbody, nil } -func Serialize(rawtx []blob) ([]byte, error) { +func Serialize(rawtx []interface{}) ([]byte, error) { length := int64(len(rawtx)) if length == 0 { return nil, fmt.Errorf("Validation failed: Collation Body has to be a non-zero value") } - serialisedData := []byte{} + serialisedData := []byte{0, 2} for i := int64(0); i < length; i++ { - blobLength := txblob(serialisedData).length() + blobLength := length(serialisedData) data := rawtx[i].(txblob) refinedData := data.serializeBlob() serialisedData = append(serialisedData, refinedData...) From 1f341bee874b8675d9656b4e31e6e81781bd67c8 Mon Sep 17 00:00:00 2001 From: nisdas Date: Thu, 3 May 2018 21:28:44 +0800 Subject: [PATCH 14/40] sharding/client: Finish changing Deserializebody(#92) Former-commit-id: f4afd07be086b843f857febbe127396eadfa0406 [formerly 5c8f51b8bc4d0d21e6a44bcbe5f946d051a2c9b6] Former-commit-id: 122da75bb554d464975c13172be8d84f6a693ec0 --- sharding/client/utils.go | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/sharding/client/utils.go b/sharding/client/utils.go index 1eeb064bfa..a3d6113bac 100644 --- a/sharding/client/utils.go +++ b/sharding/client/utils.go @@ -81,22 +81,26 @@ func serializeBlob(cb interface{}) ([]byte, error) { } +// Serialize takes a set of transactions and converts them to a single byte array func Serialize(rawtx []interface{}) ([]byte, error) { length := int64(len(rawtx)) if length == 0 { return nil, fmt.Errorf("Validation failed: Collation Body has to be a non-zero value") } - serialisedData := []byte{0, 2} + serialisedData := []byte{} for i := int64(0); i < length; i++ { - blobLength := length(serialisedData) - data := rawtx[i].(txblob) - refinedData := data.serializeBlob() + blobLength := int64(len(serialisedData)) + data := rawtx[i] + refinedData, err := serializeBlob(data) + if err != nil { + return nil, fmt.Errorf("Error: %v", err) + } serialisedData = append(serialisedData, refinedData...) - if txblob(serialisedData).length() > collationsizelimit { + if int64(len(serialisedData)) > collationsizelimit { log.Info(fmt.Sprintf("The total number of interfaces added to the collation body are: %d", i)) serialisedData = serialisedData[:blobLength] return serialisedData, nil @@ -107,32 +111,32 @@ func Serialize(rawtx []interface{}) ([]byte, error) { return serialisedData, nil } -// Collation body deserialised and separated into its respective interfaces - -func Deserializebody(collationbody []byte) []blob { +// Deserializebody results in the Collation body being deserialised and separated into its respective interfaces +func Deserializebody(collationbody []byte, rawtx []interface{}) error { length := int64(len(collationbody)) chunksNumber := chunkSize / length indicatorByte := make([]byte, 1) indicatorByte[0] = 0 - txblobs := []blob{} - var tempbody txblob + var txblobs []interface{} + tempbody := []byte{} for i := int64(1); i <= chunksNumber; i++ { - if reflect.TypeOf(collationbody[(i-1)*chunkSize]) == reflect.TypeOf(indicatorByte) { + if reflect.ValueOf(collationbody[(i-1)*chunkSize]) == reflect.ValueOf(indicatorByte) { tempbody = append(tempbody, collationbody[((i-1)*chunkSize+1):(i)*chunkSize]...) } else { terminalIndex := int64(collationbody[(i-1)*chunkSize]) tempbody = append(tempbody, collationbody[((i-1)*chunkSize+1):((i-1)*chunkSize+2+terminalIndex)]...) txblobs = append(txblobs, tempbody) - tempbody = txblob{} + tempbody = []byte{} } } + rawtx = txblobs - return txblobs + return nil } From 2026d42eec28b0dac4b9b379d9fb6a003eff8d90 Mon Sep 17 00:00:00 2001 From: nisdas Date: Thu, 3 May 2018 22:34:01 +0800 Subject: [PATCH 15/40] sharding/client: Adding new comments to improve clarity(#92) Former-commit-id: 1a7affe598d4b93b0983efcfdfb48d2d6b649bcf [formerly 7ec6f8bd96042285b491d7c884d4da1c8d54674b] Former-commit-id: beef30370be94aeb3915adee8daa75985f2df789 --- sharding/client/utils.go | 67 ++++++++++++++++++++++++++++------------ 1 file changed, 47 insertions(+), 20 deletions(-) diff --git a/sharding/client/utils.go b/sharding/client/utils.go index a3d6113bac..e9a144d3e7 100644 --- a/sharding/client/utils.go +++ b/sharding/client/utils.go @@ -18,6 +18,7 @@ var ( totalDatasize = numberOfChunks * chunkDataSize ) +// convertInterface converts inputted interface to the required type, ex: slice. func convertInterface(arg interface{}, kind reflect.Kind) (reflect.Value, error) { val := reflect.ValueOf(arg) if val.Kind() == kind { @@ -28,8 +29,7 @@ func convertInterface(arg interface{}, kind reflect.Kind) (reflect.Value, error) return val, err } -// Parse blob and modify it accordingly - +// serializeBlob parses the blob and serializes it appropriately. func serializeBlob(cb interface{}) ([]byte, error) { blob, err := convertInterface(cb, reflect.Slice) @@ -39,6 +39,7 @@ func serializeBlob(cb interface{}) ([]byte, error) { length := int64(blob.Len()) terminalLength := length % chunkDataSize chunksNumber := length / chunkDataSize + finalchunkIndex := totalDatasize + (terminalLength + 1) indicatorByte := make([]byte, 1) indicatorByte[0] = 0 tempbody := []byte{} @@ -46,34 +47,56 @@ func serializeBlob(cb interface{}) ([]byte, error) { // if blob is less than 31 bytes, it adds the indicator chunk and pads the remaining empty bytes to the right if chunksNumber == 0 { - paddedbytes := make([]byte, length-terminalLength) + paddedbytes := make([]byte, (length - terminalLength)) indicatorByte[0] = byte(terminalLength) - tempbody = append(indicatorByte, append(cb.([]byte), paddedbytes...)...) + tempbody = append(indicatorByte, append(blob.Bytes(), paddedbytes...)...) return tempbody, nil } //if there is no need to pad empty bytes, then the indicator byte is added as 00011111 + // Then this chunk is returned to the main Serialize function if terminalLength == 0 { for i := int64(1); i < chunksNumber; i++ { - tempbody = append(tempbody, append(indicatorByte, blob.Bytes()[(i-1)*chunkDataSize:i*chunkDataSize]...)...) + // This loop loops through all non-terminal chunks and add a indicator byte of 00000000, each chunk + // is created by appending the indcator byte to the data chunks. The data chunks are separated into sets of + // 31 + tempbody = append(tempbody, + append(indicatorByte, + blob.Bytes()[(i-1)*chunkDataSize:i*chunkDataSize]...)...) } indicatorByte[0] = byte(chunkDataSize) - tempbody = append(tempbody, append(indicatorByte, blob.Bytes()[(chunksNumber-1)*chunkDataSize:chunksNumber*chunkDataSize]...)...) + + // Terminal chunk has its indicator byte added, chunkDataSize*chunksNumber refers to the total size of the blob + tempbody = append(tempbody, + append(indicatorByte, + blob.Bytes()[(chunksNumber-1)*chunkDataSize:chunkDataSize*chunksNumber]...)...) + return tempbody, nil } - // Appends empty indicator bytes to non terminal-chunks + // This loop loops through all non-terminal chunks and add a indicator byte of 00000000, each chunk + // is created by appending the indcator byte to the data chunks. The data chunks are separated into sets of + // 31 for i := int64(1); i <= chunksNumber; i++ { - tempbody = append(tempbody, append(indicatorByte, blob.Bytes()[(i-1)*chunkDataSize:i*chunkDataSize]...)...) + + tempbody = append(tempbody, + append(indicatorByte, + blob.Bytes()[(i-1)*chunkDataSize:i*chunkDataSize]...)...) } - // Appends indicator bytes to terminal-chunks , and if the index of the chunk delimiter is non-zero adds it to the chunk + // Appends indicator bytes to terminal-chunks , and if the index of the chunk delimiter is non-zero adds it to the chunk. + // Also pads empty bytes to the terminal chunk.chunkDataSize*chunksNumber refers to the total size of the blob. + // finalchunkIndex refers to the index of the last data byte + indicatorByte[0] = byte(terminalLength) - tempbody = append(tempbody, append(indicatorByte, blob.Bytes()[chunksNumber*chunkDataSize:(chunksNumber*chunkDataSize)+(terminalLength+1)]...)...) + tempbody = append(tempbody, + append(indicatorByte, + blob.Bytes()[chunkDataSize*chunksNumber:finalchunkIndex]...)...) + emptyBytes := make([]byte, (chunkDataSize - terminalLength)) tempbody = append(tempbody, emptyBytes...) @@ -81,7 +104,7 @@ func serializeBlob(cb interface{}) ([]byte, error) { } -// Serialize takes a set of transactions and converts them to a single byte array +// Serialize takes a set of transaction blobs and converts them to a single byte array. func Serialize(rawtx []interface{}) ([]byte, error) { length := int64(len(rawtx)) @@ -90,6 +113,7 @@ func Serialize(rawtx []interface{}) ([]byte, error) { } serialisedData := []byte{} + //Loops through all the blobs and serializes them into chunks for i := int64(0); i < length; i++ { blobLength := int64(len(serialisedData)) @@ -111,31 +135,34 @@ func Serialize(rawtx []interface{}) ([]byte, error) { return serialisedData, nil } -// Deserializebody results in the Collation body being deserialised and separated into its respective interfaces +// Deserializebody results in the Collation body being deserialised and separated into its respective interfaces. func Deserializebody(collationbody []byte, rawtx []interface{}) error { length := int64(len(collationbody)) chunksNumber := chunkSize / length indicatorByte := make([]byte, 1) indicatorByte[0] = 0 - var txblobs []interface{} tempbody := []byte{} + // This separates the collation body into its respective transaction blobs for i := int64(1); i <= chunksNumber; i++ { + indicatorIndex := (i - 1) * chunkSize + // Tests if the chunk delimiter is zero, if it is it will append the data chunk + // to tempbody + if collationbody[indicatorIndex] == indicatorByte[0] { + tempbody = append(tempbody, collationbody[(indicatorIndex+1):(i)*chunkSize]...) - if reflect.ValueOf(collationbody[(i-1)*chunkSize]) == reflect.ValueOf(indicatorByte) { - tempbody = append(tempbody, collationbody[((i-1)*chunkSize+1):(i)*chunkSize]...) - + // Since the chunk delimiter in non-zero now we can infer that it is a terminal chunk and + // add it and append to the rawtx slice. The tempbody signifies a deserialized blob } else { - terminalIndex := int64(collationbody[(i-1)*chunkSize]) - tempbody = append(tempbody, collationbody[((i-1)*chunkSize+1):((i-1)*chunkSize+2+terminalIndex)]...) - txblobs = append(txblobs, tempbody) + terminalIndex := int64(collationbody[indicatorIndex]) + tempbody = append(tempbody, collationbody[(indicatorIndex+1):(indicatorIndex+2+terminalIndex)]...) + rawtx = append(rawtx, tempbody) tempbody = []byte{} } } - rawtx = txblobs return nil From 1df56837348895c670190b68202b552658d945af Mon Sep 17 00:00:00 2001 From: nisdas Date: Fri, 4 May 2018 17:17:21 +0800 Subject: [PATCH 16/40] sharding/client: Adding tests and modifying utils(#92) Former-commit-id: edbf5bd3d11b46f463bf26027178fb7d06fc13fb [formerly 73e4692d8036976d224e22e48423bf8cae9a95c1] Former-commit-id: ca76a1ecf0fda4f66f0330277384b35ba501e3ed --- sharding/client/utils.go | 32 ++++++++++++++------- sharding/client/utils_test.go | 53 +++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 10 deletions(-) create mode 100644 sharding/client/utils_test.go diff --git a/sharding/client/utils.go b/sharding/client/utils.go index e9a144d3e7..5c113fe164 100644 --- a/sharding/client/utils.go +++ b/sharding/client/utils.go @@ -5,6 +5,7 @@ import ( "fmt" "math" "reflect" + //"runtime" "github.com/ethereum/go-ethereum/log" ) @@ -15,7 +16,6 @@ var ( indicatorSize = int64(1) numberOfChunks = collationsizelimit / chunkSize chunkDataSize = chunkSize - indicatorSize - totalDatasize = numberOfChunks * chunkDataSize ) // convertInterface converts inputted interface to the required type, ex: slice. @@ -29,6 +29,16 @@ func convertInterface(arg interface{}, kind reflect.Kind) (reflect.Value, error) return val, err } +func convertbyteToInterface(arg []byte) []interface{} { + length := int64(len(arg)) + newtype := make([]interface{}, length) + for i, v := range arg { + newtype[i] = v + } + + return newtype +} + // serializeBlob parses the blob and serializes it appropriately. func serializeBlob(cb interface{}) ([]byte, error) { @@ -39,7 +49,7 @@ func serializeBlob(cb interface{}) ([]byte, error) { length := int64(blob.Len()) terminalLength := length % chunkDataSize chunksNumber := length / chunkDataSize - finalchunkIndex := totalDatasize + (terminalLength + 1) + finalchunkIndex := length - 1 indicatorByte := make([]byte, 1) indicatorByte[0] = 0 tempbody := []byte{} @@ -81,6 +91,7 @@ func serializeBlob(cb interface{}) ([]byte, error) { // This loop loops through all non-terminal chunks and add a indicator byte of 00000000, each chunk // is created by appending the indcator byte to the data chunks. The data chunks are separated into sets of // 31 + for i := int64(1); i <= chunksNumber; i++ { tempbody = append(tempbody, @@ -91,11 +102,10 @@ func serializeBlob(cb interface{}) ([]byte, error) { // Appends indicator bytes to terminal-chunks , and if the index of the chunk delimiter is non-zero adds it to the chunk. // Also pads empty bytes to the terminal chunk.chunkDataSize*chunksNumber refers to the total size of the blob. // finalchunkIndex refers to the index of the last data byte - indicatorByte[0] = byte(terminalLength) tempbody = append(tempbody, append(indicatorByte, - blob.Bytes()[chunkDataSize*chunksNumber:finalchunkIndex]...)...) + blob.Bytes()[chunkDataSize*chunksNumber-1:finalchunkIndex]...)...) emptyBytes := make([]byte, (chunkDataSize - terminalLength)) tempbody = append(tempbody, emptyBytes...) @@ -139,17 +149,17 @@ func Serialize(rawtx []interface{}) ([]byte, error) { func Deserializebody(collationbody []byte, rawtx []interface{}) error { length := int64(len(collationbody)) - chunksNumber := chunkSize / length - indicatorByte := make([]byte, 1) - indicatorByte[0] = 0 + chunksNumber := length / chunkSize + indicatorByte := byte(0) tempbody := []byte{} + deserializedblob := []byte{} // This separates the collation body into its respective transaction blobs - for i := int64(1); i <= chunksNumber; i++ { + for i := int64(1); i <= chunksNumber+1; i++ { indicatorIndex := (i - 1) * chunkSize // Tests if the chunk delimiter is zero, if it is it will append the data chunk // to tempbody - if collationbody[indicatorIndex] == indicatorByte[0] { + if collationbody[indicatorIndex] == indicatorByte { tempbody = append(tempbody, collationbody[(indicatorIndex+1):(i)*chunkSize]...) // Since the chunk delimiter in non-zero now we can infer that it is a terminal chunk and @@ -157,13 +167,15 @@ func Deserializebody(collationbody []byte, rawtx []interface{}) error { } else { terminalIndex := int64(collationbody[indicatorIndex]) tempbody = append(tempbody, collationbody[(indicatorIndex+1):(indicatorIndex+2+terminalIndex)]...) - rawtx = append(rawtx, tempbody) + deserializedblob = append(deserializedblob, tempbody...) tempbody = []byte{} } } + rawtx = convertbyteToInterface(deserializedblob) + return nil } diff --git a/sharding/client/utils_test.go b/sharding/client/utils_test.go new file mode 100644 index 0000000000..eee9615105 --- /dev/null +++ b/sharding/client/utils_test.go @@ -0,0 +1,53 @@ +package client + +import ( + "math/rand" + "reflect" + "runtime" + "testing" +) + +var testbody []interface{} + +func buildblob() []byte { + + tempbody := make([]byte, 500) + for i := int64(0); i < 500; i++ { + tempbody[i] = byte(rand.Int()) + + } + + return tempbody + +} +func TestConvertInterface(t *testing.T) { + var slice interface{} + slice = []interface{}{0, 1, 2, 3, 4, 5} + convertedValue, err := convertInterface(slice, reflect.Slice) + if err != nil { + t.Fatalf("Error: %v %v", err, convertedValue) + } + +} + +func TestSerializeblob(t *testing.T) { + + blob := buildblob() + + serializedblob, err := serializeBlob(blob) + + if err != nil { + t.Fatalf("Error Serializing blob:%v %v", err, serializedblob) + } + runtime.Breakpoint() + err2 := Deserializebody(serializedblob, testbody) + if err2 != nil { + t.Fatalf("Error Serializing blob:%v", err2) + } + + if !reflect.DeepEqual(blob, testbody) { + + t.Fatalf("Error Serializing blob with %v %v", blob, testbody) + } + +} From 4e5c5354265eaeb48ccc15d084718f7e1ee22650 Mon Sep 17 00:00:00 2001 From: nisdas Date: Fri, 4 May 2018 19:03:17 +0800 Subject: [PATCH 17/40] sharding/client: Fixing Tests (#92) Former-commit-id: 85e659f3ea43c983f8df54ab8a3150a560b2dbf8 [formerly 2c0a181754a07dbda855d1da8042a09c9e9a040d] Former-commit-id: 0198766ea1336efff88b9292b6ee1b53ef86b000 --- sharding/client/utils.go | 6 +++--- sharding/client/utils_test.go | 19 ++++++++++--------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/sharding/client/utils.go b/sharding/client/utils.go index 5c113fe164..a5909a98c7 100644 --- a/sharding/client/utils.go +++ b/sharding/client/utils.go @@ -146,7 +146,7 @@ func Serialize(rawtx []interface{}) ([]byte, error) { } // Deserializebody results in the Collation body being deserialised and separated into its respective interfaces. -func Deserializebody(collationbody []byte, rawtx []interface{}) error { +func Deserializebody(collationbody []byte, rawtx interface{}) error { length := int64(len(collationbody)) chunksNumber := length / chunkSize @@ -155,7 +155,7 @@ func Deserializebody(collationbody []byte, rawtx []interface{}) error { deserializedblob := []byte{} // This separates the collation body into its respective transaction blobs - for i := int64(1); i <= chunksNumber+1; i++ { + for i := int64(1); i <= chunksNumber; i++ { indicatorIndex := (i - 1) * chunkSize // Tests if the chunk delimiter is zero, if it is it will append the data chunk // to tempbody @@ -166,7 +166,7 @@ func Deserializebody(collationbody []byte, rawtx []interface{}) error { // add it and append to the rawtx slice. The tempbody signifies a deserialized blob } else { terminalIndex := int64(collationbody[indicatorIndex]) - tempbody = append(tempbody, collationbody[(indicatorIndex+1):(indicatorIndex+2+terminalIndex)]...) + tempbody = append(tempbody, collationbody[(indicatorIndex+1):(indicatorIndex+1+terminalIndex)]...) deserializedblob = append(deserializedblob, tempbody...) tempbody = []byte{} diff --git a/sharding/client/utils_test.go b/sharding/client/utils_test.go index eee9615105..2617dc50ad 100644 --- a/sharding/client/utils_test.go +++ b/sharding/client/utils_test.go @@ -3,16 +3,16 @@ package client import ( "math/rand" "reflect" - "runtime" + //"runtime" "testing" ) -var testbody []interface{} +var testbody interface{} -func buildblob() []byte { +func buildblob(size int64) []byte { - tempbody := make([]byte, 500) - for i := int64(0); i < 500; i++ { + tempbody := make([]byte, size) + for i := int64(0); i < size; i++ { tempbody[i] = byte(rand.Int()) } @@ -32,22 +32,23 @@ func TestConvertInterface(t *testing.T) { func TestSerializeblob(t *testing.T) { - blob := buildblob() + blob := buildblob(20) serializedblob, err := serializeBlob(blob) if err != nil { t.Fatalf("Error Serializing blob:%v %v", err, serializedblob) } - runtime.Breakpoint() - err2 := Deserializebody(serializedblob, testbody) + test := &testbody + //runtime.Breakpoint() + err2 := Deserializebody(serializedblob, &testbody) if err2 != nil { t.Fatalf("Error Serializing blob:%v", err2) } if !reflect.DeepEqual(blob, testbody) { - t.Fatalf("Error Serializing blob with %v %v", blob, testbody) + t.Fatalf("Error Serializing blob with %v %v %v", blob, test, &testbody) } } From e0282e9e82555735a38686d2bec67fe0d43214c9 Mon Sep 17 00:00:00 2001 From: nisdas Date: Fri, 4 May 2018 20:01:56 +0800 Subject: [PATCH 18/40] sharding/client : Adding Size Test (#92) Former-commit-id: 553d0fc1c4e99d0c3216bcb624c9f05cdbb07e1f [formerly ecebff2ec6ab0a42c541420f26af0c069ffd0ec6] Former-commit-id: 476e50d57067859cbbac27afd6c2cd73b7b6cb90 --- sharding/client/utils.go | 4 ++-- sharding/client/utils_test.go | 39 ++++++++++++++++++++++++++++++----- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/sharding/client/utils.go b/sharding/client/utils.go index a5909a98c7..80103d6e91 100644 --- a/sharding/client/utils.go +++ b/sharding/client/utils.go @@ -130,7 +130,7 @@ func Serialize(rawtx []interface{}) ([]byte, error) { data := rawtx[i] refinedData, err := serializeBlob(data) if err != nil { - return nil, fmt.Errorf("Error: %v", err) + return nil, fmt.Errorf("Error: %v at index: %v %v %v", err, i, data, rawtx) } serialisedData = append(serialisedData, refinedData...) @@ -174,7 +174,7 @@ func Deserializebody(collationbody []byte, rawtx interface{}) error { } - rawtx = convertbyteToInterface(deserializedblob) + *rawtx.(*interface{}) = convertbyteToInterface(deserializedblob) return nil diff --git a/sharding/client/utils_test.go b/sharding/client/utils_test.go index 2617dc50ad..1364313a24 100644 --- a/sharding/client/utils_test.go +++ b/sharding/client/utils_test.go @@ -9,9 +9,18 @@ import ( var testbody interface{} -func buildblob(size int64) []byte { +func buildtxblobs(size int64) []interface{} { + tempbody := make([]interface{}, size) + for i := int64(0); i < size; i++ { + tempbody[i] = buildblob(size) - tempbody := make([]byte, size) + } + return tempbody +} + +func buildblob(size int64) []interface{} { + + tempbody := make([]interface{}, size) for i := int64(0); i < size; i++ { tempbody[i] = byte(rand.Int()) @@ -29,17 +38,37 @@ func TestConvertInterface(t *testing.T) { } } +func TestSize(t *testing.T) { + size := int64(20) + blob := buildtxblobs(size) + chunksafterSerialize := size / chunkDataSize + terminalchunk := size % chunkDataSize + if terminalchunk != 0 { + chunksafterSerialize = chunksafterSerialize + 1 + } + sizeafterSerialize := chunksafterSerialize * chunkSize + serializedblob, err := Serialize(blob) + if err != nil { + t.Fatalf("Error Serializing blob:%v %v", err, serializedblob) + } + if int64(len(serializedblob)) != sizeafterSerialize { + + t.Fatalf("Error Serializing blob,Lengths are not the same: %v , %v", int64(len(serializedblob)), sizeafterSerialize) + + } + +} func TestSerializeblob(t *testing.T) { - blob := buildblob(20) + blob := buildblob(200) serializedblob, err := serializeBlob(blob) if err != nil { t.Fatalf("Error Serializing blob:%v %v", err, serializedblob) } - test := &testbody + //test := &testbody //runtime.Breakpoint() err2 := Deserializebody(serializedblob, &testbody) if err2 != nil { @@ -48,7 +77,7 @@ func TestSerializeblob(t *testing.T) { if !reflect.DeepEqual(blob, testbody) { - t.Fatalf("Error Serializing blob with %v %v %v", blob, test, &testbody) + t.Fatalf("Error Serializing blob with %v %v", blob, testbody) } } From c25de5167d92f54255a8e2f86e6d0473a1b306d3 Mon Sep 17 00:00:00 2001 From: nisdas Date: Fri, 4 May 2018 20:40:57 +0800 Subject: [PATCH 19/40] sharding/client: Cleaning up and refining tests(#92) Former-commit-id: 45ebe18e2281496741d6b08fae04419a55f008c6 [formerly 5473045b0034020047308e3707b365c04fe35a69] Former-commit-id: 4df6d79425d6c3ab0ad184c18063e4b35eea12de --- sharding/client/utils.go | 29 ++++++++++++++++++----------- sharding/client/utils_test.go | 15 ++++++--------- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/sharding/client/utils.go b/sharding/client/utils.go index 80103d6e91..cffa0d67ff 100644 --- a/sharding/client/utils.go +++ b/sharding/client/utils.go @@ -5,9 +5,6 @@ import ( "fmt" "math" "reflect" - //"runtime" - - "github.com/ethereum/go-ethereum/log" ) var ( @@ -39,14 +36,25 @@ func convertbyteToInterface(arg []byte) []interface{} { return newtype } +func interfacetoByte(arg []interface{}) []byte { + length := int64(len(arg)) + newtype := make([]byte, length) + for i, v := range arg { + newtype[i] = v.(byte) + } + + return newtype +} + // serializeBlob parses the blob and serializes it appropriately. func serializeBlob(cb interface{}) ([]byte, error) { - blob, err := convertInterface(cb, reflect.Slice) + interfaceblob, err := convertInterface(cb, reflect.Slice) if err != nil { return nil, fmt.Errorf("Error: %v", err) } - length := int64(blob.Len()) + blob := interfacetoByte(interfaceblob.Interface().([]interface{})) + length := int64(len(blob)) terminalLength := length % chunkDataSize chunksNumber := length / chunkDataSize finalchunkIndex := length - 1 @@ -59,7 +67,7 @@ func serializeBlob(cb interface{}) ([]byte, error) { if chunksNumber == 0 { paddedbytes := make([]byte, (length - terminalLength)) indicatorByte[0] = byte(terminalLength) - tempbody = append(indicatorByte, append(blob.Bytes(), paddedbytes...)...) + tempbody = append(indicatorByte, append(blob, paddedbytes...)...) return tempbody, nil } @@ -74,7 +82,7 @@ func serializeBlob(cb interface{}) ([]byte, error) { // 31 tempbody = append(tempbody, append(indicatorByte, - blob.Bytes()[(i-1)*chunkDataSize:i*chunkDataSize]...)...) + blob[(i-1)*chunkDataSize:i*chunkDataSize]...)...) } indicatorByte[0] = byte(chunkDataSize) @@ -82,7 +90,7 @@ func serializeBlob(cb interface{}) ([]byte, error) { // Terminal chunk has its indicator byte added, chunkDataSize*chunksNumber refers to the total size of the blob tempbody = append(tempbody, append(indicatorByte, - blob.Bytes()[(chunksNumber-1)*chunkDataSize:chunkDataSize*chunksNumber]...)...) + blob[(chunksNumber-1)*chunkDataSize:chunkDataSize*chunksNumber]...)...) return tempbody, nil @@ -96,7 +104,7 @@ func serializeBlob(cb interface{}) ([]byte, error) { tempbody = append(tempbody, append(indicatorByte, - blob.Bytes()[(i-1)*chunkDataSize:i*chunkDataSize]...)...) + blob[(i-1)*chunkDataSize:i*chunkDataSize]...)...) } // Appends indicator bytes to terminal-chunks , and if the index of the chunk delimiter is non-zero adds it to the chunk. @@ -105,7 +113,7 @@ func serializeBlob(cb interface{}) ([]byte, error) { indicatorByte[0] = byte(terminalLength) tempbody = append(tempbody, append(indicatorByte, - blob.Bytes()[chunkDataSize*chunksNumber-1:finalchunkIndex]...)...) + blob[chunkDataSize*chunksNumber-1:finalchunkIndex]...)...) emptyBytes := make([]byte, (chunkDataSize - terminalLength)) tempbody = append(tempbody, emptyBytes...) @@ -135,7 +143,6 @@ func Serialize(rawtx []interface{}) ([]byte, error) { serialisedData = append(serialisedData, refinedData...) if int64(len(serialisedData)) > collationsizelimit { - log.Info(fmt.Sprintf("The total number of interfaces added to the collation body are: %d", i)) serialisedData = serialisedData[:blobLength] return serialisedData, nil diff --git a/sharding/client/utils_test.go b/sharding/client/utils_test.go index 1364313a24..3b17b730c5 100644 --- a/sharding/client/utils_test.go +++ b/sharding/client/utils_test.go @@ -3,7 +3,6 @@ package client import ( "math/rand" "reflect" - //"runtime" "testing" ) @@ -39,10 +38,10 @@ func TestConvertInterface(t *testing.T) { } func TestSize(t *testing.T) { - size := int64(20) + size := int64(84) blob := buildtxblobs(size) - chunksafterSerialize := size / chunkDataSize - terminalchunk := size % chunkDataSize + chunksafterSerialize := size * size / chunkDataSize + terminalchunk := size * size % chunkDataSize if terminalchunk != 0 { chunksafterSerialize = chunksafterSerialize + 1 } @@ -59,17 +58,15 @@ func TestSize(t *testing.T) { } } -func TestSerializeblob(t *testing.T) { +func TestSerializeAndDeserializeblob(t *testing.T) { - blob := buildblob(200) + blob := buildtxblobs(31) - serializedblob, err := serializeBlob(blob) + serializedblob, err := Serialize(blob) if err != nil { t.Fatalf("Error Serializing blob:%v %v", err, serializedblob) } - //test := &testbody - //runtime.Breakpoint() err2 := Deserializebody(serializedblob, &testbody) if err2 != nil { t.Fatalf("Error Serializing blob:%v", err2) From ba48071e2defaaecd43fedbe2b93557b658856c5 Mon Sep 17 00:00:00 2001 From: nisdas Date: Fri, 4 May 2018 21:03:46 +0800 Subject: [PATCH 20/40] sharding/client: Removing errors(#92) Former-commit-id: 0c678a1778c83476c688a1bf1353aaba8cde10dc [formerly 7c5f2802600371759e1c62a64be357d05f8a2012] Former-commit-id: 34566a72446f3e902095da0b57bf4007683068f1 --- sharding/client/utils.go | 2 +- sharding/client/utils_test.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sharding/client/utils.go b/sharding/client/utils.go index cffa0d67ff..948043adc5 100644 --- a/sharding/client/utils.go +++ b/sharding/client/utils.go @@ -138,7 +138,7 @@ func Serialize(rawtx []interface{}) ([]byte, error) { data := rawtx[i] refinedData, err := serializeBlob(data) if err != nil { - return nil, fmt.Errorf("Error: %v at index: %v %v %v", err, i, data, rawtx) + return nil, fmt.Errorf("Error: %v at index: %v", i, err) } serialisedData = append(serialisedData, refinedData...) diff --git a/sharding/client/utils_test.go b/sharding/client/utils_test.go index 3b17b730c5..9f7ef00011 100644 --- a/sharding/client/utils_test.go +++ b/sharding/client/utils_test.go @@ -53,14 +53,14 @@ func TestSize(t *testing.T) { if int64(len(serializedblob)) != sizeafterSerialize { - t.Fatalf("Error Serializing blob,Lengths are not the same: %v , %v", int64(len(serializedblob)), sizeafterSerialize) + t.Fatalf("Error Serializing blobs the lengths are not the same: %v , %v", int64(len(serializedblob)), sizeafterSerialize) } } func TestSerializeAndDeserializeblob(t *testing.T) { - blob := buildtxblobs(31) + blob := buildtxblobs(30) serializedblob, err := Serialize(blob) @@ -74,7 +74,7 @@ func TestSerializeAndDeserializeblob(t *testing.T) { if !reflect.DeepEqual(blob, testbody) { - t.Fatalf("Error Serializing blob with %v %v", blob, testbody) + t.Fatalf("Error Serializing blobs, the serialized and deserialized versions are not the same: %v %v", blob, testbody) } } From 0cf0c8b8d8bd7dad7565a740f1d9f2ee28aae4ee Mon Sep 17 00:00:00 2001 From: nisdas Date: Tue, 8 May 2018 14:03:49 +0800 Subject: [PATCH 21/40] sharding/utils: Shifting to utils package(#92) Former-commit-id: 3eb0cfbf6c9c2a7ec9619d2f93c1be8ca8b670c9 [formerly 108d9e22841fc81bb11be3fd6c0f54a65c257b0c] Former-commit-id: d128c55b6437515a3cc654f502e267f43a691cc9 --- sharding/client/utils_test.go | 6 +- sharding/utils/marshal.go | 189 +++++++++++++++++++++++++++++++++ sharding/utils/marshal_test.go | 81 ++++++++++++++ 3 files changed, 273 insertions(+), 3 deletions(-) create mode 100644 sharding/utils/marshal.go create mode 100644 sharding/utils/marshal_test.go diff --git a/sharding/client/utils_test.go b/sharding/client/utils_test.go index 9f7ef00011..d87e0af2e0 100644 --- a/sharding/client/utils_test.go +++ b/sharding/client/utils_test.go @@ -6,8 +6,6 @@ import ( "testing" ) -var testbody interface{} - func buildtxblobs(size int64) []interface{} { tempbody := make([]interface{}, size) for i := int64(0); i < size; i++ { @@ -38,7 +36,7 @@ func TestConvertInterface(t *testing.T) { } func TestSize(t *testing.T) { - size := int64(84) + size := int64(2) blob := buildtxblobs(size) chunksafterSerialize := size * size / chunkDataSize terminalchunk := size * size % chunkDataSize @@ -60,6 +58,8 @@ func TestSize(t *testing.T) { } func TestSerializeAndDeserializeblob(t *testing.T) { + var testbody interface{} + blob := buildtxblobs(30) serializedblob, err := Serialize(blob) diff --git a/sharding/utils/marshal.go b/sharding/utils/marshal.go new file mode 100644 index 0000000000..bc9e70dabd --- /dev/null +++ b/sharding/utils/marshal.go @@ -0,0 +1,189 @@ +package utils + +import ( + "errors" + "fmt" + "math" + "reflect" +) + +var ( + collationsizelimit = int64(math.Pow(float64(2), float64(20))) + chunkSize = int64(32) + indicatorSize = int64(1) + numberOfChunks = collationsizelimit / chunkSize + chunkDataSize = chunkSize - indicatorSize +) + +// convertInterface converts inputted interface to the required type of interface, ex: slice. +func convertInterface(arg interface{}, kind reflect.Kind) ([]interface{}, error) { + val := reflect.ValueOf(arg) + if val.Kind() == kind { + + return val.Interface().([]interface{}), nil + + } + err := errors.New("Interface Conversion a failure") + return nil, err +} + +func convertbyteToInterface(arg []byte) []interface{} { + length := int64(len(arg)) + newtype := make([]interface{}, length) + for i, v := range arg { + newtype[i] = v + } + + return newtype +} + +func interfacetoByte(arg []interface{}) []byte { + length := int64(len(arg)) + newtype := make([]byte, length) + for i, v := range arg { + newtype[i] = v.(byte) + } + + return newtype +} + +// serializeBlob parses the blob and serializes it appropriately. +func serializeBlob(cb interface{}) ([]byte, error) { + + interfaceblob, err := convertInterface(cb, reflect.Slice) + if err != nil { + return nil, fmt.Errorf("Error: %v", err) + } + blob := interfacetoByte(interfaceblob) + length := int64(len(blob)) + terminalLength := length % chunkDataSize + chunksNumber := length / chunkDataSize + finalchunkIndex := length - 1 + indicatorByte := make([]byte, 1) + indicatorByte[0] = 0 + tempbody := []byte{} + + // if blob is less than 31 bytes, it adds the indicator chunk and pads the remaining empty bytes to the right + + if chunksNumber == 0 { + paddedbytes := make([]byte, (chunkDataSize - length)) + indicatorByte[0] = byte(terminalLength) + tempbody = append(indicatorByte, append(blob, paddedbytes...)...) + return tempbody, nil + } + + //if there is no need to pad empty bytes, then the indicator byte is added as 00011111 + // Then this chunk is returned to the main Serialize function + + if terminalLength == 0 { + + for i := int64(1); i < chunksNumber; i++ { + // This loop loops through all non-terminal chunks and add a indicator byte of 00000000, each chunk + // is created by appending the indcator byte to the data chunks. The data chunks are separated into sets of + // 31 + tempbody = append(tempbody, + append(indicatorByte, + blob[(i-1)*chunkDataSize:i*chunkDataSize]...)...) + + } + indicatorByte[0] = byte(chunkDataSize) + + // Terminal chunk has its indicator byte added, chunkDataSize*chunksNumber refers to the total size of the blob + tempbody = append(tempbody, + append(indicatorByte, + blob[(chunksNumber-1)*chunkDataSize:chunkDataSize*chunksNumber]...)...) + + return tempbody, nil + + } + + // This loop loops through all non-terminal chunks and add a indicator byte of 00000000, each chunk + // is created by appending the indcator byte to the data chunks. The data chunks are separated into sets of + // 31 + + for i := int64(1); i <= chunksNumber; i++ { + + tempbody = append(tempbody, + append(indicatorByte, + blob[(i-1)*chunkDataSize:i*chunkDataSize]...)...) + + } + // Appends indicator bytes to terminal-chunks , and if the index of the chunk delimiter is non-zero adds it to the chunk. + // Also pads empty bytes to the terminal chunk.chunkDataSize*chunksNumber refers to the total size of the blob. + // finalchunkIndex refers to the index of the last data byte + indicatorByte[0] = byte(terminalLength) + tempbody = append(tempbody, + append(indicatorByte, + blob[chunkDataSize*chunksNumber-1:finalchunkIndex]...)...) + + emptyBytes := make([]byte, (chunkDataSize - terminalLength)) + tempbody = append(tempbody, emptyBytes...) + + return tempbody, nil + +} + +// Serialize takes a set of transaction blobs and converts them to a single byte array. +func Serialize(rawtx []interface{}) ([]byte, error) { + length := int64(len(rawtx)) + + if length == 0 { + return nil, fmt.Errorf("Validation failed: Collation Body has to be a non-zero value") + } + serialisedData := []byte{} + + //Loops through all the blobs and serializes them into chunks + for i := int64(0); i < length; i++ { + + blobLength := int64(len(serialisedData)) + data := rawtx[i] + refinedData, err := serializeBlob(data) + if err != nil { + return nil, fmt.Errorf("Error: %v at index: %v", i, err) + } + serialisedData = append(serialisedData, refinedData...) + + if int64(len(serialisedData)) > collationsizelimit { + serialisedData = serialisedData[:blobLength] + return serialisedData, nil + + } + + } + return serialisedData, nil +} + +// Deserialize results in the Collation body being deserialised and separated into its respective interfaces. +func Deserialize(collationbody []byte, rawtx interface{}) error { + + length := int64(len(collationbody)) + chunksNumber := length / chunkSize + indicatorByte := byte(0) + tempbody := []byte{} + var deserializedblob []interface{} + + // This separates the collation body into its respective transaction blobs + for i := int64(1); i <= chunksNumber; i++ { + indicatorIndex := (i - 1) * chunkSize + // Tests if the chunk delimiter is zero, if it is it will append the data chunk + // to tempbody + if collationbody[indicatorIndex] == indicatorByte { + tempbody = append(tempbody, collationbody[(indicatorIndex+1):(i)*chunkSize]...) + + // Since the chunk delimiter in non-zero now we can infer that it is a terminal chunk and + // add it and append to the rawtx slice. The tempbody signifies a deserialized blob + } else { + terminalIndex := int64(collationbody[indicatorIndex]) + tempbody = append(tempbody, collationbody[(indicatorIndex+1):(indicatorIndex+1+terminalIndex)]...) + deserializedblob = append(deserializedblob, convertbyteToInterface(tempbody)) + tempbody = []byte{} + + } + + } + + *rawtx.(*interface{}) = deserializedblob + + return nil + +} diff --git a/sharding/utils/marshal_test.go b/sharding/utils/marshal_test.go new file mode 100644 index 0000000000..152531de9d --- /dev/null +++ b/sharding/utils/marshal_test.go @@ -0,0 +1,81 @@ +package utils + +import ( + "math/rand" + "reflect" + "testing" +) + +func buildtxblobs(size int64) []interface{} { + tempbody := make([]interface{}, size) + for i := int64(0); i < size; i++ { + tempbody[i] = buildblob(size) + + } + return tempbody +} + +func buildblob(size int64) []interface{} { + + tempbody := make([]interface{}, size) + for i := int64(0); i < size; i++ { + tempbody[i] = byte(rand.Int()) + + } + + return tempbody + +} +func TestConvertInterface(t *testing.T) { + var slice interface{} + slice = []interface{}{0, 1, 2, 3, 4, 5} + convertedValue, err := convertInterface(slice, reflect.Slice) + if err != nil { + t.Fatalf("Error: %v %v", err, convertedValue) + } + +} +func TestSize(t *testing.T) { + size := int64(800) + blob := buildtxblobs(size) + chunksafterSerialize := size / chunkDataSize + terminalchunk := size % chunkDataSize + if terminalchunk != 0 { + chunksafterSerialize = chunksafterSerialize + 1 + } + chunksafterSerialize = chunksafterSerialize * size + sizeafterSerialize := chunksafterSerialize * chunkSize + serializedblob, err := Serialize(blob) + if err != nil { + t.Fatalf("Error Serializing blob:%v %v", err, serializedblob) + } + + if int64(len(serializedblob)) != sizeafterSerialize { + + t.Fatalf("Error Serializing blobs the lengths are not the same: %v , %v", int64(len(serializedblob)), sizeafterSerialize) + + } + +} +func TestSerializeAndDeserializeblob(t *testing.T) { + + var testbody interface{} + + blob := buildtxblobs(31) + + serializedblob, err := Serialize(blob) + + if err != nil { + t.Fatalf("Error Serializing blob:%v %v", err, serializedblob) + } + err2 := Deserialize(serializedblob, &testbody) + if err2 != nil { + t.Fatalf("Error Serializing blob:%v", err2) + } + + if !reflect.DeepEqual(blob, testbody) { + + t.Fatalf("Error Serializing blobs, the serialized and deserialized versions are not the same: %v ------ %v", blob, testbody) + } + +} From e46714c9a88712f3fef536652a960bb1c3af3883 Mon Sep 17 00:00:00 2001 From: nisdas Date: Tue, 8 May 2018 15:34:48 +0800 Subject: [PATCH 22/40] sharding/utils: Fixed serialization tests(#92) Former-commit-id: 7bc42f4bf78f03ddecd07d309699e5771bf2e501 [formerly 9367511015a33dd13d1c605c7d4da94617a0138d] Former-commit-id: 0dd05673b86cee96f5b5d436b58584d9c78c3e0b --- sharding/utils/marshal.go | 9 +++++++-- sharding/utils/marshal_test.go | 5 +++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/sharding/utils/marshal.go b/sharding/utils/marshal.go index bc9e70dabd..e0bcf8262a 100644 --- a/sharding/utils/marshal.go +++ b/sharding/utils/marshal.go @@ -58,7 +58,6 @@ func serializeBlob(cb interface{}) ([]byte, error) { length := int64(len(blob)) terminalLength := length % chunkDataSize chunksNumber := length / chunkDataSize - finalchunkIndex := length - 1 indicatorByte := make([]byte, 1) indicatorByte[0] = 0 tempbody := []byte{} @@ -114,7 +113,7 @@ func serializeBlob(cb interface{}) ([]byte, error) { indicatorByte[0] = byte(terminalLength) tempbody = append(tempbody, append(indicatorByte, - blob[chunkDataSize*chunksNumber-1:finalchunkIndex]...)...) + blob[chunkDataSize*chunksNumber:length]...)...) emptyBytes := make([]byte, (chunkDataSize - terminalLength)) tempbody = append(tempbody, emptyBytes...) @@ -172,6 +171,12 @@ func Deserialize(collationbody []byte, rawtx interface{}) error { // Since the chunk delimiter in non-zero now we can infer that it is a terminal chunk and // add it and append to the rawtx slice. The tempbody signifies a deserialized blob + } else if collationbody[indicatorIndex] == byte(1) { + + tempbody = append(tempbody, collationbody[(indicatorIndex+1)]) + deserializedblob = append(deserializedblob, convertbyteToInterface(tempbody)) + tempbody = []byte{} + } else { terminalIndex := int64(collationbody[indicatorIndex]) tempbody = append(tempbody, collationbody[(indicatorIndex+1):(indicatorIndex+1+terminalIndex)]...) diff --git a/sharding/utils/marshal_test.go b/sharding/utils/marshal_test.go index 152531de9d..a9b8fd13a5 100644 --- a/sharding/utils/marshal_test.go +++ b/sharding/utils/marshal_test.go @@ -3,6 +3,7 @@ package utils import ( "math/rand" "reflect" + //"runtime" "testing" ) @@ -61,7 +62,7 @@ func TestSerializeAndDeserializeblob(t *testing.T) { var testbody interface{} - blob := buildtxblobs(31) + blob := buildtxblobs(1000) serializedblob, err := Serialize(blob) @@ -75,7 +76,7 @@ func TestSerializeAndDeserializeblob(t *testing.T) { if !reflect.DeepEqual(blob, testbody) { - t.Fatalf("Error Serializing blobs, the serialized and deserialized versions are not the same: %v ------ %v", blob, testbody) + t.Fatalf("Error Serializing blobs, the serialized and deserialized versions are not the same: %v ------%v ------ %v", blob, serializedblob, testbody) } } From cdf89d07ae4f2e008031382b46ec2da937e7344a Mon Sep 17 00:00:00 2001 From: nisdas Date: Tue, 8 May 2018 16:18:12 +0800 Subject: [PATCH 23/40] sharding/utils : Adding Collation Methods (#92) Former-commit-id: b59692c3ccb912131f0087a3f19cbfa88886a6e5 [formerly 7e7497f4f461a9288e83ac80633b90a25dc8a6a6] Former-commit-id: 5f5e64a4f757ae637c2148ead844c7b3970197a2 --- sharding/client/utils.go | 188 --------------------------------- sharding/client/utils_test.go | 80 -------------- sharding/collation.go | 44 ++++++++ sharding/collation_test.go | 61 +++++++++++ sharding/utils/marshal.go | 37 +++---- sharding/utils/marshal_test.go | 3 +- 6 files changed, 122 insertions(+), 291 deletions(-) delete mode 100644 sharding/client/utils.go delete mode 100644 sharding/client/utils_test.go diff --git a/sharding/client/utils.go b/sharding/client/utils.go deleted file mode 100644 index 948043adc5..0000000000 --- a/sharding/client/utils.go +++ /dev/null @@ -1,188 +0,0 @@ -package client - -import ( - "errors" - "fmt" - "math" - "reflect" -) - -var ( - collationsizelimit = int64(math.Pow(float64(2), float64(20))) - chunkSize = int64(32) - indicatorSize = int64(1) - numberOfChunks = collationsizelimit / chunkSize - chunkDataSize = chunkSize - indicatorSize -) - -// convertInterface converts inputted interface to the required type, ex: slice. -func convertInterface(arg interface{}, kind reflect.Kind) (reflect.Value, error) { - val := reflect.ValueOf(arg) - if val.Kind() == kind { - return val, nil - - } - err := errors.New("Interface Conversion a failure") - return val, err -} - -func convertbyteToInterface(arg []byte) []interface{} { - length := int64(len(arg)) - newtype := make([]interface{}, length) - for i, v := range arg { - newtype[i] = v - } - - return newtype -} - -func interfacetoByte(arg []interface{}) []byte { - length := int64(len(arg)) - newtype := make([]byte, length) - for i, v := range arg { - newtype[i] = v.(byte) - } - - return newtype -} - -// serializeBlob parses the blob and serializes it appropriately. -func serializeBlob(cb interface{}) ([]byte, error) { - - interfaceblob, err := convertInterface(cb, reflect.Slice) - if err != nil { - return nil, fmt.Errorf("Error: %v", err) - } - blob := interfacetoByte(interfaceblob.Interface().([]interface{})) - length := int64(len(blob)) - terminalLength := length % chunkDataSize - chunksNumber := length / chunkDataSize - finalchunkIndex := length - 1 - indicatorByte := make([]byte, 1) - indicatorByte[0] = 0 - tempbody := []byte{} - - // if blob is less than 31 bytes, it adds the indicator chunk and pads the remaining empty bytes to the right - - if chunksNumber == 0 { - paddedbytes := make([]byte, (length - terminalLength)) - indicatorByte[0] = byte(terminalLength) - tempbody = append(indicatorByte, append(blob, paddedbytes...)...) - return tempbody, nil - } - - //if there is no need to pad empty bytes, then the indicator byte is added as 00011111 - // Then this chunk is returned to the main Serialize function - - if terminalLength == 0 { - - for i := int64(1); i < chunksNumber; i++ { - // This loop loops through all non-terminal chunks and add a indicator byte of 00000000, each chunk - // is created by appending the indcator byte to the data chunks. The data chunks are separated into sets of - // 31 - tempbody = append(tempbody, - append(indicatorByte, - blob[(i-1)*chunkDataSize:i*chunkDataSize]...)...) - - } - indicatorByte[0] = byte(chunkDataSize) - - // Terminal chunk has its indicator byte added, chunkDataSize*chunksNumber refers to the total size of the blob - tempbody = append(tempbody, - append(indicatorByte, - blob[(chunksNumber-1)*chunkDataSize:chunkDataSize*chunksNumber]...)...) - - return tempbody, nil - - } - - // This loop loops through all non-terminal chunks and add a indicator byte of 00000000, each chunk - // is created by appending the indcator byte to the data chunks. The data chunks are separated into sets of - // 31 - - for i := int64(1); i <= chunksNumber; i++ { - - tempbody = append(tempbody, - append(indicatorByte, - blob[(i-1)*chunkDataSize:i*chunkDataSize]...)...) - - } - // Appends indicator bytes to terminal-chunks , and if the index of the chunk delimiter is non-zero adds it to the chunk. - // Also pads empty bytes to the terminal chunk.chunkDataSize*chunksNumber refers to the total size of the blob. - // finalchunkIndex refers to the index of the last data byte - indicatorByte[0] = byte(terminalLength) - tempbody = append(tempbody, - append(indicatorByte, - blob[chunkDataSize*chunksNumber-1:finalchunkIndex]...)...) - - emptyBytes := make([]byte, (chunkDataSize - terminalLength)) - tempbody = append(tempbody, emptyBytes...) - - return tempbody, nil - -} - -// Serialize takes a set of transaction blobs and converts them to a single byte array. -func Serialize(rawtx []interface{}) ([]byte, error) { - length := int64(len(rawtx)) - - if length == 0 { - return nil, fmt.Errorf("Validation failed: Collation Body has to be a non-zero value") - } - serialisedData := []byte{} - - //Loops through all the blobs and serializes them into chunks - for i := int64(0); i < length; i++ { - - blobLength := int64(len(serialisedData)) - data := rawtx[i] - refinedData, err := serializeBlob(data) - if err != nil { - return nil, fmt.Errorf("Error: %v at index: %v", i, err) - } - serialisedData = append(serialisedData, refinedData...) - - if int64(len(serialisedData)) > collationsizelimit { - serialisedData = serialisedData[:blobLength] - return serialisedData, nil - - } - - } - return serialisedData, nil -} - -// Deserializebody results in the Collation body being deserialised and separated into its respective interfaces. -func Deserializebody(collationbody []byte, rawtx interface{}) error { - - length := int64(len(collationbody)) - chunksNumber := length / chunkSize - indicatorByte := byte(0) - tempbody := []byte{} - deserializedblob := []byte{} - - // This separates the collation body into its respective transaction blobs - for i := int64(1); i <= chunksNumber; i++ { - indicatorIndex := (i - 1) * chunkSize - // Tests if the chunk delimiter is zero, if it is it will append the data chunk - // to tempbody - if collationbody[indicatorIndex] == indicatorByte { - tempbody = append(tempbody, collationbody[(indicatorIndex+1):(i)*chunkSize]...) - - // Since the chunk delimiter in non-zero now we can infer that it is a terminal chunk and - // add it and append to the rawtx slice. The tempbody signifies a deserialized blob - } else { - terminalIndex := int64(collationbody[indicatorIndex]) - tempbody = append(tempbody, collationbody[(indicatorIndex+1):(indicatorIndex+1+terminalIndex)]...) - deserializedblob = append(deserializedblob, tempbody...) - tempbody = []byte{} - - } - - } - - *rawtx.(*interface{}) = convertbyteToInterface(deserializedblob) - - return nil - -} diff --git a/sharding/client/utils_test.go b/sharding/client/utils_test.go deleted file mode 100644 index d87e0af2e0..0000000000 --- a/sharding/client/utils_test.go +++ /dev/null @@ -1,80 +0,0 @@ -package client - -import ( - "math/rand" - "reflect" - "testing" -) - -func buildtxblobs(size int64) []interface{} { - tempbody := make([]interface{}, size) - for i := int64(0); i < size; i++ { - tempbody[i] = buildblob(size) - - } - return tempbody -} - -func buildblob(size int64) []interface{} { - - tempbody := make([]interface{}, size) - for i := int64(0); i < size; i++ { - tempbody[i] = byte(rand.Int()) - - } - - return tempbody - -} -func TestConvertInterface(t *testing.T) { - var slice interface{} - slice = []interface{}{0, 1, 2, 3, 4, 5} - convertedValue, err := convertInterface(slice, reflect.Slice) - if err != nil { - t.Fatalf("Error: %v %v", err, convertedValue) - } - -} -func TestSize(t *testing.T) { - size := int64(2) - blob := buildtxblobs(size) - chunksafterSerialize := size * size / chunkDataSize - terminalchunk := size * size % chunkDataSize - if terminalchunk != 0 { - chunksafterSerialize = chunksafterSerialize + 1 - } - sizeafterSerialize := chunksafterSerialize * chunkSize - serializedblob, err := Serialize(blob) - if err != nil { - t.Fatalf("Error Serializing blob:%v %v", err, serializedblob) - } - - if int64(len(serializedblob)) != sizeafterSerialize { - - t.Fatalf("Error Serializing blobs the lengths are not the same: %v , %v", int64(len(serializedblob)), sizeafterSerialize) - - } - -} -func TestSerializeAndDeserializeblob(t *testing.T) { - - var testbody interface{} - - blob := buildtxblobs(30) - - serializedblob, err := Serialize(blob) - - if err != nil { - t.Fatalf("Error Serializing blob:%v %v", err, serializedblob) - } - err2 := Deserializebody(serializedblob, &testbody) - if err2 != nil { - t.Fatalf("Error Serializing blob:%v", err2) - } - - if !reflect.DeepEqual(blob, testbody) { - - t.Fatalf("Error Serializing blobs, the serialized and deserialized versions are not the same: %v %v", blob, testbody) - } - -} diff --git a/sharding/collation.go b/sharding/collation.go index 5528453e46..571eb8ddda 100644 --- a/sharding/collation.go +++ b/sharding/collation.go @@ -1,12 +1,16 @@ package sharding import ( + "fmt" + "math" "math/big" + "reflect" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto/sha3" "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/sharding/utils" ) // Collation base struct. @@ -90,6 +94,12 @@ func (c *Collation) Header() *CollationHeader { return c.header } func (c *Collation) Body() []byte { return c.body } // Transactions returns an array of tx's in the collation. +var ( + collationsizelimit = int64(math.Pow(float64(2), float64(20))) + chunkSize = int64(32) + numberOfChunks = collationsizelimit / chunkSize +) + func (c *Collation) Transactions() []*types.Transaction { return c.transactions } // ProposerAddress is the coinbase addr of the creator for the collation. @@ -105,3 +115,37 @@ func (c *Collation) CalculateChunkRoot() { chunkRoot := common.BytesToHash(c.body) c.header.data.ChunkRoot = &chunkRoot } + +// Serialize method serializes the collation body +func (c *Collation) Serialize() ([]byte, error) { + + blob, err := utils.ConvertInterface(c.transactions, reflect.Slice) + if err != nil { + return nil, fmt.Errorf("%v", err) + } + serializedtx, err := utils.Serialize(blob) + + if err != nil { + return nil, fmt.Errorf("%v", err) + } + + if int64(len(serializedtx)) > collationsizelimit { + serializedtx = serializedtx[0:collationsizelimit] + + } + + return serializedtx, nil + +} + +func (c *Collation) GasUsed() *big.Int { + g := uint64(0) + for _, tx := range c.transactions { + if g > math.MaxUint64-(g+tx.Gas()) { + g = math.MaxUint64 + break + } + g += tx.Gas() + } + return big.NewInt(0).SetUint64(g) +} diff --git a/sharding/collation_test.go b/sharding/collation_test.go index c94ec71996..101bf375bb 100644 --- a/sharding/collation_test.go +++ b/sharding/collation_test.go @@ -38,6 +38,67 @@ func TestCollation_ProposerAddress(t *testing.T) { t.Errorf("initialized collation does not contain correct proposer address") } } +func TestCollation_GasUsed(t *testing.T) { + tests := []struct { + transactions []*types.Transaction + gasUsed *big.Int + }{ + { + transactions: []*types.Transaction{ + makeTxWithGasLimit(100), + makeTxWithGasLimit(100000), + makeTxWithGasLimit(899900), + }, + gasUsed: big.NewInt(1000000), + }, { + transactions: []*types.Transaction{}, + gasUsed: big.NewInt(0), + }, + { + transactions: []*types.Transaction{ + makeTxWithGasLimit(math.MaxUint64), + makeTxWithGasLimit(9001), + makeTxWithGasLimit(math.MaxUint64), + }, + gasUsed: big.NewInt(0).SetUint64(math.MaxUint64), + }, + } + + for _, tt := range tests { + got := (&Collation{transactions: tt.transactions}).GasUsed() + if tt.gasUsed.Cmp(got) != 0 { + t.Errorf("Returned unexpected gasUsed. Got=%v, wanted=%v", got, tt.gasUsed) + } + } +} + +func TestSerialize(t *testing.T) { + tests := []struct { + transactions []*types.Transaction + }{ + { + transactions: []*types.Transaction{ + makeTxWithGasLimit(0), + makeTxWithGasLimit(1), + makeTxWithGasLimit(2), + makeTxWithGasLimit(3), + }, + }, { + transactions: []*types.Transaction{}, + }, + } + + for _, tt := range tests { + c := &Collation{} + + results, err := c.Serialize() + if err != nil { + t.Fatalf("%v --- %v ----%v", err, tt, results) + } + + } + +} func makeTxWithGasLimit(gl uint64) *types.Transaction { return types.NewTransaction(0 /*nonce*/, common.HexToAddress("0x0") /*to*/, nil /*amount*/, gl, nil /*gasPrice*/, nil /*data*/) diff --git a/sharding/utils/marshal.go b/sharding/utils/marshal.go index e0bcf8262a..69f1fc01bd 100644 --- a/sharding/utils/marshal.go +++ b/sharding/utils/marshal.go @@ -3,24 +3,26 @@ package utils import ( "errors" "fmt" - "math" "reflect" ) var ( - collationsizelimit = int64(math.Pow(float64(2), float64(20))) - chunkSize = int64(32) - indicatorSize = int64(1) - numberOfChunks = collationsizelimit / chunkSize - chunkDataSize = chunkSize - indicatorSize + chunkSize = int64(32) + indicatorSize = int64(1) + chunkDataSize = chunkSize - indicatorSize ) -// convertInterface converts inputted interface to the required type of interface, ex: slice. -func convertInterface(arg interface{}, kind reflect.Kind) ([]interface{}, error) { +// ConvertInterface converts inputted interface to the required type of interface, ex: slice. +func ConvertInterface(arg interface{}, kind reflect.Kind) ([]interface{}, error) { val := reflect.ValueOf(arg) if val.Kind() == kind { - return val.Interface().([]interface{}), nil + newtype := make([]interface{}, val.Len()) + for i := 1; i < val.Len(); i++ { + newtype[i] = val.Index(i) + } + + return newtype, nil } err := errors.New("Interface Conversion a failure") @@ -50,7 +52,7 @@ func interfacetoByte(arg []interface{}) []byte { // serializeBlob parses the blob and serializes it appropriately. func serializeBlob(cb interface{}) ([]byte, error) { - interfaceblob, err := convertInterface(cb, reflect.Slice) + interfaceblob, err := ConvertInterface(cb, reflect.Slice) if err != nil { return nil, fmt.Errorf("Error: %v", err) } @@ -134,7 +136,6 @@ func Serialize(rawtx []interface{}) ([]byte, error) { //Loops through all the blobs and serializes them into chunks for i := int64(0); i < length; i++ { - blobLength := int64(len(serialisedData)) data := rawtx[i] refinedData, err := serializeBlob(data) if err != nil { @@ -142,17 +143,11 @@ func Serialize(rawtx []interface{}) ([]byte, error) { } serialisedData = append(serialisedData, refinedData...) - if int64(len(serialisedData)) > collationsizelimit { - serialisedData = serialisedData[:blobLength] - return serialisedData, nil - - } - } return serialisedData, nil } -// Deserialize results in the Collation body being deserialised and separated into its respective interfaces. +// Deserialize results in the byte array being deserialised and separated into its respective interfaces. func Deserialize(collationbody []byte, rawtx interface{}) error { length := int64(len(collationbody)) @@ -161,7 +156,7 @@ func Deserialize(collationbody []byte, rawtx interface{}) error { tempbody := []byte{} var deserializedblob []interface{} - // This separates the collation body into its respective transaction blobs + // This separates the byte array into its separate blobs for i := int64(1); i <= chunksNumber; i++ { indicatorIndex := (i - 1) * chunkSize // Tests if the chunk delimiter is zero, if it is it will append the data chunk @@ -169,8 +164,6 @@ func Deserialize(collationbody []byte, rawtx interface{}) error { if collationbody[indicatorIndex] == indicatorByte { tempbody = append(tempbody, collationbody[(indicatorIndex+1):(i)*chunkSize]...) - // Since the chunk delimiter in non-zero now we can infer that it is a terminal chunk and - // add it and append to the rawtx slice. The tempbody signifies a deserialized blob } else if collationbody[indicatorIndex] == byte(1) { tempbody = append(tempbody, collationbody[(indicatorIndex+1)]) @@ -178,6 +171,8 @@ func Deserialize(collationbody []byte, rawtx interface{}) error { tempbody = []byte{} } else { + // Since the chunk delimiter in non-zero now we can infer that it is a terminal chunk and + // add it and append to the deserializedblob slice. The tempbody signifies a single deserialized blob terminalIndex := int64(collationbody[indicatorIndex]) tempbody = append(tempbody, collationbody[(indicatorIndex+1):(indicatorIndex+1+terminalIndex)]...) deserializedblob = append(deserializedblob, convertbyteToInterface(tempbody)) diff --git a/sharding/utils/marshal_test.go b/sharding/utils/marshal_test.go index a9b8fd13a5..33cdd91da1 100644 --- a/sharding/utils/marshal_test.go +++ b/sharding/utils/marshal_test.go @@ -3,7 +3,6 @@ package utils import ( "math/rand" "reflect" - //"runtime" "testing" ) @@ -30,7 +29,7 @@ func buildblob(size int64) []interface{} { func TestConvertInterface(t *testing.T) { var slice interface{} slice = []interface{}{0, 1, 2, 3, 4, 5} - convertedValue, err := convertInterface(slice, reflect.Slice) + convertedValue, err := ConvertInterface(slice, reflect.Slice) if err != nil { t.Fatalf("Error: %v %v", err, convertedValue) } From 3749175c31f579eb4912e844f3fee85f93082767 Mon Sep 17 00:00:00 2001 From: nisdas Date: Tue, 8 May 2018 17:12:11 +0800 Subject: [PATCH 24/40] sharding: Cleaning up tests (#92) Former-commit-id: 143739c759abdd01fb74539a85c5cd4abf5d1b79 [formerly d375cec6b8bfb6c08946e6bad5d423a02e554333] Former-commit-id: 302a456f3e6045605fa32790933f999844793d68 --- sharding/collation_test.go | 12 ++++++++---- sharding/utils/marshal.go | 10 +++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/sharding/collation_test.go b/sharding/collation_test.go index 101bf375bb..af5779f79e 100644 --- a/sharding/collation_test.go +++ b/sharding/collation_test.go @@ -90,11 +90,15 @@ func TestSerialize(t *testing.T) { for _, tt := range tests { c := &Collation{} - - results, err := c.Serialize() - if err != nil { - t.Fatalf("%v --- %v ----%v", err, tt, results) + for _, tx := range tt.transactions { + c.AddTransaction(tx) } + /* + results, err := c.Serialize() + if err != nil { + t.Fatalf("%v ----%v---%v", err, results, c.transactions) + } + */ } diff --git a/sharding/utils/marshal.go b/sharding/utils/marshal.go index 69f1fc01bd..a8e0980079 100644 --- a/sharding/utils/marshal.go +++ b/sharding/utils/marshal.go @@ -17,16 +17,12 @@ func ConvertInterface(arg interface{}, kind reflect.Kind) ([]interface{}, error) val := reflect.ValueOf(arg) if val.Kind() == kind { - newtype := make([]interface{}, val.Len()) - for i := 1; i < val.Len(); i++ { - newtype[i] = val.Index(i) - } - - return newtype, nil + return val.Interface().([]interface{}), nil } err := errors.New("Interface Conversion a failure") return nil, err + } func convertbyteToInterface(arg []byte) []interface{} { @@ -124,7 +120,7 @@ func serializeBlob(cb interface{}) ([]byte, error) { } -// Serialize takes a set of transaction blobs and converts them to a single byte array. +// Serialize takes a set of blobs and converts them to a single byte array. func Serialize(rawtx []interface{}) ([]byte, error) { length := int64(len(rawtx)) From 17c0e79f1c6359a1e24010db8e18e1d1a1cab8df Mon Sep 17 00:00:00 2001 From: nisdas Date: Tue, 8 May 2018 17:33:03 +0800 Subject: [PATCH 25/40] sharding: Removing Test (#92) Former-commit-id: 24cbfacd178e77721dc50a36bec4d8c8aa0ff1cf [formerly ff1f6be335a2d30b5c7ccd309261b187fa833e4d] Former-commit-id: 1540df8a127da2d1d12f1761022dad3162814aa1 --- sharding/collation_test.go | 45 -------------------------------------- 1 file changed, 45 deletions(-) diff --git a/sharding/collation_test.go b/sharding/collation_test.go index af5779f79e..226bf04ae9 100644 --- a/sharding/collation_test.go +++ b/sharding/collation_test.go @@ -27,51 +27,6 @@ func TestCollation_Transactions(t *testing.T) { } } -func TestCollation_ProposerAddress(t *testing.T) { - proposerAddr := common.BytesToAddress([]byte("proposer")) - header := NewCollationHeader(big.NewInt(1), nil, big.NewInt(1), &proposerAddr, []byte{}) - body := []byte{} - - collation := NewCollation(header, body, nil) - - if collation.ProposerAddress().String() != proposerAddr.String() { - t.Errorf("initialized collation does not contain correct proposer address") - } -} -func TestCollation_GasUsed(t *testing.T) { - tests := []struct { - transactions []*types.Transaction - gasUsed *big.Int - }{ - { - transactions: []*types.Transaction{ - makeTxWithGasLimit(100), - makeTxWithGasLimit(100000), - makeTxWithGasLimit(899900), - }, - gasUsed: big.NewInt(1000000), - }, { - transactions: []*types.Transaction{}, - gasUsed: big.NewInt(0), - }, - { - transactions: []*types.Transaction{ - makeTxWithGasLimit(math.MaxUint64), - makeTxWithGasLimit(9001), - makeTxWithGasLimit(math.MaxUint64), - }, - gasUsed: big.NewInt(0).SetUint64(math.MaxUint64), - }, - } - - for _, tt := range tests { - got := (&Collation{transactions: tt.transactions}).GasUsed() - if tt.gasUsed.Cmp(got) != 0 { - t.Errorf("Returned unexpected gasUsed. Got=%v, wanted=%v", got, tt.gasUsed) - } - } -} - func TestSerialize(t *testing.T) { tests := []struct { transactions []*types.Transaction From ef997fe2f3d101ff51396cbecf607f018b80f0e0 Mon Sep 17 00:00:00 2001 From: nisdas Date: Tue, 8 May 2018 17:43:47 +0800 Subject: [PATCH 26/40] sharding/utils: Fix Lint (#92) Former-commit-id: eeb3e8626b2956c37b5730e4ea70898cb9a70c70 [formerly e93169c79c888f006c35b3790f7ae3707c6b2249] Former-commit-id: 0ccbdb8530e85e13366755615b2c1d251d10f2e8 --- sharding/utils/marshal_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sharding/utils/marshal_test.go b/sharding/utils/marshal_test.go index 33cdd91da1..b1f69bff7c 100644 --- a/sharding/utils/marshal_test.go +++ b/sharding/utils/marshal_test.go @@ -27,8 +27,7 @@ func buildblob(size int64) []interface{} { } func TestConvertInterface(t *testing.T) { - var slice interface{} - slice = []interface{}{0, 1, 2, 3, 4, 5} + slice := []interface{}{0, 1, 2, 3, 4, 5} convertedValue, err := ConvertInterface(slice, reflect.Slice) if err != nil { t.Fatalf("Error: %v %v", err, convertedValue) From 52cd2b04ed44c6aad33a7cb708b4a5a3455e841c Mon Sep 17 00:00:00 2001 From: nisdas Date: Mon, 14 May 2018 18:13:45 +0800 Subject: [PATCH 27/40] sharding/utils: Changing from interfaces to rawblobs(#92) Former-commit-id: 5901594a02b82d0cd11e85593f7021d0cb0a6d62 [formerly dd57899b147bfb258557c20917b971338bc79f84] Former-commit-id: 047157dcdaf452f1e9d329684dd632c4a44c8af6 --- sharding/collation.go | 19 +----- sharding/collation_test.go | 11 ++-- sharding/utils/marshal.go | 114 ++++++++++++++++++++------------- sharding/utils/marshal_test.go | 33 +++++++--- 4 files changed, 101 insertions(+), 76 deletions(-) diff --git a/sharding/collation.go b/sharding/collation.go index 571eb8ddda..21fdc1912b 100644 --- a/sharding/collation.go +++ b/sharding/collation.go @@ -4,7 +4,6 @@ import ( "fmt" "math" "math/big" - "reflect" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" @@ -119,10 +118,8 @@ func (c *Collation) CalculateChunkRoot() { // Serialize method serializes the collation body func (c *Collation) Serialize() ([]byte, error) { - blob, err := utils.ConvertInterface(c.transactions, reflect.Slice) - if err != nil { - return nil, fmt.Errorf("%v", err) - } + blob := utils.ConvertToInterface(c.transactions) + serializedtx, err := utils.Serialize(blob) if err != nil { @@ -137,15 +134,3 @@ func (c *Collation) Serialize() ([]byte, error) { return serializedtx, nil } - -func (c *Collation) GasUsed() *big.Int { - g := uint64(0) - for _, tx := range c.transactions { - if g > math.MaxUint64-(g+tx.Gas()) { - g = math.MaxUint64 - break - } - g += tx.Gas() - } - return big.NewInt(0).SetUint64(g) -} diff --git a/sharding/collation_test.go b/sharding/collation_test.go index 226bf04ae9..229e6a7064 100644 --- a/sharding/collation_test.go +++ b/sharding/collation_test.go @@ -48,12 +48,11 @@ func TestSerialize(t *testing.T) { for _, tx := range tt.transactions { c.AddTransaction(tx) } - /* - results, err := c.Serialize() - if err != nil { - t.Fatalf("%v ----%v---%v", err, results, c.transactions) - } - */ + + results, err := c.Serialize() + if err != nil { + t.Fatalf("%v ----%v---%v", err, results, c.transactions) + } } diff --git a/sharding/utils/marshal.go b/sharding/utils/marshal.go index a8e0980079..252109aa1a 100644 --- a/sharding/utils/marshal.go +++ b/sharding/utils/marshal.go @@ -12,12 +12,27 @@ var ( chunkDataSize = chunkSize - indicatorSize ) -// ConvertInterface converts inputted interface to the required type of interface, ex: slice. -func ConvertInterface(arg interface{}, kind reflect.Kind) ([]interface{}, error) { - val := reflect.ValueOf(arg) - if val.Kind() == kind { +type Flags struct { + skipEvmExecution bool +} - return val.Interface().([]interface{}), nil +type RawBlob struct { + flags Flags + data []byte +} + +// ConvertInterface converts inputted interface to the required type of interface, ex: slice. +func ConvertInterfacetoBytes(arg interface{}) ([]byte, error) { + val := reflect.ValueOf(arg) + if val.Kind() == reflect.Slice { + + length := val.Len() + newtype := make([]byte, length) + for i := 0; i < length; i++ { + newtype[i] = val.Index(i).Interface().(byte) + } + + return newtype, nil } err := errors.New("Interface Conversion a failure") @@ -35,29 +50,28 @@ func convertbyteToInterface(arg []byte) []interface{} { return newtype } -func interfacetoByte(arg []interface{}) []byte { - length := int64(len(arg)) - newtype := make([]byte, length) - for i, v := range arg { - newtype[i] = v.(byte) +func ConvertToInterface(arg interface{}) []interface{} { + val := reflect.ValueOf(arg) + length := val.Len() + newtype := make([]interface{}, length) + for i := 0; i < length; i++ { + newtype[i] = val.Index(i) } return newtype } // serializeBlob parses the blob and serializes it appropriately. -func serializeBlob(cb interface{}) ([]byte, error) { +func serializeBlob(cb RawBlob) ([]byte, error) { - interfaceblob, err := ConvertInterface(cb, reflect.Slice) - if err != nil { - return nil, fmt.Errorf("Error: %v", err) - } - blob := interfacetoByte(interfaceblob) - length := int64(len(blob)) + length := int64(len(cb.data)) terminalLength := length % chunkDataSize chunksNumber := length / chunkDataSize indicatorByte := make([]byte, 1) indicatorByte[0] = 0 + if cb.flags.skipEvmExecution { + indicatorByte[0] |= (1 << 7) + } tempbody := []byte{} // if blob is less than 31 bytes, it adds the indicator chunk and pads the remaining empty bytes to the right @@ -65,7 +79,7 @@ func serializeBlob(cb interface{}) ([]byte, error) { if chunksNumber == 0 { paddedbytes := make([]byte, (chunkDataSize - length)) indicatorByte[0] = byte(terminalLength) - tempbody = append(indicatorByte, append(blob, paddedbytes...)...) + tempbody = append(indicatorByte, append(cb.data, paddedbytes...)...) return tempbody, nil } @@ -78,17 +92,21 @@ func serializeBlob(cb interface{}) ([]byte, error) { // This loop loops through all non-terminal chunks and add a indicator byte of 00000000, each chunk // is created by appending the indcator byte to the data chunks. The data chunks are separated into sets of // 31 + tempbody = append(tempbody, append(indicatorByte, - blob[(i-1)*chunkDataSize:i*chunkDataSize]...)...) + cb.data[(i-1)*chunkDataSize:i*chunkDataSize]...)...) } indicatorByte[0] = byte(chunkDataSize) + if cb.flags.skipEvmExecution { + indicatorByte[0] |= (1 << 7) + } // Terminal chunk has its indicator byte added, chunkDataSize*chunksNumber refers to the total size of the blob tempbody = append(tempbody, append(indicatorByte, - blob[(chunksNumber-1)*chunkDataSize:chunkDataSize*chunksNumber]...)...) + cb.data[(chunksNumber-1)*chunkDataSize:chunkDataSize*chunksNumber]...)...) return tempbody, nil @@ -102,16 +120,19 @@ func serializeBlob(cb interface{}) ([]byte, error) { tempbody = append(tempbody, append(indicatorByte, - blob[(i-1)*chunkDataSize:i*chunkDataSize]...)...) + cb.data[(i-1)*chunkDataSize:i*chunkDataSize]...)...) } // Appends indicator bytes to terminal-chunks , and if the index of the chunk delimiter is non-zero adds it to the chunk. // Also pads empty bytes to the terminal chunk.chunkDataSize*chunksNumber refers to the total size of the blob. // finalchunkIndex refers to the index of the last data byte indicatorByte[0] = byte(terminalLength) + if cb.flags.skipEvmExecution { + indicatorByte[0] |= (1 << 7) + } tempbody = append(tempbody, append(indicatorByte, - blob[chunkDataSize*chunksNumber:length]...)...) + cb.data[chunkDataSize*chunksNumber:length]...)...) emptyBytes := make([]byte, (chunkDataSize - terminalLength)) tempbody = append(tempbody, emptyBytes...) @@ -121,8 +142,8 @@ func serializeBlob(cb interface{}) ([]byte, error) { } // Serialize takes a set of blobs and converts them to a single byte array. -func Serialize(rawtx []interface{}) ([]byte, error) { - length := int64(len(rawtx)) +func Serialize(rawblobs []RawBlob) ([]byte, error) { + length := int64(len(rawblobs)) if length == 0 { return nil, fmt.Errorf("Validation failed: Collation Body has to be a non-zero value") @@ -132,10 +153,10 @@ func Serialize(rawtx []interface{}) ([]byte, error) { //Loops through all the blobs and serializes them into chunks for i := int64(0); i < length; i++ { - data := rawtx[i] + data := rawblobs[i] refinedData, err := serializeBlob(data) if err != nil { - return nil, fmt.Errorf("Error: %v at index: %v", i, err) + return nil, fmt.Errorf("Index %v : %v", i, err) } serialisedData = append(serialisedData, refinedData...) @@ -144,42 +165,49 @@ func Serialize(rawtx []interface{}) ([]byte, error) { } // Deserialize results in the byte array being deserialised and separated into its respective interfaces. -func Deserialize(collationbody []byte, rawtx interface{}) error { +func Deserialize(collationbody []byte) ([]RawBlob, error) { length := int64(len(collationbody)) chunksNumber := length / chunkSize indicatorByte := byte(0) - tempbody := []byte{} - var deserializedblob []interface{} + var tempbody RawBlob + var deserializedblob []RawBlob // This separates the byte array into its separate blobs for i := int64(1); i <= chunksNumber; i++ { indicatorIndex := (i - 1) * chunkSize + var terminalIndex int64 // Tests if the chunk delimiter is zero, if it is it will append the data chunk // to tempbody - if collationbody[indicatorIndex] == indicatorByte { - tempbody = append(tempbody, collationbody[(indicatorIndex+1):(i)*chunkSize]...) + if collationbody[indicatorIndex] == indicatorByte || collationbody[indicatorIndex] == byte(128) { + tempbody.data = append(tempbody.data, collationbody[(indicatorIndex+1):(i)*chunkSize]...) - } else if collationbody[indicatorIndex] == byte(1) { - - tempbody = append(tempbody, collationbody[(indicatorIndex+1)]) - deserializedblob = append(deserializedblob, convertbyteToInterface(tempbody)) - tempbody = []byte{} + } else if collationbody[indicatorIndex] == byte(31) || collationbody[indicatorIndex] == byte(159) { + if collationbody[indicatorIndex] == byte(159) { + tempbody.flags.skipEvmExecution = true + } + tempbody.data = append(tempbody.data, collationbody[(indicatorIndex+1)]) + deserializedblob = append(deserializedblob, tempbody) + tempbody = RawBlob{} } else { // Since the chunk delimiter in non-zero now we can infer that it is a terminal chunk and // add it and append to the deserializedblob slice. The tempbody signifies a single deserialized blob - terminalIndex := int64(collationbody[indicatorIndex]) - tempbody = append(tempbody, collationbody[(indicatorIndex+1):(indicatorIndex+1+terminalIndex)]...) - deserializedblob = append(deserializedblob, convertbyteToInterface(tempbody)) - tempbody = []byte{} + flagindex := collationbody[indicatorIndex] >> 7 + if flagindex == byte(1) { + terminalIndex = int64(collationbody[indicatorIndex]) - 128 + tempbody.flags.skipEvmExecution = true + } else { + terminalIndex = int64(collationbody[indicatorIndex]) + } + tempbody.data = append(tempbody.data, collationbody[(indicatorIndex+1):(indicatorIndex+1+terminalIndex)]...) + deserializedblob = append(deserializedblob, tempbody) + tempbody = RawBlob{} } } - *rawtx.(*interface{}) = deserializedblob - - return nil + return deserializedblob, nil } diff --git a/sharding/utils/marshal_test.go b/sharding/utils/marshal_test.go index b1f69bff7c..7594c6db19 100644 --- a/sharding/utils/marshal_test.go +++ b/sharding/utils/marshal_test.go @@ -6,18 +6,27 @@ import ( "testing" ) -func buildtxblobs(size int64) []interface{} { - tempbody := make([]interface{}, size) +func buildrawblob(size int64) []RawBlob { + tempbody := make([]RawBlob, size) for i := int64(0); i < size; i++ { - tempbody[i] = buildblob(size) + var rawblob RawBlob + rawblob.data = buildblob(size) + flagset := byte(rand.Int()) >> 7 + if flagset == byte(1) { + rawblob.flags.skipEvmExecution = true + + } + + tempbody[i] = rawblob } return tempbody + } -func buildblob(size int64) []interface{} { +func buildblob(size int64) []byte { - tempbody := make([]interface{}, size) + tempbody := make([]byte, size) for i := int64(0); i < size; i++ { tempbody[i] = byte(rand.Int()) @@ -26,6 +35,10 @@ func buildblob(size int64) []interface{} { return tempbody } + +/* +Might be required in the future for part 2 of serialization + func TestConvertInterface(t *testing.T) { slice := []interface{}{0, 1, 2, 3, 4, 5} convertedValue, err := ConvertInterface(slice, reflect.Slice) @@ -33,10 +46,10 @@ func TestConvertInterface(t *testing.T) { t.Fatalf("Error: %v %v", err, convertedValue) } -} +} */ func TestSize(t *testing.T) { size := int64(800) - blob := buildtxblobs(size) + blob := buildrawblob(size) chunksafterSerialize := size / chunkDataSize terminalchunk := size % chunkDataSize if terminalchunk != 0 { @@ -60,16 +73,16 @@ func TestSerializeAndDeserializeblob(t *testing.T) { var testbody interface{} - blob := buildtxblobs(1000) + blob := buildrawblob(10) serializedblob, err := Serialize(blob) if err != nil { t.Fatalf("Error Serializing blob:%v %v", err, serializedblob) } - err2 := Deserialize(serializedblob, &testbody) + raw, err2 := Deserialize(serializedblob) if err2 != nil { - t.Fatalf("Error Serializing blob:%v", err2) + t.Fatalf("Error Serializing blob:%v due to %v", raw, err2) } if !reflect.DeepEqual(blob, testbody) { From 2ad6e31004fe4cc1f8917bc605b81b796e92a265 Mon Sep 17 00:00:00 2001 From: nisdas Date: Mon, 14 May 2018 19:11:50 +0800 Subject: [PATCH 28/40] sharding/utils: Got Tests working again (#92) Former-commit-id: 08366f9b544217367dac936d2bf651a26f3a05aa [formerly 01ac4ab902c77b80226ade667d4b955e3c4dec4c] Former-commit-id: 4dcaf518aebda008e04f27043067ffa0eb6dc5e3 --- sharding/utils/marshal.go | 11 +++++++---- sharding/utils/marshal_test.go | 10 ++++------ 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/sharding/utils/marshal.go b/sharding/utils/marshal.go index 252109aa1a..f0e0f00cce 100644 --- a/sharding/utils/marshal.go +++ b/sharding/utils/marshal.go @@ -79,6 +79,9 @@ func serializeBlob(cb RawBlob) ([]byte, error) { if chunksNumber == 0 { paddedbytes := make([]byte, (chunkDataSize - length)) indicatorByte[0] = byte(terminalLength) + if cb.flags.skipEvmExecution { + indicatorByte[0] |= (1 << 7) + } tempbody = append(indicatorByte, append(cb.data, paddedbytes...)...) return tempbody, nil } @@ -170,13 +173,13 @@ func Deserialize(collationbody []byte) ([]RawBlob, error) { length := int64(len(collationbody)) chunksNumber := length / chunkSize indicatorByte := byte(0) - var tempbody RawBlob + tempbody := RawBlob{} var deserializedblob []RawBlob // This separates the byte array into its separate blobs for i := int64(1); i <= chunksNumber; i++ { indicatorIndex := (i - 1) * chunkSize - var terminalIndex int64 + // Tests if the chunk delimiter is zero, if it is it will append the data chunk // to tempbody if collationbody[indicatorIndex] == indicatorByte || collationbody[indicatorIndex] == byte(128) { @@ -193,12 +196,12 @@ func Deserialize(collationbody []byte) ([]RawBlob, error) { } else { // Since the chunk delimiter in non-zero now we can infer that it is a terminal chunk and // add it and append to the deserializedblob slice. The tempbody signifies a single deserialized blob + terminalIndex := int64(collationbody[indicatorIndex]) + //Check if EVM flag is equal to 1 flagindex := collationbody[indicatorIndex] >> 7 if flagindex == byte(1) { terminalIndex = int64(collationbody[indicatorIndex]) - 128 tempbody.flags.skipEvmExecution = true - } else { - terminalIndex = int64(collationbody[indicatorIndex]) } tempbody.data = append(tempbody.data, collationbody[(indicatorIndex+1):(indicatorIndex+1+terminalIndex)]...) deserializedblob = append(deserializedblob, tempbody) diff --git a/sharding/utils/marshal_test.go b/sharding/utils/marshal_test.go index 7594c6db19..c523129b16 100644 --- a/sharding/utils/marshal_test.go +++ b/sharding/utils/marshal_test.go @@ -48,7 +48,7 @@ func TestConvertInterface(t *testing.T) { } */ func TestSize(t *testing.T) { - size := int64(800) + size := int64(8) blob := buildrawblob(size) chunksafterSerialize := size / chunkDataSize terminalchunk := size % chunkDataSize @@ -71,9 +71,7 @@ func TestSize(t *testing.T) { } func TestSerializeAndDeserializeblob(t *testing.T) { - var testbody interface{} - - blob := buildrawblob(10) + blob := buildrawblob(330) serializedblob, err := Serialize(blob) @@ -85,9 +83,9 @@ func TestSerializeAndDeserializeblob(t *testing.T) { t.Fatalf("Error Serializing blob:%v due to %v", raw, err2) } - if !reflect.DeepEqual(blob, testbody) { + if !reflect.DeepEqual(blob, raw) { - t.Fatalf("Error Serializing blobs, the serialized and deserialized versions are not the same: %v ------%v ------ %v", blob, serializedblob, testbody) + t.Fatalf("Error Serializing blobs, the serialized and deserialized versions are not the same:\n\n %v \n\n %v \n\n %v", blob, serializedblob, raw) } } From be6051d8006844120add3a4f18f82358b0690c3d Mon Sep 17 00:00:00 2001 From: nisdas Date: Mon, 14 May 2018 20:03:18 +0800 Subject: [PATCH 29/40] sharding: Fix tests and clean up(#92) Former-commit-id: 7f58a5eb03fa6347ff5fae94443bf896f9db0486 [formerly ba599c2e0344c9e416cc5fea735407f3df3624d9] Former-commit-id: 13abf7e8ce922cb3a81e43b03d1793af2c9d6557 --- sharding/collation.go | 5 ++- sharding/collation_test.go | 5 ++- sharding/utils/marshal.go | 57 +++++++++++++--------------------- sharding/utils/marshal_test.go | 57 ++++++++++++++++++---------------- 4 files changed, 60 insertions(+), 64 deletions(-) diff --git a/sharding/collation.go b/sharding/collation.go index 21fdc1912b..d4cb96ae53 100644 --- a/sharding/collation.go +++ b/sharding/collation.go @@ -118,7 +118,10 @@ func (c *Collation) CalculateChunkRoot() { // Serialize method serializes the collation body func (c *Collation) Serialize() ([]byte, error) { - blob := utils.ConvertToInterface(c.transactions) + blob, err := utils.ConvertToRawBlob(c.transactions) + if err != nil { + return nil, fmt.Errorf("%v", err) + } serializedtx, err := utils.Serialize(blob) diff --git a/sharding/collation_test.go b/sharding/collation_test.go index 229e6a7064..9f47237e8a 100644 --- a/sharding/collation_test.go +++ b/sharding/collation_test.go @@ -27,6 +27,9 @@ func TestCollation_Transactions(t *testing.T) { } } +//TODO: Add test for converting *types.Transaction into raw blobs + +//Tests thta Transactions can be serialised func TestSerialize(t *testing.T) { tests := []struct { transactions []*types.Transaction @@ -51,7 +54,7 @@ func TestSerialize(t *testing.T) { results, err := c.Serialize() if err != nil { - t.Fatalf("%v ----%v---%v", err, results, c.transactions) + t.Fatalf("%v\n %v\n %v", err, results, c.transactions) } } diff --git a/sharding/utils/marshal.go b/sharding/utils/marshal.go index f0e0f00cce..387ed21f8a 100644 --- a/sharding/utils/marshal.go +++ b/sharding/utils/marshal.go @@ -1,9 +1,7 @@ package utils import ( - "errors" "fmt" - "reflect" ) var ( @@ -12,16 +10,20 @@ var ( chunkDataSize = chunkSize - indicatorSize ) +// Flags to add to chunk delimiter. type Flags struct { skipEvmExecution bool } +// RawBlob type which will contain flags and data for serialization. type RawBlob struct { flags Flags data []byte } -// ConvertInterface converts inputted interface to the required type of interface, ex: slice. +/* +* Might be needed in part 2 of serialisation +* ConvertInterface converts inputted interface to the required type of interface, ex: slice. func ConvertInterfacetoBytes(arg interface{}) ([]byte, error) { val := reflect.ValueOf(arg) if val.Kind() == reflect.Slice { @@ -39,26 +41,12 @@ func ConvertInterfacetoBytes(arg interface{}) ([]byte, error) { return nil, err } +*/ -func convertbyteToInterface(arg []byte) []interface{} { - length := int64(len(arg)) - newtype := make([]interface{}, length) - for i, v := range arg { - newtype[i] = v - } - - return newtype -} - -func ConvertToInterface(arg interface{}) []interface{} { - val := reflect.ValueOf(arg) - length := val.Len() - newtype := make([]interface{}, length) - for i := 0; i < length; i++ { - newtype[i] = val.Index(i) - } - - return newtype +// ConvertToRawBlob will convert any supported type into a the RawBlob type. +func ConvertToRawBlob(arg interface{}) ([]RawBlob, error) { + //TODO: will be done in part 2 to convert any type to a rawBlob + return nil, nil } // serializeBlob parses the blob and serializes it appropriately. @@ -148,9 +136,6 @@ func serializeBlob(cb RawBlob) ([]byte, error) { func Serialize(rawblobs []RawBlob) ([]byte, error) { length := int64(len(rawblobs)) - if length == 0 { - return nil, fmt.Errorf("Validation failed: Collation Body has to be a non-zero value") - } serialisedData := []byte{} //Loops through all the blobs and serializes them into chunks @@ -168,9 +153,9 @@ func Serialize(rawblobs []RawBlob) ([]byte, error) { } // Deserialize results in the byte array being deserialised and separated into its respective interfaces. -func Deserialize(collationbody []byte) ([]RawBlob, error) { +func Deserialize(data []byte) ([]RawBlob, error) { - length := int64(len(collationbody)) + length := int64(len(data)) chunksNumber := length / chunkSize indicatorByte := byte(0) tempbody := RawBlob{} @@ -182,28 +167,28 @@ func Deserialize(collationbody []byte) ([]RawBlob, error) { // Tests if the chunk delimiter is zero, if it is it will append the data chunk // to tempbody - if collationbody[indicatorIndex] == indicatorByte || collationbody[indicatorIndex] == byte(128) { - tempbody.data = append(tempbody.data, collationbody[(indicatorIndex+1):(i)*chunkSize]...) + if data[indicatorIndex] == indicatorByte || data[indicatorIndex] == byte(128) { + tempbody.data = append(tempbody.data, data[(indicatorIndex+1):(i)*chunkSize]...) - } else if collationbody[indicatorIndex] == byte(31) || collationbody[indicatorIndex] == byte(159) { - if collationbody[indicatorIndex] == byte(159) { + } else if data[indicatorIndex] == byte(31) || data[indicatorIndex] == byte(159) { + if data[indicatorIndex] == byte(159) { tempbody.flags.skipEvmExecution = true } - tempbody.data = append(tempbody.data, collationbody[(indicatorIndex+1)]) + tempbody.data = append(tempbody.data, data[(indicatorIndex+1):indicatorIndex+1+chunkDataSize]...) deserializedblob = append(deserializedblob, tempbody) tempbody = RawBlob{} } else { // Since the chunk delimiter in non-zero now we can infer that it is a terminal chunk and // add it and append to the deserializedblob slice. The tempbody signifies a single deserialized blob - terminalIndex := int64(collationbody[indicatorIndex]) + terminalIndex := int64(data[indicatorIndex]) //Check if EVM flag is equal to 1 - flagindex := collationbody[indicatorIndex] >> 7 + flagindex := data[indicatorIndex] >> 7 if flagindex == byte(1) { - terminalIndex = int64(collationbody[indicatorIndex]) - 128 + terminalIndex = int64(data[indicatorIndex]) - 128 tempbody.flags.skipEvmExecution = true } - tempbody.data = append(tempbody.data, collationbody[(indicatorIndex+1):(indicatorIndex+1+terminalIndex)]...) + tempbody.data = append(tempbody.data, data[(indicatorIndex+1):(indicatorIndex+1+terminalIndex)]...) deserializedblob = append(deserializedblob, tempbody) tempbody = RawBlob{} diff --git a/sharding/utils/marshal_test.go b/sharding/utils/marshal_test.go index c523129b16..2048b8a398 100644 --- a/sharding/utils/marshal_test.go +++ b/sharding/utils/marshal_test.go @@ -48,44 +48,49 @@ func TestConvertInterface(t *testing.T) { } */ func TestSize(t *testing.T) { - size := int64(8) - blob := buildrawblob(size) - chunksafterSerialize := size / chunkDataSize - terminalchunk := size % chunkDataSize - if terminalchunk != 0 { - chunksafterSerialize = chunksafterSerialize + 1 - } - chunksafterSerialize = chunksafterSerialize * size - sizeafterSerialize := chunksafterSerialize * chunkSize - serializedblob, err := Serialize(blob) - if err != nil { - t.Fatalf("Error Serializing blob:%v %v", err, serializedblob) - } + for i := 0; i < 300; i++ { + size := int64(i) + blob := buildrawblob(size) + chunksafterSerialize := size / chunkDataSize + terminalchunk := size % chunkDataSize + if terminalchunk != 0 { + chunksafterSerialize = chunksafterSerialize + 1 + } + chunksafterSerialize = chunksafterSerialize * size + sizeafterSerialize := chunksafterSerialize * chunkSize + serializedblob, err := Serialize(blob) + if err != nil { + t.Errorf("Error Serializing blob:%v\n %v", err, serializedblob) + } - if int64(len(serializedblob)) != sizeafterSerialize { + if int64(len(serializedblob)) != sizeafterSerialize { - t.Fatalf("Error Serializing blobs the lengths are not the same: %v , %v", int64(len(serializedblob)), sizeafterSerialize) + t.Errorf("Error Serializing blobs the lengths are not the same:\n %d \n %d", int64(len(serializedblob)), sizeafterSerialize) + } } } func TestSerializeAndDeserializeblob(t *testing.T) { - blob := buildrawblob(330) + for i := 1; i < 300; i++ { - serializedblob, err := Serialize(blob) + blob := buildrawblob(int64(i)) - if err != nil { - t.Fatalf("Error Serializing blob:%v %v", err, serializedblob) - } - raw, err2 := Deserialize(serializedblob) - if err2 != nil { - t.Fatalf("Error Serializing blob:%v due to %v", raw, err2) - } + serializedblob, err := Serialize(blob) - if !reflect.DeepEqual(blob, raw) { + if err != nil { + t.Errorf("Error Serializing blob at index %d:\n%v\n%v", i, err, serializedblob) + } + raw, err2 := Deserialize(serializedblob) + if err2 != nil { + t.Errorf("Error Serializing blob at index %d:\n%v due to \n%v", i, raw, err2) + } - t.Fatalf("Error Serializing blobs, the serialized and deserialized versions are not the same:\n\n %v \n\n %v \n\n %v", blob, serializedblob, raw) + if !reflect.DeepEqual(blob, raw) { + + t.Errorf("Error Serializing blobs at index %d, the serialized and deserialized versions are not the same:\n\n %v \n\n %v \n\n %v", i, blob, serializedblob, raw) + } } } From cc271a84f64ae6892853465774c81cf7ebfe269f Mon Sep 17 00:00:00 2001 From: nisdas Date: Tue, 15 May 2018 20:35:47 +0800 Subject: [PATCH 30/40] sharding/utils : Adding RLP encoding (#92) Former-commit-id: a18ff8c27d7ec87cd38e112f6322a753fb892307 [formerly 228bddfcec86cc2b94ce0694ca14faeeb5c926ae] Former-commit-id: 375f57b067e4ec5abd0e3f90c9a7fbf96c741c29 --- sharding/collation.go | 39 +++++++++++++++++++++++++++++++-- sharding/collation_test.go | 21 +++++++++++++++++- sharding/utils/marshal.go | 40 +++++++++++++++------------------- sharding/utils/marshal_test.go | 11 ---------- 4 files changed, 75 insertions(+), 36 deletions(-) diff --git a/sharding/collation.go b/sharding/collation.go index d4cb96ae53..0984e124dc 100644 --- a/sharding/collation.go +++ b/sharding/collation.go @@ -115,15 +115,41 @@ func (c *Collation) CalculateChunkRoot() { c.header.data.ChunkRoot = &chunkRoot } +func (c *Collation) CreateRawBlobs() ([]*utils.RawBlob, error) { + + // It does not skip evm execution by default + + blobs := make([]*utils.RawBlob, len(c.transactions)) + for i, v := range c.transactions { + + err := error(nil) + blobs[i], err = utils.NewRawBlob(v, false) + + if err != nil { + return nil, fmt.Errorf("Creation of raw blobs from transactions failed %v", err) + } + + } + + return blobs, nil + +} + // Serialize method serializes the collation body func (c *Collation) Serialize() ([]byte, error) { - blob, err := utils.ConvertToRawBlob(c.transactions) + /*blob, err := utils.ConvertToRawBlob(c.transactions) + if err != nil { + return nil, fmt.Errorf("%v", err) + }**/ + + blobs, err := c.CreateRawBlobs() + if err != nil { return nil, fmt.Errorf("%v", err) } - serializedtx, err := utils.Serialize(blob) + serializedtx, err := utils.Serialize(blobs) if err != nil { return nil, fmt.Errorf("%v", err) @@ -137,3 +163,12 @@ func (c *Collation) Serialize() ([]byte, error) { return serializedtx, nil } + +func (c *Collation) Deserialize(serialisedblob []byte) error { + var blobs []utils.RawBlob + + deserializedblobs, err := utils.Deserialize(serialisedblob) + if err != nil { + return fmt.Errorf("%v", err) + } +} diff --git a/sharding/collation_test.go b/sharding/collation_test.go index 9f47237e8a..f4184b7da0 100644 --- a/sharding/collation_test.go +++ b/sharding/collation_test.go @@ -2,6 +2,8 @@ package sharding import ( "math/big" + //"github.com/ethereum/go-ethereum/rlp" + //"reflect" "testing" "github.com/ethereum/go-ethereum/common" @@ -52,8 +54,25 @@ func TestSerialize(t *testing.T) { c.AddTransaction(tx) } - results, err := c.Serialize() + /*var tests *types.Transaction + yadd := reflect.ValueOf(*c.transactions[3]) + d := yadd.FieldByName("data").FieldByName("Hash") + + test, err := rlp.EncodeToBytes(c.transactions[3]) if err != nil { + t.Fatalf("%v\n %v\n %v", err, test, *(c.transactions[0])) + } + erx := rlp.DecodeBytes(test, &tests) + + dd := reflect.ValueOf(*tests) + cv := dd.FieldByName("data").FieldByName("Hash") + + if cv != d { + t.Fatalf("%v\n %v\n %v", erx, cv, d) + } */ + + results, err := c.Serialize() + if err == nil { t.Fatalf("%v\n %v\n %v", err, results, c.transactions) } diff --git a/sharding/utils/marshal.go b/sharding/utils/marshal.go index 387ed21f8a..267327dfe8 100644 --- a/sharding/utils/marshal.go +++ b/sharding/utils/marshal.go @@ -2,6 +2,7 @@ package utils import ( "fmt" + "github.com/ethereum/go-ethereum/rlp" ) var ( @@ -21,27 +22,22 @@ type RawBlob struct { data []byte } -/* -* Might be needed in part 2 of serialisation -* ConvertInterface converts inputted interface to the required type of interface, ex: slice. -func ConvertInterfacetoBytes(arg interface{}) ([]byte, error) { - val := reflect.ValueOf(arg) - if val.Kind() == reflect.Slice { - - length := val.Len() - newtype := make([]byte, length) - for i := 0; i < length; i++ { - newtype[i] = val.Index(i).Interface().(byte) - } - - return newtype, nil - +func NewRawBlob(i interface{}, skipevm bool) (*RawBlob, error) { + data, err := rlp.EncodeToBytes(i) + if err != nil { + return nil, fmt.Errorf("RLP encoding was a failure:%v", err) } - err := errors.New("Interface Conversion a failure") - return nil, err - + return &RawBlob{data: data, flags: Flags{skipEvmExecution: skipevm}}, nil +} + +func ConvertfromRawBlob(blob RawBlob, i interface{}) error { + err := rlp.DecodeBytes(blob.data, i) + if err != nil { + return fmt.Errorf("RLP decoding was a failure:%v", err) + } + + return nil } -*/ // ConvertToRawBlob will convert any supported type into a the RawBlob type. func ConvertToRawBlob(arg interface{}) ([]RawBlob, error) { @@ -50,7 +46,7 @@ func ConvertToRawBlob(arg interface{}) ([]RawBlob, error) { } // serializeBlob parses the blob and serializes it appropriately. -func serializeBlob(cb RawBlob) ([]byte, error) { +func SerializeBlob(cb RawBlob) ([]byte, error) { length := int64(len(cb.data)) terminalLength := length % chunkDataSize @@ -133,7 +129,7 @@ func serializeBlob(cb RawBlob) ([]byte, error) { } // Serialize takes a set of blobs and converts them to a single byte array. -func Serialize(rawblobs []RawBlob) ([]byte, error) { +func Serialize(rawblobs []*RawBlob) ([]byte, error) { length := int64(len(rawblobs)) serialisedData := []byte{} @@ -142,7 +138,7 @@ func Serialize(rawblobs []RawBlob) ([]byte, error) { for i := int64(0); i < length; i++ { data := rawblobs[i] - refinedData, err := serializeBlob(data) + refinedData, err := SerializeBlob(*data) if err != nil { return nil, fmt.Errorf("Index %v : %v", i, err) } diff --git a/sharding/utils/marshal_test.go b/sharding/utils/marshal_test.go index 2048b8a398..6cb508a785 100644 --- a/sharding/utils/marshal_test.go +++ b/sharding/utils/marshal_test.go @@ -36,17 +36,6 @@ func buildblob(size int64) []byte { } -/* -Might be required in the future for part 2 of serialization - -func TestConvertInterface(t *testing.T) { - slice := []interface{}{0, 1, 2, 3, 4, 5} - convertedValue, err := ConvertInterface(slice, reflect.Slice) - if err != nil { - t.Fatalf("Error: %v %v", err, convertedValue) - } - -} */ func TestSize(t *testing.T) { for i := 0; i < 300; i++ { size := int64(i) From 92102262e832ee5032ac937c64509182f1595255 Mon Sep 17 00:00:00 2001 From: nisdas Date: Tue, 15 May 2018 22:15:33 +0800 Subject: [PATCH 31/40] sharding: Fixing tests in util package (#92) Former-commit-id: 4feb34fa32cf0a3495a4fc8f8feb6336b567fc15 [formerly 9e5f85247891697f33c8f832538d46d59334524d] Former-commit-id: df3014bfd85e11af88bfc4d359a4ffb26bc34551 --- sharding/collation.go | 30 +++++++++++++++++++++++++++--- sharding/collation_test.go | 9 ++++++--- sharding/utils/marshal.go | 9 +++++---- sharding/utils/marshal_test.go | 16 ++++++++++++++-- 4 files changed, 52 insertions(+), 12 deletions(-) diff --git a/sharding/collation.go b/sharding/collation.go index 0984e124dc..c3c1002283 100644 --- a/sharding/collation.go +++ b/sharding/collation.go @@ -120,10 +120,10 @@ func (c *Collation) CreateRawBlobs() ([]*utils.RawBlob, error) { // It does not skip evm execution by default blobs := make([]*utils.RawBlob, len(c.transactions)) - for i, v := range c.transactions { + for i := 0; i < len(c.transactions); i++ { err := error(nil) - blobs[i], err = utils.NewRawBlob(v, false) + blobs[i], err = utils.NewRawBlob(c.transactions[i], false) if err != nil { return nil, fmt.Errorf("Creation of raw blobs from transactions failed %v", err) @@ -135,6 +135,23 @@ func (c *Collation) CreateRawBlobs() ([]*utils.RawBlob, error) { } +func (c *Collation) ConvertBacktoTx(rawblobs []utils.RawBlob) ([]*types.Transaction, error) { + + tx := make([]*types.Transaction, len(rawblobs)) + for i := 0; i < len(rawblobs); i++ { + + err := utils.ConvertfromRawBlob(&rawblobs[i], tx[i]) + + if err != nil { + return nil, fmt.Errorf("Creation of transactions from raw blobs failed %v", err) + } + + } + + return tx, nil + +} + // Serialize method serializes the collation body func (c *Collation) Serialize() ([]byte, error) { @@ -165,10 +182,17 @@ func (c *Collation) Serialize() ([]byte, error) { } func (c *Collation) Deserialize(serialisedblob []byte) error { - var blobs []utils.RawBlob deserializedblobs, err := utils.Deserialize(serialisedblob) if err != nil { return fmt.Errorf("%v", err) } + + c.transactions, err = c.ConvertBacktoTx(deserializedblobs) + + if err != nil { + return fmt.Errorf("%v", err) + } + + return nil } diff --git a/sharding/collation_test.go b/sharding/collation_test.go index f4184b7da0..b2e4790640 100644 --- a/sharding/collation_test.go +++ b/sharding/collation_test.go @@ -72,9 +72,12 @@ func TestSerialize(t *testing.T) { } */ results, err := c.Serialize() - if err == nil { - t.Fatalf("%v\n %v\n %v", err, results, c.transactions) - } + + t.Errorf("%v\n %v\n %v", err, results, c.transactions) + + err = c.Deserialize(results) + + t.Errorf("%v\n %v\n %v", err, results, c.transactions) } diff --git a/sharding/utils/marshal.go b/sharding/utils/marshal.go index 267327dfe8..d1e4c20e37 100644 --- a/sharding/utils/marshal.go +++ b/sharding/utils/marshal.go @@ -30,8 +30,9 @@ func NewRawBlob(i interface{}, skipevm bool) (*RawBlob, error) { return &RawBlob{data: data, flags: Flags{skipEvmExecution: skipevm}}, nil } -func ConvertfromRawBlob(blob RawBlob, i interface{}) error { - err := rlp.DecodeBytes(blob.data, i) +func ConvertfromRawBlob(blob *RawBlob, i interface{}) error { + data := (*blob).data + err := rlp.DecodeBytes(data, i) if err != nil { return fmt.Errorf("RLP decoding was a failure:%v", err) } @@ -137,8 +138,8 @@ func Serialize(rawblobs []*RawBlob) ([]byte, error) { //Loops through all the blobs and serializes them into chunks for i := int64(0); i < length; i++ { - data := rawblobs[i] - refinedData, err := SerializeBlob(*data) + data := *rawblobs[i] + refinedData, err := SerializeBlob(data) if err != nil { return nil, fmt.Errorf("Index %v : %v", i, err) } diff --git a/sharding/utils/marshal_test.go b/sharding/utils/marshal_test.go index 6cb508a785..2d734d6236 100644 --- a/sharding/utils/marshal_test.go +++ b/sharding/utils/marshal_test.go @@ -47,7 +47,13 @@ func TestSize(t *testing.T) { } chunksafterSerialize = chunksafterSerialize * size sizeafterSerialize := chunksafterSerialize * chunkSize - serializedblob, err := Serialize(blob) + + drefbody := make([]*RawBlob, len(blob)) + for s := 0; s < len(blob); s++ { + drefbody[s] = &(blob[s]) + + } + serializedblob, err := Serialize(drefbody) if err != nil { t.Errorf("Error Serializing blob:%v\n %v", err, serializedblob) } @@ -66,7 +72,13 @@ func TestSerializeAndDeserializeblob(t *testing.T) { blob := buildrawblob(int64(i)) - serializedblob, err := Serialize(blob) + drefbody := make([]*RawBlob, len(blob)) + for s := 0; s < len(blob); s++ { + drefbody[s] = &(blob[s]) + + } + + serializedblob, err := Serialize(drefbody) if err != nil { t.Errorf("Error Serializing blob at index %d:\n%v\n%v", i, err, serializedblob) From d7bd24225619da4c4ede7f119b416c07cf091c3c Mon Sep 17 00:00:00 2001 From: nisdas Date: Wed, 16 May 2018 00:38:24 +0800 Subject: [PATCH 32/40] sharding: Adding roundtrip tests for tx serialization(#92) Former-commit-id: 71f037aae070dcc5761f06591e889b31075c7b40 [formerly 8158b9365e8cfa148149afb695bbd982c677cb80] Former-commit-id: b6a4bc45fde9a7c92dc53733d3440ba4d9b8334c --- sharding/collation.go | 12 ++--- sharding/collation_test.go | 105 ++++++++++++++++++++++++++++--------- 2 files changed, 86 insertions(+), 31 deletions(-) diff --git a/sharding/collation.go b/sharding/collation.go index c3c1002283..342f11ec9f 100644 --- a/sharding/collation.go +++ b/sharding/collation.go @@ -135,20 +135,20 @@ func (c *Collation) CreateRawBlobs() ([]*utils.RawBlob, error) { } -func (c *Collation) ConvertBacktoTx(rawblobs []utils.RawBlob) ([]*types.Transaction, error) { +func (c *Collation) ConvertBacktoTx(rawblobs []utils.RawBlob) error { - tx := make([]*types.Transaction, len(rawblobs)) + // tx := make([]*types.Transaction, len(rawblobs)) for i := 0; i < len(rawblobs); i++ { - err := utils.ConvertfromRawBlob(&rawblobs[i], tx[i]) + err := utils.ConvertfromRawBlob(&rawblobs[i], c.transactions[i]) if err != nil { - return nil, fmt.Errorf("Creation of transactions from raw blobs failed %v", err) + return fmt.Errorf("Creation of transactions from raw blobs failed %v", err) } } - return tx, nil + return nil } @@ -188,7 +188,7 @@ func (c *Collation) Deserialize(serialisedblob []byte) error { return fmt.Errorf("%v", err) } - c.transactions, err = c.ConvertBacktoTx(deserializedblobs) + err = c.ConvertBacktoTx(deserializedblobs) if err != nil { return fmt.Errorf("%v", err) diff --git a/sharding/collation_test.go b/sharding/collation_test.go index b2e4790640..ac4229469c 100644 --- a/sharding/collation_test.go +++ b/sharding/collation_test.go @@ -4,12 +4,22 @@ import ( "math/big" //"github.com/ethereum/go-ethereum/rlp" //"reflect" + "reflect" "testing" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" ) +// fieldAccess is to access unexported fields in structs in another package +func fieldAccess(i interface{}, fields []string) reflect.Value { + val := reflect.ValueOf(i) + for i := 0; i < len(fields); i++ { + val = reflect.Indirect(val).FieldByName(fields[i]) + } + return val + +} func TestCollation_Transactions(t *testing.T) { header := NewCollationHeader(big.NewInt(1), nil, big.NewInt(1), nil, []byte{}) body := []byte{} @@ -31,20 +41,19 @@ func TestCollation_Transactions(t *testing.T) { //TODO: Add test for converting *types.Transaction into raw blobs -//Tests thta Transactions can be serialised -func TestSerialize(t *testing.T) { +//Tests that Transactions can be serialised +func TestSerialize_Deserialize(t *testing.T) { + tests := []struct { transactions []*types.Transaction }{ { transactions: []*types.Transaction{ makeTxWithGasLimit(0), - makeTxWithGasLimit(1), - makeTxWithGasLimit(2), - makeTxWithGasLimit(3), + makeTxWithGasLimit(5), + makeTxWithGasLimit(20), + makeTxWithGasLimit(100), }, - }, { - transactions: []*types.Transaction{}, }, } @@ -54,30 +63,76 @@ func TestSerialize(t *testing.T) { c.AddTransaction(tx) } - /*var tests *types.Transaction - yadd := reflect.ValueOf(*c.transactions[3]) - d := yadd.FieldByName("data").FieldByName("Hash") - - test, err := rlp.EncodeToBytes(c.transactions[3]) - if err != nil { - t.Fatalf("%v\n %v\n %v", err, test, *(c.transactions[0])) - } - erx := rlp.DecodeBytes(test, &tests) - - dd := reflect.ValueOf(*tests) - cv := dd.FieldByName("data").FieldByName("Hash") - - if cv != d { - t.Fatalf("%v\n %v\n %v", erx, cv, d) - } */ + tx := c.transactions results, err := c.Serialize() - t.Errorf("%v\n %v\n %v", err, results, c.transactions) + if err != nil { + t.Errorf("Unable to Serialize transactions, %v", err) + } err = c.Deserialize(results) - t.Errorf("%v\n %v\n %v", err, results, c.transactions) + if err != nil { + t.Errorf("Unable to deserialize collation body, %v", err) + } + + if len(tx) != len(c.transactions) { + t.Errorf("Transaction length is different before and after serialization: %v, %v", len(tx), len(c.transactions)) + } + + for i := 0; i < len(tx); i++ { + + aval := fieldAccess(tx[i], []string{"data", "AccountNonce"}) + aval2 := fieldAccess(c.transactions[i], []string{"data", "AccountNonce"}) + + if aval != aval2 { + + t.Errorf("Data before serialization and after deserialization are not the same: %v, %v", aval, aval2) + + } + + gval := fieldAccess(tx[i], []string{"data", "GasLimit"}) + gval2 := fieldAccess(c.transactions[i], []string{"data", "GasLimit"}) + if gval != gval2 { + + t.Errorf("Data before serialization and after deserialization are not the same: %v, %v", gval, gval2) + + } + + pval := fieldAccess(tx[i], []string{"data", "Price"}) + pval2 := fieldAccess(c.transactions[i], []string{"data", "Price"}) + if pval != pval2 { + + t.Errorf("Data before serialization and after deserialization are not the same: %v, %v", pval, pval2) + + } + + rval := fieldAccess(tx[i], []string{"data", "Recipient"}) + rval2 := fieldAccess(c.transactions[i], []string{"data", "Recipient"}) + if rval != rval2 { + + t.Errorf("Data before serialization and after deserialization are not the same: %v, %v", rval, rval2) + + } + + amval := fieldAccess(tx[i], []string{"data", "Amount"}) + amval2 := fieldAccess(c.transactions[i], []string{"data", "Amount"}) + if amval != amval2 { + + t.Errorf("Data before serialization and after deserialization are not the same: %v, %v", amval, amval2) + + } + + paval := fieldAccess(tx[i], []string{"data", "Payload"}) + paval2 := fieldAccess(c.transactions[i], []string{"data", "Payload"}) + if paval != paval2 { + + t.Errorf("Data before serialization and after deserialization are not the same: %v, %v", paval, paval2) + + } + + } } From 48977ba2f4c6d4026d9a57bd77a1a52c67834a32 Mon Sep 17 00:00:00 2001 From: nisdas Date: Wed, 16 May 2018 00:51:28 +0800 Subject: [PATCH 33/40] sharding: Adding comments to functions(#92) Former-commit-id: 5b0e344d84b8d1ff8d12205376c9ae8715c7089e [formerly fe3697b50f74dc0b7fe43139c2d03744dfa32e80] Former-commit-id: dfd0123b09906dae10bf4be245c5119ce958a236 --- sharding/collation.go | 16 +++++----------- sharding/utils/marshal.go | 2 ++ sharding/utils/marshal_test.go | 4 ---- 3 files changed, 7 insertions(+), 15 deletions(-) diff --git a/sharding/collation.go b/sharding/collation.go index 342f11ec9f..56367e39e2 100644 --- a/sharding/collation.go +++ b/sharding/collation.go @@ -99,6 +99,7 @@ var ( numberOfChunks = collationsizelimit / chunkSize ) +// Transactions returns an array of tx's in the collation. func (c *Collation) Transactions() []*types.Transaction { return c.transactions } // ProposerAddress is the coinbase addr of the creator for the collation. @@ -115,10 +116,10 @@ func (c *Collation) CalculateChunkRoot() { c.header.data.ChunkRoot = &chunkRoot } +// CreateRawBlobs creates raw blobs from transactions. func (c *Collation) CreateRawBlobs() ([]*utils.RawBlob, error) { // It does not skip evm execution by default - blobs := make([]*utils.RawBlob, len(c.transactions)) for i := 0; i < len(c.transactions); i++ { @@ -135,31 +136,23 @@ func (c *Collation) CreateRawBlobs() ([]*utils.RawBlob, error) { } +// ConvertBacktoTx converts raw blobs back to their original transactions. func (c *Collation) ConvertBacktoTx(rawblobs []utils.RawBlob) error { - // tx := make([]*types.Transaction, len(rawblobs)) for i := 0; i < len(rawblobs); i++ { err := utils.ConvertfromRawBlob(&rawblobs[i], c.transactions[i]) - if err != nil { return fmt.Errorf("Creation of transactions from raw blobs failed %v", err) } - } - return nil } -// Serialize method serializes the collation body +// Serialize method serializes the collation body to a byte array. func (c *Collation) Serialize() ([]byte, error) { - /*blob, err := utils.ConvertToRawBlob(c.transactions) - if err != nil { - return nil, fmt.Errorf("%v", err) - }**/ - blobs, err := c.CreateRawBlobs() if err != nil { @@ -181,6 +174,7 @@ func (c *Collation) Serialize() ([]byte, error) { } +// Deserialize takes a byte array and converts its back to its original transactions. func (c *Collation) Deserialize(serialisedblob []byte) error { deserializedblobs, err := utils.Deserialize(serialisedblob) diff --git a/sharding/utils/marshal.go b/sharding/utils/marshal.go index d1e4c20e37..1dce87cb16 100644 --- a/sharding/utils/marshal.go +++ b/sharding/utils/marshal.go @@ -22,6 +22,7 @@ type RawBlob struct { data []byte } +// NewRawBlob builds a raw blob from any interface by using RLP encoding func NewRawBlob(i interface{}, skipevm bool) (*RawBlob, error) { data, err := rlp.EncodeToBytes(i) if err != nil { @@ -30,6 +31,7 @@ func NewRawBlob(i interface{}, skipevm bool) (*RawBlob, error) { return &RawBlob{data: data, flags: Flags{skipEvmExecution: skipevm}}, nil } +// ConvertfromRawBlob converts raw blob back from a byte array to its interface func ConvertfromRawBlob(blob *RawBlob, i interface{}) error { data := (*blob).data err := rlp.DecodeBytes(data, i) diff --git a/sharding/utils/marshal_test.go b/sharding/utils/marshal_test.go index 2d734d6236..cbf8d23eb6 100644 --- a/sharding/utils/marshal_test.go +++ b/sharding/utils/marshal_test.go @@ -16,9 +16,7 @@ func buildrawblob(size int64) []RawBlob { rawblob.flags.skipEvmExecution = true } - tempbody[i] = rawblob - } return tempbody @@ -29,9 +27,7 @@ func buildblob(size int64) []byte { tempbody := make([]byte, size) for i := int64(0); i < size; i++ { tempbody[i] = byte(rand.Int()) - } - return tempbody } From 97785d23229eaecc95502071ab44b84114f6a077 Mon Sep 17 00:00:00 2001 From: nisdas Date: Wed, 16 May 2018 01:12:58 +0800 Subject: [PATCH 34/40] sharding: Cleaning up (#92) Former-commit-id: be930e31eea223934f0fa9aea28e306817746bc1 [formerly e8988d068f485c08bd3b30b6c53a39ce60864ac6] Former-commit-id: 6d2dbb09d7fb965ee202c9834526f28a0be58ae8 --- sharding/collation.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/sharding/collation.go b/sharding/collation.go index 56367e39e2..4948674cbb 100644 --- a/sharding/collation.go +++ b/sharding/collation.go @@ -94,9 +94,9 @@ func (c *Collation) Body() []byte { return c.body } // Transactions returns an array of tx's in the collation. var ( - collationsizelimit = int64(math.Pow(float64(2), float64(20))) + collationSizelimit = int64(math.Pow(float64(2), float64(20))) chunkSize = int64(32) - numberOfChunks = collationsizelimit / chunkSize + numberOfChunks = collationSizelimit / chunkSize ) // Transactions returns an array of tx's in the collation. @@ -159,30 +159,31 @@ func (c *Collation) Serialize() ([]byte, error) { return nil, fmt.Errorf("%v", err) } - serializedtx, err := utils.Serialize(blobs) + serializedTx, err := utils.Serialize(blobs) if err != nil { return nil, fmt.Errorf("%v", err) } - if int64(len(serializedtx)) > collationsizelimit { - serializedtx = serializedtx[0:collationsizelimit] + if int64(len(serializedTx)) > collationSizelimit { + + return nil, fmt.Errorf("The serialized body exceeded the collation size limit", serializedTx) } - return serializedtx, nil + return serializedTx, nil } // Deserialize takes a byte array and converts its back to its original transactions. -func (c *Collation) Deserialize(serialisedblob []byte) error { +func (c *Collation) Deserialize(serialisedBlob []byte) error { - deserializedblobs, err := utils.Deserialize(serialisedblob) + deserializedBlobs, err := utils.Deserialize(serialisedBlob) if err != nil { return fmt.Errorf("%v", err) } - err = c.ConvertBacktoTx(deserializedblobs) + err = c.ConvertBacktoTx(deserializedBlobs) if err != nil { return fmt.Errorf("%v", err) From 26359799344aa7095df96200d03af1780960a76d Mon Sep 17 00:00:00 2001 From: nisdas Date: Wed, 16 May 2018 11:07:31 +0800 Subject: [PATCH 35/40] sharding: Made Requested Changes(#92) Former-commit-id: a87c10e58acc0af3364f1de4b67856e084cd8a5a [formerly cef802b4046ea9fc8f5e1d46a4654218dc9b5410] Former-commit-id: 5b463c6e066e3dac2a8a2f488f35855d579ee879 --- sharding/collation.go | 28 ++++++++------- sharding/collation_test.go | 51 +++++++++++++------------- sharding/utils/marshal.go | 66 ++++++++++++++++------------------ sharding/utils/marshal_test.go | 10 +++--- 4 files changed, 75 insertions(+), 80 deletions(-) diff --git a/sharding/collation.go b/sharding/collation.go index 4948674cbb..39a009b6ba 100644 --- a/sharding/collation.go +++ b/sharding/collation.go @@ -136,17 +136,21 @@ func (c *Collation) CreateRawBlobs() ([]*utils.RawBlob, error) { } -// ConvertBacktoTx converts raw blobs back to their original transactions. -func (c *Collation) ConvertBacktoTx(rawblobs []utils.RawBlob) error { +// ConvertBackToTx converts raw blobs back to their original transactions. +func ConvertBackToTx(rawBlobs []utils.RawBlob) ([]*types.Transaction, error) { - for i := 0; i < len(rawblobs); i++ { + blobs := make([]*types.Transaction, len(rawBlobs)) - err := utils.ConvertfromRawBlob(&rawblobs[i], c.transactions[i]) + for i := 0; i < len(rawBlobs); i++ { + + blobs[i] = types.NewTransaction(0, common.HexToAddress("0x"), nil, 0, nil, nil) + + err := utils.ConvertFromRawBlob(&rawBlobs[i], blobs[i]) if err != nil { - return fmt.Errorf("Creation of transactions from raw blobs failed %v", err) + return nil, fmt.Errorf("Creation of transactions from raw blobs failed %v", err) } } - return nil + return blobs, nil } @@ -167,7 +171,7 @@ func (c *Collation) Serialize() ([]byte, error) { if int64(len(serializedTx)) > collationSizelimit { - return nil, fmt.Errorf("The serialized body exceeded the collation size limit", serializedTx) + return nil, fmt.Errorf("The serialized body exceeded the collation size limit: %v", serializedTx) } @@ -176,18 +180,18 @@ func (c *Collation) Serialize() ([]byte, error) { } // Deserialize takes a byte array and converts its back to its original transactions. -func (c *Collation) Deserialize(serialisedBlob []byte) error { +func Deserialize(serialisedBlob []byte) (*[]*types.Transaction, error) { deserializedBlobs, err := utils.Deserialize(serialisedBlob) if err != nil { - return fmt.Errorf("%v", err) + return nil, fmt.Errorf("%v", err) } - err = c.ConvertBacktoTx(deserializedBlobs) + txs, err := ConvertBackToTx(deserializedBlobs) if err != nil { - return fmt.Errorf("%v", err) + return nil, fmt.Errorf("%v", err) } - return nil + return &txs, nil } diff --git a/sharding/collation_test.go b/sharding/collation_test.go index ac4229469c..fe1cd28823 100644 --- a/sharding/collation_test.go +++ b/sharding/collation_test.go @@ -4,6 +4,7 @@ import ( "math/big" //"github.com/ethereum/go-ethereum/rlp" //"reflect" + "bytes" "reflect" "testing" @@ -71,64 +72,60 @@ func TestSerialize_Deserialize(t *testing.T) { t.Errorf("Unable to Serialize transactions, %v", err) } - err = c.Deserialize(results) + deserializedTxs, err := Deserialize(results) if err != nil { t.Errorf("Unable to deserialize collation body, %v", err) } - if len(tx) != len(c.transactions) { - t.Errorf("Transaction length is different before and after serialization: %v, %v", len(tx), len(c.transactions)) + if len(tx) != len(*deserializedTxs) { + t.Errorf("Transaction length is different before and after serialization: %v, %v", len(tx), len(*deserializedTxs)) } for i := 0; i < len(tx); i++ { - aval := fieldAccess(tx[i], []string{"data", "AccountNonce"}) - aval2 := fieldAccess(c.transactions[i], []string{"data", "AccountNonce"}) + beforeSerialization := tx[i] + afterDeserialization := (*deserializedTxs)[i] - if aval != aval2 { + if beforeSerialization.Nonce() != afterDeserialization.Nonce() { - t.Errorf("Data before serialization and after deserialization are not the same: %v, %v", aval, aval2) + t.Errorf("Data before serialization and after deserialization are not the same ,AccountNonce: %v, %v", beforeSerialization.Nonce(), afterDeserialization.Nonce()) } - gval := fieldAccess(tx[i], []string{"data", "GasLimit"}) - gval2 := fieldAccess(c.transactions[i], []string{"data", "GasLimit"}) - if gval != gval2 { + if beforeSerialization.Gas() != afterDeserialization.Gas() { - t.Errorf("Data before serialization and after deserialization are not the same: %v, %v", gval, gval2) + t.Errorf("Data before serialization and after deserialization are not the same ,GasLimit: %v, %v", beforeSerialization.Gas(), afterDeserialization.Gas()) } - pval := fieldAccess(tx[i], []string{"data", "Price"}) - pval2 := fieldAccess(c.transactions[i], []string{"data", "Price"}) - if pval != pval2 { + if beforeSerialization.GasPrice().Cmp(afterDeserialization.GasPrice()) != 0 { - t.Errorf("Data before serialization and after deserialization are not the same: %v, %v", pval, pval2) + t.Errorf("Data before serialization and after deserialization are not the same ,Price: %v, %v", beforeSerialization.GasPrice(), afterDeserialization.GasPrice()) } - rval := fieldAccess(tx[i], []string{"data", "Recipient"}) - rval2 := fieldAccess(c.transactions[i], []string{"data", "Recipient"}) - if rval != rval2 { + beforeAddress := reflect.ValueOf(beforeSerialization.To()) + afterAddress := reflect.ValueOf(afterDeserialization.To()) - t.Errorf("Data before serialization and after deserialization are not the same: %v, %v", rval, rval2) + if reflect.DeepEqual(beforeAddress, afterAddress) { + + t.Errorf("Data before serialization and after deserialization are not the same ,Recipient: %v, %v", beforeAddress, afterAddress) } - amval := fieldAccess(tx[i], []string{"data", "Amount"}) - amval2 := fieldAccess(c.transactions[i], []string{"data", "Amount"}) - if amval != amval2 { + if beforeSerialization.Value().Cmp(afterDeserialization.Value()) != 0 { - t.Errorf("Data before serialization and after deserialization are not the same: %v, %v", amval, amval2) + t.Errorf("Data before serialization and after deserialization are not the same ,Amount: %v, %v", beforeSerialization.Value(), afterDeserialization.Value()) } - paval := fieldAccess(tx[i], []string{"data", "Payload"}) - paval2 := fieldAccess(c.transactions[i], []string{"data", "Payload"}) - if paval != paval2 { + beforeData := beforeSerialization.Data() + afterData := afterDeserialization.Data() - t.Errorf("Data before serialization and after deserialization are not the same: %v, %v", paval, paval2) + if !bytes.Equal(beforeData, afterData) { + + t.Errorf("Data before serialization and after deserialization are not the same ,Payload: %v, %v", beforeData, afterData) } diff --git a/sharding/utils/marshal.go b/sharding/utils/marshal.go index 1dce87cb16..aaf47650c4 100644 --- a/sharding/utils/marshal.go +++ b/sharding/utils/marshal.go @@ -23,16 +23,16 @@ type RawBlob struct { } // NewRawBlob builds a raw blob from any interface by using RLP encoding -func NewRawBlob(i interface{}, skipevm bool) (*RawBlob, error) { +func NewRawBlob(i interface{}, skipEvm bool) (*RawBlob, error) { data, err := rlp.EncodeToBytes(i) if err != nil { return nil, fmt.Errorf("RLP encoding was a failure:%v", err) } - return &RawBlob{data: data, flags: Flags{skipEvmExecution: skipevm}}, nil + return &RawBlob{data: data, flags: Flags{skipEvmExecution: skipEvm}}, nil } -// ConvertfromRawBlob converts raw blob back from a byte array to its interface -func ConvertfromRawBlob(blob *RawBlob, i interface{}) error { +// ConvertFromRawBlob converts raw blob back from a byte array to its interface +func ConvertFromRawBlob(blob *RawBlob, i interface{}) error { data := (*blob).data err := rlp.DecodeBytes(data, i) if err != nil { @@ -42,13 +42,7 @@ func ConvertfromRawBlob(blob *RawBlob, i interface{}) error { return nil } -// ConvertToRawBlob will convert any supported type into a the RawBlob type. -func ConvertToRawBlob(arg interface{}) ([]RawBlob, error) { - //TODO: will be done in part 2 to convert any type to a rawBlob - return nil, nil -} - -// serializeBlob parses the blob and serializes it appropriately. +// SerializeBlob parses the blob and serializes it appropriately. func SerializeBlob(cb RawBlob) ([]byte, error) { length := int64(len(cb.data)) @@ -59,18 +53,18 @@ func SerializeBlob(cb RawBlob) ([]byte, error) { if cb.flags.skipEvmExecution { indicatorByte[0] |= (1 << 7) } - tempbody := []byte{} + tempBody := []byte{} // if blob is less than 31 bytes, it adds the indicator chunk and pads the remaining empty bytes to the right if chunksNumber == 0 { - paddedbytes := make([]byte, (chunkDataSize - length)) + paddedBytes := make([]byte, (chunkDataSize - length)) indicatorByte[0] = byte(terminalLength) if cb.flags.skipEvmExecution { indicatorByte[0] |= (1 << 7) } - tempbody = append(indicatorByte, append(cb.data, paddedbytes...)...) - return tempbody, nil + tempBody = append(indicatorByte, append(cb.data, paddedBytes...)...) + return tempBody, nil } //if there is no need to pad empty bytes, then the indicator byte is added as 00011111 @@ -83,7 +77,7 @@ func SerializeBlob(cb RawBlob) ([]byte, error) { // is created by appending the indcator byte to the data chunks. The data chunks are separated into sets of // 31 - tempbody = append(tempbody, + tempBody = append(tempBody, append(indicatorByte, cb.data[(i-1)*chunkDataSize:i*chunkDataSize]...)...) @@ -94,11 +88,11 @@ func SerializeBlob(cb RawBlob) ([]byte, error) { } // Terminal chunk has its indicator byte added, chunkDataSize*chunksNumber refers to the total size of the blob - tempbody = append(tempbody, + tempBody = append(tempBody, append(indicatorByte, cb.data[(chunksNumber-1)*chunkDataSize:chunkDataSize*chunksNumber]...)...) - return tempbody, nil + return tempBody, nil } @@ -108,7 +102,7 @@ func SerializeBlob(cb RawBlob) ([]byte, error) { for i := int64(1); i <= chunksNumber; i++ { - tempbody = append(tempbody, + tempBody = append(tempBody, append(indicatorByte, cb.data[(i-1)*chunkDataSize:i*chunkDataSize]...)...) @@ -120,14 +114,14 @@ func SerializeBlob(cb RawBlob) ([]byte, error) { if cb.flags.skipEvmExecution { indicatorByte[0] |= (1 << 7) } - tempbody = append(tempbody, + tempBody = append(tempBody, append(indicatorByte, cb.data[chunkDataSize*chunksNumber:length]...)...) emptyBytes := make([]byte, (chunkDataSize - terminalLength)) - tempbody = append(tempbody, emptyBytes...) + tempBody = append(tempBody, emptyBytes...) - return tempbody, nil + return tempBody, nil } @@ -157,44 +151,44 @@ func Deserialize(data []byte) ([]RawBlob, error) { length := int64(len(data)) chunksNumber := length / chunkSize indicatorByte := byte(0) - tempbody := RawBlob{} - var deserializedblob []RawBlob + tempBody := RawBlob{} + var deserializedBlob []RawBlob // This separates the byte array into its separate blobs for i := int64(1); i <= chunksNumber; i++ { indicatorIndex := (i - 1) * chunkSize // Tests if the chunk delimiter is zero, if it is it will append the data chunk - // to tempbody + // to tempBody if data[indicatorIndex] == indicatorByte || data[indicatorIndex] == byte(128) { - tempbody.data = append(tempbody.data, data[(indicatorIndex+1):(i)*chunkSize]...) + tempBody.data = append(tempBody.data, data[(indicatorIndex+1):(i)*chunkSize]...) } else if data[indicatorIndex] == byte(31) || data[indicatorIndex] == byte(159) { if data[indicatorIndex] == byte(159) { - tempbody.flags.skipEvmExecution = true + tempBody.flags.skipEvmExecution = true } - tempbody.data = append(tempbody.data, data[(indicatorIndex+1):indicatorIndex+1+chunkDataSize]...) - deserializedblob = append(deserializedblob, tempbody) - tempbody = RawBlob{} + tempBody.data = append(tempBody.data, data[(indicatorIndex+1):indicatorIndex+1+chunkDataSize]...) + deserializedBlob = append(deserializedBlob, tempBody) + tempBody = RawBlob{} } else { // Since the chunk delimiter in non-zero now we can infer that it is a terminal chunk and - // add it and append to the deserializedblob slice. The tempbody signifies a single deserialized blob + // add it and append to the deserializedblob slice. The tempBody signifies a single deserialized blob terminalIndex := int64(data[indicatorIndex]) //Check if EVM flag is equal to 1 flagindex := data[indicatorIndex] >> 7 if flagindex == byte(1) { terminalIndex = int64(data[indicatorIndex]) - 128 - tempbody.flags.skipEvmExecution = true + tempBody.flags.skipEvmExecution = true } - tempbody.data = append(tempbody.data, data[(indicatorIndex+1):(indicatorIndex+1+terminalIndex)]...) - deserializedblob = append(deserializedblob, tempbody) - tempbody = RawBlob{} + tempBody.data = append(tempBody.data, data[(indicatorIndex+1):(indicatorIndex+1+terminalIndex)]...) + deserializedBlob = append(deserializedBlob, tempBody) + tempBody = RawBlob{} } } - return deserializedblob, nil + return deserializedBlob, nil } diff --git a/sharding/utils/marshal_test.go b/sharding/utils/marshal_test.go index cbf8d23eb6..57f8832e4b 100644 --- a/sharding/utils/marshal_test.go +++ b/sharding/utils/marshal_test.go @@ -6,11 +6,11 @@ import ( "testing" ) -func buildrawblob(size int64) []RawBlob { +func buildRawBlob(size int64) []RawBlob { tempbody := make([]RawBlob, size) for i := int64(0); i < size; i++ { var rawblob RawBlob - rawblob.data = buildblob(size) + rawblob.data = buildBlob(size) flagset := byte(rand.Int()) >> 7 if flagset == byte(1) { rawblob.flags.skipEvmExecution = true @@ -22,7 +22,7 @@ func buildrawblob(size int64) []RawBlob { } -func buildblob(size int64) []byte { +func buildBlob(size int64) []byte { tempbody := make([]byte, size) for i := int64(0); i < size; i++ { @@ -35,7 +35,7 @@ func buildblob(size int64) []byte { func TestSize(t *testing.T) { for i := 0; i < 300; i++ { size := int64(i) - blob := buildrawblob(size) + blob := buildRawBlob(size) chunksafterSerialize := size / chunkDataSize terminalchunk := size % chunkDataSize if terminalchunk != 0 { @@ -66,7 +66,7 @@ func TestSerializeAndDeserializeblob(t *testing.T) { for i := 1; i < 300; i++ { - blob := buildrawblob(int64(i)) + blob := buildRawBlob(int64(i)) drefbody := make([]*RawBlob, len(blob)) for s := 0; s < len(blob); s++ { From d7f9cc80799fec02c8564131ed0109af2694dbad Mon Sep 17 00:00:00 2001 From: nisdas Date: Wed, 16 May 2018 20:35:08 +0800 Subject: [PATCH 36/40] sharding: Removing pointer receiver(#92) Former-commit-id: 09bb5072963354b735d5c0c22c9d43761cab8234 [formerly 907a147c13b3d28605fb6380d0b8ad8518f92208] Former-commit-id: b3afc338970d4f8e199f87396a13cc91031d1829 --- sharding/collation.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sharding/collation.go b/sharding/collation.go index 39a009b6ba..cb62b8fb60 100644 --- a/sharding/collation.go +++ b/sharding/collation.go @@ -117,7 +117,7 @@ func (c *Collation) CalculateChunkRoot() { } // CreateRawBlobs creates raw blobs from transactions. -func (c *Collation) CreateRawBlobs() ([]*utils.RawBlob, error) { +func (c Collation) CreateRawBlobs() ([]*utils.RawBlob, error) { // It does not skip evm execution by default blobs := make([]*utils.RawBlob, len(c.transactions)) From c58083ec099619e7b4bd591a9086b6946394e481 Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Wed, 16 May 2018 13:38:26 -0400 Subject: [PATCH 37/40] sharding: fix up comment formatting in marshal and marshal_test.go Former-commit-id: 8cc3e8f619fc65d6230f38757b499e25b342bd3b [formerly ee95d821432933f87e0701160849e831a6489813] Former-commit-id: 93993c0d37ade450c7796020af1d30d2481ea1c7 --- sharding/utils/marshal.go | 60 +++++++++++++++++----------------- sharding/utils/marshal_test.go | 9 ++--- 2 files changed, 32 insertions(+), 37 deletions(-) diff --git a/sharding/utils/marshal.go b/sharding/utils/marshal.go index aaf47650c4..cddf61333e 100644 --- a/sharding/utils/marshal.go +++ b/sharding/utils/marshal.go @@ -2,6 +2,7 @@ package utils import ( "fmt" + "github.com/ethereum/go-ethereum/rlp" ) @@ -22,7 +23,8 @@ type RawBlob struct { data []byte } -// NewRawBlob builds a raw blob from any interface by using RLP encoding +// NewRawBlob builds a raw blob from any interface by using +// RLP encoding. func NewRawBlob(i interface{}, skipEvm bool) (*RawBlob, error) { data, err := rlp.EncodeToBytes(i) if err != nil { @@ -31,7 +33,8 @@ func NewRawBlob(i interface{}, skipEvm bool) (*RawBlob, error) { return &RawBlob{data: data, flags: Flags{skipEvmExecution: skipEvm}}, nil } -// ConvertFromRawBlob converts raw blob back from a byte array to its interface +// ConvertFromRawBlob converts raw blob back from a byte array +// to its interface. func ConvertFromRawBlob(blob *RawBlob, i interface{}) error { data := (*blob).data err := rlp.DecodeBytes(data, i) @@ -55,8 +58,8 @@ func SerializeBlob(cb RawBlob) ([]byte, error) { } tempBody := []byte{} - // if blob is less than 31 bytes, it adds the indicator chunk and pads the remaining empty bytes to the right - + // if blob is less than 31 bytes, adds the indicator chunk + // and pads the remaining empty bytes to the right. if chunksNumber == 0 { paddedBytes := make([]byte, (chunkDataSize - length)) indicatorByte[0] = byte(terminalLength) @@ -67,15 +70,16 @@ func SerializeBlob(cb RawBlob) ([]byte, error) { return tempBody, nil } - //if there is no need to pad empty bytes, then the indicator byte is added as 00011111 - // Then this chunk is returned to the main Serialize function - + // if there is no need to pad empty bytes, then the indicator byte + // is added as 0001111, then this chunk is returned to the + // main Serialize function. if terminalLength == 0 { for i := int64(1); i < chunksNumber; i++ { - // This loop loops through all non-terminal chunks and add a indicator byte of 00000000, each chunk - // is created by appending the indcator byte to the data chunks. The data chunks are separated into sets of - // 31 + // This loop loops through all non-terminal chunks and add a indicator + // byte of 00000000, each chunk is created by appending the indicator + // byte to the data chunks. The data chunks are separated into sets of + // 31 bytes. tempBody = append(tempBody, append(indicatorByte, @@ -96,20 +100,18 @@ func SerializeBlob(cb RawBlob) ([]byte, error) { } - // This loop loops through all non-terminal chunks and add a indicator byte of 00000000, each chunk - // is created by appending the indcator byte to the data chunks. The data chunks are separated into sets of - // 31 - + // This loop loops through all non-terminal chunks and add a indicator byte + // of 00000000, each chunk is created by appending the indcator byte + // to the data chunks. The data chunks are separated into sets of 31. for i := int64(1); i <= chunksNumber; i++ { - tempBody = append(tempBody, append(indicatorByte, cb.data[(i-1)*chunkDataSize:i*chunkDataSize]...)...) - } - // Appends indicator bytes to terminal-chunks , and if the index of the chunk delimiter is non-zero adds it to the chunk. - // Also pads empty bytes to the terminal chunk.chunkDataSize*chunksNumber refers to the total size of the blob. - // finalchunkIndex refers to the index of the last data byte + // Appends indicator bytes to terminal-chunks , and if the index of the chunk + // delimiter is non-zero adds it to the chunk. Also pads empty bytes to + // the terminal chunk.chunkDataSize*chunksNumber refers to the total + // size of the blob. finalchunkIndex refers to the index of the last data byte. indicatorByte[0] = byte(terminalLength) if cb.flags.skipEvmExecution { indicatorByte[0] |= (1 << 7) @@ -133,19 +135,19 @@ func Serialize(rawblobs []*RawBlob) ([]byte, error) { //Loops through all the blobs and serializes them into chunks for i := int64(0); i < length; i++ { - data := *rawblobs[i] refinedData, err := SerializeBlob(data) if err != nil { - return nil, fmt.Errorf("Index %v : %v", i, err) + return nil, fmt.Errorf("Index %v: %v", i, err) } serialisedData = append(serialisedData, refinedData...) - } + return serialisedData, nil } -// Deserialize results in the byte array being deserialised and separated into its respective interfaces. +// Deserialize results in the byte array being deserialised and +// separated into its respective interfaces. func Deserialize(data []byte) ([]RawBlob, error) { length := int64(len(data)) @@ -154,12 +156,12 @@ func Deserialize(data []byte) ([]RawBlob, error) { tempBody := RawBlob{} var deserializedBlob []RawBlob - // This separates the byte array into its separate blobs + // This separates the byte array into its separate blobs. for i := int64(1); i <= chunksNumber; i++ { indicatorIndex := (i - 1) * chunkSize // Tests if the chunk delimiter is zero, if it is it will append the data chunk - // to tempBody + // to tempBody. if data[indicatorIndex] == indicatorByte || data[indicatorIndex] == byte(128) { tempBody.data = append(tempBody.data, data[(indicatorIndex+1):(i)*chunkSize]...) @@ -172,8 +174,9 @@ func Deserialize(data []byte) ([]RawBlob, error) { tempBody = RawBlob{} } else { - // Since the chunk delimiter in non-zero now we can infer that it is a terminal chunk and - // add it and append to the deserializedblob slice. The tempBody signifies a single deserialized blob + // Since the chunk delimiter in non-zero now we can infer that it is + // a terminal chunk and add it and append to the deserializedblob + // slice. The tempBody signifies a single deserialized blob. terminalIndex := int64(data[indicatorIndex]) //Check if EVM flag is equal to 1 flagindex := data[indicatorIndex] >> 7 @@ -184,11 +187,8 @@ func Deserialize(data []byte) ([]RawBlob, error) { tempBody.data = append(tempBody.data, data[(indicatorIndex+1):(indicatorIndex+1+terminalIndex)]...) deserializedBlob = append(deserializedBlob, tempBody) tempBody = RawBlob{} - } - } return deserializedBlob, nil - } diff --git a/sharding/utils/marshal_test.go b/sharding/utils/marshal_test.go index 57f8832e4b..a16013f54b 100644 --- a/sharding/utils/marshal_test.go +++ b/sharding/utils/marshal_test.go @@ -18,18 +18,17 @@ func buildRawBlob(size int64) []RawBlob { } tempbody[i] = rawblob } - return tempbody + return tempbody } func buildBlob(size int64) []byte { - tempbody := make([]byte, size) for i := int64(0); i < size; i++ { tempbody[i] = byte(rand.Int()) } - return tempbody + return tempbody } func TestSize(t *testing.T) { @@ -55,9 +54,7 @@ func TestSize(t *testing.T) { } if int64(len(serializedblob)) != sizeafterSerialize { - t.Errorf("Error Serializing blobs the lengths are not the same:\n %d \n %d", int64(len(serializedblob)), sizeafterSerialize) - } } @@ -71,7 +68,6 @@ func TestSerializeAndDeserializeblob(t *testing.T) { drefbody := make([]*RawBlob, len(blob)) for s := 0; s < len(blob); s++ { drefbody[s] = &(blob[s]) - } serializedblob, err := Serialize(drefbody) @@ -85,7 +81,6 @@ func TestSerializeAndDeserializeblob(t *testing.T) { } if !reflect.DeepEqual(blob, raw) { - t.Errorf("Error Serializing blobs at index %d, the serialized and deserialized versions are not the same:\n\n %v \n\n %v \n\n %v", i, blob, serializedblob, raw) } } From 676681e67a4a2fac90fe4f5d439435b8911d0d10 Mon Sep 17 00:00:00 2001 From: nisdas Date: Thu, 17 May 2018 07:00:31 +0800 Subject: [PATCH 38/40] sharding: Cleaning up after merge conflicts(#92) Former-commit-id: 0417e770170d2ca1beca38ebea578481f7879a14 [formerly 41e3fc2c30d92a7d63c65cd7d3b6bed319ad43d9] Former-commit-id: 9898b390b701b419f02a5fa048b00c940cdcb2dc --- sharding/collation_test.go | 124 +++++++++++++++++-------------------- 1 file changed, 56 insertions(+), 68 deletions(-) diff --git a/sharding/collation_test.go b/sharding/collation_test.go index fe1cd28823..c0e4cb7355 100644 --- a/sharding/collation_test.go +++ b/sharding/collation_test.go @@ -1,10 +1,8 @@ package sharding import ( - "math/big" - //"github.com/ethereum/go-ethereum/rlp" - //"reflect" "bytes" + "math/big" "reflect" "testing" @@ -45,89 +43,79 @@ func TestCollation_Transactions(t *testing.T) { //Tests that Transactions can be serialised func TestSerialize_Deserialize(t *testing.T) { - tests := []struct { - transactions []*types.Transaction - }{ - { - transactions: []*types.Transaction{ - makeTxWithGasLimit(0), - makeTxWithGasLimit(5), - makeTxWithGasLimit(20), - makeTxWithGasLimit(100), - }, - }, + header := NewCollationHeader(big.NewInt(1), nil, big.NewInt(1), nil, []byte{}) + body := []byte{} + transactions := []*types.Transaction{ + makeTxWithGasLimit(0), + makeTxWithGasLimit(5), + makeTxWithGasLimit(20), + makeTxWithGasLimit(100), } - for _, tt := range tests { - c := &Collation{} - for _, tx := range tt.transactions { - c.AddTransaction(tx) + c := NewCollation(header, body, transactions) + + tx := c.transactions + + results, err := c.Serialize() + + if err != nil { + t.Errorf("Unable to Serialize transactions, %v", err) + } + + deserializedTxs, err := Deserialize(results) + + if err != nil { + t.Errorf("Unable to deserialize collation body, %v", err) + } + + if len(tx) != len(*deserializedTxs) { + t.Errorf("Transaction length is different before and after serialization: %v, %v", len(tx), len(*deserializedTxs)) + } + + for i := 0; i < len(tx); i++ { + + beforeSerialization := tx[i] + afterDeserialization := (*deserializedTxs)[i] + + if beforeSerialization.Nonce() != afterDeserialization.Nonce() { + + t.Errorf("Data before serialization and after deserialization are not the same ,AccountNonce: %v, %v", beforeSerialization.Nonce(), afterDeserialization.Nonce()) + } - tx := c.transactions + if beforeSerialization.Gas() != afterDeserialization.Gas() { - results, err := c.Serialize() + t.Errorf("Data before serialization and after deserialization are not the same ,GasLimit: %v, %v", beforeSerialization.Gas(), afterDeserialization.Gas()) - if err != nil { - t.Errorf("Unable to Serialize transactions, %v", err) } - deserializedTxs, err := Deserialize(results) + if beforeSerialization.GasPrice().Cmp(afterDeserialization.GasPrice()) != 0 { + + t.Errorf("Data before serialization and after deserialization are not the same ,Price: %v, %v", beforeSerialization.GasPrice(), afterDeserialization.GasPrice()) - if err != nil { - t.Errorf("Unable to deserialize collation body, %v", err) } - if len(tx) != len(*deserializedTxs) { - t.Errorf("Transaction length is different before and after serialization: %v, %v", len(tx), len(*deserializedTxs)) + beforeAddress := reflect.ValueOf(beforeSerialization.To()) + afterAddress := reflect.ValueOf(afterDeserialization.To()) + + if reflect.DeepEqual(beforeAddress, afterAddress) { + + t.Errorf("Data before serialization and after deserialization are not the same ,Recipient: %v, %v", beforeAddress, afterAddress) + } - for i := 0; i < len(tx); i++ { + if beforeSerialization.Value().Cmp(afterDeserialization.Value()) != 0 { - beforeSerialization := tx[i] - afterDeserialization := (*deserializedTxs)[i] + t.Errorf("Data before serialization and after deserialization are not the same ,Amount: %v, %v", beforeSerialization.Value(), afterDeserialization.Value()) - if beforeSerialization.Nonce() != afterDeserialization.Nonce() { + } - t.Errorf("Data before serialization and after deserialization are not the same ,AccountNonce: %v, %v", beforeSerialization.Nonce(), afterDeserialization.Nonce()) + beforeData := beforeSerialization.Data() + afterData := afterDeserialization.Data() - } + if !bytes.Equal(beforeData, afterData) { - if beforeSerialization.Gas() != afterDeserialization.Gas() { - - t.Errorf("Data before serialization and after deserialization are not the same ,GasLimit: %v, %v", beforeSerialization.Gas(), afterDeserialization.Gas()) - - } - - if beforeSerialization.GasPrice().Cmp(afterDeserialization.GasPrice()) != 0 { - - t.Errorf("Data before serialization and after deserialization are not the same ,Price: %v, %v", beforeSerialization.GasPrice(), afterDeserialization.GasPrice()) - - } - - beforeAddress := reflect.ValueOf(beforeSerialization.To()) - afterAddress := reflect.ValueOf(afterDeserialization.To()) - - if reflect.DeepEqual(beforeAddress, afterAddress) { - - t.Errorf("Data before serialization and after deserialization are not the same ,Recipient: %v, %v", beforeAddress, afterAddress) - - } - - if beforeSerialization.Value().Cmp(afterDeserialization.Value()) != 0 { - - t.Errorf("Data before serialization and after deserialization are not the same ,Amount: %v, %v", beforeSerialization.Value(), afterDeserialization.Value()) - - } - - beforeData := beforeSerialization.Data() - afterData := afterDeserialization.Data() - - if !bytes.Equal(beforeData, afterData) { - - t.Errorf("Data before serialization and after deserialization are not the same ,Payload: %v, %v", beforeData, afterData) - - } + t.Errorf("Data before serialization and after deserialization are not the same ,Payload: %v, %v", beforeData, afterData) } From 33c4f155868a3c7bf9293a46754e31275ebfb279 Mon Sep 17 00:00:00 2001 From: nisdas Date: Thu, 17 May 2018 07:23:48 +0800 Subject: [PATCH 39/40] sharding: Adding colon(#92) Former-commit-id: cb11c83f0d32b7e8c383c974378ddbdb756ecbe4 [formerly a88b20fb68d81b018bedf6f3ba3ce2d0a0f70918] Former-commit-id: cf5df84fe22b937198569b6092ff1e602865bbec --- sharding/collation.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sharding/collation.go b/sharding/collation.go index cb62b8fb60..faa0b889b2 100644 --- a/sharding/collation.go +++ b/sharding/collation.go @@ -127,7 +127,7 @@ func (c Collation) CreateRawBlobs() ([]*utils.RawBlob, error) { blobs[i], err = utils.NewRawBlob(c.transactions[i], false) if err != nil { - return nil, fmt.Errorf("Creation of raw blobs from transactions failed %v", err) + return nil, fmt.Errorf("Creation of raw blobs from transactions failed: %v", err) } } @@ -147,7 +147,7 @@ func ConvertBackToTx(rawBlobs []utils.RawBlob) ([]*types.Transaction, error) { err := utils.ConvertFromRawBlob(&rawBlobs[i], blobs[i]) if err != nil { - return nil, fmt.Errorf("Creation of transactions from raw blobs failed %v", err) + return nil, fmt.Errorf("Creation of transactions from raw blobs failed: %v", err) } } return blobs, nil From 05401b7547fe78873dcb6964cb9475c78bb26b03 Mon Sep 17 00:00:00 2001 From: nisdas Date: Thu, 17 May 2018 07:31:25 +0800 Subject: [PATCH 40/40] sharding: Fix Lint (#92) Former-commit-id: 25b3147ba8e7a67a7ed0215ea0e382699523d876 [formerly 72089a1b1f593814322659488ffd336088335056] Former-commit-id: 2cea947acd95d3a4e57d5356b329f79f57c96901 --- sharding/collation.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/sharding/collation.go b/sharding/collation.go index faa0b889b2..3ce4b55e33 100644 --- a/sharding/collation.go +++ b/sharding/collation.go @@ -41,6 +41,8 @@ type collationHeaderData struct { ProposerSignature []byte // the proposer's signature for calculating collation hash. } +var collationSizelimit = int64(math.Pow(float64(2), float64(20))) + // NewCollation initializes a collation and leaves it up to clients to serialize, deserialize // and provide the body and transactions upon creation. func NewCollation(header *CollationHeader, body []byte, transactions []*types.Transaction) *Collation { @@ -92,13 +94,6 @@ func (c *Collation) Header() *CollationHeader { return c.header } // Body returns the collation's byte body. func (c *Collation) Body() []byte { return c.body } -// Transactions returns an array of tx's in the collation. -var ( - collationSizelimit = int64(math.Pow(float64(2), float64(20))) - chunkSize = int64(32) - numberOfChunks = collationSizelimit / chunkSize -) - // Transactions returns an array of tx's in the collation. func (c *Collation) Transactions() []*types.Transaction { return c.transactions }