looking for diff algo

beacon-chain/db/kv/state_test.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	"crypto/rand"
 	"encoding/binary"
+	"fmt"
 	mathRand "math/rand"
+	"os"
 	"strconv"
 	"testing"
 	"time"
@@ -21,6 +23,8 @@ import (
 	"github.com/OffchainLabs/prysm/v6/testing/assert"
 	"github.com/OffchainLabs/prysm/v6/testing/require"
 	"github.com/OffchainLabs/prysm/v6/testing/util"
+	"github.com/balacode/go-delta"
+	"github.com/golang/snappy"
 	bolt "go.etcd.io/bbolt"
 )
 
@@ -1287,3 +1291,111 @@ func BenchmarkState_CheckStateSaveTime_10(b *testing.B) { checkStateSaveTime(b,
 
 func BenchmarkState_CheckStateReadTime_1(b *testing.B)  { checkStateReadTime(b, 1) }
 func BenchmarkState_CheckStateReadTime_10(b *testing.B) { checkStateReadTime(b, 10) }
+
+func TestDiff_ReadAndUnmarshal(t *testing.T) {
+	d1, err := os.ReadFile("/home/mohamad/Desktop/state1.ssz")
+	require.NoError(t, err)
+	d2, err := os.ReadFile("/home/mohamad/Desktop/state2.ssz")
+	require.NoError(t, err)
+
+	var state1, state2 ethpb.BeaconStateElectra
+	err = state1.UnmarshalSSZ(d1)
+	require.NoError(t, err)
+	err = state2.UnmarshalSSZ(d2)
+	require.NoError(t, err)
+
+	require.DeepNotEqual(t, state1, state2)
+	state1.Balances = []uint64{}
+	state2.Balances = []uint64{}
+	state1.Validators = []*ethpb.Validator{}
+	state2.Validators = []*ethpb.Validator{}
+
+	dd1, err := state1.MarshalSSZ()
+	require.NoError(t, err)
+	dd2, err := state2.MarshalSSZ()
+	require.NoError(t, err)
+
+	d1snappy := snappy.Encode(nil, dd1)
+	d2snappy := snappy.Encode(nil, dd2)
+
+	//var patch []byte
+
+	start := time.Now()
+
+	dlta := delta.Make(d1snappy, d2snappy)
+
+	elapsed := time.Since(start)
+	fmt.Printf("Function took %s\n", elapsed)
+
+	dltabytes := dlta.Bytes()
+	p, err := delta.Load(dltabytes)
+	require.NoError(t, err)
+
+	fmt.Println(len(p.Bytes()))
+
+	start = time.Now()
+
+	newd, err := p.Apply(d1snappy)
+	require.NoError(t, err)
+
+	elapsed = time.Since(start)
+	fmt.Printf("Function2 took %s\n", elapsed)
+
+	newSSZ, err := snappy.Decode(nil, newd)
+	require.NoError(t, err)
+	var state3 ethpb.BeaconStateElectra
+	err = state3.UnmarshalSSZ(newSSZ)
+	require.NoError(t, err)
+
+	//
+	//fmt.Println(state1.Slot, len(d1), state2.Slot, len(d2))
+	//
+	//fmt.Println(state1.Validators[1923842])
+	//fmt.Println(state2.Validators[1923842])
+}
+
+// BeaconStateDiff holds separate compressed diffs for balances, validators, and the rest of the state.
+//type BeaconStateDiff struct {
+//	BalancesDiff   []byte // snappy-compressed XOR diff of balances SSZ
+//	ValidatorsDiff []byte // snappy-compressed XOR diff of validators SSZ
+//	RestDiff       []byte // snappy-compressed XOR diff of the rest of the state SSZ
+//}
+//
+//func stateDiff(s1, s2 *ethpb.BeaconStateElectra) (*BeaconStateDiff, error) {
+//	// 1) Balances
+//	baseBalBytes, err := bytesutil.tobytes
+//	targetBalBytes := ssz.Marshal(s2.Balances)
+//	balDiff, err := xorAndCompress(baseBalBytes, targetBalBytes)
+//	if err != nil {
+//		return nil, fmt.Errorf("balance diff: %w", err)
+//	}
+//
+//	// 2) Validators
+//	baseValBytes, _ := ssz.MarshalSSZ(s1.Validators)
+//	targetValBytes := ssz.Marshal(s2.Validators)
+//	valDiff, err := xorAndCompress(baseValBytes, targetValBytes)
+//	if err != nil {
+//		return nil, fmt.Errorf("validator diff: %w", err)
+//	}
+//
+//	// 3) Rest of state: zero out balances & validators
+//	baseCopy := *s1
+//	baseCopy.Balances = nil
+//	baseCopy.Validators = nil
+//	targCopy := *s2
+//	targCopy.Balances = nil
+//	targCopy.Validators = nil
+//
+//	baseRestBytes := ssz.Marshal(&baseCopy)
+//	targRestBytes := ssz.Marshal(&targCopy)
+//	restDiff, err := xorAndCompress(baseRestBytes, targRestBytes)
+//	if err != nil {
+//		return nil, fmt.Errorf("rest diff: %w", err)
+//	}
+//
+//	return &BeaconStateDiff{
+//		BalancesDiff:   balDiff,
+//		ValidatorsDiff: valDiff,
+//		RestDiff:       restDiff,
+//	}, nil
+//}

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/MariusVanDerWijden/FuzzyVM v0.0.0-20240516070431-7828990cad7d
 	github.com/MariusVanDerWijden/tx-fuzz v1.4.0
 	github.com/aristanetworks/goarista v0.0.0-20200805130819-fd197cf57d96
+	github.com/balacode/go-delta v0.1.0
 	github.com/bazelbuild/rules_go v0.23.2
 	github.com/btcsuite/btcd/btcec/v2 v2.3.4
 	github.com/consensys/gnark-crypto v0.14.0
@@ -40,6 +41,7 @@ require (
 	github.com/json-iterator/go v1.1.12
 	github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213
 	github.com/kisielk/errcheck v1.8.0
+	github.com/kr/binarydist v0.1.0
 	github.com/kr/pretty v0.3.1
 	github.com/libp2p/go-libp2p v0.36.5
 	github.com/libp2p/go-libp2p-mplex v0.9.0
@@ -108,6 +110,7 @@ require (
 	github.com/DataDog/zstd v1.5.5 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/VictoriaMetrics/fastcache v1.12.2 // indirect
+	github.com/balacode/zr v1.0.0 // indirect
 	github.com/benbjohnson/clock v1.3.5 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bits-and-blooms/bitset v1.17.0 // indirect

go.sum

@@ -89,6 +89,10 @@ github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6l
 github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU=
 github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
+github.com/balacode/go-delta v0.1.0 h1:pwz4CMn06P2bIaIfAx3GSabMPwJp/Ww4if+7SgPYa3I=
+github.com/balacode/go-delta v0.1.0/go.mod h1:wLNrwTI3lHbPBvnLzqbHmA7HVVlm1u22XLvhbeA6t3o=
+github.com/balacode/zr v1.0.0 h1:MCupkEoXvrnCljc4KddiDOhR04ZLUAACgtKuo3o+9vc=
+github.com/balacode/zr v1.0.0/go.mod h1:pLeSAL3DhZ9L0JuiRkUtIX3mLOCtzBLnDhfmykbSmkE=
 github.com/bazelbuild/rules_go v0.23.2 h1:Wxu7JjqnF78cKZbsBsARLSXx/jlGaSLCnUV3mTlyHvM=
 github.com/bazelbuild/rules_go v0.23.2/go.mod h1:MC23Dc/wkXEyk3Wpq6lCqz0ZAYOZDw2DR5y3N1q2i7M=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
@@ -564,6 +568,8 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxv
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/koron/go-ssdp v0.0.4 h1:1IDwrghSKYM7yLf7XCzbByg2sJ/JcNOZRXS2jczTwz0=
 github.com/koron/go-ssdp v0.0.4/go.mod h1:oDXq+E5IL5q0U8uSBcoAXzTzInwy5lEgC91HoKtbmZk=
+github.com/kr/binarydist v0.1.0 h1:6kAoLA9FMMnNGSehX0s1PdjbEaACznAv/W219j2uvyo=
+github.com/kr/binarydist v0.1.0/go.mod h1:DY7S//GCoz1BCd0B0EVrinCKAZN3pXe+MDaIZbXQVgM=
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=