[research/ourobors] add support for delayed endorsing]

script/research/dpos/ouroboros/beacon.py

@@ -23,6 +23,7 @@ class TrustedBeacon(SynchedNTPClock, threading.Thread):
         self.current_slot = self.slot
         self.log = Logger(self, genesis_time)
         self.log.info(f"constructed for node {str(node)}")
+        self.bb=0 # epoch counts since genesis (big bang)
 
     def __repr__(self):
         return f"trustedbeacon"
@@ -43,11 +44,16 @@ class TrustedBeacon(SynchedNTPClock, threading.Thread):
     def __callback(self):
         self.current_slot = self.slot
         if self.current_slot%self.epoch_length!=0:
+            if self.bb==0:
+                # new nodes attached to the network, need to either request old blocks, or wait for next epoch's broadcst
+                # it's temporarily, and or simplicity set to the latter 
+                return
             self.log.info(f"callback: new slot of idx: {self.current_slot}")
             y, pi = self.vrf.sign(self.current_slot)
             self.log.info(f"callback: signature calculated for {str(self.node)}")
             self.node.new_slot(self.current_slot, y, pi)
         else:
+            self.bb+=1
             sigmas = []
             proofs = []
             #TODO since it's expensive, but to generate single (y,pi) pair as seed 

script/research/dpos/ouroboros/block.py

@@ -1,4 +1,5 @@
 import json
+import time
 from ouroboros.utils import encode_genesis_data, decode_gensis_data, state_hash
 from ouroboros.consts import *
 from ouroboros.logger import Logger
@@ -17,7 +18,7 @@ class Block(object):
         it's one-based
     @param gensis: boolean, True for gensis block
     '''
-    def __init__(self, previous_block, data, slot_uid, genesis_time, genesis=False):
+    def __init__(self, previous_block, data, slot_uid, genesis_time=time.time(), genesis=False):
         # state is hte hash of the previous block in the blockchain
         self.state=''
         if slot_uid>1:
@@ -80,7 +81,7 @@ class GensisBlock(Block):
         and stake respectively of the corresponding stakeholder U_i,
         seed of the leader election function.
     '''
-    def __init__(self, previous_block, data, slot_uid):
+    def __init__(self, previous_block, data, slot_uid, genesis_time=time.time()):
         # stakeholders is list of tuple (pk_i, s_i) for the ith stakeholder
         self.stakeholders = data[STAKEHOLDERS]
         self.distribution = data[STAKEHOLDERS_DISTRIBUTIONS]
@@ -90,7 +91,7 @@ class GensisBlock(Block):
             shd_buff +=str(shd)
         #data = encode_genesis_data(shd_buff)
         data_dict = {'seed':self.seed, 'distribution':shd_buff}
-        Block.__init__(self, previous_block, str(data_dict), slot_uid, True)
+        Block.__init__(self, previous_block, str(data_dict), slot_uid, genesis_time, True)
     '''
     @return: the number of participating stakeholders in the blockchain
     '''
@@ -109,5 +110,5 @@ lead by offline leader
 is an empty Block
 '''
 class EmptyBlock(Block):
-    def __init__(self, genesis_time):
+    def __init__(self, genesis_time=time.time()):
         Block.__init__(self, '', -1, genesis_time, False)

script/research/dpos/ouroboros/clock.py

@@ -8,7 +8,7 @@ import math
 
 class SynchedNTPClock(object):
 
-    def __init__(self, slot_length=1, ntp_server='europe.pool.ntp.org'):
+    def __init__(self, slot_length=60, ntp_server='europe.pool.ntp.org'):
         #TODO how long should be the slot length
         self.slot_length=slot_length
         self.ntp_server = ntp_server
@@ -16,7 +16,7 @@ class SynchedNTPClock(object):
         #TODO validate the server
         # when was darkfi birthday? as seconds since the epoch 
         self.darkfi_epoch=0
-        
+        self.offline_cnt=0
     def __repr__(self):
         return 'darkfi time: '+ ctime(self.darkfi_time) + ', current synched time: ' + ctime(self.synched_time)
 
@@ -44,5 +44,10 @@ class SynchedNTPClock(object):
         return self.synched_time - self.darkfi_epoch
 
     @property
-    def slot(self):   
+    def offline_time(self):
+        self.offline_cnt+=1
+        return self.offline_cnt
+
+    @property
+    def slot(self):
         return math.floor(self.darkfi_time/self.slot_length)

script/research/dpos/ouroboros/environment.py

@@ -1,4 +1,3 @@
-from debugpy import configure
 import numpy as np
 import math
 import random
@@ -23,7 +22,7 @@ class Z(object):
         self.current_blk_endorser_sig=None
     
     def __repr__(self):
-        buff= f"envirnment of {self.length} stakholders"
+        buff= f"envirnment of {self.length} stakholders\tcurrent leader's id: {self.current_leader_id}\tepoch_slot: {self.epoch_slot}\tendorser_id: {self.current_endorser_id}"
         for sh in self.stakeholders:
             buff+=str(sh)+"\n"
         return buff
@@ -53,7 +52,7 @@ class Z(object):
 
     @property
     def current_endorser_id(self):
-        return self.current_endorser_id[self.epoch_slot]
+        return self.current_epoch_endorsers[self.epoch_slot]
 
     @property
     def current_endorser(self):
@@ -76,6 +75,26 @@ class Z(object):
     def current_endorser_sig_pk(self):
         return self.stakeholders[self.current_endorser_id].sig_pk
 
+    def endorser(self, epoch_slot):
+        assert epoch_slot >= 0 and epoch_slot < self.epoch_length
+        return self.stakeholders[epoch_slot]
+    
+    def endorser_sig_pk(self, epoch_slot):
+        return self.endorser(epoch_slot).sig_pk
+
+    def endorser_vrf_pk(self, epoch_slot):
+        return self.endorser(epoch_slot).vrf_pk
+
+    def leader(self, epoch_slot):
+        assert epoch_slot >= 0 and epoch_slot < self.epoch_length
+        return self.stakeholders[epoch_slot]
+
+    def leader_sig_pk(self, epoch_slot):
+        return self.leader(epoch_slot).sig_pk
+
+    def leader_vrf_pk(self, epoch_slot):
+        return self.leader(epoch_slot).vrf_pk
+        
     #TODO complete
     def obfuscate_idx(self, i):
         return i
@@ -117,11 +136,11 @@ class Z(object):
             seed = leader_selection_hash(sigma)
             random.seed(seed)
             leader_idx=seed%self.length
-            endorser_idx=random.randint(0,self.length)
+            endorser_idx=random.randint(0,self.length-1)
             # only select an honest leaders
             while leader_idx==endorser_idx or not self.adversary_mask[leader_idx] or not self.adversary_mask[endorser_idx]:
-                leader_idx=random.randint(0,self.length)
-                enderser_idx=random.randint(0,self.length)
+                leader_idx=random.randint(0,self.length-1)
+                endorser_idx=random.randint(0,self.length-1)
 
             #TODO select the following leader for this epoch, note, 
             # under a single condition that no one is able to predict who is next
@@ -133,30 +152,41 @@ class Z(object):
         self.current_slot=slot
         self.log.info(f"stakeholders: {self.stakeholders}")
         current_leader = self.stakeholders[self.current_leader_id]
-        assert current_leader!=None, "current leader cant be None"
-        if current_leader.is_leader:
-            #pass leadership to the current slot leader from the epoch leader
-            self.stakeholders[self.current_epoch_leaders[self.current_leader_id]].set_leader()
-            self.stakeholders[self.current_epoch_endorsers[self.current_endorser_id]].set_endorser()
-    
+        assert current_leader is not None, "current leader cant be None"
+        self.log.highlight('selecting epochs leaders, and ensorsers ---->')
+        self.stakeholders[self.current_epoch_endorsers[self.current_endorser_id]].set_endorser()
+        self.stakeholders[self.current_epoch_leaders[self.current_leader_id]].set_leader()
+        self.log.highlight('selected epochs leaders, and ensorsers <----')
+        
     def new_epoch(self, slot, sigmas, proofs):
         self.current_slot=slot
         leaders, endorsers = self.select_epoch_leaders(sigmas, proofs)
         return leaders, endorsers
 
     def broadcast_block(self, signed_block, slot_uid):
+        while self.current_blk_endorser_sig is None:
+            self.log.info('pending endorsing...')
+            time.sleep(1)
+            #wait for it untill it gets endorsed
+            pass
         for stakeholder in self.stakeholders:
             if not stakeholder.is_leader:
                 stakeholder.receive_block(signed_block, self.current_blk_endorser_sig, slot_uid)
         self.print_blockchain()
 
-    def encorse_block(self, sig, slot_uid):
+    @property
+    def block_id(self):
+        return self.current_slot%self.epoch_length
+ 
+    def endorse_block(self, sig, slot_uid):
         #TODO commit this step to handshake phases
-        self.current_blk_endorser_sig=sig
-        confirmed = self.stakeholders[self.current_leader_id].confirm_endorsing(sig, slot_uid)
+        self.current_blk_endorser_sig=None
+        self.log.info(f"endorsing block for current_leader_id: {self.current_leader_id}")
+        confirmed = self.stakeholders[self.current_leader_id].confirm_endorsing(sig, self.block_id, self.epoch_slot)
         if confirmed:
             self.current_blk_endorser_sig=sig
-
+        else:
+            self.log.warn("unconfirmed endorsed siganture")
 
     def start(self):
         for sh in self.stakeholders:

script/research/dpos/ouroboros/stakeholder.py

@@ -1,3 +1,5 @@
+from copy import deepcopy
+import time
 from ouroboros.block import Block, GensisBlock, EmptyBlock
 from ouroboros.blockchain import Blockchain
 from ouroboros.epoch import Epoch
@@ -11,8 +13,7 @@ from ouroboros.consts import *
 \class Stakeholder
 '''
 class Stakeholder(object):
-
-    def __init__(self, epoch_length=2, passwd='password'):
+    def __init__(self, epoch_length=10, passwd='password'):
         #TODO (fix) remove redundant variables reley on environment
         self.passwd=passwd
         self.stake=0
@@ -32,11 +33,10 @@ class Stakeholder(object):
         self.tx=''
         self.current_epoch = None
         self.am_current_leader=False
-        self.am_current_endorder=False
+        self.am_current_endorser=False
         self.am_corrupt=False
         #
         self.blockchain=None
-        self.current_blk_endorser_sig = None
 
     @property
     def is_leader(self):
@@ -51,7 +51,13 @@ class Stakeholder(object):
         return self.__vrf_base
     
     def __repr__(self):
-        buff = f"\tstakeholder with stake:{self.stake}\t"
+        buff=''
+        if self.am_current_leader:
+            buff = f"\tleader {(hash(self.passwd))} with stake:{self.stake}\nsig_sk: {self.sig_pk}"
+        elif self.am_current_endorser:
+            buff = f"\tendorser {(hash(self.passwd))} with stake:{self.stake}\nsig_sk: {self.sig_pk}"
+        else:
+            buff = f"\thonest committee memeber {(hash(self.passwd))} with stake:{self.stake}\nsig_sk: {self.sig_pk}"
         return buff
 
     def __call__(self, env):
@@ -63,9 +69,9 @@ class Stakeholder(object):
 
 
     def start(self):
-        self.log.info("start [started]")
+        self.log.info("thread [started]")
         self.beacon.start()
-        self.log.info("start [ended]")
+        self.log.info("thread [ended]")
 
     @property
     def epoch_index(self):
@@ -78,6 +84,7 @@ class Stakeholder(object):
         self.tx[TX]=self.uncommited_tx
         self.uncommited_tx=''
         self.current_block=GensisBlock(self.current_block, self.tx, self.current_slot_uid, self.env.genesis_time)
+        assert self.current_block is not None
         self.current_epoch=Epoch(self.current_block, self.epoch_length, self.epoch_index, self.env.genesis_time)
    
     '''
@@ -97,10 +104,17 @@ class Stakeholder(object):
         if self.current_slot_uid > 1 and self.current_epoch!=None and len(self.current_epoch)>0:
             self.blockchain.add_epoch(self.current_epoch)  
         #if leader, you need to broadcast the block
-        self.__gen_genesis_epoch() 
+        self.__gen_genesis_epoch()
         if self.am_current_leader:
             self.broadcast_block()
-        self.am_current_leader=False
+            self.end_leadership()
+        elif self.am_current_endorser:
+            assert self.current_slot_uid==self.env.current_slot, f' current slot: {self.current_slot_uid}, env current slot {self.env.current_slot}'
+            #assert self.sig_pk==self.env.current_endorser_sig_pk, f'current sig_pk: {self.sig_pk}\nZ sig_pk:{self.env.current_endorser_sig_pk}'
+            if not self.sig_pk==self.env.current_endorser_sig_pk:
+                return
+            self.endorse_block()
+            self.end_endorsing()
 
     '''
     it's a callback function, and called by the diffuser
@@ -118,9 +132,9 @@ class Stakeholder(object):
         if not verify(slot, sigma, proof, vrf_pk,vrf_g) :
             #TODO the leader is corrupted, action to be taken against the corrupt stakeholder
             #in this case this slot is empty
+            self.log.warn(f"<new_slot> leader verification fails")
             self.current_block=EmptyBlock(self.env.genesis_time) 
             if self.current_epoch==None:
-                self.log.warn(f"<new_slot> leader verification fails, current_epoch is None!")
                 self.__gen_genesis_epoch()
             self.current_epoch.add_block(self.current_block)
             return
@@ -131,87 +145,123 @@ class Stakeholder(object):
         prev_blk = self.blockchain[-1] if len(self.blockchain)>0 else EmptyBlock(self.env.genesis_time)
         self.current_block=Block(prev_blk, self.tx, self.current_slot_uid, self.env.genesis_time)
         self.current_epoch.add_block(self.current_block)
+        assert self.current_block is not None
         if self.am_current_leader:
+            self.log.highlight(f"leader {str(self)} is broadcasting block")
             self.broadcast_block()
             self.end_leadership()
-        elif self.am_current_endorder:
+        elif self.am_current_endorser:
+            self.log.highlight(f"endorser {str(self)} is endorsing block")
+            assert self.sig_pk==self.env.current_endorser_sig_pk
             self.endorse_block()
             self.end_endorsing()
+        else:
+            self.log.highlight(f"committee memeber is listening...")
 
 
     def end_leadership(self):
-        self.log(f"stakeholder:{str(self)} ending leadership for slot{self.current_slot_uid}")
+        self.log.info(f"stakeholder:{str(self)} ending leadership for slot{self.current_slot_uid}")
         self.am_current_leader=False
 
     def end_endorsing(self):
-        self.log(f"stakeholder:{str(self)} ending endorsing for slot{self.current_slot_uid}")
-        self.am_current_endorder=False
+        self.log.info(f"stakeholder:{str(self)} ending endorsing for slot{self.current_slot_uid}")
+        self.am_current_endorser=False
 
     def set_leader(self):
         self.am_current_leader=True
 
     def set_endorser(self):
-        self.am_endorser=True
+        self.am_current_endorser=True
 
     def set_corrupt(self):
         self.am_corrupt=False
 
     def broadcast_block(self):
         self.log.highlight("broadcasting block")
-        assert(self.am_current_leader and self.current_block is not None)
+        assert self.am_current_leader and self.current_block is not None
         signed_block=None
         #TODO should wait for l slot until block is endorsed
+        endorsing_cnt=10
+        while not self.current_block.endorsed:
+            time.sleep(1)
+            self.log.info("...waiting for endorsment..")
+            endorsing_cnt-=1
+        if not self.current_block.endorsed:
+            self.log.warn("failure endorsing the block...")
         if not self.current_block.endorsed:
             self.current_block = EmptyBlock(self.env.genesis_time)
-        else:
-            signed_block = sign_message(self.passwd, self.sig_sk, self.current_block)
+        signed_block = sign_message(self.passwd, self.sig_sk, self.current_block)
         self.env.broadcast_block(signed_block, self.current_slot_uid)
-        self.current_block=None
     
     def endorse_block(self):
-        if not self.am_current_endorder:
+        if not self.am_current_endorser:
             return
+        self.log.info(f"endorsing block for current_leader_id: {self.env.current_leader_id}")
+        if not self.am_current_endorser:
+            self.log.warn("not endorser")
+            return
+        assert self.current_block is not None
         sig = sign_message(self.passwd, self.sig_sk, self.current_block)
+        self.log.highlight(f'block to be endorsed  {str(self.current_block)}')
+        self.log.highlight(f'block to be endorsed has slot_uid: {self.current_slot_uid}')
+        self.log.highlight(f'block to be endorsed has sig_pk: {str(self.sig_pk)}')
+        assert self.env.current_endorser_sig_pk==self.sig_pk
         self.env.endorse_block(sig, self.current_slot_uid)
 
     def __get_blk(self, blk_uid):
-        cur_blk = None
-        assert(blk_uid>0)
+        assert(blk_uid>=0)
         stashed=True
-        if blk_uid>= len(self.blockchain):
-            cur_blk = self.current_block
-        else:
+        cur_blk = self.current_block
+        if blk_uid < len(self.blockchain):
             #TODO this assumes synced blockchain
             cur_blk = self.blockchain[blk_uid]
+            self.log.warn(f"current block from blockchain: {(cur_blk)}")
             stashed=False
+        self.log.info(f"current block : {str(cur_blk)}\tblock uid: {blk_uid}\tstashed: {stashed}")
+        if cur_blk is None:
+            self.log.warn(f'block is none, current block is {str(self.current_block)} and current slot {self.current_slot_uid}, current block uid {blk_uid}, env slot {self.env.current_slot}, env blk {self.env.block_id}')
+        assert cur_blk is not None and self.current_block is not None
         return cur_blk, stashed
 
     def receive_block(self, signed_block, endorser_sig, blk_uid):
         self.log.highlight("receiving block")
         cur_blk, stashed = self.__get_blk(blk_uid)
-        
+        assert cur_blk is not None
         #TODO to consider deley should retrive leader_pk of corresponding blk_uid
-        if verify_signature(self.env.current_leader_sig_pk, cur_blk, signed_block) \
-            and  verify_signature(self.env.current_endorser_sig_pk, cur_blk, endorser_sig):
+        self.log.highlight(f'receiving block  {str(cur_blk)}')
+        self.log.highlight(f'receiving block has slot_uid: {self.current_slot_uid}')
+        self.log.highlight(f'receiving block has sig_pk: {self.env.current_endorser_sig_pk}')
+        blk_verified = verify_signature(self.env.current_leader_sig_pk, cur_blk, signed_block)
+        self.log.info("endorser sig_pk {self.env.current_endorser_sig_pk}, cur_blk: {cur_blk}, endorser_sig: {endorser_sig}")
+        blk_edrs_verified = verify_signature(self.env.current_endorser_sig_pk, cur_blk, endorser_sig)
+        if blk_verified and blk_edrs_verified:
             if stashed:
                 self.current_epoch.add_block(cur_blk)
         else:
+            if not blk_verified:
+                self.log.warn("block verification failed")
+            elif not blk_edrs_verified:
+                self.log.warn("block endorsing verification failed")
             self.env.corrupt_blk()
 
-    def confirm_endorsing(self, signed_endorsed_block, blk_uid):
-        confirmed=False
-        if self.am_current_leader:
-            cur_blk, _ = self.__get_blk(blk_uid)
-            if verify_signature(self.env.current_endorser_sig_pk, cur_blk, signed_endorsed_block):
-                if blk_uid==self.current_slot_uid:
-                    self.current_block.set_endorsed()
-                else:
-                    self.blockchain[blk_uid].set_endorsed()
-                confirmed=True
-                #self.env.confirm_endorsing(signed_endorsed_block, blk_uid)
+    def confirm_endorsing(self, endorser_sig, blk_uid, epoch_slot):
+        while not self.current_slot_uid == self.env.current_slot:
+            self.log.info(" ...pending start of slot...")
+            time.sleep(1)
+        self.log.highlight("receiving block")
+        confirmed = False
+        cur_blk, _ = self.__get_blk(blk_uid)
+        assert cur_blk is not None
+        self.log.highlight(f'confirming endorsed block  {str(cur_blk)}')
+        self.log.highlight(f'confirming endorsed has slot_uid: {self.current_slot_uid}')
+        self.log.highlight(f'confirming endorsed has sig_pk: {self.env.current_endorser_sig_pk}')
+        if verify_signature(self.env.current_endorser_sig_pk, cur_blk, endorser_sig):
+            if self.current_slot_uid==self.env.current_slot:
+                self.current_block.set_endorsed()
             else:
-                confirmed=False
-                #self.env.confirm_endorsing(None, blk_uid)
-        self.log.info(f"<confirm_endorsing enderser signature: {self.current_blk_endorser_sig}")
+                self.blockchain[blk_uid].set_endorsed()
+            confirmed=True
+        else:
+            self.log.warn(f"confirmed enderser signature failure for pk: {str(self.env.current_endorser_sig_pk)} on block {str(cur_blk)}  of signature {str(endorser_sig)}")
+            confirmed=False
         return confirmed
-

script/research/dpos/simulation.py

@@ -6,11 +6,12 @@ EPOCH_LENGTH = 2
 stakeholders = []
 
 for i in range(2):
-    stakeholders.append(Stakeholder(EPOCH_LENGTH))
+    stakeholders.append(Stakeholder(EPOCH_LENGTH, 'passwd'+str(i)))
 
 stakeholders[0].set_leader()
+stakeholders[1].set_endorser()
 environment = Z(stakeholders, EPOCH_LENGTH, genesis_time=time.time())
 environment.start()
 
 for sh in environment.stakeholders:
-    sh.beacon.join()
+    sh.beacon.join()