diff --git a/Makefile b/Makefile index 2ddc82e68..a262ac5e1 100644 --- a/Makefile +++ b/Makefile @@ -67,11 +67,11 @@ phase0: $(PY_SPEC_PHASE_0_TARGETS) # "make phase1" to create pyspec for phase1 phase1: $(PY_SPEC_PHASE_1_TARGETS) -$(PY_SPEC_DIR)/eth2spec/phase0/spec.py: $(PY_SPEC_PHASE_0_DEPS) - python3 $(SCRIPT_DIR)/phase0/build_spec.py $(SPEC_DIR)/core/0_beacon-chain.md $@ +$(PY_SPEC_PHASE_0_TARGETS): $(PY_SPEC_PHASE_0_DEPS) + python3 $(SCRIPT_DIR)/build_spec.py -p0 $(SPEC_DIR)/core/0_beacon-chain.md $@ $(PY_SPEC_DIR)/eth2spec/phase1/spec.py: $(PY_SPEC_PHASE_1_DEPS) - python3 $(SCRIPT_DIR)/phase0/build_spec.py -p1 $(SPEC_DIR)/core/1_custody-game.md $@ + python3 $(SCRIPT_DIR)/build_spec.py -p1 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/1_custody-game.md $@ CURRENT_DIR = ${CURDIR} diff --git a/scripts/phase0/__init__.py b/scripts/__init__.py similarity index 100% rename from scripts/phase0/__init__.py rename to scripts/__init__.py diff --git a/scripts/build_spec.py b/scripts/build_spec.py new file mode 100644 index 000000000..c77a8b82f --- /dev/null +++ b/scripts/build_spec.py @@ -0,0 +1,196 @@ +import sys +import re +import function_puller +from argparse import ArgumentParser +from typing import Tuple, List + + +def split_retain_delimiter(regex_pattern: str, text: str) -> List[str]: + ''' + Splits a string based on regex, but down not remove the matched text + ''' + find_pattern = r'%s.*?(?=%s|$)' % (regex_pattern, regex_pattern) + return re.findall(find_pattern, text, re.DOTALL) + + +def inserter(oldfile: str, newfile: str) -> Tuple[str, str]: + ''' + Searches for occurrences of @LabelName in oldfile and replaces them with instances of code wraped as follows: + # begin insert @LabelName + def foo(bar): + return bar + # end insert @LabelName + ''' + new_insert_objects = re.split(r"(# begin insert |# end insert @[a-zA-Z0-9_]*\n)", newfile) + # Retrieve label from insert objects + def get_labeled_object(labeled_text): + label = re.match(r"@[a-zA-Z0-9_]*\n", labeled_text) + if label is not None: + label = label.group(0) + labeled_text = re.sub(label, '', labeled_text) + return {'label': label, 'text': labeled_text} + new_insert_objects = map(get_labeled_object, new_insert_objects) + # Find and replace labels + newfile = "" + for item in new_insert_objects: + if item['label'] is not None: + oldfile, insertions = re.subn('# %s' % item['label'], item['text'], oldfile) + if insertions == 0: + newfile.join('# begin insert %s/n%s# end insert %s' % (item['label'], item['text'], item['label'])) + elif re.match(r"(# begin insert |# end insert )", item['text']) is None: + newfile += item['text'] + return oldfile, newfile + + +def merger(oldfile:str, newfile:str) -> str: + ''' + Seeks out functions and objects in new and old files. + Replaces old objects with new ones if they exist. + ''' + old_objects = split_retain_delimiter('\n[a-zA-Z]', oldfile) + new_objects = split_retain_delimiter('\n[a-zA-Z]', newfile) + object_regex = r"\n[#@a-zA-Z_0-9]+[\sa-zA-Z_0-9]*[(){}=:'" "]*" + old_object_tuples = list(map(lambda x: [re.match(object_regex, x).group(0),x], old_objects)) + for new_item in new_objects: + found_old = False + for old_item in old_object_tuples: + if old_item[0] == re.match(object_regex, new_item).group(0): + print(old_item[0]) + old_item[1] = new_item + found_old = True + break + if not found_old: + old_object_tuples += [[re.match(object_regex, new_item).group(0), new_item]] + return ''.join(elem for elem in map(lambda x: x[1], old_object_tuples)) + + +def build_phase0_spec(sourcefile, outfile=None): + code_lines = [] + code_lines.append(""" + +from typing import ( + Any, + Dict, + List, + NewType, + Tuple, +) +from eth2spec.utils.minimal_ssz import ( + SSZType, + hash_tree_root, + signing_root, +) +from eth2spec.utils.bls_stub import ( + bls_aggregate_pubkeys, + bls_verify, + bls_verify_multiple, +) +from eth2spec.utils.hash_function import hash + + +# stub, will get overwritten by real var +SLOTS_PER_EPOCH = 64 + +Slot = NewType('Slot', int) # uint64 +Epoch = NewType('Epoch', int) # uint64 +Shard = NewType('Shard', int) # uint64 +ValidatorIndex = NewType('ValidatorIndex', int) # uint64 +Gwei = NewType('Gwei', int) # uint64 +Bytes32 = NewType('Bytes32', bytes) # bytes32 +BLSPubkey = NewType('BLSPubkey', bytes) # bytes48 +BLSSignature = NewType('BLSSignature', bytes) # bytes96 +Store = None +""") + + code_lines += function_puller.get_spec(sourcefile) + + code_lines.append(""" +# Monkey patch validator compute committee code +_compute_committee = compute_committee +committee_cache = {} + + +def compute_committee(indices: List[ValidatorIndex], seed: Bytes32, index: int, count: int) -> List[ValidatorIndex]: + param_hash = (hash_tree_root(indices), seed, index, count) + + if param_hash in committee_cache: + return committee_cache[param_hash] + else: + ret = _compute_committee(indices, seed, index, count) + committee_cache[param_hash] = ret + return ret + + +# Monkey patch hash cache +_hash = hash +hash_cache = {} + + +def hash(x): + if x in hash_cache: + return hash_cache[x] + else: + ret = _hash(x) + hash_cache[x] = ret + return ret + + +# Access to overwrite spec constants based on configuration +def apply_constants_preset(preset: Dict[str, Any]): + global_vars = globals() + for k, v in preset.items(): + global_vars[k] = v + + # Deal with derived constants + global_vars['GENESIS_EPOCH'] = slot_to_epoch(GENESIS_SLOT) + + # Initialize SSZ types again, to account for changed lengths + init_SSZ_types() +""") + + if outfile is not None: + with open(outfile, 'w') as out: + out.write("\n".join(code_lines)) + else: + return "\n".join(code_lines) + + +def build_phase1_spec(phase0_sourcefile, phase1_sourcefile, outfile=None): + phase0_code = build_phase0_spec(phase0_sourcefile) + phase1_code = build_phase0_spec(phase1_sourcefile) + phase0_code, phase1_code = inserter(phase0_code, phase1_code) + phase1_code = merger(phase0_code, phase1_code) + + if outfile is not None: + with open(outfile, 'w') as out: + out.write(phase1_code) + else: + return phase1_code + + +if __name__ == '__main__': + description = ''' +Build the specs from the md docs. +If building phase 0: + 1st argument is input spec.md + 2nd argument is output spec.py + +If building phase 1: + 1st argument is input spec_phase0.md + 2nd argument is input spec_phase1.md + 3rd argument is output spec.py +''' + parser = ArgumentParser(description=description) + parser.add_argument("-p", "--phase", dest="phase", type=int, default=0, help="Build for phase #") + parser.add_argument(dest="files", help="Input and output files", nargs="+") + + args = parser.parse_args() + if args.phase == 0: + build_phase0_spec(*args.files) + elif args.phase == 1: + if len(args.files) == 3: + build_phase1_spec(*args.files) + else: + print(" Phase 1 requires an output as well as 2 input files (phase0.md and phase1.md)") + else: + print("Invalid phase: {0}".format(args.phase)) diff --git a/scripts/phase0/function_puller.py b/scripts/function_puller.py similarity index 100% rename from scripts/phase0/function_puller.py rename to scripts/function_puller.py diff --git a/scripts/phase0/build_spec.py b/scripts/phase0/build_spec.py deleted file mode 100644 index 4faf8cbf6..000000000 --- a/scripts/phase0/build_spec.py +++ /dev/null @@ -1,143 +0,0 @@ -import sys -import function_puller -from optparse import OptionParser - -def build_phase0_spec(sourcefile, outfile): - code_lines = [] - code_lines.append(""" - -from typing import ( - Any, - Dict, - List, - NewType, - Tuple, -) -from eth2spec.utils.minimal_ssz import ( - SSZType, - SSZTypeExtension, - hash_tree_root, - signing_root, -) -from eth2spec.utils.bls_stub import ( - bls_aggregate_pubkeys, - bls_verify, - bls_verify_multiple, -) -from eth2spec.utils.hash_function import hash - - -# stub, will get overwritten by real var -SLOTS_PER_EPOCH = 64 - -Slot = NewType('Slot', int) # uint64 -Epoch = NewType('Epoch', int) # uint64 -Shard = NewType('Shard', int) # uint64 -ValidatorIndex = NewType('ValidatorIndex', int) # uint64 -Gwei = NewType('Gwei', int) # uint64 -Bytes32 = NewType('Bytes32', bytes) # bytes32 -BLSPubkey = NewType('BLSPubkey', bytes) # bytes48 -BLSSignature = NewType('BLSSignature', bytes) # bytes96 -Store = None -""") - - code_lines += function_puller.get_spec(sourcefile) - - code_lines.append(""" -# Monkey patch validator compute committee code -_compute_committee = compute_committee -committee_cache = {} - - -def compute_committee(indices: List[ValidatorIndex], seed: Bytes32, index: int, count: int) -> List[ValidatorIndex]: - param_hash = (hash_tree_root(indices), seed, index, count) - - if param_hash in committee_cache: - return committee_cache[param_hash] - else: - ret = _compute_committee(indices, seed, index, count) - committee_cache[param_hash] = ret - return ret - - -# Monkey patch hash cache -_hash = hash -hash_cache = {} - - -def hash(x): - if x in hash_cache: - return hash_cache[x] - else: - ret = _hash(x) - hash_cache[x] = ret - return ret - - -# Access to overwrite spec constants based on configuration -def apply_constants_preset(preset: Dict[str, Any]): - global_vars = globals() - for k, v in preset.items(): - global_vars[k] = v - - # Deal with derived constants - global_vars['GENESIS_EPOCH'] = slot_to_epoch(GENESIS_SLOT) - - # Initialize SSZ types again, to account for changed lengths - init_SSZ_types() -""") - - with open(outfile, 'w') as out: - out.write("\n".join(code_lines)) - - -def build_phase1_spec(sourcefile, outfile): - code_lines = [] - code_lines.append(""" -from eth2spec.phase0.spec import * -from eth2spec.phase0.spec import apply_constants_preset as apply_constants_preset_phase0 - -""") - - code_lines += function_puller.get_spec(sourcefile) - - code_lines.append(""" -# Access to overwrite spec constants based on configuration -def apply_constants_preset(preset: Dict[str, Any]): - - apply_constants_preset_phase0(preset) - - global_vars = globals() - for k, v in preset.items(): - global_vars[k] = v - - # Deal with derived constants - global_vars['GENESIS_EPOCH'] = slot_to_epoch(GENESIS_SLOT) - - # Initialize SSZ types again, to account for changed lengths - init_SSZ_types() -""") - - with open(outfile, 'w') as out: - out.write("\n".join(code_lines)) - - -if __name__ == '__main__': - parser = OptionParser() - parser.add_option("-p", "--phase", dest="phase", type="int", default=0, - help="Build for phase #") - - (options, args) = parser.parse_args() - - if len(args) < 2: - parser.print_help() - - if options.phase == 0: - build_phase0_spec(args[0], args[1]) - print(args[0]) - print(args[1]) - elif options.phase == 1: - build_phase1_spec(args[0], args[1]) - else: - print("Invalid phase: {0}".format(options["phase"])) -