mirror of
https://github.com/ethereum/consensus-specs.git
synced 2026-01-09 01:38:01 -05:00
301 lines
10 KiB
Python
301 lines
10 KiB
Python
import copy
|
|
import logging
|
|
import os
|
|
import string
|
|
import warnings
|
|
from collections import OrderedDict
|
|
from collections.abc import Sequence
|
|
from distutils import dir_util
|
|
from distutils.util import convert_path
|
|
from functools import cache
|
|
from pathlib import Path
|
|
from typing import cast
|
|
|
|
from ruamel.yaml import YAML
|
|
from setuptools import Command, find_packages, setup
|
|
from setuptools.command.build_py import build_py
|
|
|
|
from pysetup.constants import (
|
|
PHASE0,
|
|
)
|
|
from pysetup.helpers import (
|
|
combine_spec_objects,
|
|
dependency_order_class_objects,
|
|
objects_to_spec,
|
|
parse_config_vars,
|
|
)
|
|
from pysetup.md_doc_paths import get_md_doc_paths
|
|
from pysetup.md_to_spec import MarkdownToSpec
|
|
from pysetup.spec_builders import spec_builders
|
|
from pysetup.typing import (
|
|
BuildTarget,
|
|
SpecObject,
|
|
)
|
|
|
|
# Ignore '1.5.0-alpha.*' to '1.5.0a*' messages.
|
|
warnings.filterwarnings("ignore", message="Normalizing .* to .*")
|
|
|
|
|
|
# Ignore 'running' and 'creating' messages
|
|
class PyspecFilter(logging.Filter):
|
|
def filter(self, record: logging.LogRecord) -> bool:
|
|
return not record.getMessage().startswith(("running ", "creating "))
|
|
|
|
|
|
logging.getLogger().addFilter(PyspecFilter())
|
|
|
|
|
|
def get_spec(
|
|
file_name: Path,
|
|
preset: dict[str, str],
|
|
config: dict[str, str | list[dict[str, str]]],
|
|
preset_name: str,
|
|
) -> SpecObject:
|
|
return MarkdownToSpec(file_name, preset, config, preset_name).run()
|
|
|
|
|
|
@cache
|
|
def load_preset(preset_files: Sequence[Path]) -> dict[str, str]:
|
|
"""
|
|
Loads a directory of preset files, merges the result into one preset.
|
|
"""
|
|
preset: dict[str, str] = {}
|
|
for fork_file in preset_files:
|
|
yaml = YAML(typ="base")
|
|
fork_preset: dict = yaml.load(fork_file)
|
|
if fork_preset is None: # for empty YAML files
|
|
continue
|
|
if not set(fork_preset.keys()).isdisjoint(preset.keys()):
|
|
duplicates = set(fork_preset.keys()).intersection(set(preset.keys()))
|
|
raise Exception(f"duplicate config var(s) in preset files: {', '.join(duplicates)}")
|
|
preset.update(fork_preset)
|
|
assert preset != {}
|
|
return cast(dict[str, str], parse_config_vars(preset))
|
|
|
|
|
|
@cache
|
|
def load_config(config_path: Path) -> dict[str, str | list[dict[str, str]]]:
|
|
"""
|
|
Loads the given configuration file.
|
|
"""
|
|
yaml = YAML(typ="base")
|
|
config_data = yaml.load(config_path)
|
|
return parse_config_vars(config_data)
|
|
|
|
|
|
def build_spec(
|
|
fork: str,
|
|
preset_name: str,
|
|
source_files: Sequence[Path],
|
|
preset_files: Sequence[Path],
|
|
config_file: Path,
|
|
) -> str:
|
|
preset = load_preset(tuple(preset_files))
|
|
config = load_config(config_file)
|
|
all_specs = [get_spec(spec, preset, config, preset_name) for spec in source_files]
|
|
|
|
spec_object = all_specs[0]
|
|
for value in all_specs[1:]:
|
|
spec_object = combine_spec_objects(spec_object, value)
|
|
|
|
class_objects = {**spec_object.ssz_objects, **spec_object.dataclasses}
|
|
|
|
# Ensure it's ordered after multiple forks
|
|
new_objects: dict[str, str] = {}
|
|
while OrderedDict(new_objects) != OrderedDict(class_objects):
|
|
new_objects = copy.deepcopy(class_objects)
|
|
dependency_order_class_objects(
|
|
class_objects,
|
|
spec_object.custom_types | spec_object.preset_dep_custom_types,
|
|
)
|
|
|
|
return objects_to_spec(preset_name, spec_object, fork, class_objects)
|
|
|
|
|
|
class PySpecCommand(Command):
|
|
"""Convert spec markdown files to a spec python file"""
|
|
|
|
description = "Convert spec markdown files to a spec python file"
|
|
|
|
spec_fork: str
|
|
md_doc_paths: str
|
|
parsed_md_doc_paths: list[Path]
|
|
build_targets: str
|
|
parsed_build_targets: list[BuildTarget]
|
|
out_dir: str
|
|
|
|
# The format is (long option, short option, description).
|
|
user_options = [
|
|
("spec-fork=", None, "Spec fork to tag build with. Used to select md-docs defaults."),
|
|
("md-doc-paths=", None, "List of paths of markdown files to build spec with"),
|
|
(
|
|
"build-targets=",
|
|
None,
|
|
"Names, directory paths of compile-time presets, and default config paths.",
|
|
),
|
|
("out-dir=", None, "Output directory to write spec package to"),
|
|
]
|
|
|
|
def initialize_options(self) -> None:
|
|
"""Set default values for options."""
|
|
# Each user option must be listed here with their default value.
|
|
self.spec_fork = PHASE0
|
|
self.md_doc_paths = ""
|
|
self.out_dir = "pyspec_output"
|
|
self.build_targets = """
|
|
minimal:presets/minimal:configs/minimal.yaml
|
|
mainnet:presets/mainnet:configs/mainnet.yaml
|
|
"""
|
|
|
|
def finalize_options(self) -> None:
|
|
"""Post-process options."""
|
|
if len(self.md_doc_paths) == 0:
|
|
self.md_doc_paths = get_md_doc_paths(self.spec_fork)
|
|
if len(self.md_doc_paths) == 0:
|
|
raise Exception(
|
|
f"No markdown files specified, and spec fork {self.spec_fork!r} is unknown"
|
|
)
|
|
|
|
self.parsed_md_doc_paths = [Path(p) for p in self.md_doc_paths.split()]
|
|
|
|
for filename in self.parsed_md_doc_paths:
|
|
if not os.path.exists(filename):
|
|
raise Exception(f"Pyspec markdown input file {filename!r} does not exist")
|
|
|
|
self.parsed_build_targets = []
|
|
for target in self.build_targets.split():
|
|
target = target.strip()
|
|
data = target.split(":")
|
|
if len(data) != 3:
|
|
raise Exception(
|
|
f"invalid target, expected 'name:preset_dir:config_file' format, but got: {target}"
|
|
)
|
|
name, preset_dir_path, config_path = data
|
|
if any((c not in string.digits + string.ascii_letters) for c in name):
|
|
raise Exception(f"invalid target name: {name!r}")
|
|
if not os.path.exists(preset_dir_path):
|
|
raise Exception(f"Preset dir {preset_dir_path!r} does not exist")
|
|
_, _, preset_file_names = next(os.walk(preset_dir_path))
|
|
preset_paths = [(Path(preset_dir_path) / name) for name in preset_file_names]
|
|
|
|
if not os.path.exists(config_path):
|
|
raise Exception(f"Config file {config_path!r} does not exist")
|
|
self.parsed_build_targets.append(BuildTarget(name, preset_paths, Path(config_path)))
|
|
|
|
def run(self) -> None:
|
|
if not self.dry_run:
|
|
dir_util.mkpath(self.out_dir)
|
|
|
|
print(f"Building pyspec: {self.spec_fork}")
|
|
for name, preset_paths, config_path in self.parsed_build_targets:
|
|
spec_str = build_spec(
|
|
spec_builders[self.spec_fork].fork,
|
|
name,
|
|
self.parsed_md_doc_paths,
|
|
preset_paths,
|
|
config_path,
|
|
)
|
|
if self.dry_run:
|
|
self.announce(
|
|
"dry run successfully prepared contents for spec."
|
|
f' out dir: "{self.out_dir}", spec fork: "{self.spec_fork}", build target: "{name}"'
|
|
)
|
|
self.debug_print(spec_str)
|
|
else:
|
|
with open(os.path.join(self.out_dir, name + ".py"), "w") as out:
|
|
out.write(spec_str)
|
|
|
|
if not self.dry_run:
|
|
with open(os.path.join(self.out_dir, "__init__.py"), "w") as out:
|
|
# `mainnet` is the default spec.
|
|
out.write("from . import mainnet as spec # noqa:F401\n")
|
|
|
|
|
|
class BuildPyCommand(build_py):
|
|
"""Customize the build command to run the spec-builder on setup.py build"""
|
|
|
|
def initialize_options(self) -> None:
|
|
super().initialize_options()
|
|
|
|
def run_pyspec_cmd(self, spec_fork: str) -> None:
|
|
cmd_obj = cast(PySpecCommand, self.distribution.reinitialize_command("pyspec"))
|
|
cmd_obj.spec_fork = spec_fork
|
|
cmd_obj.out_dir = os.path.join(self.build_lib, "eth2spec", spec_fork)
|
|
self.run_command("pyspec")
|
|
|
|
def run(self) -> None:
|
|
for spec_fork in spec_builders:
|
|
self.run_pyspec_cmd(spec_fork=spec_fork)
|
|
|
|
super().run()
|
|
|
|
|
|
class PyspecDevCommand(Command):
|
|
"""Build the markdown files in-place to their source location for testing."""
|
|
|
|
description = "Build the markdown files in-place to their source location for testing."
|
|
|
|
def initialize_options(self) -> None:
|
|
pass
|
|
|
|
def finalize_options(self) -> None:
|
|
pass
|
|
|
|
def run_pyspec_cmd(self, spec_fork: str) -> None:
|
|
cmd_obj = cast(PySpecCommand, self.distribution.reinitialize_command("pyspec"))
|
|
cmd_obj.spec_fork = spec_fork
|
|
eth2spec_dir = convert_path(self.distribution.package_dir["eth2spec"])
|
|
cmd_obj.out_dir = os.path.join(eth2spec_dir, spec_fork)
|
|
self.run_command("pyspec")
|
|
|
|
def run(self) -> None:
|
|
for spec_fork in spec_builders:
|
|
self.run_pyspec_cmd(spec_fork=spec_fork)
|
|
|
|
|
|
commands = {
|
|
"pyspec": PySpecCommand,
|
|
"build_py": BuildPyCommand,
|
|
"pyspecdev": PyspecDevCommand,
|
|
}
|
|
|
|
with open("README.md", encoding="utf8") as f:
|
|
readme = f.read()
|
|
|
|
# How to use "VERSION.txt" file:
|
|
# - dev branch contains "X.Y.Z.dev", where "X.Y.Z" is the target version to release dev into.
|
|
# -> Changed as part of 'master' backport to 'dev'
|
|
# - master branch contains "X.Y.Z", where "X.Y.Z" is the current version.
|
|
# -> Changed as part of 'dev' release (or other branch) into 'master'
|
|
# -> In case of a commit on master without git tag, target the next version
|
|
# with ".postN" (release candidate, numbered) suffixed.
|
|
# See https://www.python.org/dev/peps/pep-0440/#public-version-identifiers
|
|
with open(os.path.join("tests", "core", "pyspec", "eth2spec", "VERSION.txt")) as f:
|
|
spec_version = f.read().strip()
|
|
|
|
setup(
|
|
version=spec_version,
|
|
long_description=readme,
|
|
long_description_content_type="text/markdown",
|
|
url="https://github.com/ethereum/consensus-specs",
|
|
include_package_data=False,
|
|
package_data={
|
|
"configs": ["*.yaml"],
|
|
"eth2spec": ["VERSION.txt"],
|
|
"presets": ["**/*.yaml", "**/*.json"],
|
|
"specs": ["**/*.md"],
|
|
"sync": ["optimistic.md"],
|
|
},
|
|
package_dir={
|
|
"configs": "configs",
|
|
"eth2spec": "tests/core/pyspec/eth2spec",
|
|
"presets": "presets",
|
|
"specs": "specs",
|
|
"sync": "sync",
|
|
},
|
|
packages=find_packages(where="tests/core/pyspec")
|
|
+ ["configs", "presets", "specs", "presets", "sync"],
|
|
py_modules=["eth2spec"],
|
|
cmdclass=commands,
|
|
)
|