feat: Reproducible builds and *.deb packages (#19678)

Signed-off-by: bakhtin <a@bakhtin.net>

.dockerignore

@@ -4,6 +4,7 @@
 # include source files
 !/bin
 !/crates
+!/pkg
 !/testing
 !book.toml
 !Cargo.lock
@@ -11,6 +12,7 @@
 !Cross.toml
 !deny.toml
 !Makefile
+!README.md
 
 # include for vergen constants
 !/.git

.github/workflows/release-reproducible.yml

@@ -1,11 +1,11 @@
-# This workflow is for building and pushing reproducible Docker images for releases.
+# This workflow is for building and pushing reproducible artifacts for releases
 
 name: release-reproducible
 
 on:
-  push:
+  workflow_run:
-    tags:
+    workflows: [release]
-      - v*
+    types: [completed]
 
 env:
   DOCKER_REPRODUCIBLE_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/reth-reproducible
@@ -13,23 +13,41 @@ env:
 jobs:
   extract-version:
     name: extract version
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
     runs-on: ubuntu-latest
     steps:
-      - name: Extract version
+      - name: Extract version from triggering tag
-        run: echo "VERSION=$(echo ${GITHUB_REF#refs/tags/})" >> $GITHUB_OUTPUT
         id: extract_version
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          # Get the tag that points to the head SHA of the triggering workflow
+          TAG=$(gh api /repos/${{ github.repository }}/git/refs/tags \
+            --jq '.[] | select(.object.sha == "${{ github.event.workflow_run.head_sha }}") | .ref' \
+            | head -1 \
+            | sed 's|refs/tags/||')
+
+          if [ -z "$TAG" ]; then
+            echo "No tag found for SHA ${{ github.event.workflow_run.head_sha }}"
+            exit 1
+          fi
+
+          echo "VERSION=$TAG" >> $GITHUB_OUTPUT
     outputs:
       VERSION: ${{ steps.extract_version.outputs.VERSION }}
 
   build-reproducible:
-    name: build and push reproducible image
+    name: build and push reproducible image and binaries
     runs-on: ubuntu-latest
-    needs: extract-version
+    needs: [extract-version]
     permissions:
       packages: write
-      contents: read
+      contents: write
     steps:
       - uses: actions/checkout@v6
+        with:
+          ref: ${{ needs.extract-version.outputs.VERSION }}
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
@@ -40,12 +58,37 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Build and push reproducible image
+      - name: Extract Rust version
+        id: rust_version
+        run: |
+          RUST_TOOLCHAIN=$(rustc --version | cut -d' ' -f2)
+          echo "RUST_TOOLCHAIN=$RUST_TOOLCHAIN" >> $GITHUB_OUTPUT
+
+      - name: Build reproducible artifacts
+        uses: docker/build-push-action@v6
+        id: docker_build
+        with:
+          context: .
+          file: ./Dockerfile.reproducible
+          build-args: |
+            RUST_TOOLCHAIN=${{ steps.rust_version.outputs.RUST_TOOLCHAIN }}
+            VERSION=${{ needs.extract-version.outputs.VERSION }}
+          target: artifacts
+          outputs: type=local,dest=./docker-artifacts
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+        env:
+          DOCKER_BUILD_RECORD_UPLOAD: false
+
+      - name: Build and push final image
         uses: docker/build-push-action@v6
         with:
           context: .
           file: ./Dockerfile.reproducible
           push: true
+          build-args: |
+            RUST_TOOLCHAIN=${{ steps.rust_version.outputs.RUST_TOOLCHAIN }}
+            VERSION=${{ needs.extract-version.outputs.VERSION }}
           tags: |
             ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${{ needs.extract-version.outputs.VERSION }}
             ${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:latest
@@ -54,3 +97,30 @@ jobs:
           provenance: false
         env:
           DOCKER_BUILD_RECORD_UPLOAD: false
+
+      - name: Prepare artifacts from Docker build
+        run: |
+          mkdir reproducible-artifacts
+          cp docker-artifacts/reth reproducible-artifacts/reth-reproducible-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu
+          cp docker-artifacts/*.deb reproducible-artifacts/reth-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu-reproducible.deb
+
+      - name: Configure GPG and create artifacts
+        env:
+          GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }}
+          GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
+        run: |
+          export GPG_TTY=$(tty)
+          echo -n "$GPG_SIGNING_KEY" | base64 --decode | gpg --batch --import
+
+          cd reproducible-artifacts
+          tar -czf reth-reproducible-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu.tar.gz reth-reproducible-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu --remove-files
+          echo "$GPG_PASSPHRASE" | gpg --passphrase-fd 0 --pinentry-mode loopback --batch -ab reth-reproducible-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu.tar.gz
+          echo "$GPG_PASSPHRASE" | gpg --passphrase-fd 0 --pinentry-mode loopback --batch -ab reth-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu-reproducible.deb
+
+      - name: Upload reproducible artifacts to release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh release upload ${{ needs.extract-version.outputs.VERSION }} \
+            reproducible-artifacts/*
+

.github/workflows/reproducible-build.yml

@@ -8,31 +8,73 @@ on:
 jobs:
   build:
     name: build reproducible binaries
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      matrix:
+        include:
+          - runner: ubuntu-latest
+            machine: machine-1
+          - runner: ubuntu-22.04
+            machine: machine-2
     steps:
       - uses: actions/checkout@v6
-      - uses: rui314/setup-mold@v1
+
       - uses: dtolnay/rust-toolchain@stable
         with:
           target: x86_64-unknown-linux-gnu
-      - name: Install cross main
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build reproducible binary with Docker
         run: |
-          cargo install cross --git https://github.com/cross-rs/cross
+          RUST_TOOLCHAIN=$(rustc --version | cut -d' ' -f2)
-      - name: Install cargo-cache
+          docker build \
+            --build-arg "RUST_TOOLCHAIN=${RUST_TOOLCHAIN}" \
+            -f Dockerfile.reproducible -t reth:release \
+            --target artifacts \
+            --output type=local,dest=./target .
+
+      - name: Calculate SHA256
+        id: sha256
         run: |
-          cargo install cargo-cache
+          sha256sum target/reth > checksum.sha256
-      - uses: Swatinem/rust-cache@v2
+          echo "Binaries SHA256 on ${{ matrix.machine }}: $(cat checksum.sha256)"
+
+      - name: Upload the hash
+        uses: actions/upload-artifact@v4
         with:
-          cache-on-failure: true
+          name: checksum-${{ matrix.machine }}
-      - name: Build Reth
+          path: |
+            checksum.sha256
+          retention-days: 1
+
+  compare:
+    name: compare reproducible binaries
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download artifacts from machine-1
+        uses: actions/download-artifact@v4
+        with:
+          name: checksum-machine-1
+          path: machine-1/
+      - name: Download artifacts from machine-2
+        uses: actions/download-artifact@v4
+        with:
+          name: checksum-machine-2
+          path: machine-2/
+      - name: Compare SHA256 hashes
         run: |
-          make build-reproducible
+          echo "=== SHA256 Comparison ==="
-          mv target/x86_64-unknown-linux-gnu/release/reth reth-build-1
+          echo "Machine 1 hash:"
-      - name: Clean cache
+          cat machine-1/checksum.sha256
-        run: make clean && cargo cache -a
+          echo "Machine 2 hash:"
-      - name: Build Reth again
+          cat machine-2/checksum.sha256
-        run: |
+
-          make build-reproducible
+          if cmp -s machine-1/checksum.sha256 machine-2/checksum.sha256; then
-          mv target/x86_64-unknown-linux-gnu/release/reth reth-build-2
+            echo "✅ SUCCESS: Binaries are identical (reproducible build verified)"
-      - name: Compare binaries
+          else
-        run: cmp reth-build-1 reth-build-2
+            echo "❌ FAILURE: Binaries differ (reproducible build failed)"
+            exit 1
+          fi

Cargo.toml

@@ -328,6 +328,12 @@ inherits = "release"
 lto = "fat"
 codegen-units = 1
 
+[profile.reproducible]
+inherits = "release"
+panic = "abort"
+codegen-units = 1
+incremental = false
+
 [workspace.dependencies]
 # reth
 op-reth = { path = "crates/optimism/bin" }

Dockerfile.reproducible

@@ -1,20 +1,25 @@
-# Use the Rust 1.88 image based on Debian Bookworm
+ARG RUST_TOOLCHAIN=1.89.0
-FROM rust:1.88-bookworm AS builder
+FROM docker.io/rust:$RUST_TOOLCHAIN-trixie AS builder
 
-# Install specific version of libclang-dev
+ARG PROFILE
-RUN apt-get update && apt-get install -y libclang-dev=1:14.0-55.7~deb12u1
+ARG VERSION
-
+# Switch to snapshot repository to pin dependencies
-# Copy the project to the container
+RUN sed -i '/^# http/{N;s|^# \(http[^ ]*\)\nURIs: .*|# \1\nURIs: \1|}' /etc/apt/sources.list.d/debian.sources
-COPY ./ /app
+RUN apt-get -o Acquire::Check-Valid-Until=false update && \
+    apt-get install -y \
+        libjemalloc-dev \
+        libclang-dev \
+        mold
 WORKDIR /app
+COPY . .
+RUN RUSTFLAGS_REPRODUCIBLE_EXTRA="-Clink-arg=-fuse-ld=mold" make build-reth-reproducible && \
+  PROFILE=${PROFILE:-reproducible} VERSION=$VERSION make build-deb-x86_64-unknown-linux-gnu
 
-# Build the project with the reproducible settings
+FROM scratch AS artifacts
-RUN make build-reproducible
+COPY --from=builder /app/target/x86_64-unknown-linux-gnu/reproducible/reth /reth
+COPY --from=builder /app/target/x86_64-unknown-linux-gnu/reproducible/*.deb /
 
-RUN mv /app/target/x86_64-unknown-linux-gnu/release/reth /reth
+FROM gcr.io/distroless/cc-debian13:nonroot-239cdd2c8a6b275b6a6f6ed1428c57de2fff3e50
-
+COPY --from=artifacts /reth /reth
-# Create a minimal final image with just the binary
-FROM gcr.io/distroless/cc-debian12:nonroot-6755e21ccd99ddead6edc8106ba03888cbeed41a
-COPY --from=builder /reth /reth
 EXPOSE 30303 30303/udp 9001 8545 8546
 ENTRYPOINT [ "/reth" ]

Makefile

@@ -65,33 +65,24 @@ build: ## Build the reth binary into `target` directory.
 	cargo build --bin reth --features "$(FEATURES)" --profile "$(PROFILE)"
 
 # Environment variables for reproducible builds
-# Initialize RUSTFLAGS
-RUST_BUILD_FLAGS =
-# Enable static linking to ensure reproducibility across builds
-RUST_BUILD_FLAGS += --C target-feature=+crt-static
-# Set the linker to use static libgcc to ensure reproducibility across builds
-RUST_BUILD_FLAGS += -C link-arg=-static-libgcc
-# Remove build ID from the binary to ensure reproducibility across builds
-RUST_BUILD_FLAGS += -C link-arg=-Wl,--build-id=none
-# Remove metadata hash from symbol names to ensure reproducible builds
-RUST_BUILD_FLAGS += -C metadata=''
 # Set timestamp from last git commit for reproducible builds
 SOURCE_DATE ?= $(shell git log -1 --pretty=%ct)
-# Disable incremental compilation to avoid non-deterministic artifacts
-CARGO_INCREMENTAL_VAL = 0
-# Set C locale for consistent string handling and sorting
-LOCALE_VAL = C
-# Set UTC timezone for consistent time handling across builds
-TZ_VAL = UTC
 
-.PHONY: build-reproducible
+# Extra RUSTFLAGS for reproducible builds. Can be overridden via the environment.
-build-reproducible: ## Build the reth binary into `target` directory with reproducible builds. Only works for x86_64-unknown-linux-gnu currently
+RUSTFLAGS_REPRODUCIBLE_EXTRA ?=
+
+# `reproducible` only supports reth on x86_64-unknown-linux-gnu
+build-%-reproducible:
+	@if [ "$*" != "reth" ]; then \
+		echo "Error: Reproducible builds are only supported for reth, not $*"; \
+		exit 1; \
+	fi
 	SOURCE_DATE_EPOCH=$(SOURCE_DATE) \
-	RUSTFLAGS="${RUST_BUILD_FLAGS} --remap-path-prefix $$(pwd)=." \
+	RUSTFLAGS="-C symbol-mangling-version=v0 -C strip=none -C link-arg=-Wl,--build-id=none -C metadata='' --remap-path-prefix $$(pwd)=. $(RUSTFLAGS_REPRODUCIBLE_EXTRA)" \
-	CARGO_INCREMENTAL=${CARGO_INCREMENTAL_VAL} \
+	LC_ALL=C \
-	LC_ALL=${LOCALE_VAL} \
+	TZ=UTC \
-	TZ=${TZ_VAL} \
+	JEMALLOC_OVERRIDE=/usr/lib/x86_64-linux-gnu/libjemalloc.a \
-	cargo build --bin reth --features "$(FEATURES)" --profile "release" --locked --target x86_64-unknown-linux-gnu
+	cargo build --bin reth --features "$(FEATURES) jemalloc-unprefixed" --profile "reproducible" --locked --target x86_64-unknown-linux-gnu
 
 .PHONY: build-debug
 build-debug: ## Build the reth binary into `target/debug` directory.
@@ -155,6 +146,22 @@ op-build-x86_64-apple-darwin:
 op-build-aarch64-apple-darwin:
 	$(MAKE) op-build-native-aarch64-apple-darwin
 
+build-deb-%:
+	@case "$*" in \
+		x86_64-unknown-linux-gnu|aarch64-unknown-linux-gnu|riscv64gc-unknown-linux-gnu) \
+			echo "Building debian package for $*"; \
+			;; \
+		*) \
+			echo "Error: Debian packages are only supported for x86_64-unknown-linux-gnu, aarch64-unknown-linux-gnu, and riscv64gc-unknown-linux-gnu, not $*"; \
+			exit 1; \
+			;; \
+	esac
+	cargo install cargo-deb@3.6.0 --locked
+	cargo deb --profile $(PROFILE) --no-build --no-dbgsym --no-strip \
+		--target $* \
+		$(if $(VERSION),--deb-version "1~$(VERSION)") \
+		$(if $(VERSION),--output "target/$*/$(PROFILE)/reth-$(VERSION)-$*-$(PROFILE).deb")
+
 # Create a `.tar.gz` containing a binary for a specific target.
 define tarball_release_binary
 	cp $(CARGO_TARGET_DIR)/$(1)/$(PROFILE)/$(2) $(BIN_DIR)/$(2)

bin/reth/Cargo.toml

@@ -9,6 +9,20 @@ repository.workspace = true
 description = "Reth node implementation"
 default-run = "reth"
 
+[package.metadata.deb]
+maintainer = "reth team"
+depends = "$auto"
+section = "network"
+priority = "optional"
+maintainer-scripts = "../../pkg/reth/debian/"
+assets = [
+    "$auto",
+    ["../../README.md", "usr/share/doc/reth/", "644"],
+    ["../../LICENSE-APACHE", "usr/share/doc/reth/", "644"],
+    ["../../LICENSE-MIT", "usr/share/doc/reth/", "644"],
+]
+systemd-units = { enable = false, start = false, unit-name = "reth", unit-scripts = "../../pkg/reth/debian" }
+
 [lints]
 workspace = true
 
@@ -100,6 +114,12 @@ jemalloc-prof = [
     "reth-cli-util/jemalloc-prof",
     "reth-ethereum-cli/jemalloc-prof",
 ]
+jemalloc-unprefixed = [
+    "reth-cli-util/jemalloc-unprefixed",
+    "reth-node-core/jemalloc",
+    "reth-node-metrics/jemalloc",
+    "reth-ethereum-cli/jemalloc",
+]
 tracy-allocator = [
     "reth-cli-util/tracy-allocator",
     "reth-ethereum-cli/tracy-allocator",

crates/cli/util/Cargo.toml

@@ -42,6 +42,9 @@ jemalloc = ["dep:tikv-jemallocator"]
 # Enables jemalloc profiling features
 jemalloc-prof = ["jemalloc", "tikv-jemallocator?/profiling"]
 
+# Enables unprefixed malloc (reproducible builds support)
+jemalloc-unprefixed = ["jemalloc", "tikv-jemallocator?/unprefixed_malloc_on_supported_platforms"]
+
 # Wraps the selected allocator in the tracy profiling allocator
 tracy-allocator = ["dep:tracy-client"]
 

pkg/reth/debian/reth.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=Modular, contributor-friendly and blazing-fast implementation of the Ethereum protocol
+Wants=network-online.target
+After=network.target network-online.target
+
+[Service]
+Type=exec
+DynamicUser=yes
+StateDirectory=reth
+ExecStart=/usr/bin/reth node --datadir %S/reth --log.file.max-files 0
+
+[Install]
+WantedBy=multi-user.target