Merge commit 'b517b69' into multihash-poseidon2

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

.github/actions/add_comment/action.yml

@@ -0,0 +1,41 @@
+name: Add Comment
+description: "Add or update comment in the PR"
+inputs:
+  marker:
+    description: "Text used to find the comment to update"
+    required: true
+  markdown_path:
+    description: "Path to the file containing markdown"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Add/Update Comment
+      uses: actions/github-script@v7
+      with:
+        script: |
+          const fs = require('fs');
+          const marker = "${{ inputs.marker }}";
+          const body = fs.readFileSync("${{ inputs.markdown_path }}", 'utf8');
+          const { data: comments } = await github.rest.issues.listComments({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            issue_number: context.issue.number,
+          });
+          const existing = comments.find(c => c.body && c.body.startsWith(marker));
+          if (existing) {
+            await github.rest.issues.updateComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              comment_id: existing.id,
+              body,
+            });
+          } else {
+            await github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+              body,
+            });
+          }

.github/workflows/daily_amd64.yml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   test_amd64_latest:
-    name: Daily amd64 (latest dependencies)
+    name: Daily test amd64 (latest dependencies)
     uses: ./.github/workflows/daily_common.yml
     with:
       nim: "[
@@ -17,7 +17,7 @@ jobs:
         ]"
       cpu: "['amd64']"
   test_amd64_pinned:
-    name: Daily amd64 (pinned dependencies)
+    name: Daily test amd64 (pinned dependencies)
     uses: ./.github/workflows/daily_common.yml
     with:
       pinned_deps: true

.github/workflows/daily_i386.yml

@@ -6,8 +6,8 @@ on:
   workflow_dispatch:
 
 jobs:
-  test_i386:
-    name: Daily i386 (Linux)
+  test_i386_latest:
+    name: Daily i386 (latest dependencies)
     uses: ./.github/workflows/daily_common.yml
     with:
       nim: "[
@@ -20,9 +20,24 @@ jobs:
           {'platform': {'os':'macos'}},
           {'platform': {'os':'windows'}},
         ]"
+  test_i386_pinned:
+    name: Daily i386 (pinned dependencies)
+    uses: ./.github/workflows/daily_common.yml
+    with:
+      pinned_deps: true
+      nim: "[
+          {'ref': 'version-2-0', 'memory_management': 'refc'},
+          {'ref': 'version-2-2', 'memory_management': 'refc'},
+          {'ref': 'devel', 'memory_management': 'refc'},
+        ]"
+      cpu: "['i386']"
+      exclude: "[
+          {'platform': {'os':'macos'}},
+          {'platform': {'os':'windows'}},
+        ]"
   notify-on-failure:
     name: Notify Discord on Failure
-    needs: [test_i386]
+    needs: [test_i386_latest, test_i386_pinned]
     if: failure()
     runs-on: ubuntu-latest
     steps:

.github/workflows/daily_nimbus.yml

@@ -0,0 +1,39 @@
+name: Daily Nimbus
+
+on:
+  schedule:
+    - cron: "30 6 * * *"
+  workflow_dispatch:
+
+jobs:
+  compile_nimbus:
+    timeout-minutes: 80
+    name: 'Compile Nimbus (linux-amd64)'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Compile nimbus using nim-libp2p
+        run: |
+          git clone --branch unstable --single-branch https://github.com/status-im/nimbus-eth2.git
+          cd nimbus-eth2
+          git submodule set-branch --branch ${{ github.sha }} vendor/nim-libp2p
+
+          make -j"$(nproc)"
+          make -j"$(nproc)" nimbus_beacon_node
+
+  notify-on-failure:
+    name: Notify Discord on Failure
+    needs: compile_nimbus
+    if: failure()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Discord notification
+        uses: ./.github/actions/discord_notify
+        with:
+          webhook_url: ${{ secrets.DISCORD_WEBHOOK_URL }}
+

.github/workflows/performance.yml

@@ -0,0 +1,69 @@
+name: Performance
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+  merge_group:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  examples:
+    timeout-minutes: 10
+    strategy:
+      fail-fast: false
+
+    defaults:
+      run:
+        shell: bash
+
+    name: "Performance"
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build Docker Image with cache
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: performance/Dockerfile
+          tags: test-node:latest
+          load: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Run
+        run: |
+          ./performance/runner.sh
+
+      - name: Set up Nim for aggragate script
+        uses: jiro4989/setup-nim-action@v2
+        with:
+          nim-version: "2.x"
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Aggregate and display summary
+        env:
+          MARKER: "<!-- perf-summary-marker -->"
+          PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+          COMMENT_SUMMARY_PATH: "/tmp/perf-summary.md"
+        run: |
+          nim c -r -d:release -o:/tmp/aggregate_stats ./performance/aggregate_stats.nim
+
+      - name: Post/Update PR Performance Comment
+        if: github.event_name == 'pull_request'
+        uses: ./.github/actions/add_comment
+        with:
+          marker: "<!-- perf-summary-marker -->"
+          markdown_path: "/tmp/perf-summary.md"

.pinned

@@ -1,4 +1,5 @@
 bearssl;https://github.com/status-im/nim-bearssl@#34d712933a4e0f91f5e66bc848594a581504a215
+blscurve;https://github.com/status-im/nim-blscurve@#52ae4332c749d89fa05226f5493decae568f682c
 chronicles;https://github.com/status-im/nim-chronicles@#61759a5e8df8f4d68bcd1b4b8c1adab3e72bbd8d
 chronos;https://github.com/status-im/nim-chronos@#b55e2816eb45f698ddaca8d8473e401502562db2
 dnsclient;https://github.com/ba0f3/dnsclient.nim@#23214235d4784d24aceed99bbfe153379ea557c8
@@ -19,3 +20,5 @@ websock;https://github.com/status-im/nim-websock@#d5cd89062cd2d168ef35193c7d29d2
 zlib;https://github.com/status-im/nim-zlib@#daa8723fd32299d4ca621c837430c29a5a11e19a
 jwt;https://github.com/vacp2p/nim-jwt@#18f8378de52b241f321c1f9ea905456e89b95c6f
 bearssl_pkey_decoder;https://github.com/vacp2p/bearssl_pkey_decoder@#21dd3710df9345ed2ad8bf8f882761e07863b8e0
+constantine;https://github.com/mratsim/constantine@#v0.2.0
+poseidon2;https://github.com/codex-storage/nim-poseidon2@#e173dd817b794d2bdadaa7ed45583798aaa91f0d

config.nims

@@ -11,6 +11,7 @@ switch("warning", "LockLevel:off")
 --styleCheck:
   usages
 switch("warningAsError", "UseBase:on")
+--excludePath:nimbledeps/
 --styleCheck:
   error
 --mm:

libp2p.nimble

@@ -1,7 +1,7 @@
 mode = ScriptMode.Verbose
 
 packageName = "libp2p"
-version = "1.11.0"
+version = "1.12.0"
 author = "Status Research & Development GmbH"
 description = "LibP2P implementation"
 license = "MIT"
@@ -10,7 +10,8 @@ skipDirs = @["tests", "examples", "Nim", "tools", "scripts", "docs"]
 requires "nim >= 2.0.0",
   "nimcrypto >= 0.6.0 & < 0.7.0", "dnsclient >= 0.3.0 & < 0.4.0", "bearssl >= 0.2.5",
   "chronicles >= 0.11.0 & < 0.12.0", "chronos >= 4.0.4", "metrics", "secp256k1",
-  "stew >= 0.4.0", "websock >= 0.2.0", "unittest2", "results", "quic >= 0.2.10",
+  "stew >= 0.4.0", "websock >= 0.2.0", "unittest2", "blscurve >= 0.0.1", "results",
+  "quic >= 0.2.10",
   "https://github.com/vacp2p/nim-jwt.git#18f8378de52b241f321c1f9ea905456e89b95c6f"
 
 let nimc = getEnv("NIMC", "nim") # Which nim compiler to use

libp2p/cid.nim

@@ -38,7 +38,7 @@ type
     hpos*: int
     data*: VBuffer
 
-const ContentIdsList = [
+const ContentIdsList* = [
   multiCodec("raw"),
   multiCodec("dag-pb"),
   multiCodec("dag-cbor"),
@@ -67,6 +67,12 @@ const ContentIdsList = [
   multiCodec("torrent-info"),
   multiCodec("torrent-file"),
   multiCodec("ed25519-pub"),
+  multiCodec("codex-root"),
+  multiCodec("codex-manifest"),
+  multiCodec("codex-block"),
+  multiCodec("codex-slot-root"),
+  multiCodec("codex-proving-root"),
+  multiCodec("codex-slot-cell"),
 ]
 
 proc initCidCodeTable(): Table[int, MultiCodec] {.compileTime.} =

libp2p/multibase.nim

@@ -17,6 +17,7 @@
 
 import tables
 import results
+import sequtils
 import stew/[base32, base58, base64]
 import ./utils/sequninit
 
@@ -387,6 +388,7 @@ proc initMultiBaseNameTable(): Table[string, MBCodec] {.compileTime.} =
 const
   CodeMultiBases = initMultiBaseCodeTable()
   NameMultiBases = initMultiBaseNameTable()
+  MultibaseList* = MultiBaseCodecs.mapIt(it.name)
 
 proc encodedLength*(mbtype: typedesc[MultiBase], encoding: string, length: int): int =
   ## Return estimated size of buffer to store MultiBase encoded value with

libp2p/multicodec.nim

@@ -370,6 +370,10 @@ const MultiCodecList = [
   ("skein1024-1008", 0xB3DE),
   ("skein1024-1016", 0xB3DF),
   ("skein1024-1024", 0xB3E0),
+  # poseidon2
+  ("poseidon2-alt_bn_128-sponge-r2", 0xCD10), # bn128 rate 2 sponge
+  ("poseidon2-alt_bn_128-merkle-2kb", 0xCD11), # bn128 2kb compress & merkleize
+  ("poseidon2-alt_bn_128-keyed-compress", 0xCD12), # bn128 keyed compress
   # multiaddrs
   ("ip4", 0x04),
   ("ip6", 0x29),
@@ -430,6 +434,12 @@ const MultiCodecList = [
   ("torrent-info", 0x7B),
   ("torrent-file", 0x7C),
   ("ed25519-pub", 0xED),
+  ("codex-manifest", 0xCD01),
+  ("codex-block", 0xCD02),
+  ("codex-root", 0xCD03),
+  ("codex-slot-root", 0xCD04),
+  ("codex-proving-root", 0xCD05),
+  ("codex-slot-cell", 0xCD06),
 ]
 
 type

libp2p/multihash.nim

@@ -25,9 +25,13 @@
 {.used.}
 
 import tables
+import sequtils
 import nimcrypto/[sha, sha2, keccak, blake2, hash, utils]
+import poseidon2
 import varint, vbuffer, multicodec, multibase
 import stew/base58
+import blscurve/bls_public_exports
+
 import results
 export results
 # This is workaround for Nim `import` bug.
@@ -110,13 +114,9 @@ proc blake2Shash(data: openArray[byte], output: var openArray[byte]) =
 
 proc sha2_256hash(data: openArray[byte], output: var openArray[byte]) =
   if len(output) > 0:
-    var digest = sha256.digest(data)
-    var length =
-      if sha256.sizeDigest > len(output):
-        len(output)
-      else:
-        sha256.sizeDigest
-    copyMem(addr output[0], addr digest.data[0], length)
+    var digest: array[32, byte]
+    digest.bls_sha256_digest(data)
+    copyMem(addr output[0], addr digest[0], 32)
 
 proc sha2_512hash(data: openArray[byte], output: var openArray[byte]) =
   if len(output) > 0:
@@ -226,7 +226,17 @@ proc shake_256hash(data: openArray[byte], output: var openArray[byte]) =
     discard sctx.output(addr output[0], uint(len(output)))
     sctx.clear()
 
-const HashesList = [
+proc poseidon2_sponge_rate2(data: openArray[byte], output: var openArray[byte]) =
+  if len(output) > 0:
+    var digest = poseidon2.Sponge.digest(data).toBytes()
+    copyMem(addr output[0], addr digest[0], uint(len(output)))
+
+proc poseidon2_merkle_2kb_sponge(data: openArray[byte], output: var openArray[byte]) =
+  if len(output) > 0:
+    var digest = poseidon2.SpongeMerkle.digest(data, 2048).toBytes()
+    copyMem(addr output[0], addr digest[0], uint(len(output)))
+
+const HashesList* = [
   MHash(mcodec: multiCodec("identity"), size: 0, coder: identhash),
   MHash(mcodec: multiCodec("sha1"), size: sha1.sizeDigest, coder: sha1hash),
   MHash(
@@ -348,6 +358,16 @@ const HashesList = [
   MHash(mcodec: multiCodec("blake2s-240"), size: 30, coder: blake2Shash),
   MHash(mcodec: multiCodec("blake2s-248"), size: 31, coder: blake2Shash),
   MHash(mcodec: multiCodec("blake2s-256"), size: 32, coder: blake2Shash),
+  MHash(
+    mcodec: multiCodec("poseidon2-alt_bn_128-sponge-r2"),
+    size: 32,
+    coder: poseidon2_sponge_rate2,
+  ),
+  MHash(
+    mcodec: multiCodec("poseidon2-alt_bn_128-merkle-2kb"),
+    size: 32,
+    coder: poseidon2_merkle_2kb_sponge,
+  ),
 ]
 
 proc initMultiHashCodeTable(): Table[MultiCodec, MHash] {.compileTime.} =

libp2p/protocols/kademlia/kademlia.nim

@@ -68,6 +68,8 @@ proc waitRepliesOrTimeouts(
 
   return (receivedReplies, failedPeers)
 
+# Helper function forward declaration
+proc checkConvergence(state: LookupState, me: PeerId): bool {.raises: [], gcsafe.}
 proc findNode*(
     kad: KadDHT, targetId: Key
 ): Future[seq[PeerId]] {.async: (raises: [CancelledError]).} =
@@ -81,12 +83,13 @@ proc findNode*(
 
   while not state.done:
     let toQuery = state.selectAlphaPeers()
+    debug "queries", list = toQuery.mapIt(it.shortLog()), addrTab = addrTable
     var pendingFutures = initTable[PeerId, Future[Message]]()
 
-    for peer in toQuery:
-      if pendingFutures.hasKey(peer):
-        continue
-
+    # TODO: pending futures always empty here, no?
+    for peer in toQuery.filterIt(
+      kad.switch.peerInfo.peerId != it or pendingFutures.hasKey(it)
+    ):
       state.markPending(peer)
 
       pendingFutures[peer] = kad
@@ -112,10 +115,16 @@ proc findNode*(
     for timedOut in timedOutPeers:
       state.markFailed(timedOut)
 
-    state.done = state.checkConvergence()
+    # Check for covergence: no active queries, and no other peers to be selected
+    state.done = checkConvergence(state, kad.switch.peerInfo.peerId)
 
   return state.selectClosestK()
 
+proc checkConvergence(state: LookupState, me: PeerId): bool {.raises: [], gcsafe.} =
+  let ready = state.activeQueries == 0
+  let noNew = selectAlphaPeers(state).filterIt(me != it).len == 0
+  return ready and noNew
+
 proc bootstrap*(
     kad: KadDHT, bootstrapNodes: seq[PeerInfo]
 ) {.async: (raises: [CancelledError]).} =

libp2p/protocols/kademlia/lookupstate.nim

@@ -103,11 +103,6 @@ proc init*(T: type LookupState, targetId: Key, initialPeers: seq[PeerId]): T =
   )
   return res
 
-proc checkConvergence*(state: LookupState): bool =
-  let ready = state.activeQueries == 0
-  let noNew = selectAlphaPeers(state).len == 0
-  return ready and noNew
-
 proc selectClosestK*(state: LookupState): seq[PeerId] =
   var res: seq[PeerId] = @[]
   for p in state.shortlist.filterIt(not it.failed):

performance/Dockerfile

@@ -17,7 +17,8 @@ WORKDIR /node
 
 COPY --from=build /node/performance/main /node/main
 
-RUN chmod +x main
+RUN chmod +x main \
+  && apk add --no-cache curl iproute2
 
 VOLUME ["/output"]
 

performance/aggregate_stats.nim

@@ -0,0 +1,130 @@
+import json
+import os
+import sequtils
+import strutils
+import strformat
+import tables
+import ./types
+
+const unknownFloat = -1.0
+
+proc parseJsonFiles*(outputDir: string): seq[JsonNode] =
+  var jsons: seq[JsonNode]
+
+  for kind, path in walkDir(outputDir):
+    if kind == pcFile and path.endsWith(".json"):
+      let content = readFile(path)
+      let json = parseJson(content)
+
+      jsons.add(json)
+
+  return jsons
+
+proc extractStats(scenario: JsonNode): Stats =
+  Stats(
+    scenarioName: scenario["scenarioName"].getStr(""),
+    totalSent: scenario["totalSent"].getInt(0),
+    totalReceived: scenario["totalReceived"].getInt(0),
+    latency: LatencyStats(
+      minLatencyMs: scenario["minLatencyMs"].getStr($unknownFloat).parseFloat(),
+      maxLatencyMs: scenario["maxLatencyMs"].getStr($unknownFloat).parseFloat(),
+      avgLatencyMs: scenario["avgLatencyMs"].getStr($unknownFloat).parseFloat(),
+    ),
+  )
+
+proc getJsonResults*(jsons: seq[JsonNode]): seq[Table[string, Stats]] =
+  jsons.mapIt(
+    it["results"]
+    .getElems(@[])
+    .mapIt(it.extractStats())
+    .mapIt((it.scenarioName, it)).toTable
+  )
+
+proc aggregateResults*(
+    jsonResults: seq[Table[string, Stats]]
+): (Table[string, Stats], Table[string, int]) =
+  var aggragated: Table[string, Stats]
+  var validNodes: Table[string, int]
+
+  for jsonResult in jsonResults:
+    for scenarioName, stats in jsonResult.pairs:
+      let startingStats = Stats(
+        scenarioName: scenarioName,
+        totalSent: 0,
+        totalReceived: 0,
+        latency: LatencyStats(minLatencyMs: Inf, maxLatencyMs: 0, avgLatencyMs: 0),
+      )
+      discard aggragated.hasKeyOrPut(scenarioName, startingStats)
+      discard validNodes.hasKeyOrPut(scenarioName, 0)
+
+      aggragated[scenarioName].totalSent += stats.totalSent
+      aggragated[scenarioName].totalReceived += stats.totalReceived
+
+      let minL = stats.latency.minLatencyMs
+      let maxL = stats.latency.maxLatencyMs
+      let avgL = stats.latency.avgLatencyMs
+      if minL != unknownFloat and maxL != unknownFloat and avgL != unknownFloat:
+        if minL < aggragated[scenarioName].latency.minLatencyMs:
+          aggragated[scenarioName].latency.minLatencyMs = minL
+
+        if maxL > aggragated[scenarioName].latency.maxLatencyMs:
+          aggragated[scenarioName].latency.maxLatencyMs = maxL
+
+        aggragated[scenarioName].latency.avgLatencyMs += avgL
+          # used to store sum of averages
+
+        validNodes[scenarioName] += 1
+
+  for scenarioName, stats in aggragated.mpairs:
+    let nodes = validNodes[scenarioName]
+    let globalAvgLatency = stats.latency.avgLatencyMs / float(nodes)
+    stats.latency.avgLatencyMs = globalAvgLatency
+
+  return (aggragated, validNodes)
+
+proc getMarkdownReport*(
+    results: Table[string, Stats],
+    validNodes: Table[string, int],
+    marker: string,
+    commitSha: string,
+): string =
+  var output: seq[string]
+
+  output.add marker & "\n"
+  output.add "# 🏁 **Performance Summary**\n"
+
+  output.add fmt"**Commit:** `{commitSha}`"
+
+  output.add "| Scenario | Nodes | Total messages sent | Total messages received | Latency min (ms) | Latency max (ms) | Latency avg (ms) |"
+  output.add "|:---:|:---:|:---:|:---:|:---:|:---:|:---:|"
+
+  for scenarioName, stats in results.pairs:
+    let nodes = validNodes[scenarioName]
+    output.add fmt"| {stats.scenarioName} | {nodes} | {stats.totalSent} | {stats.totalReceived} | {stats.latency.minLatencyMs:.3f} | {stats.latency.maxLatencyMs:.3f} | {stats.latency.avgLatencyMs:.3f} |"
+
+  let markdown = output.join("\n")
+
+  return markdown
+
+proc main() =
+  let outputDir = "performance/output"
+  let parsedJsons = parseJsonFiles(outputDir)
+
+  let jsonResults = getJsonResults(parsedJsons)
+  let (aggregatedResults, validNodes) = aggregateResults(jsonResults)
+
+  let marker = getEnv("MARKER", "<!-- marker -->")
+  let commitSha = getEnv("PR_HEAD_SHA", getEnv("GITHUB_SHA", "unknown"))
+  let markdown = getMarkdownReport(aggregatedResults, validNodes, marker, commitSha)
+
+  echo markdown
+
+  # For GitHub summary
+  let summaryPath = getEnv("GITHUB_STEP_SUMMARY", "/tmp/summary.txt")
+  writeFile(summaryPath, markdown & "\n")
+
+  # For PR comment
+  let commentPath = getEnv("COMMENT_SUMMARY_PATH", "/tmp/summary.txt")
+  writeFile(commentPath, markdown & "\n")
+
+main()

performance/main.nim

@@ -1,4 +1 @@
-import chronos
 import ./scenarios
-
-waitFor(baseTest())

performance/runner.sh

@@ -21,10 +21,12 @@ for ((i = 0; i < $PEERS; i++)); do
     hostname="$hostname_prefix$i"
 
     docker run -d \
+      --cap-add=NET_ADMIN \
       --name "$hostname" \
       -e NODE_ID="$i" \
       -e HOSTNAME_PREFIX="$hostname_prefix" \
       -v "$output_dir:/output" \
+      -v /var/run/docker.sock:/var/run/docker.sock \
       --hostname="$hostname" \
       --network="$network" \
       test-node > /dev/null

performance/scenarios.nim

@@ -1,23 +1,36 @@
+# Nim-LibP2P
+# Copyright (c) 2025 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+{.used.}
+
 import metrics
 import metrics/chronos_httpserver
 import os
+import strformat
 import strutils
 import ../libp2p
 import ../libp2p/protocols/ping
+import ../tests/helpers
 import ./utils
 from nativesockets import getHostname
 
-proc baseTest*() {.async.} =
+proc baseTest*(scenarioName = "Base test") {.async.} =
+  # --- Scenario ---
+  let scenario = scenarioName
   const
-    # --- Scenario ---
-    scenario = "Base test"
     nodeCount = 10
-    publisherCount = 10
+    publisherCount = 5
     peerLimit = 5
-    msgCount = 200
-    msgInterval = 20 # ms
-    msgSize = 500 # bytes
-    warmupCount = 20
+    msgCount = 100
+    msgInterval = 100 # ms
+    msgSize = 200 # bytes
+    warmupCount = 10
 
   # --- Node Setup ---
   let
@@ -26,6 +39,9 @@ proc baseTest*() {.async.} =
     hostname = getHostname()
     rng = libp2p.newRng()
 
+  if nodeId == 0:
+    clearSyncFiles()
+
   let (switch, gossipSub, pingProtocol) = setupNode(nodeId, rng)
   gossipSub.setGossipSubParams()
 
@@ -41,13 +57,15 @@ proc baseTest*() {.async.} =
   defer:
     await switch.stop()
 
-  info "Node started, waiting 5s",
+  info "Node started, synchronizing",
+    scenario,
     nodeId,
     address = switch.peerInfo.addrs,
     peerId = switch.peerInfo.peerId,
     isPublisher = nodeId <= publisherCount,
     hostname = hostname
-  await sleepAsync(5.seconds)
+
+  await syncNodes("started", nodeId, nodeCount)
 
   # --- Peer Discovery & Connection ---
   var peersAddresses = resolvePeersAddresses(nodeCount, hostnamePrefix, nodeId)
@@ -55,21 +73,115 @@ proc baseTest*() {.async.} =
 
   await connectPeers(switch, peersAddresses, peerLimit, nodeId)
 
-  info "Mesh populated, waiting 5s",
+  info "Mesh populated, synchronizing",
     nodeId, meshSize = gossipSub.mesh.getOrDefault(topic).len
-  await sleepAsync(5.seconds)
+
+  await syncNodes("mesh", nodeId, nodeCount)
 
   # --- Message Publishing ---
   let sentMessages = await publishMessagesWithWarmup(
     gossipSub, warmupCount, msgCount, msgInterval, msgSize, publisherCount, nodeId
   )
 
-  info "Waiting 2 seconds for message delivery"
-  await sleepAsync(2.seconds)
+  info "Waiting for message delivery, synchronizing"
+
+  await syncNodes("published", nodeId, nodeCount)
 
   # --- Performance summary  ---
-  let stats = getStats(receivedMessages[], sentMessages)
+  let stats = getStats(scenario, receivedMessages[], sentMessages)
   info "Performance summary", nodeId, stats = $stats
 
   let outputPath = "/output/" & hostname & ".json"
   writeResultsToJson(outputPath, scenario, stats)
+
+  await syncNodes("finished", nodeId, nodeCount)
+
+suite "Network Performance Tests":
+  teardown:
+    checkTrackers()
+
+asyncTest "Base Test":
+  await baseTest()
+
+asyncTest "Latency Test":
+  const
+    latency = 100
+    jitter = 20
+
+  discard execShellCommand(
+    fmt"{enableTcCommand} netem delay {latency}ms {jitter}ms distribution normal"
+  )
+  await baseTest(fmt"Latency {latency}ms {jitter}ms")
+  discard execShellCommand(disableTcCommand)
+
+asyncTest "Packet Loss Test":
+  const packetLoss = 5
+
+  discard execShellCommand(fmt"{enableTcCommand} netem loss {packetLoss}%")
+  await baseTest(fmt"Packet Loss {packetLoss}%")
+  discard execShellCommand(disableTcCommand)
+
+asyncTest "Low Bandwidth Test":
+  const
+    rate = "256kbit"
+    burst = "8kbit"
+    limit = "5000"
+
+  discard
+    execShellCommand(fmt"{enableTcCommand} tbf rate {rate} burst {burst} limit {limit}")
+  await baseTest(fmt"Low Bandwidth rate {rate} burst {burst} limit {limit}")
+  discard execShellCommand(disableTcCommand)
+
+asyncTest "Packet Reorder Test":
+  const
+    reorderPercent = 15
+    reorderCorr = 40
+    delay = 2
+
+  discard execShellCommand(
+    fmt"{enableTcCommand} netem delay {delay}ms reorder {reorderPercent}% {reorderCorr}%"
+  )
+  await baseTest(
+    fmt"Packet Reorder {reorderPercent}% {reorderCorr}% with {delay}ms delay"
+  )
+  discard execShellCommand(disableTcCommand)
+
+asyncTest "Burst Loss Test":
+  const
+    lossPercent = 8
+    lossCorr = 30
+
+  discard execShellCommand(fmt"{enableTcCommand} netem loss {lossPercent}% {lossCorr}%")
+  await baseTest(fmt"Burst Loss {lossPercent}% {lossCorr}%")
+  discard execShellCommand(disableTcCommand)
+
+asyncTest "Duplication Test":
+  const duplicatePercent = 2
+
+  discard execShellCommand(fmt"{enableTcCommand} netem duplicate {duplicatePercent}%")
+  await baseTest(fmt"Duplication {duplicatePercent}%")
+  discard execShellCommand(disableTcCommand)
+
+asyncTest "Corruption Test":
+  const corruptPercent = 0.5
+
+  discard execShellCommand(fmt"{enableTcCommand} netem corrupt {corruptPercent}%")
+  await baseTest(fmt"Corruption {corruptPercent}%")
+  discard execShellCommand(disableTcCommand)
+
+asyncTest "Queue Limit Test":
+  const queueLimit = 5
+
+  discard execShellCommand(fmt"{enableTcCommand} netem limit {queueLimit}")
+  await baseTest(fmt"Queue Limit {queueLimit}")
+  discard execShellCommand(disableTcCommand)
+
+asyncTest "Combined Network Conditions Test":
+  discard execShellCommand(
+    "tc qdisc add dev eth0 root handle 1:0 tbf rate 2mbit burst 32kbit limit 25000"
+  )
+  discard execShellCommand(
+    "tc qdisc add dev eth0 parent 1:1 handle 10: netem delay 100ms 20ms distribution normal loss 5% 20% reorder 10% 30% duplicate 0.5% corrupt 0.05% limit 20"
+  )
+  await baseTest("Combined Network Conditions")
+  discard execShellCommand(disableTcCommand)

performance/types.nim

@@ -0,0 +1,10 @@
+type LatencyStats* = object
+  minLatencyMs*: float
+  maxLatencyMs*: float
+  avgLatencyMs*: float
+
+type Stats* = object
+  scenarioName*: string
+  totalSent*: int
+  totalReceived*: int
+  latency*: LatencyStats

performance/utils.nim

@@ -1,18 +1,22 @@
+import chronos
+import hashes
+import json
+import metrics
+import metrics/chronos_httpserver
+import os
+import osproc
+import sequtils
 import stew/byteutils
 import stew/endians2
 import strutils
 import strformat
-import sequtils
 import tables
-import hashes
-import metrics
-import metrics/chronos_httpserver
-import chronos
-import json
 import ../libp2p
 import ../libp2p/protocols/pubsub/rpc/messages
 import ../libp2p/muxers/mplex/lpchannel
 import ../libp2p/protocols/ping
+import ../tests/helpers
+import ./types
 
 const
   topic* = "test"
@@ -97,7 +101,7 @@ proc createMessageHandler*(
     let latency = getLatency(sentNs)
 
     receivedMessages[msgId] = latency
-    info "Message delivered", msgId = msgId, latency = formatLatencyMs(latency), nodeId
+    debug "Message delivered", msgId = msgId, latency = formatLatencyMs(latency), nodeId
 
   return (messageHandler, receivedMessages)
 
@@ -124,22 +128,19 @@ proc resolvePeersAddresses*(
 proc connectPeers*(
     switch: Switch, peersAddresses: seq[MultiAddress], peerLimit: int, nodeId: int
 ) {.async.} =
-  var
-    connected = 0
-    index = 0
-  while connected < peerLimit:
-    while true:
-      let address = peersAddresses[index]
-      try:
-        let peerId =
-          await switch.connect(address, allowUnknownPeerId = true).wait(5.seconds)
-        connected.inc()
-        index.inc()
-        debug "Connected peer", nodeId, address = address
-        break
-      except CatchableError as exc:
-        warn "Failed to dial, waiting 5s", nodeId, address = address, error = exc.msg
-        await sleepAsync(5.seconds)
+  proc connectPeer(address: MultiAddress): Future[bool] {.async.} =
+    try:
+      let peerId =
+        await switch.connect(address, allowUnknownPeerId = true).wait(5.seconds)
+      debug "Connected peer", nodeId, address, peerId
+      return true
+    except CatchableError as exc:
+      warn "Failed to dial, waiting 1s", nodeId, address = address, error = exc.msg
+      return false
+
+  for index in 0 ..< peerLimit:
+    checkUntilTimeoutCustom(5.seconds, 500.milliseconds):
+      await connectPeer(peersAddresses[index])
 
 proc publishMessagesWithWarmup*(
     gossipSub: GossipSub,
@@ -150,8 +151,9 @@ proc publishMessagesWithWarmup*(
     publisherCount: int,
     nodeId: int,
 ): Future[seq[uint64]] {.async.} =
+  info "Publishing messages", nodeId
   # Warm-up phase
-  info "Sending warmup messages", nodeId
+  debug "Sending warmup messages", nodeId
   for msg in 0 ..< warmupCount:
     await sleepAsync(msgInterval)
     discard await gossipSub.publish(topic, warmupData)
@@ -164,17 +166,12 @@ proc publishMessagesWithWarmup*(
       let timestamp = Moment.now().epochNanoSeconds()
       var data = @(toBytesLE(uint64(timestamp))) & newSeq[byte](msgSize)
 
-      info "Sending message", msgId = timestamp, nodeId = nodeId
+      debug "Sending message", msgId = timestamp, nodeId = nodeId
       doAssert((await gossipSub.publish(topic, data)) > 0)
       sentMessages.add(uint64(timestamp))
 
   return sentMessages
 
-type LatencyStats* = object
-  minLatencyMs*: float
-  maxLatencyMs*: float
-  avgLatencyMs*: float
-
 proc getLatencyStats*(latencies: seq[float]): LatencyStats =
   var
     minLatencyMs = 0.0
@@ -192,16 +189,20 @@ proc getLatencyStats*(latencies: seq[float]): LatencyStats =
   )
 
 type Stats* = object
+  scenarioName*: string
   totalSent*: int
   totalReceived*: int
   latency*: LatencyStats
 
 proc getStats*(
-    receivedMessages: Table[uint64, float], sentMessages: seq[uint64]
+    scenarioName: string,
+    receivedMessages: Table[uint64, float],
+    sentMessages: seq[uint64],
 ): Stats =
   let latencyStats = getLatencyStats(receivedMessages.values().toSeq())
 
   let stats = Stats(
+    scenarioName: scenarioName,
     totalSent: sentMessages.len,
     totalReceived: receivedMessages.len,
     latency: latencyStats,
@@ -217,17 +218,64 @@ proc `$`*(stats: Stats): string =
     fmt"avg={formatLatencyMs(stats.latency.avgLatencyMs)}"
 
 proc writeResultsToJson*(outputPath: string, scenario: string, stats: Stats) =
-  let json =
+  var resultsArr: JsonNode = newJArray()
+  if fileExists(outputPath):
+    try:
+      let existing = parseFile(outputPath)
+      resultsArr = existing["results"]
+    except:
+      discard
+
+  let newResult =
     %*{
-      "results": [
-        {
-          "scenario": scenario,
-          "totalSent": stats.totalSent,
-          "totalReceived": stats.totalReceived,
-          "minLatency": formatLatencyMs(stats.latency.minLatencyMs),
-          "maxLatency": formatLatencyMs(stats.latency.maxLatencyMs),
-          "avgLatency": formatLatencyMs(stats.latency.avgLatencyMs),
-        }
-      ]
+      "scenarioName": scenario,
+      "totalSent": stats.totalSent,
+      "totalReceived": stats.totalReceived,
+      "minLatencyMs": formatLatencyMs(stats.latency.minLatencyMs),
+      "maxLatencyMs": formatLatencyMs(stats.latency.maxLatencyMs),
+      "avgLatencyMs": formatLatencyMs(stats.latency.avgLatencyMs),
     }
+  resultsArr.add(newResult)
+
+  let json = %*{"results": resultsArr}
   writeFile(outputPath, json.pretty)
+
+const
+  enableTcCommand* = "tc qdisc add dev eth0 root"
+  disableTcCommand* = "tc qdisc del dev eth0 root"
+
+proc execShellCommand*(cmd: string): string =
+  try:
+    let output = execProcess(
+        "/bin/sh", args = ["-c", cmd], options = {poUsePath, poStdErrToStdOut}
+      )
+      .strip()
+    debug "Shell command executed", cmd, output
+    return output
+  except OSError as e:
+    raise newException(OSError, "Shell command failed")
+
+const syncDir = "/output/sync"
+
+proc syncNodes*(stage: string, nodeId, nodeCount: int) {.async.} =
+  # initial wait
+  await sleepAsync(2.seconds)
+
+  let prefix = "sync_"
+  let myFile = syncDir / (prefix & stage & "_" & $nodeId)
+  writeFile(myFile, "ok")
+
+  let expectedFiles = (0 ..< nodeCount).mapIt(syncDir / (prefix & stage & "_" & $it))
+  checkUntilTimeoutCustom(5.seconds, 100.milliseconds):
+    expectedFiles.allIt(fileExists(it))
+
+  # final wait
+  await sleepAsync(500.milliseconds)
+
+proc clearSyncFiles*() =
+  if not dirExists(syncDir):
+    createDir(syncDir)
+  else:
+    for f in walkDir(syncDir):
+      if fileExists(f.path):
+        removeFile(f.path)

tests/testcid.nim

@@ -10,8 +10,11 @@
 # those terms.
 
 import unittest2
+import std/sequtils
 import ../libp2p/[cid, multihash, multicodec]
 
+const MultiHashCodecsList* = HashesList.mapIt(it.mcodec)
+
 suite "Content identifier CID test suite":
   test "CIDv0 test vector":
     var cid0Text = "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n"
@@ -66,3 +69,25 @@ suite "Content identifier CID test suite":
       cid1 != cid5
       cid2 != cid4
       cid3 != cid6
+
+  test "Check all cids and hashes":
+    var msg = cast[seq[byte]]("Hello World!")
+    for cidCodec in ContentIdsList:
+      for mhashCodec in MultiHashCodecsList:
+        let cid = Cid
+          .init(CidVersion.CIDv1, cidCodec, MultiHash.digest($mhashCodec, msg).get())
+          .get()
+        check:
+          cid.mcodec == cidCodec
+          cid.mhash().get().mcodec == mhashCodec
+
+  test "Check all cids and hashes base encode":
+    var msg = cast[seq[byte]]("Hello World!")
+    for cidCodec in ContentIdsList:
+      for mhashCodec in MultiHashCodecsList:
+        let cid = Cid
+          .init(CidVersion.CIDv1, cidCodec, MultiHash.digest($mhashCodec, msg).get())
+          .get()
+        check:
+          cid.mcodec == cidCodec
+          cid.mhash().get().mcodec == mhashCodec

tests/testmultihash.nim

@@ -67,16 +67,27 @@ const RustTestVectors = [
          B7C42181F40AA1046F39E2EF9EFC6910782A998E0013D172458957957FAC9405
          B67D""",
   ],
+  [
+    "poseidon2-alt_bn_128-sponge-r2", "hello world",
+    """909A0320823F7FB71C0998153E73AC734AE4870518F5FE324BD2484B68B565C288CF1E1E""",
+  ],
+  [
+    "poseidon2-alt_bn_128-merkle-2kb", "hello world",
+    """919A0320D9A6AE0CBF28C5E9CBE28D7231D3A4DEDF8B3826B0F8C3C002CA95C21253E614""",
+  ],
 ]
 
 suite "MultiHash test suite":
+  template checkTestVector(vector) =
+    var msg = vector[1]
+    var bmsg = cast[seq[byte]](msg)
+    var mh1 = MultiHash.digest(vector[0], bmsg).get()
+    var mh2 = MultiHash.init(stripSpaces(vector[2])).get()
+    check:
+      hex(mh1) == stripSpaces(vector[2])
+      hex(mh1) == hex(mh2)
+      mh1 == mh2
+
   test "rust-multihash test vectors":
     for item in RustTestVectors:
-      var msg = item[1]
-      var bmsg = cast[seq[byte]](msg)
-      var mh1 = MultiHash.digest(item[0], bmsg).get()
-      var mh2 = MultiHash.init(stripSpaces(item[2])).get()
-      check:
-        hex(mh1) == stripSpaces(item[2])
-        hex(mh1) == hex(mh2)
-        mh1 == mh2
+      checkTestVector(item)