Dandelion proof-of-concept implementation

move -d:nimRawSetjmp to nim.cfg
switch .cfg to .nims
add --skipParentCfg

Co-authored-by: Tanguy <tanguy@status.im>

.assets/small-logo.svg

@@ -0,0 +1,96 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:v="https://vecta.io/nano" xmlns:xlink="http://www.w3.org/1999/xlink" width="172.071" height="196.414" viewBox="0 0 45.527 51.968">
+   <g transform="matrix(.2822 0 0 .2822 -212.833275 -150.656248)">
+      <path d="M835.432 533.821l-12.483 9.783c-6.482-.207-19.197 1.251-26.086 3.769-6.346-4.04-11.923-8.5-11.923-8.5l-7.762 13.071c-4.444 2.375-8.906 5.046-12.883 8.58l-10.162-4.17c6.125 12.414 10.243 24.844 21.445 32.316 17.834-28.299 100.705-25.691 118.907-.16 11.764-6.165 16.339-19.429 20.965-31.674-.507.168-6.802 2.285-10.882 3.849-2.436-2.665-8.179-6.763-11.443-8.741-3.096-5.696-7.602-13.391-7.602-13.391s-5.337 3.988-11.523 8.34c-8.357-1.55-18.465-3.433-26.966-2.967-5.787-4.779-11.603-10.104-11.603-10.104z" fill="#f3d400" />
+      <g opacity=".9" transform="matrix(.9375 0 0 .9375 765.1166 550.13225)">
+         <path d="M99.952 106.898l.215-.107 24.755-14.248-24.97-14.535-24.97 14.374z" fill="#cc2a65" />
+         <use xlink:href="#B" fill="#a21d4c" />
+         <path d="M124.922 92.542l-24.755 14.248-.215.107v28.89l24.97-14.356z" fill="#b62454" />
+         <path d="M50.012 106.737l.215-.125 24.755-14.248-24.97-14.517-24.97 14.356z" fill="#c8d92b" />
+         <path d="M50.012 135.609v-28.872l-24.97-14.535v28.89h.018z" fill="#c2d02f" />
+         <path d="M74.982 92.381l-24.755 14.23-.215.125v28.872.018h.018l24.952-14.356v-.018z" fill="#b9be33" />
+         <path d="M74.982 121.253l.215-.107 24.755-14.248-24.97-14.535-24.97 14.374z" fill="#cc2a65" />
+         <use xlink:href="#B" x="-24.97" y="14.356" fill="#a21d4c" />
+         <path d="M99.952 106.898l-24.755 14.248-.215.107v28.89H75l24.952-14.356z" fill="#b62454" />
+         <path d="M124.905 121.415l.215-.125 24.737-14.23-24.952-14.535-24.97 14.356z" fill="#a159a2" />
+         <path d="M124.905 150.305v-28.89l-24.97-14.535v28.89h.018z" fill="#772a86" />
+         <path d="M149.875 107.059l-24.755 14.23-.215.125v28.89l24.97-14.356z" fill="#8e3b95" />
+         <path d="M74.982 92.345l.215-.125 24.737-14.248-24.952-14.517-24.97 14.356z" fill="#bec831" />
+         <path d="M74.982 121.217V92.345L50.012 77.81v28.89h.018z" fill="#a1a938" />
+         <path d="M99.952 77.989l-24.755 14.23-.215.125v28.872.018l24.97-14.356v-.018z" fill="#999b37" />
+         <path d="M75 60.645l.197-.125 24.755-14.23L75 31.755 50.029 46.11l24.952 14.535z" fill="#bec831" />
+         <path d="M74.982 89.535L75 60.645 50.029 46.11 50.012 75h.018z" fill="#a1a938" />
+         <path d="M99.97 46.307L75.197 60.52l-.197.125h-.018v28.89l24.97-14.338v-.018z" fill="#999b37" />
+         <path d="M99.952 75.179l.215-.107 24.755-14.23L99.97 46.306 75 60.644z" fill="#ee539a" />
+         <path d="M99.952 104.069v-28.89L75 60.644l-.018 28.89H75z" fill="#d01b68" />
+         <path d="M124.922 60.841l-24.755 14.23-.215.107v28.89l24.97-14.338.018-28.89z" fill="#ec0f68" />
+         <path d="M124.923 89.731l.215-.125 24.755-14.23-24.952-14.535h-.018l-24.97 14.338z" fill="#a159a2" />
+         <path d="M124.905 118.622l.018-28.89-24.97-14.535v28.872.018z" fill="#772a86" />
+         <path d="M149.893 75.376l-24.755 14.23-.215.125-.018 28.89h.018l24.97-14.356z" fill="#8e3b95" />
+         <path d="M50.03 75l.197-.125 24.755-14.23L50.03 46.109 25.06 60.447l24.952 14.535z" fill="#c8d92b" />
+         <path d="M50.012 103.872L50.03 75 25.06 60.447l-.018 28.89h.018z" fill="#c2d02f" />
+         <path d="M75 60.644l-24.773 14.23-.197.125-.018 28.872 24.97-14.338z" fill="#b9be33" />
+         <path d="M74.982 89.534l.215-.125 24.755-14.23L75 60.644l-24.97 14.338z" fill="#f7af19" />
+         <path d="M74.982 118.425v-.018.018-28.89L50.029 75l-.018 28.872.018.018z" fill="#f2901f" />
+         <path d="M99.952 75.179l-24.755 14.23-.215.125v28.89l24.97-14.356.018-28.89z" fill="#f9a120" />
+         <path d="M99.934 135.769l.215-.125 24.684-14.356-25.042-14.409L74.91 121.36z" fill="#833593" />
+         <path d="M100.077 164.66l-.143-28.89-25.042-14.409.143 28.89h.018z" fill="#652977" />
+         <path d="M124.833 121.288l-24.684 14.356-.215.125.143 28.89 24.899-14.481z" fill="#4d1f5b" />
+         <path d="M99.952 104.069l.215-.107 24.755-14.23L99.97 75.179h-.018l-24.97 14.356z" fill="#a159a2" />
+         <path d="M99.934 132.959l.018-28.89-24.97-14.535v28.89z" fill="#772a86" />
+         <path d="M124.922 89.732l-24.755 14.23-.215.107-.018 28.89h.018l24.97-14.338z" fill="#8e3b95" />
+         <path d="M25.042 121.074l.197-.125 24.755-14.248-24.952-14.517h-.018L.071 106.54l24.952 14.535z" fill="#f6dd03" />
+         <path d="M25.024 149.947h.018v-28.872L.071 106.54v28.89z" fill="#f9bb1d" />
+         <path d="M49.994 106.719l-24.755 14.23-.197.125v28.872.018l24.952-14.356h.018v-.018-28.872z" fill="#e9ae20" />
+         <path d="M25.06 89.338l.197-.125 24.755-14.23L25.06 60.447.089 74.803l24.952 14.535z" fill="#f6dd03" />
+         <path d="M25.042 118.228l.018-28.89L.089 74.803.072 103.675l.018.018z" fill="#f9bb1d" />
+         <path d="M50.03 75L25.257 89.212l-.197.125-.018 28.89 24.97-14.356v.018-.018z" fill="#e9ae20" />
+         <path d="M50.012 135.59l.215-.107 24.737-14.248L50.012 106.7l-24.97 14.374z" fill="#f7af19" />
+         <path d="M50.012 164.481v-28.89l-24.97-14.517v28.872.018z" fill="#f2901f" />
+         <path d="M74.964 121.235l-24.755 14.248-.197.107v28.89l24.97-14.356v-28.89z" fill="#f9a120" />
+         <path d="M50.012 103.872l.215-.107 24.755-14.23L50.03 74.982 25.06 89.338z" fill="#f7af19" />
+         <path d="M50.012 132.763v-28.89L25.06 89.338l-.018 28.89h.018z" fill="#f2901f" />
+         <path d="M74.982 89.535l-24.755 14.23-.215.107v28.89l24.97-14.338.018-28.89z" fill="#f9a120" />
+         <path d="M74.982 150.125l.197-.125 24.755-14.23-24.952-14.535h-.018l-24.952 14.356 24.952 14.535z" fill="#f7af19" />
+         <path d="M74.964 179.015h.018v-28.89l-24.97-14.517v28.872.018z" fill="#f2901f" />
+         <path d="M99.934 135.77L75.179 150l-.197.125v28.89l24.97-14.356v-28.89z" fill="#f9a120" />
+         <path d="M74.982 118.425l.215-.125 24.755-14.23L75 89.535h-.018l-24.97 14.338z" fill="#31838b" />
+         <path d="M74.964 147.297l.018-28.872-24.97-14.535v-.018 28.89.018z" fill="#22626c" />
+         <path d="M99.952 104.069L75.197 118.3l-.215.125-.018 28.872v.018h.018l24.97-14.356z" fill="#1b4b56" />
+         <path d="M74.982 28.962l.215-.125 24.737-14.248L74.982.072l-24.97 14.356 24.97 14.517z" fill="#bec831" />
+         <path d="M74.982 57.834V28.962l-24.97-14.535v28.89h.018z" fill="#a1a938" />
+         <path d="M99.952 14.606l-24.755 14.23-.215.125v28.872l24.97-14.356z" fill="#999b37" />
+         <path d="M74.964 28.944l.215-.125 24.755-14.23L74.982.054h-.018l-24.97 14.338z" fill="#a159a2" />
+         <path d="M74.946 57.835l.018-28.89-24.97-14.535v28.872.018z" fill="#772a86" />
+         <path d="M99.934 14.589l-24.755 14.23-.215.125-.018 28.89h.018l24.97-14.356z" fill="#8e3b95" />
+         <path d="M99.952 43.479l.215-.107 24.755-14.248-24.97-14.535-24.97 14.356z" fill="#ee539a" />
+         <use xlink:href="#B" y="-63.419" fill="#d01b68" />
+         <path d="M124.922 29.123l-24.755 14.248-.215.107v28.89l24.97-14.356z" fill="#ec0f68" />
+         <path d="M50.03 43.317l.215-.125L75 28.961 50.048 14.427h-.018L25.06 28.765z" fill="#31838b" />
+         <path d="M50.012 72.189l.018-28.872-24.97-14.535v-.018 28.89.018z" fill="#22626c" />
+         <path d="M75 28.961l-24.755 14.23-.215.125-.018 28.872v.018h.018L75 57.852z" fill="#1b4b56" />
+         <path d="M124.923 58.013l.215-.125 24.737-14.23-24.952-14.535-24.97 14.356z" fill="#cc2a65" />
+         <use xlink:href="#B" x="24.971" y="-48.884" fill="#a21d4c" />
+         <path d="M149.893 43.658l-24.755 14.23-.215.125v28.89l24.97-14.356z" fill="#b62454" />
+         <path d="M74.982 57.835l.215-.107 24.755-14.248-24.97-14.535L50.012 43.3z" fill="#c8d92b" />
+         <path d="M74.982 86.725v-28.89l-24.97-14.517V72.19l.018.018z" fill="#c2d02f" />
+         <path d="M99.952 43.479L75.197 57.727l-.215.107v28.89H75l24.952-14.356z" fill="#b9be33" />
+         <path d="M99.952 72.369l.215-.125 24.755-14.23-24.97-14.535-24.97 14.356z" fill="#33b4d7" />
+         <use xlink:href="#B" y="-34.529" fill="#209ac5" />
+         <path d="M124.922 58.014l-24.755 14.23-.215.125v28.89h.018l24.952-14.356z" fill="#0f8cae" />
+         <path d="M25.06 57.673l.197-.125L50.012 43.3 25.06 28.783h-.018L.089 43.139l24.952 14.535z" fill="#94d6e3" />
+         <path d="M25.042 86.546h.018V57.673L.089 43.139v28.89z" fill="#73ccdd" />
+         <path d="M50.012 43.318l-24.755 14.23-.197.125v28.872.018l24.952-14.356h.018v-.018-28.872z" fill="#3bafbb" />
+         <path d="M50.03 72.19l.215-.107 24.737-14.248L50.03 43.318 25.06 57.674z" fill="#94d6e3" />
+         <path d="M50.03 101.08V72.208v-.018L25.06 57.674v28.872.018z" fill="#73ccdd" />
+         <path d="M74.982 57.835L50.227 72.083l-.197.107v28.89L75 86.725v-28.89z" fill="#3bafbb" />
+         <path d="M75 86.724l.197-.107 24.755-14.248L75 57.834h-.018L50.029 72.189l24.952 14.535z" fill="#33b4d7" />
+         <path d="M74.982 115.614H75v-28.89l-24.97-14.517v28.872.018z" fill="#209ac5" />
+         <path d="M99.952 72.368L75.197 86.617l-.197.107v28.89l24.97-14.356v-28.89z" fill="#0f8cae" />
+      </g>
+      <path d="M759.126 567.007s10.273 21.02 16.364 35.698c25.549 33.869 90.792 36.224 119.235.656 9.484-17.619 16.733-36.357 16.733-36.357-7.297 10.862-20.094 18.056-27.408 22.095-5.197 2.861-17.189 4.59-17.189 4.59l-31.482-16.393-31.663 16.065s-11.832-1.91-17.189-4.426c-10.811-5.799-19.735-12.549-27.401-21.928z" fill="#ffe953" />
+   </g>
+   <defs>
+      <path id="B" d="M99.952 135.788v-28.89l-24.97-14.517v28.872l.018.018z" />
+   </defs>
+</svg>

.github/workflows/bumper.yml

@@ -0,0 +1,36 @@
+name: Bumper
+on:
+  push:
+    branches:
+      - unstable
+      - bumper
+  workflow_dispatch:
+
+jobs:
+  bumpNimbus:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Clone NBC
+        uses: actions/checkout@v2
+        with:
+          repository: status-im/nimbus-eth2
+          ref: unstable
+          path: nbc
+          submodules: true
+          fetch-depth: 0
+          token:  ${{ secrets.ACTIONS_GITHUB_TOKEN }}
+
+      - name: Checkout this ref
+        run: |
+          cd nbc/vendor/nim-libp2p
+          git checkout $GITHUB_SHA
+
+      - name: Commit this bump
+        run: |
+          cd nbc
+          git config --global user.email "${{ github.actor }}@users.noreply.github.com"
+          git config --global user.name = "${{ github.actor }}"
+          git commit -a -m "auto-bump nim-libp2p"
+          git branch -D nim-libp2p-auto-bump-${GITHUB_REF##*/} || true
+          git switch -c nim-libp2p-auto-bump-${GITHUB_REF##*/}
+          git push -f origin nim-libp2p-auto-bump-${GITHUB_REF##*/}

.github/workflows/ci.yml

@@ -24,7 +24,7 @@ jobs:
             cpu: amd64
           #- os: windows
             #cpu: i386
-        branch: [version-1-2, devel]
+        branch: [version-1-2, version-1-6]
         include:
           - target:
               os: linux
@@ -32,7 +32,7 @@ jobs:
             shell: bash
           - target:
               os: macos
-            builder: macos-10.15
+            builder: macos-12
             shell: bash
           - target:
               os: windows
@@ -162,94 +162,7 @@ jobs:
 
       - name: Run tests
         run: |
-          if [[ "${{ matrix.target.os }}" == "windows" ]]; then
-            # https://github.com/status-im/nimbus-eth2/issues/3121
-            export NIMFLAGS="-d:nimRawSetjmp"
-          fi
           nim --version
           nimble --version
           nimble install_pinned
           nimble test
-
-  bumpNBC-stable:
-    if: github.ref == 'refs/heads/master'
-    needs: build
-    runs-on: ubuntu-latest
-    steps:
-      - uses: status-im/github-app-token@v1
-        name: Generate token
-        id: generate-token
-        with:
-          app_id: ${{ secrets.BUMP_BOT_APP_ID }}
-          private_key: ${{ secrets.BUMP_BOT_APP_PRIVATE_KEY }}
-
-      - name: Clone NBC
-        uses: actions/checkout@v2
-        with:
-          repository: status-im/nimbus-eth2
-          ref: unstable
-          path: nbc
-          submodules: true
-          fetch-depth: 0
-
-      - name: Checkout this ref
-        run: |
-          cd nbc/vendor/nim-libp2p
-          git checkout $GITHUB_SHA
-
-      - name: Commit this bump
-        run: |
-          cd nbc
-          git config --global user.email "${{ github.actor }}@users.noreply.github.com"
-          git config --global user.name = "${{ github.actor }}"
-          git commit -a -m "auto-bump nim-libp2p"
-
-      - name: Make PR
-        uses: peter-evans/create-pull-request@v3.5.0
-        with:
-          branch: nim-libp2p-auto-bump
-          path: nbc
-          token: ${{ steps.generate-token.outputs.token }}
-          title: nim-libp2p auto bump
-
-  bumpNBC-unstable:
-    if: github.ref == 'refs/heads/unstable'
-    needs: build
-    runs-on: ubuntu-latest
-    steps:
-      - uses: status-im/github-app-token@v1
-        name: Generate token
-        id: generate-token
-        with:
-          app_id: ${{ secrets.BUMP_BOT_APP_ID }}
-          private_key: ${{ secrets.BUMP_BOT_APP_PRIVATE_KEY }}
-
-      - name: Clone NBC
-        uses: actions/checkout@v2
-        with:
-          repository: status-im/nimbus-eth2
-          ref: unstable
-          path: nbc
-          submodules: true
-          fetch-depth: 0
-
-      - name: Checkout this ref
-        run: |
-          cd nbc/vendor/nim-libp2p
-          git checkout $GITHUB_SHA
-
-      - name: Commit this bump
-        run: |
-          cd nbc
-          git config --global user.email "${{ github.actor }}@users.noreply.github.com"
-          git config --global user.name = "${{ github.actor }}"
-          git commit -a -m "auto-bump nim-libp2p"
-
-      - name: Make PR
-        uses: peter-evans/create-pull-request@v3.5.0
-        with:
-          branch: nim-libp2p-auto-bump-unstable
-          path: nbc
-          token: ${{ steps.generate-token.outputs.token }}
-          title: nim-libp2p unstable auto bump
-          draft: true

.github/workflows/codecov.yml

@@ -25,6 +25,8 @@ jobs:
         ]
     steps:
       - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
       - name: Run
         run: |
           sudo apt-get update
@@ -63,6 +65,8 @@ jobs:
         ]
     steps:
       - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
       - name: Run
         run: |
           sudo apt-get update
@@ -104,6 +108,8 @@ jobs:
         ]
     steps:
       - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
       - name: Run
         run: |
           sudo apt-get update

.github/workflows/doc.yml

@@ -0,0 +1,99 @@
+name: Docgen
+on:
+  push:
+  workflow_dispatch:
+
+
+jobs:
+  build:
+    timeout-minutes: 20
+
+    name: 'Generate & upload documentation'
+    runs-on: 'ubuntu-20.04'
+    continue-on-error: true
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          submodules: true
+
+      - uses: jiro4989/setup-nim-action@v1
+        with:
+          nim-version: 'stable'
+
+      - name: Generate doc
+        run: |
+          nim --version
+          nimble --version
+          nimble install_pinned
+          # nim doc can "fail", but the doc is still generated
+          nim doc --git.url:https://github.com/status-im/nim-libp2p --git.commit:${GITHUB_REF##*/} --outdir:${GITHUB_REF##*/} --project libp2p || true
+
+          # check that the folder exists
+          ls ${GITHUB_REF##*/}
+
+      - name: Clone the gh-pages branch
+        uses: actions/checkout@v2
+        with:
+          repository: status-im/nim-libp2p
+          ref: gh-pages
+          path: subdoc
+          submodules: true
+          fetch-depth: 0
+
+      - name: Commit & push
+        run: |
+          cd subdoc
+
+          # Delete merged branches doc's
+          for branch in $(git branch -vv | grep ': gone]' | awk '{print $1}'); do rm -rf $branch; done
+
+          # Update / create this branch doc
+          rm -rf ${GITHUB_REF##*/}
+          mv ../${GITHUB_REF##*/} .
+
+          # Remove .idx files
+          # NOTE: git also uses idx files in his
+          # internal folder, hence the `*` instead of `.`
+          find * -name "*.idx" -delete
+          git add .
+          git config --global user.email "${{ github.actor }}@users.noreply.github.com"
+          git config --global user.name = "${{ github.actor }}"
+          git commit -a -m "update docs for ${GITHUB_REF##*/}"
+          git push origin gh-pages
+
+  update_site:
+    if: github.ref == 'refs/heads/master'
+    name: 'Rebuild website'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.x
+
+      - name: Generate website
+        run: pip install mkdocs-material && mkdocs build
+
+      - name: Clone the gh-pages branch
+        uses: actions/checkout@v2
+        with:
+          repository: status-im/nim-libp2p
+          ref: gh-pages
+          path: subdoc
+          fetch-depth: 0
+
+      - name: Commit & push
+        run: |
+          cd subdoc
+
+          rm -rf docs
+          mv ../site docs
+
+          git add .
+          git config --global user.email "${{ github.actor }}@users.noreply.github.com"
+          git config --global user.name = "${{ github.actor }}"
+          git commit -a -m "update website"
+          git push origin gh-pages

.github/workflows/multi_nim.yml

@@ -29,7 +29,7 @@ jobs:
             shell: bash
           - target:
               os: macos
-            builder: macos-10.15
+            builder: macos-12
             shell: bash
           - target:
               os: windows
@@ -160,10 +160,6 @@ jobs:
 
       - name: Run tests
         run: |
-          if [[ "${{ matrix.target.os }}" == "windows" ]]; then
-            # https://github.com/status-im/nimbus-eth2/issues/3121
-            export NIMFLAGS="-d:nimRawSetjmp"
-          fi
           nim --version
           nimble --version
           nimble install -y --depsOnly

.gitignore

@@ -13,3 +13,5 @@ build/
 .vscode/
 .DS_Store
 tests/pubsub/testgossipsub
+nimble.develop
+nimble.paths

.pinned

@@ -1,17 +1,16 @@
-asynctest;https://github.com/markspanbroek/asynctest@#5347c59b4b057443a014722aa40800cd8bb95c69
-bearssl;https://github.com/status-im/nim-bearssl@#0ebb1d7a4af5f4b4d4756a9b6dbfe5d411fa55d9
-chronicles;https://github.com/status-im/nim-chronicles@#2a2681b60289aaf7895b7056f22616081eb1a882
-chronos;https://github.com/status-im/nim-chronos@#875d7d8e6ef0803ae1c331dbf76b1981b0caeb15
-dnsclient;https://github.com/ba0f3/dnsclient.nim@#fbb76f8af8a33ab818184a7d4406d9fee20993be
+bearssl;https://github.com/status-im/nim-bearssl@#25009951ff8e0006171d566e3c7dc73a8231c2ed
+chronicles;https://github.com/status-im/nim-chronicles@#32ac8679680ea699f7dbc046e8e0131cac97d41a
+chronos;https://github.com/status-im/nim-chronos@#41b82cdea34744148600b67a9154331b76181189
+dnsclient;https://github.com/ba0f3/dnsclient.nim@#4960de2b345f567b12f09a08e9967af104ab39a3
 faststreams;https://github.com/status-im/nim-faststreams@#49e2c52eb5dda46b1c9c10d079abe7bffe6cea89
 httputils;https://github.com/status-im/nim-http-utils@#f83fbce4d6ec7927b75be3f85e4fa905fcb69788
-json_serialization;https://github.com/status-im/nim-json-serialization@#3509706517f3562cbcbe9d94988eccdd80474ab8
-metrics;https://github.com/status-im/nim-metrics@#11edec862f96e42374bc2d584c84cc88d5d1f95f
-nimcrypto;https://github.com/cheatfate/nimcrypto@#a5742a9a214ac33f91615f3862c7b099aec43b00
-secp256k1;https://github.com/status-im/nim-secp256k1@#e092373a5cbe1fa25abfc62e0f2a5f138dc3fb13
-serialization;https://github.com/status-im/nim-serialization@#9631fbd1c81c8b25ff8740df440ca7ba87fa6131
-stew;https://github.com/status-im/nim-stew@#cdb1f213d073fd2ecbdaf35a866417657da9294c
-testutils;https://github.com/status-im/nim-testutils@#aa6e5216f4b4ab5aa971cdcdd70e1ec1203cedf2
-unittest2;https://github.com/status-im/nim-unittest2@#4e2893eacb916c7678fdc4935ff7420f13bf3a9c
-websock;https://github.com/status-im/nim-websock@#8927db93f6ca96abaacfea39f8ca50ce9d41bcdb
-zlib;https://github.com/status-im/nim-zlib@#74cdeb54b21bededb5a515d36f608bc1850555a2
+json_serialization;https://github.com/status-im/nim-json-serialization@#e5b18fb710c3d0167ec79f3b892f5a7a1bc6d1a4
+metrics;https://github.com/status-im/nim-metrics@#9070af9c830e93e5239ddc488cd65aa6f609ba73
+nimcrypto;https://github.com/cheatfate/nimcrypto@#24e006df85927f64916e60511620583b11403178
+secp256k1;https://github.com/status-im/nim-secp256k1@#5340cf188168d6afcafc8023770d880f067c0b2f
+serialization;https://github.com/status-im/nim-serialization@#493d18b8292fc03aa4f835fd825dea1183f97466
+stew;https://github.com/status-im/nim-stew@#598246620da5c41d0e92a8dd6aab0755381b21cd
+testutils;https://github.com/status-im/nim-testutils@#dfc4c1b39f9ded9baf6365014de2b4bfb4dafc34
+unittest2;https://github.com/status-im/nim-unittest2@#f180f596c88dfd266f746ed6f8dbebce39c824db
+websock;https://github.com/status-im/nim-websock@#8a72c0f7690802753b1d59887745b1ce1f0c8b3d
+zlib;https://github.com/status-im/nim-zlib@#6a6670afba6b97b29b920340e2641978c05ab4d8

README.md

@@ -1,5 +1,5 @@
 <h1 align="center">
-  <a href="https://libp2p.io"><img width="250" src="https://github.com/libp2p/libp2p/blob/master/logo/black-bg-2.png?raw=true" alt="libp2p hex logo" /></a>
+  <a href="https://libp2p.io"><img width="250" src="./.assets/full-logo.svg?raw=true" alt="nim-libp2p logo" /></a>
 </h1>
 
 <h3 align="center">The Nim implementation of the libp2p Networking Stack.</h3>
@@ -7,7 +7,7 @@
 <p align="center">
 <a href="https://github.com/status-im/nim-libp2p/actions"><img src="https://github.com/status-im/nim-libp2p/actions/workflows/ci.yml/badge.svg" /></a>
 <a href="https://codecov.io/gh/status-im/nim-libp2p"><img src="https://codecov.io/gh/status-im/nim-libp2p/branch/master/graph/badge.svg?token=UR5JRQ249W"/></a>
-    
+
 </p>
 
 <p align="center">
@@ -18,28 +18,16 @@
 
 ## Introduction
 
-An implementation of [libp2p](https://libp2p.io/) in Nim.
-
-## Project Status
-libp2p is used in production by a few projects at [Status](https://github.com/status-im), including [Nimbus](https://github.com/status-im/nimbus-eth2).
-
-While far from complete, currently available components are stable.
-
-Check our [examples folder](/examples) to get started!
+An implementation of [libp2p](https://libp2p.io/) in [Nim](https://nim-lang.org/).
 
 # Table of Contents
 - [Background](#background)
 - [Install](#install)
-  - [Prerequisite](#prerequisite)
-- [Usage](#usage)
-  - [API](#api)
-  - [Getting Started](#getting-started)
-  - [Tutorials and Examples](#tutorials-and-examples)
-  - [Using the Go Daemon](#using-the-go-daemon)
+- [Getting Started](#getting-started)
+- [Modules](#modules)
+- [Users](#users)
 - [Development](#development)
-  - [Tests](#tests)
-  - [Packages](#packages)
-- [Contribute](#contribute)
+  - [Contribute](#contribute)
   - [Core Developers](#core-developers)
 - [License](#license)
 
@@ -54,33 +42,61 @@ libp2p grew out of IPFS, but it is built so that lots of people can use it, for
 - [Here](https://github.com/libp2p/libp2p#description) is an overview of libp2p and its implementations in other programming languages.
 
 ## Install
+**Prerequisite**
+- [Nim](https://nim-lang.org/install.html)
 ```
 nimble install libp2p
 ```
-### Prerequisite
-- [Nim](https://nim-lang.org/install.html)
 
-## Usage
+## Getting Started
+You'll find the documentation [here](https://status-im.github.io/nim-libp2p/docs/).
 
-### API
-The specification is available in the [docs/api](docs/api) folder.
+**Go Daemon:**
+Please find the installation and usage intructions in [daemonapi.md](examples/go-daemon/daemonapi.md).
 
-### Getting Started
-Please read the [GETTING_STARTED.md](docs/GETTING_STARTED.md) guide.
+## Modules
 
-### Tutorials and Examples
-Example code can be found in the [examples folder](/examples).
+List of packages modules implemented in nim-libp2p:
 
-#### Direct Chat Tutorial
-- [Part I](https://our.status.im/nim-libp2p-tutorial-a-peer-to-peer-chat-example-1/): Set up the main function and use multi-thread for processing IO.
-- [Part II](https://our.status.im/nim-libp2p-tutorial-a-peer-to-peer-chat-example-2/): Dial remote peer and allow customized user input commands.
-- [Part III](https://our.status.im/nim-libp2p-tutorial-a-peer-to-peer-chat-example-3/): Configure and establish a libp2p node.
+| Name                                                       | Description                                                                                                      |
+| ---------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------- |
+| **Libp2p**                                                 |                                                                                                                  |
+| [libp2p](libp2p/switch.nim)                                | The core of the project                                                                                          |
+| [connmanager](libp2p/connmanager.nim)                      | Connection manager                                                                                               |
+| [identify / push identify](libp2p/protocols/identify.nim)  | [Identify](https://docs.libp2p.io/concepts/protocols/#identify) protocol                                         |
+| [ping](libp2p/protocols/ping.nim)                          | [Ping](https://docs.libp2p.io/concepts/protocols/#ping) protocol                                                 |
+| [libp2p-daemon-client](libp2p/daemon/daemonapi.nim)        | [go-daemon](https://github.com/libp2p/go-libp2p-daemon) nim wrapper                                              |
+| [interop-libp2p](tests/testinterop.nim)                    | Interop tests                                                                                                    |
+| **Transports**                                             |                                                                                                                  |
+| [libp2p-tcp](libp2p/transports/tcptransport.nim)           | TCP transport                                                                                                    |
+| [libp2p-ws](libp2p/transports/wstransport.nim)             | WebSocket & WebSocket Secure transport                                                                           |
+| **Secure Channels**                                        |                                                                                                                  |
+| [libp2p-secio](libp2p/protocols/secure/secio.nim)          | [Secio](https://docs.libp2p.io/concepts/protocols/#secio) secure channel                                         |
+| [libp2p-noise](libp2p/protocols/secure/noise.nim)          | [Noise](https://github.com/libp2p/specs/tree/master/noise) secure channel                                        |
+| [libp2p-plaintext](libp2p/protocols/secure/plaintext.nim)  | [Plain Text](https://github.com/libp2p/specs/tree/master/plaintext) for development purposes                     |
+| **Stream Multiplexers**                                    |                                                                                                                  |
+| [libp2p-mplex](libp2p/muxers/mplex/mplex.nim)              | [MPlex](https://github.com/libp2p/specs/tree/master/mplex) multiplexer                                           |
+| **Data Types**                                             |                                                                                                                  |
+| [peer-id](libp2p/peerid.nim)                               | [Cryptographic identifiers](https://docs.libp2p.io/concepts/peer-id/)                                            |
+| [peer-store](libp2p/peerstore.nim)                         | ["Phone book" of known peers](https://docs.libp2p.io/concepts/peer-id/#peerinfo)                                 |
+| [multiaddress](libp2p/multiaddress.nim)                    | [Composable network addresses](https://github.com/multiformats/multiaddr)                                        |
+| [signed envelope](libp2p/signed_envelope.nim)              | [Signed generic data container](https://github.com/libp2p/specs/blob/master/RFC/0002-signed-envelopes.md)        |
+| [routing record](libp2p/routing_record.nim)                | [Signed peer dialing informations](https://github.com/libp2p/specs/blob/master/RFC/0003-routing-records.md)      |
+| **Utilities**                                              |                                                                                                                  |
+| [libp2p-crypto](libp2p/crypto)                             | Cryptographic backend                                                                                            |
+| [libp2p-crypto-secp256k1](libp2p/crypto/secp.nim)          |                                                                                                                  |
+| **Pubsub**                                                 |                                                                                                                  |
+| [libp2p-pubsub](libp2p/protocols/pubsub/pubsub.nim)        | Pub-Sub generic interface                                                                                        |
+| [libp2p-floodsub](libp2p/protocols/pubsub/floodsub.nim)    | FloodSub implementation                                                                                          |
+| [libp2p-gossipsub](libp2p/protocols/pubsub/gossipsub.nim)  | [GossipSub](https://docs.libp2p.io/concepts/publish-subscribe/) implementation                                   |
 
+## Users
 
-### Using the Go Daemon
-Please find the installation and usage intructions in [daemonapi.md](docs/api/libp2p/daemonapi.md).
-
-Examples can be found in the [examples/go-daemon folder](https://github.com/status-im/nim-libp2p/tree/readme/examples/go-daemon);
+nim-libp2p is used by:
+- [Nimbus](https://github.com/status-im/nimbus-eth2), an Ethereum client
+- [nwaku](https://github.com/status-im/nwaku), a decentralized messaging application
+- [nim-codex](https://github.com/status-im/nim-codex), a decentralized storage application
+- (open a pull request if you want to be included here)
 
 ## Development
 **Clone and Install dependencies:**
@@ -90,96 +106,45 @@ git clone https://github.com/status-im/nim-libp2p
 cd nim-libp2p
 nimble install
 ```
-#### Run unit tests
+
+**Run unit tests**
 ```sh
 # run all the unit tests
 nimble test
 ```
-The code follows the [Status Nim Style Guide](https://status-im.github.io/nim-style-guide/).
 
-### Packages
-
-List of packages currently in existence for nim-libp2p:
-
-#### Libp2p
-- [libp2p](https://github.com/status-im/nim-libp2p)
-- [libp2p-daemon-client](https://github.com/status-im/nim-libp2p/blob/master/libp2p/daemon/daemonapi.nim)
-- [interop-libp2p](https://github.com/status-im/nim-libp2p/blob/master/tests/testinterop.nim)
-
-#### Transports
-- [libp2p-tcp](https://github.com/status-im/nim-libp2p/blob/master/libp2p/transports/tcptransport.nim)
-- [libp2p-ws](https://github.com/status-im/nim-libp2p/blob/master/libp2p/transports/wstransport.nim)
-
-#### Secure Channels
-- [libp2p-secio](https://github.com/status-im/nim-libp2p/blob/master/libp2p/protocols/secure/secio.nim)
-- [libp2p-noise](https://github.com/status-im/nim-libp2p/blob/master/libp2p/protocols/secure/noise.nim)
-- [libp2p-plaintext](https://github.com/status-im/nim-libp2p/blob/master/libp2p/protocols/secure/plaintext.nim)
-
-#### Stream Multiplexers
-- [libp2p-mplex](https://github.com/status-im/nim-libp2p/blob/master/libp2p/muxers/mplex/mplex.nim)
-
-#### Utilities
-- [libp2p-crypto](https://github.com/status-im/nim-libp2p/tree/master/libp2p/crypto)
-- [libp2p-crypto-secp256k1](https://github.com/status-im/nim-libp2p/blob/master/libp2p/crypto/secp.nim)
-
-#### Data Types
-- [peer-id](https://github.com/status-im/nim-libp2p/blob/master/libp2p/peer.nim)
-- [peer-info](https://github.com/status-im/nim-libp2p/blob/master/libp2p/peerinfo.nim)
-
-#### Pubsub
-- [libp2p-pubsub](https://github.com/status-im/nim-libp2p/blob/master/libp2p/protocols/pubsub/pubsub.nim)
-- [libp2p-floodsub](https://github.com/status-im/nim-libp2p/blob/master/libp2p/protocols/pubsub/floodsub.nim)
-- [libp2p-gossipsub](https://github.com/status-im/nim-libp2p/blob/master/libp2p/protocols/pubsub/gossipsub.nim)
-
-
-Packages that exist in the original libp2p specs and are under active development:
-- libp2p-daemon
-- libp2p-webrtc-direct
-- libp2p-webrtc-star
-- libp2p-spdy
-- libp2p-bootstrap
-- libp2p-kad-dht
-- libp2p-mdns
-- libp2p-webrtc-star
-- libp2p-delegated-content-routing
-- libp2p-delegated-peer-routing
-- libp2p-nat-mgnr
-- libp2p-utils
-
-** Note that the current stack reflects the minimal requirements for the upcoming Eth2 implementation.
-
-### Tips and tricks
-
-#### enable expensive metrics:
-
-```bash
-nim c -d:libp2p_expensive_metrics some_file.nim
-```
-
-#### use identify metrics
-
-```bash
-nim c -d:libp2p_agents_metrics -d:KnownLibP2PAgents=nimbus,lighthouse,prysm,teku some_file.nim
-```
-
-### specify gossipsub specific topics to measure
-
-```bash
-nim c -d:KnownLibP2PTopics=topic1,topic2,topic3 some_file.nim
-```
-
-## Contribute
+### Contribute
 
 The libp2p implementation in Nim is a work in progress. We welcome contributors to help out! Specifically, you can:
 - Go through the modules and **check out existing issues**. This would be especially useful for modules in active development. Some knowledge of IPFS/libp2p may be required, as well as the infrastructure behind it.
 - **Perform code reviews**. Feel free to let us know if you found anything that can a) speed up the project development b) ensure better quality and c) reduce possible future bugs.
-- **Add tests**. Help nim-libp2p to be more robust by adding more tests to the [tests folder](https://github.com/status-im/nim-libp2p/tree/master/tests).
+- **Add tests**. Help nim-libp2p to be more robust by adding more tests to the [tests folder](tests/).
 
 The code follows the [Status Nim Style Guide](https://status-im.github.io/nim-style-guide/).
 
 ### Core Developers
 [@cheatfate](https://github.com/cheatfate), [Dmitriy Ryajov](https://github.com/dryajov), [Tanguy](https://github.com/Menduist), [Zahary Karadjov](https://github.com/zah)
 
+### Tips and tricks
+
+**enable expensive metrics:**
+
+```bash
+nim c -d:libp2p_expensive_metrics some_file.nim
+```
+
+**use identify metrics**
+
+```bash
+nim c -d:libp2p_agents_metrics -d:KnownLibP2PAgents=nimbus,lighthouse,prysm,teku some_file.nim
+```
+
+**specify gossipsub specific topics to measure**
+
+```bash
+nim c -d:KnownLibP2PTopics=topic1,topic2,topic3 some_file.nim
+```
+
 ## License
 
 Licensed and distributed under either of
@@ -191,4 +156,3 @@ or
 * Apache License, Version 2.0, ([LICENSE-APACHEv2](LICENSE-APACHEv2) or http://www.apache.org/licenses/LICENSE-2.0)
 
 at your option. These files may not be copied, modified, or distributed except according to those terms.
-

config.nims

@@ -2,5 +2,22 @@
 if dirExists("nimbledeps/pkgs"):
   switch("NimblePath", "nimbledeps/pkgs")
 
-when (NimMajor, NimMinor) > (1, 2):
-  switch("hint", "XCannotRaiseY:off")
+switch("warning", "CaseTransition:off")
+switch("warning", "ObservableStores:off")
+switch("warning", "LockLevel:off")
+--define:chronosStrictException
+--styleCheck:usages
+if (NimMajor, NimMinor) < (1, 6):
+  --styleCheck:hint
+else:
+  --styleCheck:error
+
+# Avoid some rare stack corruption while using exceptions with a SEH-enabled
+# toolchain: https://github.com/status-im/nimbus-eth2/issues/3121
+if defined(windows) and not defined(vcc):
+  --define:nimRawSetjmp
+
+# begin Nimble config (version 1)
+when fileExists("nimble.paths"):
+  include "nimble.paths"
+# end Nimble config

docs/API.md

@@ -1,3 +0,0 @@
-# API
-
-Coming Soon... 

docs/GETTING_STARTED.md

@@ -1,7 +0,0 @@
-# Getting Started
-Welcome to nim-libp2p!
-
-
-To get started, please look at the [tutorials](../examples/tutorial_1_connect.md)
-
-For more concrete examples, you can look at the [hello world example](../examples/helloworld.nim) or the [direct chat](../examples/directchat.nim)

docs/README.md

@@ -1,29 +0,0 @@
-# Introduction
-This folder contains the documentation for each nim-libp2p module and the sample code for the tutorials. 
-
-# Table of Contents 
-### [Getting Started](GETTING_STARTED.md)
-### Tutorials
-- P2P Chat Example
-    - [part I](tutorial/directchat/start.nim)
-    - [part II](tutorial/directchat/second.nim)
-### API Specifications
-- libp2p
-    - [libp2p-daemon-client](api/libp2p/daemonapi.md)
-    - [interop-libp2p](api/libp2p/interop.md)
-- transports
-    - [libp2p-tcp](api/transports/tcptransport.md)
-- secure channels
-    - [libp2p-secio](api/secure_channels/secio.md)
-- stream multiplexers
-    - [libp2p-mplex](api/stream_multiplexers/mplex.md)
-- utilities
-    - [libp2p-crypto](api/utilities/crypto.md)
-    - [libp2p-crypto-secp256k1](api/utilities/secp256k1.md)
-- data types
-    - [peer-id](api/data_types/peer.md)
-    - [peer-info](api/data_types/peerinfo.md)
-- pubsub
-    - [libp2p-pubsub](api/pubsub/pubsub.md)
-    - [libp2p-floodsub](api/pubsub/floodsub.md)
-    - [libp2p-gossipsub](api/pubsub/gossipsub.md)

docs/tutorial/directchat/second.nim

@@ -1,149 +0,0 @@
-when not(compileOption("threads")):
-  {.fatal: "Please, compile this program with the --threads:on option!".}
-
-import tables, strformat, strutils
-import chronos
-import ../libp2p/[switch,
-                  multistream,
-                  crypto/crypto,
-                  protocols/identify,
-                  connection,
-                  transports/transport,
-                  transports/tcptransport,
-                  multiaddress,
-                  peerinfo,
-                  peerid,
-                  protocols/protocol,
-                  protocols/secure/secure,
-                  protocols/secure/secio,
-                  muxers/muxer,
-                  muxers/mplex/mplex]
-
-const ChatCodec = "/nim-libp2p/chat/1.0.0"
-const DefaultAddr = "/ip4/127.0.0.1/tcp/55505"
-
-const Help = """
-  Commands: /[?|hep|connect|disconnect|exit]
-  help: Prints this help
-  connect: dials a remote peer
-  disconnect: ends current session
-  exit: closes the chat
-"""
-
-type ChatProto = ref object of LPProtocol
-  switch: Switch          # a single entry point for dialing and listening to peer
-  transp: StreamTransport # transport streams between read & write file descriptor
-  conn: Connection        # create and close read & write stream
-  connected: bool         # if the node is connected to another peer
-  started: bool           # if the node has started
-
-# copied from https://github.com/status-im/nimbus-eth2/blob/0ed657e953740a92458f23033d47483ffa17ccb0/beacon_chain/eth2_network.nim#L109-L115
-proc initAddress(T: type MultiAddress, str: string): T =
-  let address = MultiAddress.init(str)
-  if IPFS.match(address) and matchPartial(multiaddress.TCP, address):
-    result = address
-  else:
-    raise newException(MultiAddressError,
-                         "Invalid bootstrap node multi-address")
-
-proc dialPeer(p: ChatProto, address: string) {.async.} =
-  let multiAddr = MultiAddress.initAddress(address);
-  let parts = address.split("/")
-  let remotePeer = PeerInfo.init(parts[^1],
-                                 [multiAddr])
-
-  echo &"dialing peer: {multiAddr}"
-  p.conn = await p.switch.dial(remotePeer, ChatCodec)
-  p.connected = true
-
-proc readAndPrint(p: ChatProto) {.async.} =
-  while true:
-    while p.connected:
-      echo cast[string](await p.conn.readLp(1024))
-    await sleepAsync(100.millis)
-
-proc writeAndPrint(p: ChatProto) {.async.} =
-  while true:
-    if not p.connected:
-      echo "type an address or wait for a connection:"
-      echo "type /[help|?] for help"
-
-    let line = await p.transp.readLine()
-    if line.startsWith("/help") or line.startsWith("/?") or not p.started:
-      echo Help
-      continue
-
-    if line.startsWith("/disconnect"):
-      echo "Ending current session"
-      if p.connected and p.conn.closed.not:
-        await p.conn.close()
-      p.connected = false
-    elif line.startsWith("/connect"):
-      if p.connected:
-        var yesno = "N"
-        echo "a session is already in progress, do you want end it [y/N]?"
-        yesno = await p.transp.readLine()
-        if yesno.cmpIgnoreCase("y") == 0:
-          await p.conn.close()
-          p.connected = false
-        elif yesno.cmpIgnoreCase("n") == 0:
-          continue
-        else:
-          echo "unrecognized response"
-          continue
-
-      echo "enter address of remote peer"
-      let address = await p.transp.readLine()
-      if address.len > 0:
-        await p.dialPeer(address)
-
-    elif line.startsWith("/exit"):
-      if p.connected and p.conn.closed.not:
-        await p.conn.close()
-        p.connected = false
-
-      await p.switch.stop()
-      echo "quitting..."
-      quit(0)
-    else:
-      if p.connected:
-        await p.conn.writeLp(line)
-      else:
-        try:
-          if line.startsWith("/") and "ipfs" in line:
-            await p.dialPeer(line)
-        except:
-          echo &"unable to dial remote peer {line}"
-          echo getCurrentExceptionMsg()
-
-proc readWriteLoop(p: ChatProto) {.async.} =
-  asyncSpawn p.writeAndPrint() # execute the async function but does not block
-  asyncSpawn p.readAndPrint()
-
-proc processInput(rfd: AsyncFD) {.async.} =
-  let transp = fromPipe(rfd)
-  while true:
-    let a = await transp.readLine()
-    echo "You just entered: " & a
-
-proc readInput(wfd: AsyncFD) {.thread.} =
-  ## This procedure performs reading from `stdin` and sends data over
-  ## pipe to main thread.
-  let transp = fromPipe(wfd)
-
-  while true:
-    let line = stdin.readLine()
-    discard waitFor transp.write(line & "\r\n")
-
-proc main() {.async.} =
-  let (rfd, wfd) = createAsyncPipe()
-  if rfd == asyncInvalidPipe or wfd == asyncInvalidPipe:
-    raise newException(ValueError, "Could not initialize pipe!")
-
-  var thread: Thread[AsyncFD]
-  thread.createThread(readInput, wfd)
-
-  await processInput(rfd)
-
-when isMainModule:      # isMainModule = true when the module is compiled as the main file
-  waitFor(main())

docs/tutorial/directchat/start.nim

@@ -1,39 +0,0 @@
-when not(compileOption("threads")):
-  {.fatal: "Please, compile this program with the --threads:on option!".}
-  
-import chronos          # an efficient library for async
-
-proc processInput(rfd: AsyncFD) {.async.} =
-  echo "Type something below to see if the multithread IO works:\nType 'exit' to exit."
-
-  let transp = fromPipe(rfd)
-  while true:
-    let a = await transp.readLine()
-
-    if a == "exit":
-      quit(0);
-
-    echo "You just entered: " & a
-
-proc readInput(wfd: AsyncFD) {.thread.} =
-  ## This procedure performs reading from `stdin` and sends data over
-  ## pipe to main thread.
-  let transp = fromPipe(wfd)
-
-  while true:
-    let line = stdin.readLine()
-    discard waitFor transp.write(line & "\r\n")
-
-proc main() {.async.} =
-  let (rfd, wfd) = createAsyncPipe()
-  if rfd == asyncInvalidPipe or wfd == asyncInvalidPipe:
-    raise newException(ValueError, "Could not initialize pipe!")
-  
-  var thread: Thread[AsyncFD]
-  thread.createThread(readInput, wfd)
-  
-  await processInput(rfd)
-
-when isMainModule:      # isMainModule = true when the module is compiled as the main file
-  waitFor(main())
- 

docs/tutorial/second.nim

@@ -1,205 +0,0 @@
-when not(compileOption("threads")):
-  {.fatal: "Please, compile this program with the --threads:on option!".}
-
-import tables, strformat, strutils, bearssl
-import chronos                              # an efficient library for async
-import ../libp2p/[switch,                   # manage transports, a single entry point for dialing and listening
-                  builders,                 # helper to build the switch object
-                  multistream,              # tag stream with short header to identify it
-                  multicodec,               # multicodec utilities
-                  crypto/crypto,            # cryptographic functions
-                  errors,                   # error handling utilities
-                  protocols/identify,       # identify the peer info of a peer
-                  stream/connection,        # create and close stream read / write connections
-                  transports/transport,     # listen and dial to other peers using p2p protocol
-                  transports/tcptransport,  # listen and dial to other peers using client-server protocol
-                  multiaddress,             # encode different addressing schemes. For example, /ip4/7.7.7.7/tcp/6543 means it is using IPv4 protocol and TCP
-                  peerinfo,                 # manage the information of a peer, such as peer ID and public / private key
-                  peerid,                   # Implement how peers interact
-                  protocols/protocol,       # define the protocol base type
-                  protocols/secure/secure,  # define the protocol of secure connection
-                  protocols/secure/secio,   # define the protocol of secure input / output, allows encrypted communication that uses public keys to validate signed messages instead of a certificate authority like in TLS
-                  muxers/muxer,             # define an interface for stream multiplexing, allowing peers to offer many protocols over a single connection
-                  muxers/mplex/mplex]       # define some contants and message types for stream multiplexing
-
-const ChatCodec = "/nim-libp2p/chat/1.0.0"
-const DefaultAddr = "/ip4/127.0.0.1/tcp/55505"
-
-const Help = """
-  Commands: /[?|hep|connect|disconnect|exit]
-  help: Prints this help
-  connect: dials a remote peer
-  disconnect: ends current session
-  exit: closes the chat
-"""
-
-type ChatProto = ref object of LPProtocol
-  switch: Switch          # a single entry point for dialing and listening to peer
-  transp: StreamTransport # transport streams between read & write file descriptor
-  conn: Connection        # create and close read & write stream
-  connected: bool         # if the node is connected to another peer
-  started: bool           # if the node has started
-
-proc readAndPrint(p: ChatProto) {.async.} =
-  while true:
-    var strData = await p.conn.readLp(1024)
-    strData &= '\0'.uint8
-    var str = cast[cstring](addr strdata[0])
-    echo $p.switch.peerInfo.peerId & ": " & $str
-    await sleepAsync(100.millis)
-
-proc dialPeer(p: ChatProto, address: string) {.async.} =
-  let
-    multiAddr = MultiAddress.init(address).tryGet()
-    # split the peerId part /p2p/...
-    peerIdBytes = multiAddr[multiCodec("p2p")]
-      .tryGet()
-      .protoAddress()
-      .tryGet()
-    remotePeer = PeerId.init(peerIdBytes).tryGet()
-    # split the wire address
-    ip4Addr = multiAddr[multiCodec("ip4")].tryGet()
-    tcpAddr = multiAddr[multiCodec("tcp")].tryGet()
-    wireAddr = ip4Addr & tcpAddr
-
-  echo &"dialing peer: {multiAddr}"
-  p.conn = await p.switch.dial(remotePeer, @[wireAddr], ChatCodec)
-  p.connected = true
-  asyncSpawn p.readAndPrint()
-
-proc writeAndPrint(p: ChatProto) {.async.} =
-  while true:
-    if not p.connected:
-      echo "type an address or wait for a connection:"
-      echo "type /[help|?] for help"
-
-    let line = await p.transp.readLine()
-    if line.startsWith("/help") or line.startsWith("/?") or not p.started:
-      echo Help
-      continue
-
-    if line.startsWith("/disconnect"):
-      echo "Ending current session"
-      if p.connected and p.conn.closed.not:
-        await p.conn.close()
-      p.connected = false
-    elif line.startsWith("/connect"):
-      if p.connected:
-        var yesno = "N"
-        echo "a session is already in progress, do you want end it [y/N]?"
-        yesno = await p.transp.readLine()
-        if yesno.cmpIgnoreCase("y") == 0:
-          await p.conn.close()
-          p.connected = false
-        elif yesno.cmpIgnoreCase("n") == 0:
-          continue
-        else:
-          echo "unrecognized response"
-          continue
-
-      echo "enter address of remote peer"
-      let address = await p.transp.readLine()
-      if address.len > 0:
-        await p.dialPeer(address)
-
-    elif line.startsWith("/exit"):
-      if p.connected and p.conn.closed.not:
-        await p.conn.close()
-        p.connected = false
-
-      await p.switch.stop()
-      echo "quitting..."
-      quit(0)
-    else:
-      if p.connected:
-        await p.conn.writeLp(line)
-      else:
-        try:
-          if line.startsWith("/") and "p2p" in line:
-            await p.dialPeer(line)
-        except:
-          echo &"unable to dial remote peer {line}"
-          echo getCurrentExceptionMsg()
-
-proc readWriteLoop(p: ChatProto) {.async.} =
-  await p.writeAndPrint()
-
-proc newChatProto(switch: Switch, transp: StreamTransport): ChatProto =
-  var chatproto = ChatProto(switch: switch, transp: transp, codecs: @[ChatCodec])
-
-  # create handler for incoming connection
-  proc handle(stream: Connection, proto: string) {.async.} =
-    if chatproto.connected and not chatproto.conn.closed:
-      echo "a chat session is already in progress - disconnecting!"
-      await stream.close()
-    else:
-      chatproto.conn = stream
-      chatproto.connected = true
-      await chatproto.readAndPrint()
-
-  # assign the new handler
-  chatproto.handler = handle
-  return chatproto
-
-proc readInput(wfd: AsyncFD) {.thread.} =
-  ## This procedure performs reading from `stdin` and sends data over
-  ## pipe to main thread.
-  let transp = fromPipe(wfd)
-
-  while true:
-    let line = stdin.readLine()
-    discard waitFor transp.write(line & "\r\n")
-
-proc processInput(rfd: AsyncFD, rng: ref BrHmacDrbgContext) {.async.} =
-  let transp = fromPipe(rfd)
-
-  let seckey = PrivateKey.random(RSA, rng[]).get()
-  var localAddress = DefaultAddr
-  while true:
-    echo &"Type an address to bind to or Enter to use the default {DefaultAddr}"
-    let a = await transp.readLine()
-    try:
-      if a.len > 0:
-        localAddress = a
-        break
-      # uise default
-      break
-    except:
-      echo "invalid address"
-      localAddress = DefaultAddr
-      continue
-
-  var switch = SwitchBuilder
-    .init()
-    .withRng(rng)
-    .withPrivateKey(seckey)
-    .withAddress(MultiAddress.init(localAddress).tryGet())
-    .build()
-
-  let chatProto = newChatProto(switch, transp)
-  switch.mount(chatProto)
-  let libp2pFuts = await switch.start()
-  chatProto.started = true
-
-  let id = $switch.peerInfo.peerId
-  echo "PeerId: " & id
-  echo "listening on: "
-  for a in switch.peerInfo.addrs:
-    echo &"{a}/p2p/{id}"
-
-  await chatProto.readWriteLoop()
-  await allFuturesThrowing(libp2pFuts)
-
-proc main() {.async.} =
-  let rng = newRng() # Singe random number source for the whole application
-  let (rfd, wfd) = createAsyncPipe()
-  if rfd == asyncInvalidPipe or wfd == asyncInvalidPipe:
-    raise newException(ValueError, "Could not initialize pipe!")
-
-  var thread: Thread[AsyncFD]
-  thread.createThread(readInput, wfd)
-
-  await processInput(rfd, rng)
-
-when isMainModule: # isMainModule = true when the module is compiled as the main file
-  waitFor(main())

docs/tutorial/start.nim

@@ -1,39 +0,0 @@
-when not(compileOption("threads")):
-  {.fatal: "Please, compile this program with the --threads:on option!".}
-  
-import chronos          # an efficient library for async
-
-proc processInput(rfd: AsyncFD) {.async.} =
-  echo "Type something below to see if the multithread IO works:\nType 'exit' to exit."
-
-  let transp = fromPipe(rfd)
-  while true:
-    let a = await transp.readLine()
-
-    if a == "exit":
-      quit(0);
-
-    echo "You just entered: " & a
-
-proc readInput(wfd: AsyncFD) {.thread.} =
-  ## This procedure performs reading from `stdin` and sends data over
-  ## pipe to main thread.
-  let transp = fromPipe(wfd)
-
-  while true:
-    let line = stdin.readLine()
-    discard waitFor transp.write(line & "\r\n")
-
-proc main() {.async.} =
-  let (rfd, wfd) = createAsyncPipe()
-  if rfd == asyncInvalidPipe or wfd == asyncInvalidPipe:
-    raise newException(ValueError, "Could not initialize pipe!")
-  
-  var thread: Thread[AsyncFD]
-  thread.createThread(readInput, wfd)
-  
-  await processInput(rfd)
-
-when isMainModule:      # isMainModule = true when the module is compiled as the main file
-  waitFor(main())
- 

examples/README.md

@@ -0,0 +1,6 @@
+# nim-libp2p documentation
+
+Welcome to the nim-libp2p documentation!
+
+Here, you'll find [tutorials](tutorial_1_connect.md) to help you get started, as well as [examples](directchat.nim) and
+the [full reference](https://status-im.github.io/nim-libp2p/master/libp2p.html).

examples/circuitrelay.nim

@@ -0,0 +1,76 @@
+import chronos, stew/byteutils
+import ../libp2p,
+       ../libp2p/protocols/relay/[relay, client]
+
+# Helper to create a circuit relay node
+proc createCircuitRelaySwitch(r: Relay): Switch =
+  SwitchBuilder.new()
+    .withRng(newRng())
+    .withAddresses(@[ MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet() ])
+    .withTcpTransport()
+    .withMplex()
+    .withNoise()
+    .withCircuitRelay(r)
+    .build()
+
+proc main() {.async.} =
+  # Create a custom protocol
+  let customProtoCodec = "/test"
+  var proto = new LPProtocol
+  proto.codec = customProtoCodec
+  proto.handler = proc(conn: Connection, proto: string) {.async.} =
+    var msg = string.fromBytes(await conn.readLp(1024))
+    echo "1 - Dst Received: ", msg
+    assert "test1" == msg
+    await conn.writeLp("test2")
+    msg = string.fromBytes(await conn.readLp(1024))
+    echo "2 - Dst Received: ", msg
+    assert "test3" == msg
+    await conn.writeLp("test4")
+
+  let
+    relay = Relay.new()
+    clSrc = RelayClient.new()
+    clDst = RelayClient.new()
+
+    # Create three hosts, enable relay client on two of them.
+    # The third one can relay connections for other peers.
+    # RelayClient can use a relay, Relay is a relay.
+    swRel = createCircuitRelaySwitch(relay)
+    swSrc = createCircuitRelaySwitch(clSrc)
+    swDst = createCircuitRelaySwitch(clDst)
+
+    # Create a relay address to swDst using swRel as the relay
+    addrs = MultiAddress.init($swRel.peerInfo.addrs[0] & "/p2p/" &
+                              $swRel.peerInfo.peerId & "/p2p-circuit/p2p/" &
+                              $swDst.peerInfo.peerId).get()
+
+  swDst.mount(proto)
+
+  await swRel.start()
+  await swSrc.start()
+  await swDst.start()
+
+  # Connect both Src and Dst to the relay, but not to each other.
+  await swSrc.connect(swRel.peerInfo.peerId, swRel.peerInfo.addrs)
+  await swDst.connect(swRel.peerInfo.peerId, swRel.peerInfo.addrs)
+
+  # Dst reserve a slot on the relay.
+  let rsvp = await clDst.reserve(swRel.peerInfo.peerId, swRel.peerInfo.addrs)
+
+  # Src dial Dst using the relay
+  let conn = await swSrc.dial(swDst.peerInfo.peerId, @[ addrs ], customProtoCodec)
+
+  await conn.writeLp("test1")
+  var msg = string.fromBytes(await conn.readLp(1024))
+  echo "1 - Src Received: ", msg
+  assert "test2" == msg
+  await conn.writeLp("test3")
+  msg = string.fromBytes(await conn.readLp(1024))
+  echo "2 - Src Received: ", msg
+  assert "test4" == msg
+
+  await relay.stop()
+  await allFutures(swSrc.stop(), swDst.stop(), swRel.stop())
+
+waitFor(main())

examples/directchat.nim

@@ -2,7 +2,7 @@ when not(compileOption("threads")):
   {.fatal: "Please, compile this program with the --threads:on option!".}
 
 import
-  strformat, strutils, bearssl,
+  strformat, strutils,
   stew/byteutils,
   chronos,
   ../libp2p

examples/helloworld.nim

@@ -1,4 +1,3 @@
-import bearssl
 import chronos         # an efficient library for async
 import stew/byteutils  # various utils
 import ../libp2p       # when installed through nimble, just use `import libp2p`
@@ -26,7 +25,7 @@ proc new(T: typedesc[TestProto]): T =
 ##
 # Helper to create a switch/node
 ##
-proc createSwitch(ma: MultiAddress, rng: ref BrHmacDrbgContext): Switch =
+proc createSwitch(ma: MultiAddress, rng: ref HmacDrbgContext): Switch =
   var switch = SwitchBuilder
     .new()
     .withRng(rng)       # Give the application RNG

examples/tutorial_1_connect.md

@@ -1,15 +1,18 @@
-Hi all, welcome to the first article of the nim-libp2p's tutorial series!
+# Simple ping tutorial
 
-_This tutorial is for everyone who is interested in building peer-to-peer chatting applications. No Nim programming experience is needed._
+Hi all, welcome to the first nim-libp2p tutorial!
+
+!!! tips ""
+    This tutorial is for everyone who is interested in building peer-to-peer applications. No Nim programming experience is needed.
 
 To give you a quick overview, **Nim** is the programming language we are using and **nim-libp2p** is the Nim implementation of [libp2p](https://libp2p.io/), a modular library that enables the development of peer-to-peer network applications.
 
 Hope you'll find it helpful in your journey of learning. Happy coding! ;)
 
-# Before you start
+## Before you start
 The only prerequisite here is [Nim](https://nim-lang.org/), the programming language with a Python-like syntax and a performance similar to C. Detailed information can be found [here](https://nim-lang.org/docs/tut1.html).
 
-Install Nim via their official website: [https://nim-lang.org/install.html](https://nim-lang.org/install.html)  
+Install Nim via their [official website](https://nim-lang.org/install.html).
 Check Nim's installation via `nim --version` and its package manager Nimble via `nimble --version`.
 
 You can now install the latest version of `nim-libp2p`:
@@ -17,25 +20,24 @@ You can now install the latest version of `nim-libp2p`:
 nimble install libp2p@#master
 ```
 
-# A simple ping application
+## A simple ping application
 We'll start by creating a simple application, which is starting two libp2p [switch](https://docs.libp2p.io/concepts/stream-multiplexing/#switch-swarm), and pinging each other using the [Ping](https://docs.libp2p.io/concepts/protocols/#ping) protocol.
 
-_TIP: You can extract the code from this tutorial by running `nim c -r tools/markdown_runner.nim examples/tutorial_1_connect.md` in the libp2p folder!_
+!!! tips ""
+    You can extract the code from this tutorial by running `nim c -r tools/markdown_runner.nim examples/tutorial_1_connect.md` in the libp2p folder!
 
 Let's create a `part1.nim`, and import our dependencies:
 ```nim
-import bearssl
 import chronos
 
 import libp2p
 import libp2p/protocols/ping
 ```
-[bearssl](https://github.com/status-im/nim-bearssl) is used as a [cryptographic pseudorandom number generator](https://en.wikipedia.org/wiki/Cryptographically-secure_pseudorandom_number_generator)  
 [chronos](https://github.com/status-im/nim-chronos) the asynchronous framework used by `nim-libp2p`
 
 Next, we'll create an helper procedure to create our switches. A switch needs a bit of configuration, and it will be easier to do this configuration only once:
 ```nim
-proc createSwitch(ma: MultiAddress, rng: ref BrHmacDrbgContext): Switch =
+proc createSwitch(ma: MultiAddress, rng: ref HmacDrbgContext): Switch =
   var switch = SwitchBuilder
     .new()
     .withRng(rng)       # Give the application RNG
@@ -60,17 +62,17 @@ proc main() {.async, gcsafe.} =
     localAddress = MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet()
     pingProtocol = Ping.new(rng=rng)
 ```
-We created some variables that we'll need for the rest of the application: the global `rng` instance, our `localAddress`, and an instance of the `Ping` protocol.  
+We created some variables that we'll need for the rest of the application: the global `rng` instance, our `localAddress`, and an instance of the `Ping` protocol.
 The address is in the [MultiAddress](https://github.com/multiformats/multiaddr) format. The port `0` means "take any port available".
 
-`tryGet` is procedure which is part of the [nim-result](https://github.com/arnetheduck/nim-result/), that will throw an exception if the supplied MultiAddress is not valid.
+`tryGet` is procedure which is part of [nim-result](https://github.com/arnetheduck/nim-result/), that will throw an exception if the supplied MultiAddress is invalid.
 
 We can now create our two switches:
 ```nim
   let
     switch1 = createSwitch(localAddress, rng)
     switch2 = createSwitch(localAddress, rng)
-    
+
   switch1.mount(pingProtocol)
 
   await switch1.start()
@@ -78,7 +80,7 @@ We can now create our two switches:
 ```
 We've **mounted** the `pingProtocol` on our first switch. This means that the first switch will actually listen for any ping requests coming in, and handle them accordingly.
 
-Now that we've started the nodes, they are listening for incoming peers.  
+Now that we've started the nodes, they are listening for incoming peers.
 We can find out which port was attributed, and the resulting local addresses, by using `switch1.peerInfo.addrs`.
 
 We'll **dial** the first switch from the second one, by specifying it's **Peer ID**, it's **MultiAddress** and the **`Ping` protocol codec**:
@@ -97,7 +99,7 @@ We now have a `Ping` connection setup between the second and the first switch, w
 And that's it! Just a little bit of cleanup: shutting down the switches, waiting for them to stop, and we'll call our `main` procedure:
 ```nim
   await allFutures(switch1.stop(), switch2.stop()) # close connections and shutdown all transports
-  
+
 waitFor(main())
 ```
 

examples/tutorial_2_customproto.md

@@ -1,11 +1,11 @@
+# Custom protocol in libp2p
+
 In the [previous tutorial](tutorial_1_connect.md), we've looked at how to create a simple ping program using the `nim-libp2p`.
 
 We'll now look at how to create a custom protocol inside the libp2p
 
-# Custom protocol in libp2p
 Let's create a `part2.nim`, and import our dependencies:
 ```nim
-import bearssl
 import chronos
 import stew/byteutils
 
@@ -22,7 +22,7 @@ type TestProto = ref object of LPProtocol
 
 We've set a [protocol ID](https://docs.libp2p.io/concepts/protocols/#protocol-ids), and created a custom `LPProtocol`. In a more complex protocol, we could use this structure to store interesting variables.
 
-A protocol generally has two part: and handling/server part, and a dialing/client part.  
+A protocol generally has two part: and handling/server part, and a dialing/client part.
 Theses two parts can be identical, but in our trivial protocol, the server will wait for a message from the client, and the client will send a message, so we have to handle the two cases separately.
 
 Let's start with the server part:
@@ -30,13 +30,15 @@ Let's start with the server part:
 proc new(T: typedesc[TestProto]): T =
   # every incoming connections will in be handled in this closure
   proc handle(conn: Connection, proto: string) {.async, gcsafe.} =
+    # Read up to 1024 bytes from this connection, and transform them into
+    # a string
     echo "Got from remote - ", string.fromBytes(await conn.readLp(1024))
     # We must close the connections ourselves when we're done with it
     await conn.close()
 
   return T(codecs: @[TestCodec], handler: handle)
 ```
-This is a constructor for our `TestProto`, that will specify our `codecs` and a `handler`, which will be called for each incoming peer asking for this protocol.  
+This is a constructor for our `TestProto`, that will specify our `codecs` and a `handler`, which will be called for each incoming peer asking for this protocol.
 In our handle, we simply read a message from the connection and `echo` it.
 
 We can now create our client part:
@@ -54,12 +56,12 @@ proc main() {.async, gcsafe.} =
     testProto = TestProto.new()
     switch1 = newStandardSwitch(rng=rng)
     switch2 = newStandardSwitch(rng=rng)
-  
+
   switch1.mount(testProto)
-  
+
   await switch1.start()
   await switch2.start()
-    
+
   let conn = await switch2.dial(switch1.peerInfo.peerId, switch1.peerInfo.addrs, TestCodec)
 
   await testProto.hello(conn)
@@ -70,7 +72,7 @@ proc main() {.async, gcsafe.} =
   await allFutures(switch1.stop(), switch2.stop()) # close connections and shutdown all transports
 ```
 
-This is very similar to the first tutorial's `main`, the only noteworthy difference is that we use `newStandardSwitch`, which is similar to `createSwitch` but is bundled directly in libp2p
+This is very similar to the first tutorial's `main`, the only noteworthy difference is that we use `newStandardSwitch`, which is similar to the `createSwitch` of the first tutorial, but is bundled directly in libp2p
 
 We can now wrap our program by calling our main proc:
 ```nim

libp2p.nim

@@ -1,40 +1,72 @@
-## Nim-LibP2P
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-import
-  libp2p/[protobuf/minprotobuf,
-          muxers/muxer,
-          muxers/mplex/mplex,
-          stream/lpstream,
-          stream/bufferstream,
-          stream/connection,
-          transports/transport,
-          transports/tcptransport,
-          protocols/secure/noise,
-          cid,
-          multihash,
-          multibase,
-          multicodec,
-          errors,
-          switch,
-          peerid,
-          peerinfo,
-          multiaddress,
-          builders,
-          crypto/crypto,
-          protocols/pubsub]
+when defined(nimdoc):
+  ## Welcome to the nim-libp2p reference!
+  ##
+  ## On the left, you'll find a switch that allows you to see private
+  ## procedures. By default, you'll only see the public one (marked with `{.public.}`)
+  ##
+  ## The difference between public and private procedures is that public procedure
+  ## stay backward compatible during the Major version, whereas private ones can
+  ## change at each new Minor version.
+  ##
+  ## If you're new to nim-libp2p, you can find a tutorial `here<https://github.com/status-im/nim-libp2p/blob/master/examples/tutorial_1_connect.md>`_
+  ## that can help you get started.
 
-import bearssl
+  # Import stuff for doc
+  import libp2p/[
+      protobuf/minprotobuf,
+      switch,
+      stream/lpstream,
+      builders,
+      transports/tcptransport,
+      transports/wstransport,
+      protocols/ping,
+      protocols/pubsub,
+      peerid,
+      peerinfo,
+      peerstore,
+      multiaddress]
 
-export
-  minprotobuf, switch, peerid, peerinfo,
-  connection, multiaddress, crypto, lpstream,
-  bufferstream, bearssl, muxer, mplex, transport,
-  tcptransport, noise, errors, cid, multihash,
-  multicodec, builders, pubsub
+  proc dummyPrivateProc*() =
+    ## A private proc example
+    discard
+else:
+  import
+    libp2p/[protobuf/minprotobuf,
+            muxers/muxer,
+            muxers/mplex/mplex,
+            stream/lpstream,
+            stream/bufferstream,
+            stream/connection,
+            transports/transport,
+            transports/tcptransport,
+            transports/wstransport,
+            protocols/secure/noise,
+            protocols/ping,
+            cid,
+            multihash,
+            multibase,
+            multicodec,
+            errors,
+            switch,
+            peerid,
+            peerinfo,
+            multiaddress,
+            builders,
+            crypto/crypto,
+            protocols/pubsub]
+
+  export
+    minprotobuf, switch, peerid, peerinfo,
+    connection, multiaddress, crypto, lpstream,
+    bufferstream, muxer, mplex, transport,
+    tcptransport, noise, errors, cid, multihash,
+    multicodec, builders, pubsub

libp2p.nimble

@@ -18,31 +18,21 @@ requires "nim >= 1.2.0",
          "stew#head",
          "websock"
 
-const nimflags =
-  "--verbosity:0 --hints:off " &
-  "--warning[CaseTransition]:off --warning[ObservableStores]:off " &
-  "--warning[LockLevel]:off " &
-  "-d:chronosStrictException " &
-  "--styleCheck:usages --styleCheck:hint "
-
 proc runTest(filename: string, verify: bool = true, sign: bool = true,
              moreoptions: string = "") =
-  var excstr = "nim c --opt:speed -d:debug -d:libp2p_agents_metrics -d:libp2p_protobuf_metrics -d:libp2p_network_protocols_metrics -d:libp2p_mplex_metrics "
+  var excstr = "nim c --skipParentCfg --opt:speed -d:debug -d:libp2p_agents_metrics -d:libp2p_protobuf_metrics -d:libp2p_network_protocols_metrics -d:libp2p_mplex_metrics "
+  excstr.add(" -d:chronicles_sinks=textlines[stdout],json[dynamic] -d:chronicles_log_level=TRACE ")
+  excstr.add(" -d:chronicles_runtime_filtering=TRUE ")
   excstr.add(" " & getEnv("NIMFLAGS") & " ")
-  excstr.add(" " & nimflags & " ")
+  excstr.add(" --verbosity:0 --hints:off ")
   excstr.add(" -d:libp2p_pubsub_sign=" & $sign)
   excstr.add(" -d:libp2p_pubsub_verify=" & $verify)
   excstr.add(" " & moreoptions & " ")
-  if verify and sign:
-    # build it with TRACE and JSON logs
-    exec excstr & " -d:chronicles_log_level=TRACE -d:chronicles_sinks:json" & " tests/" & filename
-  # build it again, to run it with less verbose logs
-  exec excstr & " -d:chronicles_log_level=INFO -r" & " tests/" & filename
+  exec excstr & " -r " & " tests/" & filename
   rmFile "tests/" & filename.toExe
 
 proc buildSample(filename: string, run = false) =
-  var excstr = "nim c --opt:speed --threads:on -d:debug "
-  excstr.add(" " & nimflags & " ")
+  var excstr = "nim c --opt:speed --threads:on -d:debug --verbosity:0 --hints:off "
   excstr.add(" examples/" & filename)
   exec excstr
   if run:
@@ -51,7 +41,7 @@ proc buildSample(filename: string, run = false) =
 
 proc buildTutorial(filename: string) =
   discard gorge "cat " & filename & " | nim c -r --hints:off tools/markdown_runner.nim | " &
-    " nim " & nimflags & " c -"
+    " nim --verbosity:0 --hints:off c -"
 
 task testnative, "Runs libp2p native tests":
   runTest("testnative")
@@ -99,6 +89,7 @@ task test_slim, "Runs the (slimmed down) test suite":
 task examples_build, "Build the samples":
   buildSample("directchat")
   buildSample("helloworld", true)
+  buildSample("circuitrelay", true)
   buildTutorial("examples/tutorial_1_connect.md")
   buildTutorial("examples/tutorial_2_customproto.md")
 

libp2p/builders.nim

@@ -1,45 +1,54 @@
-## Nim-Libp2p
-## Copyright (c) 2020 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-{.push raises: [Defect].}
+## This module contains a Switch Building helper.
+runnableExamples:
+  let switch =
+   SwitchBuilder.new()
+   .withRng(rng)
+   .withAddresses(multiaddress)
+   # etc
+   .build()
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 import
-  options, tables, chronos, chronicles, bearssl,
+  options, tables, chronos, chronicles, sequtils,
   switch, peerid, peerinfo, stream/connection, multiaddress,
   crypto/crypto, transports/[transport, tcptransport],
-  muxers/[muxer, mplex/mplex],
-  protocols/[identify, secure/secure, secure/noise, relay],
+  muxers/[muxer, mplex/mplex, yamux/yamux],
+  protocols/[identify, secure/secure, secure/noise, autonat],
+  protocols/relay/[relay, client, rtransport],
   connmanager, upgrademngrs/muxedupgrade,
   nameresolving/nameresolver,
-  errors
+  errors, utility
 
 export
   switch, peerid, peerinfo, connection, multiaddress, crypto, errors
 
 type
-  TransportProvider* = proc(upgr: Upgrade): Transport {.gcsafe, raises: [Defect].}
+  TransportProvider* {.public.} = proc(upgr: Upgrade): Transport {.gcsafe, raises: [Defect].}
 
   SecureProtocol* {.pure.} = enum
     Noise,
     Secio {.deprecated.}
 
-  MplexOpts = object
-    enable: bool
-    newMuxer: MuxerConstructor
-
   SwitchBuilder* = ref object
     privKey: Option[PrivateKey]
     addresses: seq[MultiAddress]
     secureManagers: seq[SecureProtocol]
-    mplexOpts: MplexOpts
+    muxers: seq[MuxerProvider]
     transports: seq[TransportProvider]
-    rng: ref BrHmacDrbgContext
+    rng: ref HmacDrbgContext
     maxConnections: int
     maxIn: int
     sendSignedPeerRecord: bool
@@ -49,10 +58,11 @@ type
     agentVersion: string
     nameResolver: NameResolver
     peerStoreCapacity: Option[int]
-    isCircuitRelay: bool
-    circuitRelayCanHop: bool
+    autonat: bool
+    circuitRelay: Relay
 
-proc new*(T: type[SwitchBuilder]): T =
+proc new*(T: type[SwitchBuilder]): T {.public.} =
+  ## Creates a SwitchBuilder
 
   let address = MultiAddress
   .init("/ip4/127.0.0.1/tcp/0")
@@ -67,22 +77,30 @@ proc new*(T: type[SwitchBuilder]): T =
     maxOut: -1,
     maxConnsPerPeer: MaxConnectionsPerPeer,
     protoVersion: ProtoVersion,
-    agentVersion: AgentVersion,
-    isCircuitRelay: false)
+    agentVersion: AgentVersion)
+
+proc withPrivateKey*(b: SwitchBuilder, privateKey: PrivateKey): SwitchBuilder {.public.} =
+  ## Set the private key of the switch. Will be used to
+  ## generate a PeerId
 
-proc withPrivateKey*(b: SwitchBuilder, privateKey: PrivateKey): SwitchBuilder =
   b.privKey = some(privateKey)
   b
 
-proc withAddress*(b: SwitchBuilder, address: MultiAddress): SwitchBuilder =
+proc withAddress*(b: SwitchBuilder, address: MultiAddress): SwitchBuilder {.public.} =
+  ## | Set the listening address of the switch
+  ## | Calling it multiple time will override the value
+
   b.addresses = @[address]
   b
 
-proc withAddresses*(b: SwitchBuilder, addresses: seq[MultiAddress]): SwitchBuilder =
+proc withAddresses*(b: SwitchBuilder, addresses: seq[MultiAddress]): SwitchBuilder {.public.} =
+  ## | Set the listening addresses of the switch
+  ## | Calling it multiple time will override the value
+
   b.addresses = addresses
   b
 
-proc withSignedPeerRecord*(b: SwitchBuilder, sendIt = true): SwitchBuilder =
+proc withSignedPeerRecord*(b: SwitchBuilder, sendIt = true): SwitchBuilder {.public.} =
   b.sendSignedPeerRecord = sendIt
   b
 
@@ -90,7 +108,9 @@ proc withMplex*(
     b: SwitchBuilder,
     inTimeout = 5.minutes,
     outTimeout = 5.minutes,
-    maxChannCount = 200): SwitchBuilder =
+    maxChannCount = 200): SwitchBuilder {.public.} =
+  ## | Uses `Mplex <https://docs.libp2p.io/concepts/stream-multiplexing/#mplex>`_ as a multiplexer
+  ## | `Timeout` is the duration after which a inactive connection will be closed
   proc newMuxer(conn: Connection): Muxer =
     Mplex.new(
       conn,
@@ -98,67 +118,84 @@ proc withMplex*(
       outTimeout,
       maxChannCount)
 
-  b.mplexOpts = MplexOpts(
-    enable: true,
-    newMuxer: newMuxer,
-  )
-
+  assert b.muxers.countIt(it.codec == MplexCodec) == 0, "Mplex build multiple times"
+  b.muxers.add(MuxerProvider.new(newMuxer, MplexCodec))
   b
 
-proc withNoise*(b: SwitchBuilder): SwitchBuilder =
+proc withYamux*(b: SwitchBuilder): SwitchBuilder =
+  proc newMuxer(conn: Connection): Muxer = Yamux.new(conn)
+
+  assert b.muxers.countIt(it.codec == YamuxCodec) == 0, "Yamux build multiple times"
+  b.muxers.add(MuxerProvider.new(newMuxer, YamuxCodec))
+  b
+
+proc withNoise*(b: SwitchBuilder): SwitchBuilder {.public.} =
   b.secureManagers.add(SecureProtocol.Noise)
   b
 
-proc withTransport*(b: SwitchBuilder, prov: TransportProvider): SwitchBuilder =
+proc withTransport*(b: SwitchBuilder, prov: TransportProvider): SwitchBuilder {.public.} =
+  ## Use a custom transport
+  runnableExamples:
+    let switch =
+      SwitchBuilder.new()
+      .withTransport(proc(upgr: Upgrade): Transport = TcpTransport.new(flags, upgr))
+      .build()
   b.transports.add(prov)
   b
 
-proc withTcpTransport*(b: SwitchBuilder, flags: set[ServerFlags] = {}): SwitchBuilder =
+proc withTcpTransport*(b: SwitchBuilder, flags: set[ServerFlags] = {}): SwitchBuilder {.public.} =
   b.withTransport(proc(upgr: Upgrade): Transport = TcpTransport.new(flags, upgr))
 
-proc withRng*(b: SwitchBuilder, rng: ref BrHmacDrbgContext): SwitchBuilder =
+proc withRng*(b: SwitchBuilder, rng: ref HmacDrbgContext): SwitchBuilder {.public.} =
   b.rng = rng
   b
 
-proc withMaxConnections*(b: SwitchBuilder, maxConnections: int): SwitchBuilder =
+proc withMaxConnections*(b: SwitchBuilder, maxConnections: int): SwitchBuilder {.public.} =
+  ## Maximum concurrent connections of the switch. You should either use this, or
+  ## `withMaxIn <#withMaxIn,SwitchBuilder,int>`_ & `withMaxOut<#withMaxOut,SwitchBuilder,int>`_
   b.maxConnections = maxConnections
   b
 
-proc withMaxIn*(b: SwitchBuilder, maxIn: int): SwitchBuilder =
+proc withMaxIn*(b: SwitchBuilder, maxIn: int): SwitchBuilder {.public.} =
+  ## Maximum concurrent incoming connections. Should be used with `withMaxOut<#withMaxOut,SwitchBuilder,int>`_
   b.maxIn = maxIn
   b
 
-proc withMaxOut*(b: SwitchBuilder, maxOut: int): SwitchBuilder =
+proc withMaxOut*(b: SwitchBuilder, maxOut: int): SwitchBuilder {.public.} =
+  ## Maximum concurrent outgoing connections. Should be used with `withMaxIn<#withMaxIn,SwitchBuilder,int>`_
   b.maxOut = maxOut
   b
 
-proc withMaxConnsPerPeer*(b: SwitchBuilder, maxConnsPerPeer: int): SwitchBuilder =
+proc withMaxConnsPerPeer*(b: SwitchBuilder, maxConnsPerPeer: int): SwitchBuilder {.public.} =
   b.maxConnsPerPeer = maxConnsPerPeer
   b
 
-proc withPeerStore*(b: SwitchBuilder, capacity: int): SwitchBuilder =
+proc withPeerStore*(b: SwitchBuilder, capacity: int): SwitchBuilder {.public.} =
   b.peerStoreCapacity = some(capacity)
   b
 
-proc withProtoVersion*(b: SwitchBuilder, protoVersion: string): SwitchBuilder =
+proc withProtoVersion*(b: SwitchBuilder, protoVersion: string): SwitchBuilder {.public.} =
   b.protoVersion = protoVersion
   b
 
-proc withAgentVersion*(b: SwitchBuilder, agentVersion: string): SwitchBuilder =
+proc withAgentVersion*(b: SwitchBuilder, agentVersion: string): SwitchBuilder {.public.} =
   b.agentVersion = agentVersion
   b
 
-proc withNameResolver*(b: SwitchBuilder, nameResolver: NameResolver): SwitchBuilder =
+proc withNameResolver*(b: SwitchBuilder, nameResolver: NameResolver): SwitchBuilder {.public.} =
   b.nameResolver = nameResolver
   b
 
-proc withRelayTransport*(b: SwitchBuilder, canHop: bool): SwitchBuilder =
-  b.isCircuitRelay = true
-  b.circuitRelayCanHop = canHop
+proc withAutonat*(b: SwitchBuilder): SwitchBuilder =
+  b.autonat = true
+  b
+
+proc withCircuitRelay*(b: SwitchBuilder, r: Relay = Relay.new()): SwitchBuilder =
+  b.circuitRelay = r
   b
 
 proc build*(b: SwitchBuilder): Switch
-  {.raises: [Defect, LPError].} =
+  {.raises: [Defect, LPError], public.} =
 
   if b.rng == nil: # newRng could fail
     raise newException(Defect, "Cannot initialize RNG")
@@ -179,18 +216,11 @@ proc build*(b: SwitchBuilder): Switch
       protoVersion = b.protoVersion,
       agentVersion = b.agentVersion)
 
-  let
-    muxers = block:
-      var muxers: Table[string, MuxerProvider]
-      if b.mplexOpts.enable:
-        muxers[MplexCodec] = MuxerProvider.new(b.mplexOpts.newMuxer, MplexCodec)
-      muxers
-
   let
     identify = Identify.new(peerInfo, b.sendSignedPeerRecord)
     connManager = ConnManager.new(b.maxConnsPerPeer, b.maxConnections, b.maxIn, b.maxOut)
     ms = MultistreamSelect.new()
-    muxedUpgrade = MuxedUpgrade.new(identify, muxers, secureManagerInstances, connManager, ms)
+    muxedUpgrade = MuxedUpgrade.new(identify, b.muxers, secureManagerInstances, connManager, ms)
 
   let
     transports = block:
@@ -215,17 +245,21 @@ proc build*(b: SwitchBuilder): Switch
     peerInfo = peerInfo,
     transports = transports,
     identity = identify,
-    muxers = muxers,
     secureManagers = secureManagerInstances,
     connManager = connManager,
     ms = ms,
     nameResolver = b.nameResolver,
     peerStore = peerStore)
 
-  if b.isCircuitRelay:
-    let relay = Relay.new(switch, b.circuitRelayCanHop)
-    switch.mount(relay)
-    switch.addTransport(RelayTransport.new(relay, muxedUpgrade))
+  if b.autonat:
+    let autonat = Autonat.new(switch)
+    switch.mount(autonat)
+
+  if not isNil(b.circuitRelay):
+    if b.circuitRelay of RelayClient:
+      switch.addTransport(RelayTransport.new(RelayClient(b.circuitRelay), muxedUpgrade))
+    b.circuitRelay.setup(switch)
+    switch.mount(b.circuitRelay)
 
   return switch
 
@@ -246,7 +280,9 @@ proc newStandardSwitch*(
   nameResolver: NameResolver = nil,
   sendSignedPeerRecord = false,
   peerStoreCapacity = 1000): Switch
-  {.raises: [Defect, LPError].} =
+  {.raises: [Defect, LPError], public.} =
+  ## Helper for common switch configurations.
+
   if SecureProtocol.Secio in secureManagers:
       quit("Secio is deprecated!") # use of secio is unsafe
 

libp2p/cid.nim

@@ -1,15 +1,18 @@
-## Nim-LibP2P
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module implementes CID (Content IDentifier).
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 import tables, hashes
 import multibase, multicodec, multihash, vbuffer, varint

libp2p/connmanager.nim

@@ -1,13 +1,16 @@
-## Nim-LibP2P
-## Copyright (c) 2020 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 import std/[options, tables, sequtils, sets]
 import pkg/[chronos, chronicles, metrics]
@@ -30,9 +33,6 @@ const
 type
   TooManyConnectionsError* = object of LPError
 
-  ConnProvider* = proc(): Future[Connection]
-    {.gcsafe, closure, raises: [Defect].}
-
   ConnEventKind* {.pure.} = enum
     Connected,    # A connection was made and securely upgraded - there may be
                   # more than one concurrent connection thus more than one upgrade
@@ -81,6 +81,10 @@ type
     peerEvents: array[PeerEventKind, OrderedSet[PeerEventHandler]]
     peerStore*: PeerStore
 
+  ConnectionSlot* = object
+    connManager: ConnManager
+    direction: Direction
+
 proc newTooManyConnectionsError(): ref TooManyConnectionsError {.inline.} =
   result = newException(TooManyConnectionsError, "Too many connections")
 
@@ -401,90 +405,39 @@ proc storeConn*(c: ConnManager, conn: Connection)
   trace "Stored connection",
     conn, direction = $conn.dir, connections = c.conns.len
 
-proc trackConn(c: ConnManager,
-               provider: ConnProvider,
-               sema: AsyncSemaphore):
-               Future[Connection] {.async.} =
-  var conn: Connection
-  try:
-    conn = await provider()
-
-    if isNil(conn):
-      return
-
-    trace "Got connection", conn
-
-    proc semaphoreMonitor() {.async.} =
-      try:
-        await conn.join()
-      except CatchableError as exc:
-        trace "Exception in semaphore monitor, ignoring", exc = exc.msg
-
-      sema.release()
-
-    asyncSpawn semaphoreMonitor()
-  except CatchableError as exc:
-    trace "Exception tracking connection", exc = exc.msg
-    if not isNil(conn):
-      await conn.close()
-
-    raise exc
-
-  return conn
-
-proc trackIncomingConn*(c: ConnManager,
-                        provider: ConnProvider):
-                        Future[Connection] {.async.} =
-  ## await for a connection slot before attempting
-  ## to call the connection provider
-  ##
-
-  var conn: Connection
-  try:
-    trace "Tracking incoming connection"
-    await c.inSema.acquire()
-    conn = await c.trackConn(provider, c.inSema)
-    if isNil(conn):
-      trace "Couldn't acquire connection, releasing semaphore slot", dir = $Direction.In
-      c.inSema.release()
-
-    return conn
-  except CatchableError as exc:
-    trace "Exception tracking connection", exc = exc.msg
-    c.inSema.release()
-    raise exc
-
-proc trackOutgoingConn*(c: ConnManager,
-                        provider: ConnProvider,
-                        forceDial = false):
-                        Future[Connection] {.async.} =
-  ## try acquiring a connection if all slots
-  ## are already taken, raise TooManyConnectionsError
-  ## exception
-  ##
-
-  trace "Tracking outgoing connection", count = c.outSema.count,
-                                        max = c.outSema.size
+proc getIncomingSlot*(c: ConnManager): Future[ConnectionSlot] {.async.} =
+  await c.inSema.acquire()
+  return ConnectionSlot(connManager: c, direction: In)
 
+proc getOutgoingSlot*(c: ConnManager, forceDial = false): Future[ConnectionSlot] {.async.} =
   if forceDial:
     c.outSema.forceAcquire()
   elif not c.outSema.tryAcquire():
     trace "Too many outgoing connections!", count = c.outSema.count,
                                             max = c.outSema.size
     raise newTooManyConnectionsError()
+  return ConnectionSlot(connManager: c, direction: Out)
 
-  var conn: Connection
-  try:
-    conn = await c.trackConn(provider, c.outSema)
-    if isNil(conn):
-      trace "Couldn't acquire connection, releasing semaphore slot", dir = $Direction.Out
-      c.outSema.release()
+proc release*(cs: ConnectionSlot) =
+  if cs.direction == In:
+    cs.connManager.inSema.release()
+  else:
+    cs.connManager.outSema.release()
 
-    return conn
-  except CatchableError as exc:
-    trace "Exception tracking connection", exc = exc.msg
-    c.outSema.release()
-    raise exc
+proc trackConnection*(cs: ConnectionSlot, conn: Connection) =
+  if isNil(conn):
+    cs.release()
+    return
+
+  proc semaphoreMonitor() {.async.} =
+    try:
+      await conn.join()
+    except CatchableError as exc:
+      trace "Exception in semaphore monitor, ignoring", exc = exc.msg
+
+    cs.release()
+
+  asyncSpawn semaphoreMonitor()
 
 proc storeMuxer*(c: ConnManager,
                  muxer: Muxer,

libp2p/crypto/chacha20poly1305.nim

@@ -1,11 +1,11 @@
-## Nim-Libp2p
-## Copyright (c) 2020-2022 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module integrates BearSSL ChaCha20+Poly1305
 ##
@@ -15,19 +15,15 @@
 
 # RFC @ https://tools.ietf.org/html/rfc7539
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
-import bearssl
+import bearssl/blockx
 from stew/assign2 import assign
 from stew/ranges/ptr_arith import baseAddr
 
-# have to do this due to a nim bug and raises[] on callbacks
-# https://github.com/nim-lang/Nim/issues/13905
-proc ourPoly1305CtmulRun*(key: pointer; iv: pointer; data: pointer; len: int;
-                      aad: pointer; aadLen: int; tag: pointer; ichacha: pointer;
-                      encrypt: cint) {.cdecl, importc: "br_poly1305_ctmul_run",
-                                     header: "bearssl_block.h".}
-
 const
   ChaChaPolyKeySize = 32
   ChaChaPolyNonceSize = 12
@@ -67,15 +63,16 @@ proc encrypt*(_: type[ChaChaPoly],
          else:
            nil
 
-  ourPoly1305CtmulRun(
+  poly1305CtmulRun(
     unsafeAddr key[0],
     unsafeAddr nonce[0],
     baseAddr(data),
-    data.len,
+    uint(data.len),
     ad,
-    aad.len,
+    uint(aad.len),
     baseAddr(tag),
-    chacha20CtRun,
+    # cast is required to workaround https://github.com/nim-lang/Nim/issues/13905
+    cast[Chacha20Run](chacha20CtRun),
     #[encrypt]# 1.cint)
 
 proc decrypt*(_: type[ChaChaPoly],
@@ -90,13 +87,14 @@ proc decrypt*(_: type[ChaChaPoly],
          else:
            nil
 
-  ourPoly1305CtmulRun(
+  poly1305CtmulRun(
     unsafeAddr key[0],
     unsafeAddr nonce[0],
     baseAddr(data),
-    data.len,
+    uint(data.len),
     ad,
-    aad.len,
+    uint(aad.len),
     baseAddr(tag),
-    chacha20CtRun,
+    # cast is required to workaround https://github.com/nim-lang/Nim/issues/13905
+    cast[Chacha20Run](chacha20CtRun),
     #[decrypt]# 0.cint)

libp2p/crypto/crypto.nim

@@ -1,14 +1,17 @@
-## Nim-Libp2p
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module implements Public Key and Private Key interface for libp2p.
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 from strutils import split, strip, cmpIgnoreCase
 
@@ -69,17 +72,17 @@ when supported(PKScheme.Secp256k1):
 # We are still importing `ecnist` because, it is used for SECIO handshake,
 # but it will be impossible to create ECNIST keys or import ECNIST keys.
 
-import ecnist, bearssl
+import ecnist, bearssl/rand, bearssl/hash as bhash
 import ../protobuf/minprotobuf, ../vbuffer, ../multihash, ../multicodec
 import nimcrypto/[rijndael, twofish, sha2, hash, hmac]
 # We use `ncrutils` for constant-time hexadecimal encoding/decoding procedures.
 import nimcrypto/utils as ncrutils
 import ../utility
 import stew/results
-export results
+export results, utility
 
 # This is workaround for Nim's `import` bug
-export rijndael, twofish, sha2, hash, hmac, ncrutils
+export rijndael, twofish, sha2, hash, hmac, ncrutils, rand
 
 type
   DigestSheme* = enum
@@ -158,26 +161,28 @@ type
 template orError*(exp: untyped, err: untyped): untyped =
   (exp.mapErr do (_: auto) -> auto: err)
 
-proc newRng*(): ref BrHmacDrbgContext =
+proc newRng*(): ref HmacDrbgContext =
   # You should only create one instance of the RNG per application / library
   # Ref is used so that it can be shared between components
   # TODO consider moving to bearssl
-  var seeder = brPrngSeederSystem(nil)
+  var seeder = prngSeederSystem(nil)
   if seeder == nil:
     return nil
 
-  var rng = (ref BrHmacDrbgContext)()
-  brHmacDrbgInit(addr rng[], addr sha256Vtable, nil, 0)
+  var rng = (ref HmacDrbgContext)()
+  hmacDrbgInit(rng[], addr sha256Vtable, nil, 0)
   if seeder(addr rng.vtable) == 0:
     return nil
   rng
 
 proc shuffle*[T](
-  rng: ref BrHmacDrbgContext,
+  rng: ref HmacDrbgContext,
   x: var openArray[T]) =
 
+  if x.len == 0: return
+
   var randValues = newSeqUninitialized[byte](len(x) * 2)
-  brHmacDrbgGenerate(rng[], randValues)
+  hmacDrbgGenerate(rng[], randValues)
 
   for i in countdown(x.high, 1):
     let
@@ -186,7 +191,7 @@ proc shuffle*[T](
     swap(x[i], x[y])
 
 proc random*(T: typedesc[PrivateKey], scheme: PKScheme,
-             rng: var BrHmacDrbgContext,
+             rng: var HmacDrbgContext,
              bits = RsaDefaultKeySize): CryptoResult[PrivateKey] =
   ## Generate random private key for scheme ``scheme``.
   ##
@@ -218,7 +223,7 @@ proc random*(T: typedesc[PrivateKey], scheme: PKScheme,
     else:
       err(SchemeError)
 
-proc random*(T: typedesc[PrivateKey], rng: var BrHmacDrbgContext,
+proc random*(T: typedesc[PrivateKey], rng: var HmacDrbgContext,
              bits = RsaDefaultKeySize): CryptoResult[PrivateKey] =
   ## Generate random private key using default public-key cryptography scheme.
   ##
@@ -242,7 +247,7 @@ proc random*(T: typedesc[PrivateKey], rng: var BrHmacDrbgContext,
     err(SchemeError)
 
 proc random*(T: typedesc[KeyPair], scheme: PKScheme,
-             rng: var BrHmacDrbgContext,
+             rng: var HmacDrbgContext,
              bits = RsaDefaultKeySize): CryptoResult[KeyPair] =
   ## Generate random key pair for scheme ``scheme``.
   ##
@@ -282,7 +287,7 @@ proc random*(T: typedesc[KeyPair], scheme: PKScheme,
     else:
       err(SchemeError)
 
-proc random*(T: typedesc[KeyPair], rng: var BrHmacDrbgContext,
+proc random*(T: typedesc[KeyPair], rng: var HmacDrbgContext,
              bits = RsaDefaultKeySize): CryptoResult[KeyPair] =
   ## Generate random private pair of keys using default public-key cryptography
   ## scheme.
@@ -870,7 +875,7 @@ proc mac*(secret: Secret, id: int): seq[byte] {.inline.} =
 
 proc ephemeral*(
     scheme: ECDHEScheme,
-    rng: var BrHmacDrbgContext): CryptoResult[EcKeyPair] =
+    rng: var HmacDrbgContext): CryptoResult[EcKeyPair] =
   ## Generate ephemeral keys used to perform ECDHE.
   var keypair: EcKeyPair
   if scheme == Secp256r1:
@@ -882,7 +887,7 @@ proc ephemeral*(
   ok(keypair)
 
 proc ephemeral*(
-    scheme: string, rng: var BrHmacDrbgContext): CryptoResult[EcKeyPair] =
+    scheme: string, rng: var HmacDrbgContext): CryptoResult[EcKeyPair] =
   ## Generate ephemeral keys used to perform ECDHE using string encoding.
   ##
   ## Currently supported encoding strings are P-256, P-384, P-521, if encoding

libp2p/crypto/curve25519.nim

@@ -1,11 +1,11 @@
-## Nim-Libp2p
-## Copyright (c) 2020-2022 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022-2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module integrates BearSSL Cyrve25519 mul and mulgen
 ##
@@ -15,9 +15,12 @@
 
 # RFC @ https://tools.ietf.org/html/rfc7748
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
-import bearssl
+import bearssl/[ec, rand, hash]
 import stew/results
 from stew/assign2 import assign
 export results
@@ -46,7 +49,7 @@ proc byteswap(buf: var Curve25519Key) {.inline.} =
     buf[31 - i] = x
 
 proc mul*(_: type[Curve25519], point: var Curve25519Key, multiplier: Curve25519Key) =
-  let defaultBrEc = brEcGetDefault()
+  let defaultBrEc = ecGetDefault()
 
   # multiplier needs to be big-endian
   var
@@ -54,15 +57,15 @@ proc mul*(_: type[Curve25519], point: var Curve25519Key, multiplier: Curve25519K
   multiplierBs.byteswap()
   let
     res = defaultBrEc.mul(
-      cast[pcuchar](addr point[0]),
+      addr point[0],
       Curve25519KeySize,
-      cast[pcuchar](addr multiplierBs[0]),
+      addr multiplierBs[0],
       Curve25519KeySize,
       EC_curve25519)
   assert res == 1
 
 proc mulgen(_: type[Curve25519], dst: var Curve25519Key, point: Curve25519Key) =
-  let defaultBrEc = brEcGetDefault()
+  let defaultBrEc = ecGetDefault()
 
   var
     rpoint = point
@@ -70,8 +73,8 @@ proc mulgen(_: type[Curve25519], dst: var Curve25519Key, point: Curve25519Key) =
 
   let
     size = defaultBrEc.mulgen(
-      cast[pcuchar](addr dst[0]),
-      cast[pcuchar](addr rpoint[0]),
+      addr dst[0],
+      addr rpoint[0],
       Curve25519KeySize,
       EC_curve25519)
   
@@ -80,10 +83,10 @@ proc mulgen(_: type[Curve25519], dst: var Curve25519Key, point: Curve25519Key) =
 proc public*(private: Curve25519Key): Curve25519Key =
   Curve25519.mulgen(result, private)
 
-proc random*(_: type[Curve25519Key], rng: var BrHmacDrbgContext): Curve25519Key =
+proc random*(_: type[Curve25519Key], rng: var HmacDrbgContext): Curve25519Key =
   var res: Curve25519Key
-  let defaultBrEc = brEcGetDefault()
-  let len = brEcKeygen(
+  let defaultBrEc = ecGetDefault()
+  let len = ecKeygen(
     addr rng.vtable, defaultBrEc, nil, addr res[0], EC_curve25519)
   # Per bearssl documentation, the keygen only fails if the curve is
   # unrecognised -

libp2p/crypto/ecnist.nim

@@ -1,11 +1,11 @@
-## Nim-Libp2p
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module implements constant-time ECDSA and ECDHE for NIST elliptic
 ## curves secp256r1, secp384r1 and secp521r1.
@@ -14,9 +14,12 @@
 ## BearSSL library <https://bearssl.org/>
 ## Copyright(C) 2018 Thomas Pornin <pornin@bolet.org>.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
-import bearssl
+import bearssl/[ec, rand, hash]
 # We use `ncrutils` for constant-time hexadecimal encoding/decoding procedures.
 import nimcrypto/utils as ncrutils
 import minasn1
@@ -40,12 +43,12 @@ const
 
 type
   EcPrivateKey* = ref object
-    buffer*: array[BR_EC_KBUF_PRIV_MAX_SIZE, byte]
-    key*: BrEcPrivateKey
+    buffer*: array[EC_KBUF_PRIV_MAX_SIZE, byte]
+    key*: ec.EcPrivateKey
 
   EcPublicKey* = ref object
-    buffer*: array[BR_EC_KBUF_PUB_MAX_SIZE, byte]
-    key*: BrEcPublicKey
+    buffer*: array[EC_KBUF_PUB_MAX_SIZE, byte]
+    key*: ec.EcPublicKey
 
   EcKeyPair* = object
     seckey*: EcPrivateKey
@@ -55,9 +58,9 @@ type
     buffer*: seq[byte]
 
   EcCurveKind* = enum
-    Secp256r1 = BR_EC_SECP256R1,
-    Secp384r1 = BR_EC_SECP384R1,
-    Secp521r1 = BR_EC_SECP521R1
+    Secp256r1 = EC_secp256r1,
+    Secp384r1 = EC_secp384r1,
+    Secp521r1 = EC_secp521r1
 
   EcPKI* = EcPrivateKey | EcPublicKey | EcSignature
 
@@ -101,15 +104,15 @@ proc checkScalar(scalar: openArray[byte], curve: cint): uint32 =
   ##   - ``scalar`` is lower than the curve ``order``.
   ##
   ## Otherwise, return ``0``.
-  var impl = brEcGetDefault()
-  var orderlen = 0
-  var order = cast[ptr UncheckedArray[byte]](impl.order(curve, addr orderlen))
+  var impl = ecGetDefault()
+  var orderlen: uint = 0
+  var order = cast[ptr UncheckedArray[byte]](impl.order(curve, orderlen))
 
   var z = 0'u32
   var c = 0'i32
   for u in scalar:
     z = z or u
-  if len(scalar) == orderlen:
+  if len(scalar) == int(orderlen):
     for i in 0..<len(scalar):
       c = c or (-(cast[int32](EQ0(c))) and CMP(scalar[i], order[i]))
   else:
@@ -119,12 +122,12 @@ proc checkScalar(scalar: openArray[byte], curve: cint): uint32 =
 proc checkPublic(key: openArray[byte], curve: cint): uint32 =
   ## Return ``1`` if public key ``key`` is on curve.
   var ckey = @key
-  var x = [0x00'u8, 0x01'u8]
-  var impl = brEcGetDefault()
-  var orderlen = 0
-  discard impl.order(curve, addr orderlen)
-  result = impl.mul(cast[ptr char](unsafeAddr ckey[0]), len(ckey),
-                    cast[ptr char](addr x[0]), len(x), curve)
+  var x = [byte 0x00, 0x01]
+  var impl = ecGetDefault()
+  var orderlen: uint = 0
+  discard impl.order(curve, orderlen)
+  result = impl.mul(unsafeAddr ckey[0], uint(len(ckey)),
+                    addr x[0], uint(len(x)), curve)
 
 proc getOffset(pubkey: EcPublicKey): int {.inline.} =
   let o = cast[uint](pubkey.key.q) - cast[uint](unsafeAddr pubkey.buffer[0])
@@ -174,7 +177,7 @@ proc copy*[T: EcPKI](dst: var T, src: T): bool =
           dst.buffer = src.buffer
           dst.key.curve = src.key.curve
           dst.key.xlen = length
-          dst.key.x = cast[ptr char](addr dst.buffer[offset])
+          dst.key.x = addr dst.buffer[offset]
           result = true
     elif T is EcPublicKey:
       let length = src.key.qlen
@@ -184,7 +187,7 @@ proc copy*[T: EcPKI](dst: var T, src: T): bool =
           dst.buffer = src.buffer
           dst.key.curve = src.key.curve
           dst.key.qlen = length
-          dst.key.q = cast[ptr char](addr dst.buffer[offset])
+          dst.key.q = addr dst.buffer[offset]
           result = true
     else:
       let length = len(src.buffer)
@@ -230,15 +233,15 @@ proc clear*[T: EcPKI|EcKeyPair](pki: var T) =
 
 proc random*(
     T: typedesc[EcPrivateKey], kind: EcCurveKind,
-    rng: var BrHmacDrbgContext): EcResult[EcPrivateKey] =
+    rng: var HmacDrbgContext): EcResult[EcPrivateKey] =
   ## Generate new random EC private key using BearSSL's HMAC-SHA256-DRBG
   ## algorithm.
   ##
   ## ``kind`` elliptic curve kind of your choice (secp256r1, secp384r1 or
   ## secp521r1).
-  var ecimp = brEcGetDefault()
+  var ecimp = ecGetDefault()
   var res = new EcPrivateKey
-  if brEcKeygen(addr rng.vtable, ecimp,
+  if ecKeygen(addr rng.vtable, ecimp,
                 addr res.key, addr res.buffer[0],
                 cast[cint](kind)) == 0:
     err(EcKeyGenError)
@@ -250,12 +253,12 @@ proc getPublicKey*(seckey: EcPrivateKey): EcResult[EcPublicKey] =
   if isNil(seckey):
     return err(EcKeyIncorrectError)
 
-  var ecimp = brEcGetDefault()
+  var ecimp = ecGetDefault()
   if seckey.key.curve in EcSupportedCurvesCint:
     var res = new EcPublicKey
     assert res.buffer.len > getPublicKeyLength(cast[EcCurveKind](seckey.key.curve))
-    if brEcComputePublicKey(ecimp, addr res.key,
-                            addr res.buffer[0], unsafeAddr seckey.key) == 0:
+    if ecComputePub(ecimp, addr res.key,
+                    addr res.buffer[0], unsafeAddr seckey.key) == 0:
       err(EcKeyIncorrectError)
     else:
       ok(res)
@@ -264,7 +267,7 @@ proc getPublicKey*(seckey: EcPrivateKey): EcResult[EcPublicKey] =
 
 proc random*(
     T: typedesc[EcKeyPair], kind: EcCurveKind,
-    rng: var BrHmacDrbgContext): EcResult[T] =
+    rng: var HmacDrbgContext): EcResult[T] =
   ## Generate new random EC private and public keypair using BearSSL's
   ## HMAC-SHA256-DRBG algorithm.
   ##
@@ -373,24 +376,24 @@ proc toBytes*(seckey: EcPrivateKey, data: var openArray[byte]): EcResult[int] =
     var p = Asn1Composite.init(Asn1Tag.Sequence)
     var c0 = Asn1Composite.init(0)
     var c1 = Asn1Composite.init(1)
-    if seckey.key.curve == BR_EC_SECP256R1:
+    if seckey.key.curve == EC_secp256r1:
       c0.write(Asn1Tag.Oid, Asn1OidSecp256r1)
-    elif seckey.key.curve == BR_EC_SECP384R1:
+    elif seckey.key.curve == EC_secp384r1:
       c0.write(Asn1Tag.Oid, Asn1OidSecp384r1)
-    elif seckey.key.curve == BR_EC_SECP521R1:
+    elif seckey.key.curve == EC_secp521r1:
       c0.write(Asn1Tag.Oid, Asn1OidSecp521r1)
     c0.finish()
     offset = pubkey.getOffset()
     if offset < 0:
       return err(EcKeyIncorrectError)
-    length = pubkey.key.qlen
+    length = int(pubkey.key.qlen)
     c1.write(Asn1Tag.BitString,
              pubkey.buffer.toOpenArray(offset, offset + length - 1))
     c1.finish()
     offset = seckey.getOffset()
     if offset < 0:
       return err(EcKeyIncorrectError)
-    length = seckey.key.xlen
+    length = int(seckey.key.xlen)
     p.write(1'u64)
     p.write(Asn1Tag.OctetString,
             seckey.buffer.toOpenArray(offset, offset + length - 1))
@@ -421,18 +424,18 @@ proc toBytes*(pubkey: EcPublicKey, data: var openArray[byte]): EcResult[int] =
     var p = Asn1Composite.init(Asn1Tag.Sequence)
     var c = Asn1Composite.init(Asn1Tag.Sequence)
     c.write(Asn1Tag.Oid, Asn1OidEcPublicKey)
-    if pubkey.key.curve == BR_EC_SECP256R1:
+    if pubkey.key.curve == EC_secp256r1:
       c.write(Asn1Tag.Oid, Asn1OidSecp256r1)
-    elif pubkey.key.curve == BR_EC_SECP384R1:
+    elif pubkey.key.curve == EC_secp384r1:
       c.write(Asn1Tag.Oid, Asn1OidSecp384r1)
-    elif pubkey.key.curve == BR_EC_SECP521R1:
+    elif pubkey.key.curve == EC_secp521r1:
       c.write(Asn1Tag.Oid, Asn1OidSecp521r1)
     c.finish()
     p.write(c)
     let offset = getOffset(pubkey)
     if offset < 0:
       return err(EcKeyIncorrectError)
-    let length = pubkey.key.qlen
+    let length = int(pubkey.key.qlen)
     p.write(Asn1Tag.BitString,
             pubkey.buffer.toOpenArray(offset, offset + length - 1))
     p.finish()
@@ -638,8 +641,8 @@ proc init*(key: var EcPrivateKey, data: openArray[byte]): Result[void, Asn1Error
   if checkScalar(raw.toOpenArray(), curve) == 1'u32:
     key = new EcPrivateKey
     copyMem(addr key.buffer[0], addr raw.buffer[raw.offset], raw.length)
-    key.key.x = cast[ptr char](addr key.buffer[0])
-    key.key.xlen = raw.length
+    key.key.x = addr key.buffer[0]
+    key.key.xlen = uint(raw.length)
     key.key.curve = curve
     ok()
   else:
@@ -697,8 +700,8 @@ proc init*(pubkey: var EcPublicKey, data: openArray[byte]): Result[void, Asn1Err
   if checkPublic(raw.toOpenArray(), curve) != 0:
     pubkey = new EcPublicKey
     copyMem(addr pubkey.buffer[0], addr raw.buffer[raw.offset], raw.length)
-    pubkey.key.q = cast[ptr char](addr pubkey.buffer[0])
-    pubkey.key.qlen = raw.length
+    pubkey.key.q = addr pubkey.buffer[0]
+    pubkey.key.qlen = uint(raw.length)
     pubkey.key.curve = curve
     ok()
   else:
@@ -785,8 +788,8 @@ proc initRaw*(key: var EcPrivateKey, data: openArray[byte]): bool =
       let length = len(data)
       key = new EcPrivateKey
       copyMem(addr key.buffer[0], unsafeAddr data[0], length)
-      key.key.x = cast[ptr char](addr key.buffer[0])
-      key.key.xlen = length
+      key.key.x = addr key.buffer[0]
+      key.key.xlen = uint(length)
       key.key.curve = curve
       result = true
 
@@ -816,8 +819,8 @@ proc initRaw*(pubkey: var EcPublicKey, data: openArray[byte]): bool =
       let length = len(data)
       pubkey = new EcPublicKey
       copyMem(addr pubkey.buffer[0], unsafeAddr data[0], length)
-      pubkey.key.q = cast[ptr char](addr pubkey.buffer[0])
-      pubkey.key.qlen = length
+      pubkey.key.q = addr pubkey.buffer[0]
+      pubkey.key.qlen = uint(length)
       pubkey.key.curve = curve
       result = true
 
@@ -883,7 +886,7 @@ proc scalarMul*(pub: EcPublicKey, sec: EcPrivateKey): EcPublicKey =
   ##
   ## Returns point in curve as ``pub * sec`` or ``nil`` otherwise.
   doAssert((not isNil(pub)) and (not isNil(sec)))
-  var impl = brEcGetDefault()
+  var impl = ecGetDefault()
   if sec.key.curve in EcSupportedCurvesCint:
     if pub.key.curve == sec.key.curve:
       var key = new EcPublicKey
@@ -891,9 +894,9 @@ proc scalarMul*(pub: EcPublicKey, sec: EcPrivateKey): EcPublicKey =
         let poffset = key.getOffset()
         let soffset = sec.getOffset()
         if poffset >= 0 and soffset >= 0:
-          let res = impl.mul(cast[ptr char](addr key.buffer[poffset]),
+          let res = impl.mul(addr key.buffer[poffset],
                              key.key.qlen,
-                             cast[ptr char](unsafeAddr sec.buffer[soffset]),
+                             unsafeAddr sec.buffer[soffset],
                              sec.key.xlen,
                              key.key.curve)
           if res != 0:
@@ -913,11 +916,11 @@ proc toSecret*(pubkey: EcPublicKey, seckey: EcPrivateKey,
   doAssert((not isNil(pubkey)) and (not isNil(seckey)))
   var mult = scalarMul(pubkey, seckey)
   if not isNil(mult):
-    if seckey.key.curve == BR_EC_SECP256R1:
+    if seckey.key.curve == EC_secp256r1:
       result = Secret256Length
-    elif seckey.key.curve == BR_EC_SECP384R1:
+    elif seckey.key.curve == EC_secp384r1:
       result = Secret384Length
-    elif seckey.key.curve == BR_EC_SECP521R1:
+    elif seckey.key.curve == EC_secp521r1:
       result = Secret521Length
     if len(data) >= result:
       var qplus1 = cast[pointer](cast[uint](mult.key.q) + 1'u)
@@ -941,20 +944,20 @@ proc sign*[T: byte|char](seckey: EcPrivateKey,
   ## Get ECDSA signature of data ``message`` using private key ``seckey``.
   if isNil(seckey):
     return err(EcKeyIncorrectError)
-  var hc: BrHashCompatContext
+  var hc: HashCompatContext
   var hash: array[32, byte]
-  var impl = brEcGetDefault()
+  var impl = ecGetDefault()
   if seckey.key.curve in EcSupportedCurvesCint:
     var sig = new EcSignature
     sig.buffer = newSeq[byte](256)
     var kv = addr sha256Vtable
     kv.init(addr hc.vtable)
     if len(message) > 0:
-      kv.update(addr hc.vtable, unsafeAddr message[0], len(message))
+      kv.update(addr hc.vtable, unsafeAddr message[0], uint(len(message)))
     else:
       kv.update(addr hc.vtable, nil, 0)
-    kv.output(addr hc.vtable, addr hash[0])
-    let res = brEcdsaSignAsn1(impl, kv, addr hash[0], addr seckey.key,
+    kv.out(addr hc.vtable, addr hash[0])
+    let res = ecdsaI31SignAsn1(impl, kv, addr hash[0], addr seckey.key,
                               addr sig.buffer[0])
     # Clear context with initial value
     kv.init(addr hc.vtable)
@@ -974,20 +977,20 @@ proc verify*[T: byte|char](sig: EcSignature, message: openArray[T],
   ## Return ``true`` if message verification succeeded, ``false`` if
   ## verification failed.
   doAssert((not isNil(sig)) and (not isNil(pubkey)))
-  var hc: BrHashCompatContext
+  var hc: HashCompatContext
   var hash: array[32, byte]
-  var impl = brEcGetDefault()
+  var impl = ecGetDefault()
   if pubkey.key.curve in EcSupportedCurvesCint:
     var kv = addr sha256Vtable
     kv.init(addr hc.vtable)
     if len(message) > 0:
-      kv.update(addr hc.vtable, unsafeAddr message[0], len(message))
+      kv.update(addr hc.vtable, unsafeAddr message[0], uint(len(message)))
     else:
       kv.update(addr hc.vtable, nil, 0)
-    kv.output(addr hc.vtable, addr hash[0])
-    let res = brEcdsaVerifyAsn1(impl, addr hash[0], len(hash),
-                                unsafeAddr pubkey.key,
-                                addr sig.buffer[0], len(sig.buffer))
+    kv.out(addr hc.vtable, addr hash[0])
+    let res = ecdsaI31VrfyAsn1(impl, addr hash[0], uint(len(hash)),
+                               unsafeAddr pubkey.key,
+                               addr sig.buffer[0], uint(len(sig.buffer)))
     # Clear context with initial value
     kv.init(addr hc.vtable)
     result = (res == 1)

libp2p/crypto/ed25519/constants.nim

@@ -1,11 +1,11 @@
-## Nim-Libp2p
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module implements ED25519.
 ## This is pure nim implementation of ED25519 ref10.

libp2p/crypto/ed25519/ed25519.nim

@@ -1,19 +1,23 @@
-## Nim-Libp2p
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module implements ED25519.
 ## This code is a port of the public domain, "ref10" implementation of ed25519
 ## from SUPERCOP.
 
-{.push raises: Defect.}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
-import constants, bearssl
+import bearssl/rand
+import constants
 import nimcrypto/[hash, sha2]
 # We use `ncrutils` for constant-time hexadecimal encoding/decoding procedures.
 import nimcrypto/utils as ncrutils
@@ -21,7 +25,7 @@ import stew/[results, ctops]
 export results
 
 # This workaround needed because of some bugs in Nim Static[T].
-export hash, sha2
+export hash, sha2, rand
 
 const
   EdPrivateKeySize* = 64
@@ -1644,14 +1648,14 @@ proc checkScalar*(scalar: openArray[byte]): uint32 =
     c = -1
   result = NEQ(z, 0'u32) and LT0(c)
 
-proc random*(t: typedesc[EdPrivateKey], rng: var BrHmacDrbgContext): EdPrivateKey =
+proc random*(t: typedesc[EdPrivateKey], rng: var HmacDrbgContext): EdPrivateKey =
   ## Generate new random ED25519 private key using the given random number generator
   var
     point: GeP3
     pk: array[EdPublicKeySize, byte]
     res: EdPrivateKey
 
-  brHmacDrbgGenerate(addr rng, addr res.data[0], 32)
+  hmacDrbgGenerate(rng, res.data.toOpenArray(0, 31))
 
   var hh = sha512.digest(res.data.toOpenArray(0, 31))
   hh.data[0] = hh.data[0] and 0xF8'u8
@@ -1663,14 +1667,14 @@ proc random*(t: typedesc[EdPrivateKey], rng: var BrHmacDrbgContext): EdPrivateKe
 
   res
 
-proc random*(t: typedesc[EdKeyPair], rng: var BrHmacDrbgContext): EdKeyPair =
+proc random*(t: typedesc[EdKeyPair], rng: var HmacDrbgContext): EdKeyPair =
   ## Generate new random ED25519 private and public keypair using OS specific
   ## CSPRNG.
   var
     point: GeP3
     res: EdKeyPair
 
-  brHmacDrbgGenerate(addr rng, addr res.seckey.data[0], 32)
+  hmacDrbgGenerate(rng, res.seckey.data.toOpenArray(0, 31))
 
   var hh = sha512.digest(res.seckey.data.toOpenArray(0, 31))
   hh.data[0] = hh.data[0] and 0xF8'u8

libp2p/crypto/hkdf.nim

@@ -1,40 +1,36 @@
-## Nim-LibP2P
-## Copyright (c) 2020 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 # https://tools.ietf.org/html/rfc5869
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 import nimcrypto
-import bearssl
+import bearssl/[kdf, rand, hash]
 
-type
-  BearHKDFContext {.importc: "br_hkdf_context", header: "bearssl_kdf.h".} = object
-  HKDFResult*[len: static int] = array[len, byte]
+type HkdfResult*[len: static int] = array[len, byte]
 
-proc br_hkdf_init(ctx: ptr BearHKDFContext; hashClass: ptr HashClass; salt: pointer; len: csize_t) {.importc: "br_hkdf_init", header: "bearssl_kdf.h", raises: [].}
-proc br_hkdf_inject(ctx: ptr BearHKDFContext; ikm: pointer; len: csize_t) {.importc: "br_hkdf_inject", header: "bearssl_kdf.h", raises: [].}
-proc br_hkdf_flip(ctx: ptr BearHKDFContext) {.importc: "br_hkdf_flip", header: "bearssl_kdf.h", raises: [].}
-proc br_hkdf_produce(ctx: ptr BearHKDFContext; info: pointer; infoLen: csize_t; output: pointer; outputLen: csize_t) {.importc: "br_hkdf_produce", header: "bearssl_kdf.h", raises: [].}
-
-proc hkdf*[T: sha256; len: static int](_: type[T]; salt, ikm, info: openArray[byte]; outputs: var openArray[HKDFResult[len]]) =
+proc hkdf*[T: sha256; len: static int](_: type[T]; salt, ikm, info: openArray[byte]; outputs: var openArray[HkdfResult[len]]) =
   var
-    ctx: BearHKDFContext
-  br_hkdf_init(
-    addr ctx, addr sha256Vtable,
+    ctx: HkdfContext
+  hkdfInit(
+    ctx, addr sha256Vtable,
     if salt.len > 0: unsafeAddr salt[0] else: nil, csize_t(salt.len))
-  br_hkdf_inject(
-    addr ctx, if ikm.len > 0: unsafeAddr ikm[0] else: nil, csize_t(ikm.len))
-  br_hkdf_flip(addr ctx)
+  hkdfInject(
+    ctx, if ikm.len > 0: unsafeAddr ikm[0] else: nil, csize_t(ikm.len))
+  hkdfFlip(ctx)
   for i in 0..outputs.high:
-    br_hkdf_produce(
-      addr ctx,
+    discard hkdfProduce(
+      ctx,
       if info.len > 0: unsafeAddr info[0]
       else: nil, csize_t(info.len),
       addr outputs[i][0], csize_t(outputs[i].len))

libp2p/crypto/minasn1.nim

@@ -1,15 +1,18 @@
-## Nim-Libp2p
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module implements minimal ASN.1 encoding/decoding primitives.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 import stew/[endians2, results, ctops]
 export results

libp2p/crypto/rsa.nim

@@ -1,11 +1,11 @@
-## Nim-Libp2p
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module implements constant-time RSA PKCS#1.5 DSA.
 ##
@@ -13,8 +13,12 @@
 ## BearSSL library <https://bearssl.org/>
 ## Copyright(C) 2018 Thomas Pornin <pornin@bolet.org>.
 
-{.push raises: Defect.}
-import bearssl
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import bearssl/[rsa, rand, hash]
 import minasn1
 import stew/[results, ctops]
 # We use `ncrutils` for constant-time hexadecimal encoding/decoding procedures.
@@ -33,41 +37,41 @@ const
     ## Default RSA key size in bits.
 
   RsaOidSha1* = [
-    0x05'u8, 0x2B'u8, 0x0E'u8, 0x03'u8, 0x02'u8, 0x1A'u8
+    byte 0x05, 0x2B, 0x0E, 0x03, 0x02, 0x1A
   ]
     ## RSA PKCS#1.5 SHA-1 hash object identifier.
   RsaOidSha224* = [
-    0x09'u8, 0x60'u8, 0x86'u8, 0x48'u8, 0x01'u8, 0x65'u8, 0x03'u8, 0x04'u8,
-    0x02'u8, 0x04'u8
+    byte 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04,
+    0x02, 0x04
   ]
     ## RSA PKCS#1.5 SHA-224 hash object identifier.
   RsaOidSha256* = [
-    0x09'u8, 0x60'u8, 0x86'u8, 0x48'u8, 0x01'u8, 0x65'u8, 0x03'u8, 0x04'u8,
-    0x02'u8, 0x01'u8
+    byte 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04,
+    0x02, 0x01
   ]
     ## RSA PKCS#1.5 SHA-256 hash object identifier.
   RsaOidSha384* = [
-    0x09'u8, 0x60'u8, 0x86'u8, 0x48'u8, 0x01'u8, 0x65'u8, 0x03'u8, 0x04'u8,
-    0x02'u8, 0x02'u8
+    byte 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04,
+    0x02, 0x02
   ]
     ## RSA PKCS#1.5 SHA-384 hash object identifier.
   RsaOidSha512* = [
-    0x09'u8, 0x60'u8, 0x86'u8, 0x48'u8, 0x01'u8, 0x65'u8, 0x03'u8, 0x04'u8,
-    0x02'u8, 0x03'u8
+    byte 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04,
+    0x02, 0x03
   ]
     ## RSA PKCS#1.5 SHA-512 hash object identifier.
 
 type
   RsaPrivateKey* = ref object
     buffer*: seq[byte]
-    seck*: BrRsaPrivateKey
-    pubk*: BrRsaPublicKey
-    pexp*: ptr char
-    pexplen*: int
+    seck*: rsa.RsaPrivateKey
+    pubk*: rsa.RsaPublicKey
+    pexp*: ptr byte
+    pexplen*: uint
 
   RsaPublicKey* = ref object
     buffer*: seq[byte]
-    key*: BrRsaPublicKey
+    key*: rsa.RsaPublicKey
 
   RsaKeyPair* = RsaPrivateKey
 
@@ -99,8 +103,8 @@ template getFinish(bs, os, ls: untyped): untyped =
   var eo = -1
   if p >= s:
     let so = cast[int](p - s)
-    if so + ls <= len(bs):
-      eo = so + ls - 1
+    if so + int(ls) <= len(bs):
+      eo = so + int(ls) - 1
   eo
 
 template getArray*(bs, os, ls: untyped): untyped =
@@ -109,12 +113,12 @@ template getArray*(bs, os, ls: untyped): untyped =
 template trimZeroes(b: seq[byte], pt, ptlen: untyped) =
   var length = ptlen
   for i in 0..<length:
-    if pt[] != cast[char](0x00'u8):
+    if pt[] != byte(0x00):
       break
-    pt = cast[ptr char](cast[uint](pt) + 1)
+    pt = cast[ptr byte](cast[uint](pt) + 1)
     ptlen -= 1
 
-proc random*[T: RsaKP](t: typedesc[T], rng: var BrHmacDrbgContext,
+proc random*[T: RsaKP](t: typedesc[T], rng: var HmacDrbgContext,
                        bits = DefaultKeySize,
                        pubexp = DefaultPublicExponent): RsaResult[T] =
   ## Generate new random RSA private key using BearSSL's HMAC-SHA256-DRBG
@@ -129,14 +133,14 @@ proc random*[T: RsaKP](t: typedesc[T], rng: var BrHmacDrbgContext,
 
   let
     sko = 0
-    pko = brRsaPrivateKeyBufferSize(bits)
-    eko = pko + brRsaPublicKeyBufferSize(bits)
+    pko = rsaKbufPrivSize(bits)
+    eko = pko + rsaKbufPubSize(bits)
     length = eko + ((bits + 7) shr 3)
 
   let res = new T
   res.buffer = newSeq[byte](length)
 
-  var keygen = brRsaKeygenGetDefault()
+  var keygen = rsaKeygenGetDefault()
 
   if keygen(addr rng.vtable,
             addr res.seck, addr res.buffer[sko],
@@ -145,12 +149,12 @@ proc random*[T: RsaKP](t: typedesc[T], rng: var BrHmacDrbgContext,
     return err(RsaGenError)
 
   let
-    compute = brRsaComputePrivexpGetDefault()
+    compute = rsaComputePrivexpGetDefault()
     computed = compute(addr res.buffer[eko], addr res.seck, pubexp)
   if computed == 0:
     return err(RsaGenError)
 
-  res.pexp = cast[ptr char](addr res.buffer[eko])
+  res.pexp = addr res.buffer[eko]
   res.pexplen = computed
 
   trimZeroes(res.buffer, res.seck.p, res.seck.plen)
@@ -169,12 +173,12 @@ proc copy*[T: RsaPKI](key: T): T =
   doAssert(not isNil(key))
   when T is RsaPrivateKey:
     if len(key.buffer) > 0:
-      let length = key.seck.plen + key.seck.qlen + key.seck.dplen +
-                   key.seck.dqlen + key.seck.iqlen + key.pubk.nlen +
-                   key.pubk.elen + key.pexplen
+      let length = key.seck.plen.uint + key.seck.qlen.uint + key.seck.dplen.uint +
+                   key.seck.dqlen.uint + key.seck.iqlen.uint + key.pubk.nlen.uint +
+                   key.pubk.elen.uint + key.pexplen.uint
       result = new RsaPrivateKey
       result.buffer = newSeq[byte](length)
-      let po = 0
+      let po: uint = 0
       let qo = po + key.seck.plen
       let dpo = qo + key.seck.qlen
       let dqo = dpo + key.seck.dplen
@@ -190,14 +194,14 @@ proc copy*[T: RsaPKI](key: T): T =
       copyMem(addr result.buffer[no], key.pubk.n, key.pubk.nlen)
       copyMem(addr result.buffer[eo], key.pubk.e, key.pubk.elen)
       copyMem(addr result.buffer[peo], key.pexp, key.pexplen)
-      result.seck.p = cast[ptr char](addr result.buffer[po])
-      result.seck.q = cast[ptr char](addr result.buffer[qo])
-      result.seck.dp = cast[ptr char](addr result.buffer[dpo])
-      result.seck.dq = cast[ptr char](addr result.buffer[dqo])
-      result.seck.iq = cast[ptr char](addr result.buffer[iqo])
-      result.pubk.n = cast[ptr char](addr result.buffer[no])
-      result.pubk.e = cast[ptr char](addr result.buffer[eo])
-      result.pexp = cast[ptr char](addr result.buffer[peo])
+      result.seck.p = addr result.buffer[po]
+      result.seck.q = addr result.buffer[qo]
+      result.seck.dp = addr result.buffer[dpo]
+      result.seck.dq = addr result.buffer[dqo]
+      result.seck.iq = addr result.buffer[iqo]
+      result.pubk.n = addr result.buffer[no]
+      result.pubk.e = addr result.buffer[eo]
+      result.pexp = addr result.buffer[peo]
       result.seck.plen = key.seck.plen
       result.seck.qlen = key.seck.qlen
       result.seck.dplen = key.seck.dplen
@@ -231,8 +235,8 @@ proc getPublicKey*(key: RsaPrivateKey): RsaPublicKey =
   let length = key.pubk.nlen + key.pubk.elen
   result = new RsaPublicKey
   result.buffer = newSeq[byte](length)
-  result.key.n = cast[ptr char](addr result.buffer[0])
-  result.key.e = cast[ptr char](addr result.buffer[key.pubk.nlen])
+  result.key.n = addr result.buffer[0]
+  result.key.e = addr result.buffer[key.pubk.nlen]
   copyMem(addr result.buffer[0], cast[pointer](key.pubk.n), key.pubk.nlen)
   copyMem(addr result.buffer[key.pubk.nlen], cast[pointer](key.pubk.e),
           key.pubk.elen)
@@ -472,22 +476,22 @@ proc init*(key: var RsaPrivateKey, data: openArray[byte]): Result[void, Asn1Erro
      len(rawdp) > 0 and len(rawdq) > 0 and len(rawiq) > 0:
     key = new RsaPrivateKey
     key.buffer = @data
-    key.pubk.n = cast[ptr char](addr key.buffer[rawn.offset])
-    key.pubk.e = cast[ptr char](addr key.buffer[rawpube.offset])
-    key.seck.p = cast[ptr char](addr key.buffer[rawp.offset])
-    key.seck.q = cast[ptr char](addr key.buffer[rawq.offset])
-    key.seck.dp = cast[ptr char](addr key.buffer[rawdp.offset])
-    key.seck.dq = cast[ptr char](addr key.buffer[rawdq.offset])
-    key.seck.iq = cast[ptr char](addr key.buffer[rawiq.offset])
-    key.pexp = cast[ptr char](addr key.buffer[rawprie.offset])
-    key.pubk.nlen = len(rawn)
-    key.pubk.elen = len(rawpube)
-    key.seck.plen = len(rawp)
-    key.seck.qlen = len(rawq)
-    key.seck.dplen = len(rawdp)
-    key.seck.dqlen = len(rawdq)
-    key.seck.iqlen = len(rawiq)
-    key.pexplen = len(rawprie)
+    key.pubk.n = addr key.buffer[rawn.offset]
+    key.pubk.e = addr key.buffer[rawpube.offset]
+    key.seck.p = addr key.buffer[rawp.offset]
+    key.seck.q = addr key.buffer[rawq.offset]
+    key.seck.dp = addr key.buffer[rawdp.offset]
+    key.seck.dq = addr key.buffer[rawdq.offset]
+    key.seck.iq = addr key.buffer[rawiq.offset]
+    key.pexp = addr key.buffer[rawprie.offset]
+    key.pubk.nlen = uint(len(rawn))
+    key.pubk.elen = uint(len(rawpube))
+    key.seck.plen = uint(len(rawp))
+    key.seck.qlen = uint(len(rawq))
+    key.seck.dplen = uint(len(rawdp))
+    key.seck.dqlen = uint(len(rawdq))
+    key.seck.iqlen = uint(len(rawiq))
+    key.pexplen = uint(len(rawprie))
     key.seck.nBitlen = cast[uint32](len(rawn) shl 3)
     ok()
   else:
@@ -554,10 +558,10 @@ proc init*(key: var RsaPublicKey, data: openArray[byte]): Result[void, Asn1Error
   if len(rawn) >= (MinKeySize shr 3) and len(rawe) > 0:
     key = new RsaPublicKey
     key.buffer = @data
-    key.key.n = cast[ptr char](addr key.buffer[rawn.offset])
-    key.key.e = cast[ptr char](addr key.buffer[rawe.offset])
-    key.key.nlen = len(rawn)
-    key.key.elen = len(rawe)
+    key.key.n = addr key.buffer[rawn.offset]
+    key.key.e = addr key.buffer[rawe.offset]
+    key.key.nlen = uint(len(rawn))
+    key.key.elen = uint(len(rawe))
     ok()
   else:
     err(Asn1Error.Incorrect)
@@ -749,22 +753,22 @@ proc sign*[T: byte|char](key: RsaPrivateKey,
   if isNil(key):
     return err(RsaKeyIncorrectError)
 
-  var hc: BrHashCompatContext
+  var hc: HashCompatContext
   var hash: array[32, byte]
-  let impl = BrRsaPkcs1SignGetDefault()
+  let impl = rsaPkcs1SignGetDefault()
   var res = new RsaSignature
   res.buffer = newSeq[byte]((key.seck.nBitlen + 7) shr 3)
   var kv = addr sha256Vtable
   kv.init(addr hc.vtable)
   if len(message) > 0:
-    kv.update(addr hc.vtable, unsafeAddr message[0], len(message))
+    kv.update(addr hc.vtable, unsafeAddr message[0], uint(len(message)))
   else:
     kv.update(addr hc.vtable, nil, 0)
-  kv.output(addr hc.vtable, addr hash[0])
+  kv.out(addr hc.vtable, addr hash[0])
   var oid = RsaOidSha256
-  let implRes = impl(cast[ptr char](addr oid[0]),
-                 cast[ptr char](addr hash[0]), len(hash),
-                 addr key.seck, cast[ptr char](addr res.buffer[0]))
+  let implRes = impl(addr oid[0],
+                 addr hash[0], uint(len(hash)),
+                 addr key.seck, addr res.buffer[0])
   if implRes == 0:
     err(RsaSignatureError)
   else:
@@ -779,20 +783,20 @@ proc verify*[T: byte|char](sig: RsaSignature, message: openArray[T],
   ## verification failed.
   doAssert((not isNil(sig)) and (not isNil(pubkey)))
   if len(sig.buffer) > 0:
-    var hc: BrHashCompatContext
+    var hc: HashCompatContext
     var hash: array[32, byte]
     var check: array[32, byte]
-    var impl = BrRsaPkcs1VrfyGetDefault()
+    var impl = rsaPkcs1VrfyGetDefault()
     var kv = addr sha256Vtable
     kv.init(addr hc.vtable)
     if len(message) > 0:
-      kv.update(addr hc.vtable, unsafeAddr message[0], len(message))
+      kv.update(addr hc.vtable, unsafeAddr message[0], uint(len(message)))
     else:
       kv.update(addr hc.vtable, nil, 0)
-    kv.output(addr hc.vtable, addr hash[0])
+    kv.out(addr hc.vtable, addr hash[0])
     var oid = RsaOidSha256
-    let res = impl(cast[ptr char](addr sig.buffer[0]), len(sig.buffer),
-                   cast[ptr char](addr oid[0]),
-                   len(check), addr pubkey.key, cast[ptr char](addr check[0]))
+    let res = impl(addr sig.buffer[0], uint(len(sig.buffer)),
+                   addr oid[0],
+                   uint(len(check)), addr pubkey.key, addr check[0])
     if res == 1:
       result = equalMem(addr check[0], addr hash[0], len(hash))

libp2p/crypto/secp.nim

@@ -1,20 +1,24 @@
-## Nim-Libp2p
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
+import bearssl/rand
 import
-  secp256k1, bearssl,
+  secp256k1,
   stew/[byteutils, results],
   nimcrypto/[hash, sha2]
 
-export sha2, results
+export sha2, results, rand
 
 const
   SkRawPrivateKeySize* = 256 div 8
@@ -34,17 +38,18 @@ type
 template pubkey*(v: SkKeyPair): SkPublicKey = SkPublicKey(secp256k1.SkKeyPair(v).pubkey)
 template seckey*(v: SkKeyPair): SkPrivateKey = SkPrivateKey(secp256k1.SkKeyPair(v).seckey)
 
-proc random*(t: typedesc[SkPrivateKey], rng: var BrHmacDrbgContext): SkPrivateKey =
-  let rngPtr = unsafeAddr rng # doesn't escape
+proc random*(t: typedesc[SkPrivateKey], rng: var HmacDrbgContext): SkPrivateKey =
+  #TODO is there a better way?
+  var rngPtr = addr rng
   proc callRng(data: var openArray[byte]) =
-    brHmacDrbgGenerate(rngPtr[], data)
+    hmacDrbgGenerate(rngPtr[], data)
 
   SkPrivateKey(SkSecretKey.random(callRng))
 
-proc random*(t: typedesc[SkKeyPair], rng: var BrHmacDrbgContext): SkKeyPair =
-  let rngPtr = unsafeAddr rng # doesn't escape
+proc random*(t: typedesc[SkKeyPair], rng: var HmacDrbgContext): SkKeyPair =
+  let rngPtr = addr rng
   proc callRng(data: var openArray[byte]) =
-    brHmacDrbgGenerate(rngPtr[], data)
+    hmacDrbgGenerate(rngPtr[], data)
 
   SkKeyPair(secp256k1.SkKeyPair.random(callRng))
 

libp2p/daemon/daemonapi.nim

@@ -1,13 +1,16 @@
-## Nim-LibP2P
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 ## This module implementes API for `go-libp2p-daemon`.
 import std/[os, osproc, strutils, tables, strtabs, sequtils]
@@ -1022,7 +1025,7 @@ proc dhtGetSinglePeerId(pb: ProtoBuffer): PeerId
   if pb.getRequiredField(3, result).isErr():
     raise newException(DaemonLocalError, "Missing field `value`!")
 
-proc enterDhtMessage(pb: ProtoBuffer, rt: DHTResponseType): Protobuffer
+proc enterDhtMessage(pb: ProtoBuffer, rt: DHTResponseType): ProtoBuffer
   {.inline, raises: [Defect, DaemonLocalError].} =
   var dhtResponse: seq[byte]
   if pb.getRequiredField(ResponseType.DHT.int, dhtResponse).isOk():

libp2p/daemon/transpool.nim

@@ -1,13 +1,16 @@
-## Nim-Libp2p
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 ## This module implements Pool of StreamTransport.
 import chronos

libp2p/debugutils.nim

@@ -1,11 +1,11 @@
-## Nim-LibP2P
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## To enable dump of all incoming and outgoing unencrypted messages you need
 ## to compile project with ``-d:libp2p_dump`` compile-time option. When this

libp2p/dial.nim

@@ -1,13 +1,16 @@
-## Nim-LibP2P
-## Copyright (c) 2021 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 import chronos
 import peerid,
@@ -55,3 +58,9 @@ method addTransport*(
   self: Dial,
   transport: Transport) {.base.} =
   doAssert(false, "Not implemented!")
+
+method tryDial*(
+  self: Dial,
+  peerId: PeerId,
+  addrs: seq[MultiAddress]): Future[MultiAddress] {.async, base.} =
+  doAssert(false, "Not implemented!")

libp2p/dialer.nim

@@ -1,11 +1,11 @@
-## Nim-LibP2P
-## Copyright (c) 2021 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 import std/[sugar, tables]
 
@@ -21,6 +21,7 @@ import dial,
        stream/connection,
        transports/transport,
        nameresolving/nameresolver,
+       upgrademngrs/upgrade,
        errors
 
 export dial, errors
@@ -47,8 +48,7 @@ type
 proc dialAndUpgrade(
   self: Dialer,
   peerId: PeerId,
-  addrs: seq[MultiAddress],
-  forceDial: bool):
+  addrs: seq[MultiAddress]):
   Future[Connection] {.async.} =
   debug "Dialing peer", peerId
 
@@ -65,20 +65,7 @@ proc dialAndUpgrade(
           trace "Dialing address", address = $a, peerId, hostname
           let dialed = try:
               libp2p_total_dial_attempts.inc()
-              # await a connection slot when the total
-              # connection count is equal to `maxConns`
-              #
-              # Need to copy to avoid "cannot be captured" errors in Nim-1.4.x.
-              let
-                transportCopy = transport
-                addressCopy = a
-              await self.connManager.trackOutgoingConn(
-                () => transportCopy.dial(hostname, addressCopy),
-                forceDial
-              )
-            except TooManyConnectionsError as exc:
-              trace "Connection limit reached!"
-              raise exc
+              await transport.dial(hostname, a)
             except CancelledError as exc:
               debug "Dialing canceled", msg = exc.msg, peerId
               raise exc
@@ -101,6 +88,7 @@ proc dialAndUpgrade(
             except CatchableError as exc:
               # If we failed to establish the connection through one transport,
               # we won't succeeded through another - no use in trying again
+              # TODO we should try another address though
               await dialed.close()
               debug "Upgrade failed", msg = exc.msg, peerId
               if exc isnot CancelledError:
@@ -139,12 +127,18 @@ proc internalConnect(
       trace "Reusing existing connection", conn, direction = $conn.dir
       return conn
 
-    conn = await self.dialAndUpgrade(peerId, addrs, forceDial)
+    let slot = await self.connManager.getOutgoingSlot(forceDial)
+    conn =
+      try:
+        await self.dialAndUpgrade(peerId, addrs)
+      except CatchableError as exc:
+        slot.release()
+        raise exc
+    slot.trackConnection(conn)
     if isNil(conn): # None of the addresses connected
       raise newException(DialFailedError, "Unable to establish outgoing link")
 
-    # We already check for this in Connection manager
-    # but a disconnect could have happened right after
+    # A disconnect could have happened right after
     # we've added the connection so we check again
     # to prevent races due to that.
     if conn.closed() or conn.atEof():
@@ -185,6 +179,27 @@ proc negotiateStream(
 
   return conn
 
+method tryDial*(
+  self: Dialer,
+  peerId: PeerId,
+  addrs: seq[MultiAddress]): Future[MultiAddress] {.async.} =
+  ## Create a protocol stream in order to check
+  ## if a connection is possible.
+  ## Doesn't use the Connection Manager to save it.
+  ##
+
+  trace "Check if it can dial", peerId, addrs
+  try:
+    let conn = await self.dialAndUpgrade(peerId, addrs)
+    if conn.isNil():
+      raise newException(DialFailedError, "No valid multiaddress")
+    await conn.close()
+    return conn.observedAddr
+  except CancelledError as exc:
+    raise exc
+  except CatchableError as exc:
+    raise newException(DialFailedError, exc.msg)
+
 method dial*(
   self: Dialer,
   peerId: PeerId,

libp2p/multiaddress.nim

@@ -1,23 +1,27 @@
-## Nim-Libp2p
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module implements MultiAddress.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+{.push public.}
 
 import pkg/chronos
 import std/[nativesockets, hashes]
 import tables, strutils, sets, stew/shims/net
 import multicodec, multihash, multibase, transcoder, vbuffer, peerid,
-       protobuf/minprotobuf, errors
+       protobuf/minprotobuf, errors, utility
 import stew/[base58, base32, endians2, results]
-export results, minprotobuf, vbuffer
+export results, minprotobuf, vbuffer, utility
 
 type
   MAKind* = enum
@@ -586,10 +590,28 @@ proc getPart(ma: MultiAddress, index: int): MaResult[MultiAddress] =
     inc(offset)
   ok(res)
 
+proc getParts[U, V](ma: MultiAddress, slice: HSlice[U, V]): MaResult[MultiAddress] =
+  when slice.a is BackwardsIndex or slice.b is BackwardsIndex:
+    let maLength = ? len(ma)
+  template normalizeIndex(index): int =
+    when index is BackwardsIndex: maLength - int(index)
+    else: int(index)
+  let
+    indexStart = normalizeIndex(slice.a)
+    indexEnd = normalizeIndex(slice.b)
+  var res: MultiAddress
+  for i in indexStart..indexEnd:
+    ? res.append(? ma[i])
+  ok(res)
+
 proc `[]`*(ma: MultiAddress, i: int): MaResult[MultiAddress] {.inline.} =
   ## Returns part with index ``i`` of MultiAddress ``ma``.
   ma.getPart(i)
 
+proc `[]`*(ma: MultiAddress, slice: HSlice): MaResult[MultiAddress] {.inline.} =
+  ## Returns parts with slice ``slice`` of MultiAddress ``ma``.
+  ma.getParts(slice)
+
 iterator items*(ma: MultiAddress): MaResult[MultiAddress] =
   ## Iterates over all addresses inside of MultiAddress ``ma``.
   var header: uint64
@@ -626,6 +648,13 @@ iterator items*(ma: MultiAddress): MaResult[MultiAddress] =
     res.data.finish()
     yield ok(MaResult[MultiAddress], res)
 
+proc len*(ma: MultiAddress): MaResult[int] =
+  var counter: int
+  for part in ma:
+    if part.isErr: return err(part.error)
+    counter.inc()
+  ok(counter)
+
 proc contains*(ma: MultiAddress, codec: MultiCodec): MaResult[bool] {.inline.} =
   ## Returns ``true``, if address with MultiCodec ``codec`` present in
   ## MultiAddress ``ma``.

libp2p/multibase.nim

@@ -1,11 +1,11 @@
-## Nim-Libp2p
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module implements MultiBase.
 ##
@@ -13,7 +13,10 @@
 ## 1. base32z
 ##
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 import tables
 import stew/[base32, base58, base64, results]

libp2p/multicodec.nim

@@ -1,15 +1,18 @@
-## Nim-Libp2p
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not BE copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not BE copied, modified, or distributed except according to
+# those terms.
 
 ## This module implements MultiCodec.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 import tables, hashes
 import varint, vbuffer

libp2p/multihash.nim

@@ -1,11 +1,11 @@
-## Nim-Libp2p
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module implements MultiHash.
 ## Supported hashes are:
@@ -21,7 +21,10 @@
 ## 1. SKEIN
 ## 2. MURMUR
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 import tables
 import nimcrypto/[sha, sha2, keccak, blake2, hash, utils]

libp2p/multistream.nim

@@ -1,15 +1,18 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
-import std/[strutils]
+import std/[strutils, sequtils]
 import chronos, chronicles, stew/byteutils
 import stream/connection,
        protocols/protocol
@@ -76,7 +79,7 @@ proc select*(m: MultistreamSelect,
     trace "reading first requested proto", conn
     if s == proto[0]:
       trace "successfully selected ", conn, proto = proto[0]
-      conn.tag = proto[0]
+      conn.protocol = proto[0]
       return proto[0]
     elif proto.len > 1:
       # Try to negotiate alternatives
@@ -89,7 +92,7 @@ proc select*(m: MultistreamSelect,
         validateSuffix(s)
         if s == p:
           trace "selected protocol", conn, protocol = s
-          conn.tag = s
+          conn.protocol = s
           return s
       return ""
     else:
@@ -167,7 +170,7 @@ proc handle*(m: MultistreamSelect, conn: Connection, active: bool = false) {.asy
           if (not isNil(h.match) and h.match(ms)) or h.protos.contains(ms):
             trace "found handler", conn, protocol = ms
             await conn.writeLp(ms & "\n")
-            conn.tag = ms
+            conn.protocol = ms
             await h.protocol.handler(conn, ms)
             return
         debug "no handlers", conn, protocol = ms
@@ -209,3 +212,9 @@ proc addHandler*(m: MultistreamSelect,
   m.handlers.add(HandlerHolder(protos: @[codec],
                                protocol: protocol,
                                match: matcher))
+
+proc start*(m: MultistreamSelect) {.async.} =
+  await allFutures(m.handlers.mapIt(it.protocol.start()))
+
+proc stop*(m: MultistreamSelect) {.async.} =
+  await allFutures(m.handlers.mapIt(it.protocol.stop()))

libp2p/muxers/mplex/coder.nim

@@ -1,13 +1,16 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 import pkg/[chronos, nimcrypto/utils, chronicles, stew/byteutils]
 import ../../stream/connection,

libp2p/muxers/mplex/lpchannel.nim

@@ -1,13 +1,16 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 import std/[oids, strformat]
 import pkg/[chronos, chronicles, metrics, nimcrypto/utils]
@@ -78,11 +81,13 @@ proc open*(s: LPChannel) {.async, gcsafe.} =
   try:
     await s.conn.writeMsg(s.id, MessageType.New, s.name)
     s.isOpen = true
+  except CancelledError as exc:
+    raise exc
   except CatchableError as exc:
     await s.conn.close()
     raise exc
 
-method closed*(s: LPChannel): bool {.raises: [Defect].} =
+method closed*(s: LPChannel): bool =
   s.closedLocal
 
 proc closeUnderlying(s: LPChannel): Future[void] {.async.} =
@@ -166,8 +171,8 @@ method readOnce*(s: LPChannel,
   try:
     let bytes = await procCall BufferStream(s).readOnce(pbytes, nbytes)
     when defined(libp2p_network_protocols_metrics):
-      if s.tag.len > 0:
-        libp2p_protocols_bytes.inc(bytes.int64, labelValues=[s.tag, "in"])
+      if s.protocol.len > 0:
+        libp2p_protocols_bytes.inc(bytes.int64, labelValues=[s.protocol, "in"])
 
     trace "readOnce", s, bytes
     if bytes == 0:
@@ -217,10 +222,15 @@ proc completeWrite(
       await fut
 
     when defined(libp2p_network_protocol_metrics):
-      if s.tag.len > 0:
-        libp2p_protocols_bytes.inc(msgLen.int64, labelValues=[s.tag, "out"])
+      if s.protocol.len > 0:
+        libp2p_protocols_bytes.inc(msgLen.int64, labelValues=[s.protocol, "out"])
 
     s.activity = true
+  except CancelledError as exc:
+    # Chronos may still send the data
+    raise exc
+  except LPStreamClosedError as exc:
+    raise exc
   except CatchableError as exc:
     trace "exception in lpchannel write handler", s, msg = exc.msg
     await s.reset()
@@ -247,6 +257,8 @@ method write*(s: LPChannel, msg: seq[byte]): Future[void] =
 
   s.completeWrite(fut, msg.len)
 
+method getWrapped*(s: LPChannel): Connection = s.conn
+
 proc init*(
   L: type LPChannel,
   id: uint64,

libp2p/muxers/mplex/mplex.nim

@@ -1,13 +1,16 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 import tables, sequtils, oids
 import chronos, chronicles, stew/byteutils, metrics
@@ -34,7 +37,6 @@ when defined(libp2p_expensive_metrics):
     "mplex channels", labels = ["initiator", "peer"])
 
 type
-  TooManyChannels* = object of MuxerError
   InvalidChannelIdError* = object of MuxerError
 
   Mplex* = ref object of Muxer

libp2p/muxers/muxer.nim

@@ -1,13 +1,16 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 import chronos, chronicles
 import ../protocols/protocol,
@@ -22,6 +25,7 @@ const
 
 type
   MuxerError* = object of LPError
+  TooManyChannels* = object of MuxerError
 
   StreamHandler* = proc(conn: Connection): Future[void] {.gcsafe, raises: [Defect].}
   MuxerHandler* = proc(muxer: Muxer): Future[void] {.gcsafe, raises: [Defect].}

libp2p/muxers/yamux/yamux.nim

@@ -0,0 +1,509 @@
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import sequtils, std/[tables]
+import chronos, chronicles, metrics, stew/[endians2, byteutils, objects]
+import ../muxer,
+       ../../stream/connection
+
+export muxer
+
+logScope:
+  topics = "libp2p yamux"
+
+const
+  YamuxCodec* = "/yamux/1.0.0"
+  YamuxVersion = 0.uint8
+  DefaultWindowSize = 256000
+  MaxChannelCount = 200
+
+when defined(libp2p_yamux_metrics):
+  declareGauge(libp2p_yamux_channels, "yamux channels", labels = ["initiator", "peer"])
+  declareHistogram libp2p_yamux_send_queue, "message send queue length (in byte)",
+    buckets = [0.0, 100.0, 250.0, 1000.0, 2000.0, 1600.0, 6400.0, 25600.0, 256000.0]
+  declareHistogram libp2p_yamux_recv_queue, "message recv queue length (in byte)",
+    buckets = [0.0, 100.0, 250.0, 1000.0, 2000.0, 1600.0, 6400.0, 25600.0, 256000.0]
+
+type
+  YamuxError* = object of CatchableError
+
+  MsgType = enum
+    Data = 0x0
+    WindowUpdate = 0x1
+    Ping = 0x2
+    GoAway = 0x3
+
+  MsgFlags {.size: 2.} = enum
+    Syn
+    Ack
+    Fin
+    Rst
+
+  GoAwayStatus = enum
+    NormalTermination = 0x0,
+    ProtocolError = 0x1,
+    InternalError = 0x2,
+
+  YamuxHeader = object
+    version: uint8
+    msgType: MsgType
+    flags: set[MsgFlags]
+    streamId: uint32
+    length: uint32
+
+proc readHeader(conn: LPStream): Future[YamuxHeader] {.async, gcsafe.} =
+  var buffer: array[12, byte]
+  await conn.readExactly(addr buffer[0], 12)
+
+  result.version = buffer[0]
+  let flags = fromBytesBE(uint16, buffer[2..3])
+  if not result.msgType.checkedEnumAssign(buffer[1]) or flags notin 0'u16..15'u16:
+     raise newException(YamuxError, "Wrong header")
+  result.flags = cast[set[MsgFlags]](flags)
+  result.streamId = fromBytesBE(uint32, buffer[4..7])
+  result.length = fromBytesBE(uint32, buffer[8..11])
+  return result
+
+proc `$`(header: YamuxHeader): string =
+  result = "{" & $header.msgType & ", "
+  result &= "{" & header.flags.foldl(if a != "": a & ", " & $b else: $b, "") & "}, "
+  result &= "streamId: " & $header.streamId & ", "
+  result &= "length: " & $header.length & "}"
+
+proc encode(header: YamuxHeader): array[12, byte] =
+  result[0] = header.version
+  result[1] = uint8(header.msgType)
+  result[2..3] = toBytesBE(cast[uint16](header.flags))
+  result[4..7] = toBytesBE(header.streamId)
+  result[8..11] = toBytesBE(header.length)
+
+proc write(conn: LPStream, header: YamuxHeader): Future[void] {.gcsafe.} =
+  trace "write directly on stream", h = $header
+  var buffer = header.encode()
+  return conn.write(@buffer)
+
+proc ping(T: type[YamuxHeader], flag: MsgFlags, pingData: uint32): T =
+  T(
+    version: YamuxVersion,
+    msgType: MsgType.Ping,
+    flags: {flag},
+    length: pingData
+  )
+
+proc goAway(T: type[YamuxHeader], status: GoAwayStatus): T =
+  T(
+    version: YamuxVersion,
+    msgType: MsgType.GoAway,
+    length: uint32(status)
+  )
+
+proc data(
+  T: type[YamuxHeader],
+  streamId: uint32,
+  length: uint32 = 0,
+  flags: set[MsgFlags] = {},
+  ): T =
+  T(
+    version: YamuxVersion,
+    msgType: MsgType.Data,
+    length: length,
+    flags: flags,
+    streamId: streamId
+  )
+
+proc windowUpdate(
+  T: type[YamuxHeader],
+  streamId: uint32,
+  delta: uint32,
+  flags: set[MsgFlags] = {},
+  ): T =
+  T(
+    version: YamuxVersion,
+    msgType: MsgType.WindowUpdate,
+    length: delta,
+    flags: flags,
+    streamId: streamId
+  )
+
+type
+  ToSend = tuple
+    data: seq[byte]
+    sent: int
+    fut: Future[void]
+  YamuxChannel* = ref object of Connection
+    id: uint32
+    recvWindow: int
+    sendWindow: int
+    maxRecvWindow: int
+    conn: Connection
+    isSrc: bool
+    opened: bool
+    isSending: bool
+    sendQueue: seq[ToSend]
+    recvQueue: seq[byte]
+    isReset: bool
+    closedRemotely: Future[void]
+    closedLocally: bool
+    receivedData: AsyncEvent
+    returnedEof: bool
+
+proc `$`(channel: YamuxChannel): string =
+  result = if channel.conn.dir == Out: "=> " else: "<= "
+  result &= $channel.id
+  var s: seq[string] = @[]
+  if channel.closedRemotely.done():
+    s.add("ClosedRemotely")
+  if channel.closedLocally:
+    s.add("ClosedLocally")
+  if channel.isReset:
+    s.add("Reset")
+  if s.len > 0:
+    result &= " {" & s.foldl(if a != "": a & ", " & b else: b, "") & "}"
+
+proc sendQueueBytes(channel: YamuxChannel, limit: bool = false): int =
+  for (elem, sent, _) in channel.sendQueue:
+    result.inc(min(elem.len - sent, if limit: channel.maxRecvWindow div 3 else: elem.len - sent))
+
+proc actuallyClose(channel: YamuxChannel) {.async.} =
+  if channel.closedLocally and channel.sendQueue.len == 0 and
+     channel.closedRemotely.done():
+    await procCall Connection(channel).closeImpl()
+
+proc remoteClosed(channel: YamuxChannel) {.async.} =
+  if not channel.closedRemotely.done():
+    channel.closedRemotely.complete()
+    await channel.actuallyClose()
+
+method closeImpl*(channel: YamuxChannel) {.async, gcsafe.} =
+  if not channel.closedLocally:
+    channel.closedLocally = true
+
+    if channel.isReset == false and channel.sendQueue.len == 0:
+      await channel.conn.write(YamuxHeader.data(channel.id, 0, {Fin}))
+    await channel.actuallyClose()
+
+proc reset(channel: YamuxChannel, isLocal: bool = false) {.async.} =
+  if not channel.isReset:
+    trace "Reset channel"
+    channel.isReset = true
+    for (d, s, fut) in channel.sendQueue:
+      fut.fail(newLPStreamEOFError())
+    channel.sendQueue = @[]
+    channel.recvQueue = @[]
+    channel.sendWindow = 0
+    if not channel.closedLocally:
+      if isLocal:
+        try: await channel.conn.write(YamuxHeader.data(channel.id, 0, {Rst}))
+        except LPStreamEOFError as exc: discard
+        except LPStreamClosedError as exc: discard
+      await channel.close()
+    if not channel.closedRemotely.done():
+      await channel.remoteClosed()
+    channel.receivedData.fire()
+  if not isLocal:
+    # If we reset locally, we want to flush up to a maximum of recvWindow
+    # bytes. We use the recvWindow in the proc cleanupChann.
+    channel.recvWindow = 0
+
+proc updateRecvWindow(channel: YamuxChannel) {.async.} =
+  let inWindow = channel.recvWindow + channel.recvQueue.len
+  if inWindow > channel.maxRecvWindow div 2:
+    return
+
+  let delta = channel.maxRecvWindow - inWindow
+  channel.recvWindow.inc(delta)
+  await channel.conn.write(YamuxHeader.windowUpdate(
+    channel.id,
+    delta.uint32
+  ))
+  trace "increasing the recvWindow", delta
+
+method readOnce*(
+  channel: YamuxChannel,
+  pbytes: pointer,
+  nbytes: int):
+  Future[int] {.async.} =
+
+  if channel.returnedEof: raise newLPStreamEOFError()
+  if channel.recvQueue.len == 0:
+    channel.receivedData.clear()
+    await channel.closedRemotely or channel.receivedData.wait()
+    if channel.closedRemotely.done() and channel.recvQueue.len == 0:
+      channel.returnedEof = true
+      return 0
+
+  let toRead = min(channel.recvQueue.len, nbytes)
+
+  var p = cast[ptr UncheckedArray[byte]](pbytes)
+  toOpenArray(p, 0, nbytes - 1)[0..<toRead] = channel.recvQueue.toOpenArray(0, toRead - 1)
+  channel.recvQueue = channel.recvQueue[toRead..^1]
+
+  # We made some room in the recv buffer let the peer know
+  await channel.updateRecvWindow()
+  channel.activity = true
+  return toRead
+
+proc gotDataFromRemote(channel: YamuxChannel, b: seq[byte]) {.async.} =
+  channel.recvWindow -= b.len
+  channel.recvQueue = channel.recvQueue.concat(b)
+  channel.receivedData.fire()
+  when defined(libp2p_yamux_metrics):
+    libp2p_yamux_recv_queue.observe(channel.recvQueue.len.int64)
+  await channel.updateRecvWindow()
+
+proc setMaxRecvWindow*(channel: YamuxChannel, maxRecvWindow: int) =
+  channel.maxRecvWindow = maxRecvWindow
+
+proc trySend(channel: YamuxChannel) {.async.} =
+  if channel.isSending:
+    return
+  channel.isSending = true
+  defer: channel.isSending = false
+  while channel.sendQueue.len != 0:
+    channel.sendQueue.keepItIf(not (it.fut.cancelled() and it.sent == 0))
+    if channel.sendWindow == 0:
+      trace "send window empty"
+      if channel.sendQueueBytes(true) > channel.maxRecvWindow:
+        debug "channel send queue too big, resetting", maxSendWindow=channel.maxRecvWindow,
+          currentQueueSize = channel.sendQueueBytes(true)
+        try:
+          await channel.reset(true)
+        except CatchableError as exc:
+          debug "failed to reset", msg=exc.msg
+      break
+
+    let
+      bytesAvailable = channel.sendQueueBytes()
+      toSend = min(channel.sendWindow, bytesAvailable)
+    var
+      sendBuffer = newSeqUninitialized[byte](toSend + 12)
+      header = YamuxHeader.data(channel.id, toSend.uint32)
+      inBuffer = 0
+
+    if toSend >= bytesAvailable and channel.closedLocally:
+      trace "last buffer we'll sent on this channel", toSend, bytesAvailable
+      header.flags.incl({Fin})
+
+    sendBuffer[0..<12] = header.encode()
+
+    var futures: seq[Future[void]]
+    while inBuffer < toSend:
+      let (data, sent, fut) = channel.sendQueue[0]
+      let bufferToSend = min(data.len - sent, toSend - inBuffer)
+      sendBuffer.toOpenArray(12, 12 + toSend - 1)[inBuffer..<(inBuffer+bufferToSend)] =
+        channel.sendQueue[0].data.toOpenArray(sent, sent + bufferToSend - 1)
+      channel.sendQueue[0].sent.inc(bufferToSend)
+      if channel.sendQueue[0].sent >= data.len:
+        futures.add(fut)
+        channel.sendQueue.delete(0)
+      inBuffer.inc(bufferToSend)
+
+    trace "build send buffer", h = $header, msg=string.fromBytes(sendBuffer[12..^1])
+    channel.sendWindow.dec(toSend)
+    try: await channel.conn.write(sendBuffer)
+    except CatchableError as exc:
+      for fut in futures.items():
+        fut.fail(exc)
+      await channel.reset()
+      break
+    for fut in futures.items():
+      fut.complete()
+    channel.activity = true
+
+method write*(channel: YamuxChannel, msg: seq[byte]): Future[void] =
+  result = newFuture[void]("Yamux Send")
+  if channel.closedLocally or channel.isReset:
+    result.fail(newLPStreamEOFError())
+    return result
+  if msg.len == 0:
+    result.complete()
+    return result
+  channel.sendQueue.add((msg, 0, result))
+  when defined(libp2p_yamux_metrics):
+    libp2p_yamux_recv_queue.observe(channel.sendQueueBytes().int64)
+  asyncSpawn channel.trySend()
+
+proc open*(channel: YamuxChannel) {.async, gcsafe.} =
+  if channel.opened:
+    trace "Try to open channel twice"
+    return
+  channel.opened = true
+  await channel.conn.write(YamuxHeader.data(channel.id, 0, {if channel.isSrc: Syn else: Ack}))
+
+type
+  Yamux* = ref object of Muxer
+    channels: Table[uint32, YamuxChannel]
+    flushed: Table[uint32, int]
+    currentId: uint32
+    isClosed: bool
+    maxChannCount: int
+
+proc lenBySrc(m: Yamux, isSrc: bool): int =
+  for v in m.channels.values():
+    if v.isSrc == isSrc: result += 1
+
+proc cleanupChann(m: Yamux, channel: YamuxChannel) {.async.} =
+  await channel.join()
+  m.channels.del(channel.id)
+  when defined(libp2p_yamux_metrics):
+    libp2p_yamux_channels.set(m.lenBySrc(channel.isSrc).int64, [$channel.isSrc, $channel.peerId])
+  if channel.isReset and channel.recvWindow > 0:
+    m.flushed[channel.id] = channel.recvWindow
+
+proc createStream(m: Yamux, id: uint32, isSrc: bool): YamuxChannel =
+  result = YamuxChannel(
+    id: id,
+    maxRecvWindow: DefaultWindowSize,
+    recvWindow: DefaultWindowSize,
+    sendWindow: DefaultWindowSize,
+    isSrc: isSrc,
+    conn: m.connection,
+    receivedData: newAsyncEvent(),
+    closedRemotely: newFuture[void]()
+  )
+  result.objName = "YamuxStream"
+  result.dir = if isSrc: Direction.Out else: Direction.In
+  result.timeoutHandler = proc(): Future[void] {.gcsafe.} =
+    trace "Idle timeout expired, resetting YamuxChannel"
+    result.reset()
+  result.initStream()
+  result.peerId = m.connection.peerId
+  result.observedAddr = m.connection.observedAddr
+  result.transportDir = m.connection.transportDir
+  when defined(libp2p_agents_metrics):
+    result.shortAgent = m.connection.shortAgent
+  m.channels[id] = result
+  asyncSpawn m.cleanupChann(result)
+  trace "created channel", id, pid=m.connection.peerId
+  when defined(libp2p_yamux_metrics):
+    libp2p_yamux_channels.set(m.lenBySrc(isSrc).int64, [$isSrc, $result.peerId])
+
+method close*(m: Yamux) {.async.} =
+  if m.isClosed == true:
+    trace "Already closed"
+    return
+  m.isClosed = true
+
+  trace "Closing yamux"
+  for channel in m.channels.values:
+    await channel.reset()
+  await m.connection.write(YamuxHeader.goAway(NormalTermination))
+  await m.connection.close()
+  trace "Closed yamux"
+
+proc handleStream(m: Yamux, channel: YamuxChannel) {.async.} =
+  ## call the muxer stream handler for this channel
+  ##
+  try:
+    await m.streamHandler(channel)
+    trace "finished handling stream"
+    doAssert(channel.isClosed, "connection not closed by handler!")
+  except CatchableError as exc:
+    trace "Exception in yamux stream handler", msg = exc.msg
+    await channel.reset()
+
+method handle*(m: Yamux) {.async, gcsafe.} =
+  trace "Starting yamux handler", pid=m.connection.peerId
+  try:
+    while not m.connection.atEof:
+      trace "waiting for header"
+      let header = await m.connection.readHeader()
+      trace "got message", h = $header
+
+      case header.msgType:
+      of Ping:
+        if MsgFlags.Syn in header.flags:
+          await m.connection.write(YamuxHeader.ping(MsgFlags.Ack, header.length))
+      of GoAway:
+        var status: GoAwayStatus
+        if status.checkedEnumAssign(header.length): trace "Received go away", status
+        else: trace "Received unexpected error go away"
+        break
+      of Data, WindowUpdate:
+        if MsgFlags.Syn in header.flags:
+          if header.streamId in m.channels:
+            debug "Trying to create an existing channel, skipping", id=header.streamId
+          else:
+            if header.streamId in m.flushed:
+              m.flushed.del(header.streamId)
+            if header.streamId mod 2 == m.currentId mod 2:
+              raise newException(YamuxError, "Peer used our reserved stream id")
+            let newStream = m.createStream(header.streamId, false)
+            if m.channels.len >= m.maxChannCount:
+              await newStream.reset()
+              continue
+            await newStream.open()
+            asyncSpawn m.handleStream(newStream)
+        elif header.streamId notin m.channels:
+          if header.streamId notin m.flushed:
+            raise newException(YamuxError, "Unknown stream ID: " & $header.streamId)
+          elif header.msgType == Data:
+            # Flush the data
+            m.flushed[header.streamId].dec(int(header.length))
+            if m.flushed[header.streamId] < 0:
+              raise newException(YamuxError, "Peer exhausted the recvWindow after reset")
+            var buffer = newSeqUninitialized[byte](header.length)
+            await m.connection.readExactly(addr buffer[0], int(header.length))
+          continue
+
+        let channel = m.channels[header.streamId]
+
+        if header.msgType == WindowUpdate:
+          channel.sendWindow += int(header.length)
+          await channel.trySend()
+        else:
+          if header.length.int > channel.recvWindow.int:
+            # check before allocating the buffer
+            raise newException(YamuxError, "Peer exhausted the recvWindow")
+
+          if header.length > 0:
+            var buffer = newSeqUninitialized[byte](header.length)
+            await m.connection.readExactly(addr buffer[0], int(header.length))
+            trace "Msg Rcv", msg=string.fromBytes(buffer)
+            await channel.gotDataFromRemote(buffer)
+
+        if MsgFlags.Fin in header.flags:
+          trace "remote closed channel"
+          await channel.remoteClosed()
+        if MsgFlags.Rst in header.flags:
+          trace "remote reset channel"
+          await channel.reset()
+  except LPStreamEOFError as exc:
+    trace "Stream EOF", msg = exc.msg
+  except YamuxError as exc:
+    trace "Closing yamux connection", error=exc.msg
+    await m.connection.write(YamuxHeader.goAway(ProtocolError))
+  finally:
+    await m.close()
+  trace "Stopped yamux handler"
+
+method newStream*(
+  m: Yamux,
+  name: string = "",
+  lazy: bool = false): Future[Connection] {.async, gcsafe.} =
+
+  if m.channels.len > m.maxChannCount - 1:
+    raise newException(TooManyChannels, "max allowed channel count exceeded")
+  let stream = m.createStream(m.currentId, true)
+  m.currentId += 2
+  if not lazy:
+    await stream.open()
+  return stream
+
+proc new*(T: type[Yamux], conn: Connection, maxChannCount: int = MaxChannelCount): T =
+  T(
+    connection: conn,
+    currentId: if conn.dir == Out: 1 else: 2,
+    maxChannCount: maxChannCount
+  )

libp2p/nameresolving/dnsresolver.nim

@@ -1,13 +1,16 @@
-## Nim-LibP2P
-## Copyright (c) 2021 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 import
   std/[streams, strutils, sets, sequtils],

libp2p/nameresolving/mockresolver.nim

@@ -1,13 +1,16 @@
-## Nim-LibP2P
-## Copyright (c) 2021 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 import
   std/[streams, strutils, tables],

libp2p/nameresolving/nameresolver.nim

@@ -1,13 +1,16 @@
-## Nim-LibP2P
-## Copyright (c) 2021 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 import std/[sugar, sets, sequtils, strutils]
 import 

libp2p/peerid.nim

@@ -1,25 +1,30 @@
-## Nim-LibP2P
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module implementes API for libp2p peer.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+{.push public.}
 
 import
   std/[hashes, strutils],
   stew/[base58, results],
   chronicles,
   nimcrypto/utils,
+  utility,
   ./crypto/crypto, ./multicodec, ./multihash, ./vbuffer,
   ./protobuf/minprotobuf
 
-export results
+export results, utility
 
 const
   maxInlineKeyLength* = 42

libp2p/peerinfo.nim

@@ -1,17 +1,21 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+{.push public.}
 
 import std/[options, sequtils]
 import pkg/[chronos, chronicles, stew/results]
-import peerid, multiaddress, crypto/crypto, routing_record, errors
+import peerid, multiaddress, crypto/crypto, routing_record, errors, utility
 
 export peerid, multiaddress, crypto, routing_record, errors, results
 
@@ -20,7 +24,7 @@ export peerid, multiaddress, crypto, routing_record, errors, results
 type
   PeerInfoError* = LPError
 
-  PeerInfo* = ref object
+  PeerInfo* {.public.} = ref object
     peerId*: PeerId
     addrs*: seq[MultiAddress]
     protocols*: seq[string]
@@ -65,7 +69,7 @@ proc new*(
     except CatchableError:
       raise newException(PeerInfoError, "invalid private key")
   
-  let peerId = PeerID.init(key).tryGet()
+  let peerId = PeerId.init(key).tryGet()
 
   let peerInfo = PeerInfo(
     peerId: peerId,

libp2p/peerstore.nim

@@ -1,13 +1,31 @@
-## Nim-LibP2P
-## Copyright (c) 2021 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-{.push raises: [Defect].}
+## Stores generic informations about peers.
+runnableExamples:
+  # Will keep info of all connected peers +
+  # last 50 disconnected peers
+  let peerStore = PeerStore.new(capacity = 50)
+
+  # Create a custom book type
+  type MoodBook = ref object of PeerBook[string]
+
+  var somePeerId: PeerId
+  discard somePeerId.init("")
+
+  peerStore[MoodBook][somePeerId] = "Happy"
+  doAssert peerStore[MoodBook][somePeerId] == "Happy"
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 import
   std/[tables, sets, options, macros],
@@ -15,7 +33,8 @@ import
   ./protocols/identify,
   ./peerid, ./peerinfo,
   ./routing_record,
-  ./multiaddress
+  ./multiaddress,
+  utility
 
 type
   #################
@@ -23,7 +42,7 @@ type
   #################
 
   PeerBookChangeHandler* = proc(peerId: PeerId) {.gcsafe, raises: [Defect].}
-  
+
   #########
   # Books #
   #########
@@ -33,29 +52,29 @@ type
     changeHandlers: seq[PeerBookChangeHandler]
     deletor: PeerBookChangeHandler
 
-  PeerBook*[T] = ref object of BasePeerBook
+  PeerBook*[T] {.public.} = ref object of BasePeerBook
     book*: Table[PeerId, T]
 
   SeqPeerBook*[T] = ref object of PeerBook[seq[T]]
-  
-  AddressBook* = ref object of SeqPeerBook[MultiAddress]
-  ProtoBook* = ref object of SeqPeerBook[string]
-  KeyBook* = ref object of PeerBook[PublicKey]
 
-  AgentBook* = ref object of PeerBook[string]
-  ProtoVersionBook* = ref object of PeerBook[string]
-  SPRBook* = ref object of PeerBook[Envelope]
-  
+  AddressBook* {.public.} = ref object of SeqPeerBook[MultiAddress]
+  ProtoBook* {.public.} = ref object of SeqPeerBook[string]
+  KeyBook* {.public.} = ref object of PeerBook[PublicKey]
+
+  AgentBook* {.public.} = ref object of PeerBook[string]
+  ProtoVersionBook* {.public.} = ref object of PeerBook[string]
+  SPRBook* {.public.} = ref object of PeerBook[Envelope]
+
   ####################
   # Peer store types #
   ####################
 
-  PeerStore* = ref object
+  PeerStore* {.public.} = ref object
     books: Table[string, BasePeerBook]
     capacity*: int
     toClean*: seq[PeerId]
-  
-proc new*(T: type PeerStore, capacity = 1000): PeerStore =
+
+proc new*(T: type PeerStore, capacity = 1000): PeerStore {.public.} =
   T(capacity: capacity)
 
 #########################
@@ -63,16 +82,15 @@ proc new*(T: type PeerStore, capacity = 1000): PeerStore =
 #########################
 
 proc `[]`*[T](peerBook: PeerBook[T],
-             peerId: PeerId): T =
-  ## Get all the known metadata of a provided peer.
+             peerId: PeerId): T {.public.} =
+  ## Get all known metadata of a provided peer, or default(T) if missing
   peerBook.book.getOrDefault(peerId)
 
 proc `[]=`*[T](peerBook: PeerBook[T],
              peerId: PeerId,
-             entry: T) =
-  ## Set metadata for a given peerId. This will replace any
-  ## previously stored metadata.
-  
+             entry: T) {.public.} =
+  ## Set metadata for a given peerId.
+
   peerBook.book[peerId] = entry
 
   # Notify clients
@@ -80,9 +98,9 @@ proc `[]=`*[T](peerBook: PeerBook[T],
     handler(peerId)
 
 proc del*[T](peerBook: PeerBook[T],
-                peerId: PeerId): bool =
-  ## Delete the provided peer from the book.
-  
+                peerId: PeerId): bool {.public.} =
+  ## Delete the provided peer from the book. Returns whether the peer was in the book
+
   if peerId notin peerBook.book:
     return false
   else:
@@ -92,15 +110,16 @@ proc del*[T](peerBook: PeerBook[T],
       handler(peerId)
     return true
 
-proc contains*[T](peerBook: PeerBook[T], peerId: PeerId): bool =
+proc contains*[T](peerBook: PeerBook[T], peerId: PeerId): bool {.public.} =
   peerId in peerBook.book
 
-proc addHandler*[T](peerBook: PeerBook[T], handler: PeerBookChangeHandler) =
+proc addHandler*[T](peerBook: PeerBook[T], handler: PeerBookChangeHandler) {.public.} =
+  ## Adds a callback that will be called everytime the book changes
   peerBook.changeHandlers.add(handler)
 
-proc len*[T](peerBook: PeerBook[T]): int = peerBook.book.len
+proc len*[T](peerBook: PeerBook[T]): int {.public.} = peerBook.book.len
 
-##################  
+##################
 # Peer Store API #
 ##################
 macro getTypeName(t: type): untyped =
@@ -108,7 +127,8 @@ macro getTypeName(t: type): untyped =
   let typ = getTypeImpl(t)[1]
   newLit(repr(typ.owner()) & "." & repr(typ))
 
-proc `[]`*[T](p: PeerStore, typ: type[T]): T =
+proc `[]`*[T](p: PeerStore, typ: type[T]): T {.public.} =
+  ## Get a book from the PeerStore (ex: peerStore[AddressBook])
   let name = getTypeName(T)
   result = T(p.books.getOrDefault(name))
   if result.isNil:
@@ -121,7 +141,7 @@ proc `[]`*[T](p: PeerStore, typ: type[T]): T =
   return result
 
 proc del*(peerStore: PeerStore,
-             peerId: PeerId) =
+             peerId: PeerId) {.public.} =
   ## Delete the provided peer from every book.
   for _, book in peerStore.books:
     book.deletor(peerId)

libp2p/protobuf/minprotobuf.nim

@@ -1,18 +1,23 @@
-## Nim-Libp2p
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module implements minimal Google's ProtoBuf primitives.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
-import ../varint, stew/[endians2, results]
-export results
+import ../varint, ../utility, stew/[endians2, results]
+export results, utility
+
+{.push public.}
 
 const
   MaxMessageSize* = 1'u shl 22

libp2p/protocols/autonat.nim

@@ -0,0 +1,301 @@
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import std/[options, sets, sequtils]
+import chronos, chronicles, stew/objects
+import ./protocol,
+       ../switch,
+       ../multiaddress,
+       ../multicodec,
+       ../peerid,
+       ../utils/semaphore,
+       ../errors
+
+logScope:
+  topics = "libp2p autonat"
+
+const
+  AutonatCodec* = "/libp2p/autonat/1.0.0"
+  AddressLimit = 8
+
+type
+  AutonatError* = object of LPError
+
+  MsgType* = enum
+    Dial = 0
+    DialResponse = 1
+
+  ResponseStatus* = enum
+    Ok = 0
+    DialError = 100
+    DialRefused = 101
+    BadRequest = 200
+    InternalError = 300
+
+  AutonatPeerInfo* = object
+    id: Option[PeerId]
+    addrs: seq[MultiAddress]
+
+  AutonatDial* = object
+    peerInfo: Option[AutonatPeerInfo]
+
+  AutonatDialResponse* = object
+    status*: ResponseStatus
+    text*: Option[string]
+    ma*: Option[MultiAddress]
+
+  AutonatMsg = object
+    msgType: MsgType
+    dial: Option[AutonatDial]
+    response: Option[AutonatDialResponse]
+
+proc encode*(msg: AutonatMsg): ProtoBuffer =
+  result = initProtoBuffer()
+  result.write(1, msg.msgType.uint)
+  if msg.dial.isSome():
+    var dial = initProtoBuffer()
+    if msg.dial.get().peerInfo.isSome():
+      var bufferPeerInfo = initProtoBuffer()
+      let peerInfo = msg.dial.get().peerInfo.get()
+      if peerInfo.id.isSome():
+        bufferPeerInfo.write(1, peerInfo.id.get())
+      for ma in peerInfo.addrs:
+        bufferPeerInfo.write(2, ma.data.buffer)
+      bufferPeerInfo.finish()
+      dial.write(1, bufferPeerInfo.buffer)
+    dial.finish()
+    result.write(2, dial.buffer)
+  if msg.response.isSome():
+    var bufferResponse = initProtoBuffer()
+    let response = msg.response.get()
+    bufferResponse.write(1, response.status.uint)
+    if response.text.isSome():
+      bufferResponse.write(2, response.text.get())
+    if response.ma.isSome():
+      bufferResponse.write(3, response.ma.get())
+    bufferResponse.finish()
+    result.write(3, bufferResponse.buffer)
+  result.finish()
+
+proc encode*(d: AutonatDial): ProtoBuffer =
+  result = initProtoBuffer()
+  result.write(1, MsgType.Dial.uint)
+  var dial = initProtoBuffer()
+  if d.peerInfo.isSome():
+    var bufferPeerInfo = initProtoBuffer()
+    let peerInfo = d.peerInfo.get()
+    if peerInfo.id.isSome():
+      bufferPeerInfo.write(1, peerInfo.id.get())
+    for ma in peerInfo.addrs:
+      bufferPeerInfo.write(2, ma.data.buffer)
+    bufferPeerInfo.finish()
+    dial.write(1, bufferPeerInfo.buffer)
+  dial.finish()
+  result.write(2, dial.buffer)
+  result.finish()
+
+proc encode*(r: AutonatDialResponse): ProtoBuffer =
+  result = initProtoBuffer()
+  result.write(1, MsgType.DialResponse.uint)
+  var bufferResponse = initProtoBuffer()
+  bufferResponse.write(1, r.status.uint)
+  if r.text.isSome():
+    bufferResponse.write(2, r.text.get())
+  if r.ma.isSome():
+    bufferResponse.write(3, r.ma.get())
+  bufferResponse.finish()
+  result.write(3, bufferResponse.buffer)
+  result.finish()
+
+proc decode(_: typedesc[AutonatMsg], buf: seq[byte]): Option[AutonatMsg] =
+  var
+    msgTypeOrd: uint32
+    pbDial: ProtoBuffer
+    pbResponse: ProtoBuffer
+    msg: AutonatMsg
+
+  let
+    pb = initProtoBuffer(buf)
+    r1 = pb.getField(1, msgTypeOrd)
+    r2 = pb.getField(2, pbDial)
+    r3 = pb.getField(3, pbResponse)
+  if r1.isErr() or r2.isErr() or r3.isErr(): return none(AutonatMsg)
+
+  if r1.get() and not checkedEnumAssign(msg.msgType, msgTypeOrd):
+    return none(AutonatMsg)
+  if r2.get():
+    var
+      pbPeerInfo: ProtoBuffer
+      dial: AutonatDial
+    let
+      r4 = pbDial.getField(1, pbPeerInfo)
+    if r4.isErr(): return none(AutonatMsg)
+
+    var peerInfo: AutonatPeerInfo
+    if r4.get():
+      var pid: PeerId
+      let
+        r5 = pbPeerInfo.getField(1, pid)
+        r6 = pbPeerInfo.getRepeatedField(2, peerInfo.addrs)
+      if r5.isErr() or r6.isErr(): return none(AutonatMsg)
+      if r5.get(): peerInfo.id = some(pid)
+      dial.peerInfo = some(peerInfo)
+    msg.dial = some(dial)
+
+  if r3.get():
+    var
+      statusOrd: uint
+      text: string
+      ma: MultiAddress
+      response: AutonatDialResponse
+
+    let
+      r4 = pbResponse.getField(1, statusOrd)
+      r5 = pbResponse.getField(2, text)
+      r6 = pbResponse.getField(3, ma)
+
+    if r4.isErr() or r5.isErr() or r6.isErr() or
+       (r4.get() and not checkedEnumAssign(response.status, statusOrd)):
+      return none(AutonatMsg)
+    if r5.get(): response.text = some(text)
+    if r6.get(): response.ma = some(ma)
+    msg.response = some(response)
+
+  return some(msg)
+
+proc sendDial(conn: Connection, pid: PeerId, addrs: seq[MultiAddress]) {.async.} =
+  let pb = AutonatDial(peerInfo: some(AutonatPeerInfo(
+                         id: some(pid),
+                         addrs: addrs
+                       ))).encode()
+  await conn.writeLp(pb.buffer)
+
+proc sendResponseError(conn: Connection, status: ResponseStatus, text: string = "") {.async.} =
+  let pb = AutonatDialResponse(
+             status: status,
+             text: if text == "": none(string) else: some(text),
+             ma: none(MultiAddress)
+           ).encode()
+  await conn.writeLp(pb.buffer)
+
+proc sendResponseOk(conn: Connection, ma: MultiAddress) {.async.} =
+  let pb = AutonatDialResponse(
+             status: ResponseStatus.Ok,
+             text: some("Ok"),
+             ma: some(ma)
+           ).encode()
+  await conn.writeLp(pb.buffer)
+
+type
+  Autonat* = ref object of LPProtocol
+    sem: AsyncSemaphore
+    switch*: Switch
+
+proc dialMe*(a: Autonat, pid: PeerId, ma: MultiAddress|seq[MultiAddress]):
+    Future[MultiAddress] {.async.} =
+  let addrs = when ma is MultiAddress: @[ma] else: ma
+  let conn = await a.switch.dial(pid, addrs, AutonatCodec)
+  defer: await conn.close()
+  await conn.sendDial(a.switch.peerInfo.peerId, a.switch.peerInfo.addrs)
+  let msgOpt = AutonatMsg.decode(await conn.readLp(1024))
+  if msgOpt.isNone() or
+     msgOpt.get().msgType != DialResponse or
+     msgOpt.get().response.isNone():
+    raise newException(AutonatError, "Unexpected response")
+  let response = msgOpt.get().response.get()
+  if response.status != ResponseStatus.Ok:
+    raise newException(AutonatError, "Bad status " &
+                                      $response.status & " " &
+                                      response.text.get(""))
+  if response.ma.isNone():
+    raise newException(AutonatError, "Missing address")
+  return response.ma.get()
+
+proc tryDial(a: Autonat, conn: Connection, addrs: seq[MultiAddress]) {.async.} =
+  try:
+    await a.sem.acquire()
+    let ma = await a.switch.dialer.tryDial(conn.peerId, addrs)
+    await conn.sendResponseOk(ma)
+  except CancelledError as exc:
+    raise exc
+  except CatchableError as exc:
+    await conn.sendResponseError(DialError, exc.msg)
+  finally:
+    a.sem.release()
+
+proc handleDial(a: Autonat, conn: Connection, msg: AutonatMsg): Future[void] =
+  if msg.dial.isNone() or msg.dial.get().peerInfo.isNone():
+    return conn.sendResponseError(BadRequest, "Missing Peer Info")
+  let peerInfo = msg.dial.get().peerInfo.get()
+  if peerInfo.id.isSome() and peerInfo.id.get() != conn.peerId:
+    return conn.sendResponseError(BadRequest, "PeerId mismatch")
+
+  var isRelayed = conn.observedAddr.contains(multiCodec("p2p-circuit"))
+  if isRelayed.isErr() or isRelayed.get():
+    return conn.sendResponseError(DialRefused, "Refused to dial a relayed observed address")
+  let hostIp = conn.observedAddr[0]
+  if hostIp.isErr() or not IP.match(hostIp.get()):
+    trace "wrong observed address", address=conn.observedAddr
+    return conn.sendResponseError(InternalError, "Expected an IP address")
+  var addrs = initHashSet[MultiAddress]()
+  addrs.incl(conn.observedAddr)
+  for ma in peerInfo.addrs:
+    isRelayed = ma.contains(multiCodec("p2p-circuit"))
+    if isRelayed.isErr() or isRelayed.get():
+      continue
+    let maFirst = ma[0]
+    if maFirst.isErr() or not IP.match(maFirst.get()):
+      continue
+
+    try:
+      addrs.incl(
+        if maFirst.get() == hostIp.get():
+          ma
+        else:
+          let maEnd = ma[1..^1]
+          if maEnd.isErr(): continue
+          hostIp.get() & maEnd.get()
+      )
+    except LPError as exc:
+      continue
+    if len(addrs) >= AddressLimit:
+      break
+
+  if len(addrs) == 0:
+    return conn.sendResponseError(DialRefused, "No dialable address")
+  return a.tryDial(conn, toSeq(addrs))
+
+proc new*(T: typedesc[Autonat], switch: Switch, semSize: int = 1): T =
+  let autonat = T(switch: switch, sem: newAsyncSemaphore(semSize))
+  autonat.init()
+  autonat
+
+method init*(a: Autonat) =
+  proc handleStream(conn: Connection, proto: string) {.async, gcsafe.} =
+    try:
+      let msgOpt = AutonatMsg.decode(await conn.readLp(1024))
+      if msgOpt.isNone() or msgOpt.get().msgType != MsgType.Dial:
+        raise newException(AutonatError, "Received malformed message")
+      let msg = msgOpt.get()
+      await a.handleDial(conn, msg)
+    except CancelledError as exc:
+      raise exc
+    except CatchableError as exc:
+      trace "exception in autonat handler", exc = exc.msg, conn
+    finally:
+      trace "exiting autonat handler", conn
+      await conn.close()
+
+  a.handler = handleStream
+  a.codec = AutonatCodec

libp2p/protocols/identify.nim

@@ -1,13 +1,19 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-{.push raises: [Defect].}
+## `Identify <https://docs.libp2p.io/concepts/protocols/#identify>`_ and
+## `Push Identify <https://docs.libp2p.io/concepts/protocols/#identify-push>`_ implementation
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 import std/[sequtils, options, strutils, sugar]
 import chronos, chronicles
@@ -36,7 +42,7 @@ type
   IdentityInvalidMsgError* = object of IdentifyError
   IdentifyNoPubKeyError* = object of IdentifyError
 
-  IdentifyInfo* = object
+  IdentifyInfo* {.public.} = object
     pubkey*: Option[PublicKey]
     peerId*: PeerId
     addrs*: seq[MultiAddress]
@@ -54,7 +60,7 @@ type
     peer: PeerId,
     newInfo: IdentifyInfo):
     Future[void]
-    {.gcsafe, raises: [Defect].}
+    {.gcsafe, raises: [Defect], public.}
 
   IdentifyPush* = ref object of LPProtocol
     identifyHandler: IdentifyPushHandler
@@ -203,7 +209,9 @@ proc identify*(p: Identify,
   else:
     raise newException(IdentityInvalidMsgError, "No pubkey in identify")
 
-proc new*(T: typedesc[IdentifyPush], handler: IdentifyPushHandler = nil): T =
+proc new*(T: typedesc[IdentifyPush], handler: IdentifyPushHandler = nil): T {.public.} =
+  ## Create a IdentifyPush protocol. `handler` will be called every time
+  ## a peer sends us new `PeerInfo`
   let identifypush = T(identifyHandler: handler)
   identifypush.init()
   identifypush
@@ -240,6 +248,7 @@ proc init*(p: IdentifyPush) =
   p.handler = handle
   p.codec = IdentifyPushCodec
 
-proc push*(p: IdentifyPush, peerInfo: PeerInfo, conn: Connection) {.async.} =
+proc push*(p: IdentifyPush, peerInfo: PeerInfo, conn: Connection) {.async, public.} =
+  ## Send new `peerInfo`s to a connection
   var pb = encodeMsg(peerInfo, conn.observedAddr, true)
   await conn.writeLp(pb.buffer)

libp2p/protocols/ping.nim

@@ -1,15 +1,21 @@
-## Nim-LibP2P
-## Copyright (c) 2021 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-{.push raises: [Defect].}
+## `Ping <https://docs.libp2p.io/concepts/protocols/#ping>`_ protocol implementation
 
-import chronos, chronicles, bearssl
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import chronos, chronicles
+import bearssl/[rand, hash]
 import ../protobuf/minprotobuf,
        ../peerinfo,
        ../stream/connection,
@@ -17,8 +23,11 @@ import ../protobuf/minprotobuf,
        ../crypto/crypto,
        ../multiaddress,
        ../protocols/protocol,
+       ../utility,
        ../errors
 
+export chronicles, rand, connection
+
 logScope:
   topics = "libp2p ping"
 
@@ -28,18 +37,18 @@ const
 
 type
   PingError* = object of LPError
-  WrongPingAckError* = object of LPError
+  WrongPingAckError* = object of PingError
 
-  PingHandler* = proc (
+  PingHandler* {.public.} = proc (
     peer: PeerId):
     Future[void]
     {.gcsafe, raises: [Defect].}
 
   Ping* = ref object of LPProtocol
     pingHandler*: PingHandler
-    rng: ref BrHmacDrbgContext
+    rng: ref HmacDrbgContext
 
-proc new*(T: typedesc[Ping], handler: PingHandler = nil, rng: ref BrHmacDrbgContext = newRng()): T =
+proc new*(T: typedesc[Ping], handler: PingHandler = nil, rng: ref HmacDrbgContext = newRng()): T {.public.} =
   let ping = Ping(pinghandler: handler, rng: rng)
   ping.init()
   ping
@@ -65,10 +74,8 @@ method init*(p: Ping) =
 proc ping*(
   p: Ping,
   conn: Connection,
-  ): Future[Duration] {.async, gcsafe.} =
-  ## Sends ping to `conn`
-  ## Returns the delay
-  ##
+  ): Future[Duration] {.async, gcsafe, public.} =
+  ## Sends ping to `conn`, returns the delay
 
   trace "initiating ping", conn
 
@@ -76,7 +83,7 @@ proc ping*(
     randomBuf: array[PingSize, byte]
     resultBuf: array[PingSize, byte]
 
-  p.rng[].brHmacDrbgGenerate(randomBuf)
+  hmacDrbgGenerate(p.rng[], randomBuf)
 
   let startTime = Moment.now()
 

libp2p/protocols/protocol.nim

@@ -1,13 +1,16 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 import chronos
 import ../stream/connection
@@ -22,8 +25,12 @@ type
   LPProtocol* = ref object of RootObj
     codecs*: seq[string]
     handler*: LPProtoHandler ## this handler gets invoked by the protocol negotiator
+    started*: bool
 
 method init*(p: LPProtocol) {.base, gcsafe.} = discard
+method start*(p: LPProtocol) {.async, base.} = p.started = true
+method stop*(p: LPProtocol) {.async, base.} = p.started = false
+
 
 func codec*(p: LPProtocol): string =
   assert(p.codecs.len > 0, "Codecs sequence was empty!")

libp2p/protocols/pubsub.nim

@@ -1,3 +1,8 @@
 import ./pubsub/[pubsub, floodsub, gossipsub]
 
+## Home of PubSub & it's implementations:
+## | **pubsub**: base interface for pubsub implementations
+## | **floodsub**: simple flood-based publishing
+## | **gossipsub**: more sophisticated gossip based publishing
+
 export pubsub, floodsub, gossipsub

libp2p/protocols/pubsub/errors.nim

@@ -1,6 +1,8 @@
 # this module will be further extended in PR
 # https://github.com/status-im/nim-libp2p/pull/107/
 
+import ../../utility
+
 type
-  ValidationResult* {.pure.} = enum
+  ValidationResult* {.pure, public.} = enum
     Accept, Reject, Ignore

libp2p/protocols/pubsub/floodsub.nim

@@ -1,16 +1,19 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 import std/[sequtils, sets, hashes, tables]
-import chronos, chronicles, metrics, bearssl
+import chronos, chronicles, metrics
 import ./pubsub,
        ./pubsubpeer,
        ./timedcache,
@@ -22,31 +25,33 @@ import ./pubsub,
        ../../peerinfo,
        ../../utility
 
+## Simple flood-based publishing.
+
 logScope:
   topics = "libp2p floodsub"
 
 const FloodSubCodec* = "/floodsub/1.0.0"
 
 type
-  FloodSub* = ref object of PubSub
+  FloodSub* {.public.} = ref object of PubSub
     floodsub*: PeerTable      # topic to remote peer map
-    seen*: TimedCache[MessageID] # message id:s already seen on the network
+    seen*: TimedCache[MessageId] # message id:s already seen on the network
     seenSalt*: seq[byte]
 
-proc hasSeen*(f: FloodSub, msgId: MessageID): bool =
+proc hasSeen*(f: FloodSub, msgId: MessageId): bool =
   f.seenSalt & msgId in f.seen
 
-proc addSeen*(f: FloodSub, msgId: MessageID): bool =
+proc addSeen*(f: FloodSub, msgId: MessageId): bool =
   # Salting the seen hash helps avoid attacks against the hash function used
   # in the nim hash table
   # Return true if the message has already been seen
   f.seen.put(f.seenSalt & msgId)
 
-proc firstSeen*(f: FloodSub, msgId: MessageID): Moment =
+proc firstSeen*(f: FloodSub, msgId: MessageId): Moment =
   f.seen.addedAt(f.seenSalt & msgId)
 
 proc handleSubscribe*(f: FloodSub,
-                      peer: PubsubPeer,
+                      peer: PubSubPeer,
                       topic: string,
                       subscribe: bool) =
   logScope:
@@ -137,7 +142,7 @@ method rpcHandler*(f: FloodSub,
       discard
 
     var toSendPeers = initHashSet[PubSubPeer]()
-    for t in msg.topicIDs:                     # for every topic in the message
+    for t in msg.topicIds:                     # for every topic in the message
       if t notin f.topics:
         continue
       f.floodsub.withValue(t, peers): toSendPeers.incl(peers[])
@@ -187,21 +192,17 @@ method publish*(f: FloodSub,
     debug "No peers for topic, skipping publish", topic
     return 0
 
-  inc f.msgSeqno
   let
     msg =
       if f.anonymize:
         Message.init(none(PeerInfo), data, topic, none(uint64), false)
       else:
+        inc f.msgSeqno
         Message.init(some(f.peerInfo), data, topic, some(f.msgSeqno), f.sign)
-    msgIdResult = f.msgIdProvider(msg)
-
-  if msgIdResult.isErr:
-    trace "Error generating message id, skipping publish",
-      error = msgIdResult.error
-    return 0
-
-  let msgId = msgIdResult.get
+    msgId = f.msgIdProvider(msg).valueOr:
+      trace "Error generating message id, skipping publish",
+        error = error
+      return 0
 
   trace "Created new message",
     msg = shortLog(msg), peers = peers.len, topic, msgId
@@ -224,8 +225,8 @@ method publish*(f: FloodSub,
 method initPubSub*(f: FloodSub)
   {.raises: [Defect, InitializationError].} =
   procCall PubSub(f).initPubSub()
-  f.seen = TimedCache[MessageID].init(2.minutes)
+  f.seen = TimedCache[MessageId].init(2.minutes)
   f.seenSalt = newSeqUninitialized[byte](sizeof(Hash))
-  brHmacDrbgGenerate(f.rng[], f.seenSalt)
+  hmacDrbgGenerate(f.rng[], f.seenSalt)
 
   f.init()

libp2p/protocols/pubsub/gossipsub.nim

@@ -1,15 +1,20 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-{.push raises: [Defect].}
+## Gossip based publishing
 
-import std/[tables, sets, options, sequtils]
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import std/[tables, sets, options, random, sequtils]
 import chronos, chronicles, metrics
 import ./pubsub,
        ./floodsub,
@@ -156,7 +161,7 @@ method onNewPeer(g: GossipSub, peer: PubSubPeer) =
     peer.iWantBudget = IWantPeerBudget
     peer.iHaveBudget = IHavePeerBudget
 
-method onPubSubPeerEvent*(p: GossipSub, peer: PubsubPeer, event: PubSubPeerEvent) {.gcsafe.} =
+method onPubSubPeerEvent*(p: GossipSub, peer: PubSubPeer, event: PubSubPeerEvent) {.gcsafe.} =
   case event.kind
   of PubSubPeerEventKind.Connected:
     discard
@@ -259,7 +264,7 @@ proc handleControl(g: GossipSub, peer: PubSubPeer, control: ControlMessage) =
 
   var respControl: ControlMessage
   let iwant = g.handleIHave(peer, control.ihave)
-  if iwant.messageIDs.len > 0:
+  if iwant.messageIds.len > 0:
     respControl.iwant.add(iwant)
   respControl.prune.add(g.handleGraft(peer, control.graft))
   let messages = g.handleIWant(peer, control.iwant)
@@ -271,7 +276,7 @@ proc handleControl(g: GossipSub, peer: PubSubPeer, control: ControlMessage) =
     # iwant and prunes from here, also messages
 
     for smsg in messages:
-      for topic in smsg.topicIDs:
+      for topic in smsg.topicIds:
         if g.knownTopics.contains(topic):
           libp2p_pubsub_broadcast_messages.inc(labelValues = [topic])
         else:
@@ -280,8 +285,8 @@ proc handleControl(g: GossipSub, peer: PubSubPeer, control: ControlMessage) =
     libp2p_pubsub_broadcast_iwant.inc(respControl.iwant.len.int64)
 
     for prune in respControl.prune:
-      if g.knownTopics.contains(prune.topicID):
-        libp2p_pubsub_broadcast_prune.inc(labelValues = [prune.topicID])
+      if g.knownTopics.contains(prune.topicId):
+        libp2p_pubsub_broadcast_prune.inc(labelValues = [prune.topicId])
       else:
         libp2p_pubsub_broadcast_prune.inc(labelValues = ["generic"])
 
@@ -290,10 +295,26 @@ proc handleControl(g: GossipSub, peer: PubSubPeer, control: ControlMessage) =
       peer,
       RPCMsg(control: some(respControl), messages: messages))
 
+proc sampleDandelionPeers(peers: var HashSet[PubSubPeer]) =
+  const dandelionD = 2
+  if peers.len <= dandelionD:
+    return
+
+  trace "Sampling peers", availablePeers = peers.len, dandelionD
+  var
+    remainingPeers = peers.toSeq()
+    randomPeers: HashSet[PubSubPeer]
+  while randomPeers.len < dandelionD:
+    let index = random(remainingPeers.len)
+    randomPeers.incl remainingPeers[index]
+    remainingPeers.delete index
+  peers = randomPeers
+
 proc validateAndRelay(g: GossipSub,
-                      msg: Message,
+                      m: Message,
                       msgId, msgIdSalted: MessageId,
                       peer: PubSubPeer) {.async.} =
+  var msg = m
   try:
     let validation = await g.validate(msg)
 
@@ -305,7 +326,7 @@ proc validateAndRelay(g: GossipSub,
     of ValidationResult.Reject:
       debug "Dropping message after validation, reason: reject",
         msgId = shortLog(msgId), peer
-      g.punishInvalidMessage(peer, msg.topicIDs)
+      g.punishInvalidMessage(peer, msg.topicIds)
       return
     of ValidationResult.Ignore:
       debug "Dropping message after validation, reason: ignore",
@@ -314,13 +335,16 @@ proc validateAndRelay(g: GossipSub,
     of ValidationResult.Accept:
       discard
 
+    if msg.stem.get(0) > 0:
+      msg.stem = some(msg.stem.get - 1)
+
     # store in cache only after validation
     g.mcache.put(msgId, msg)
 
-    g.rewardDelivered(peer, msg.topicIDs, true)
+    g.rewardDelivered(peer, msg.topicIds, true)
 
     var toSendPeers = HashSet[PubSubPeer]()
-    for t in msg.topicIDs:                      # for every topic in the message
+    for t in msg.topicIds:                      # for every topic in the message
       if t notin g.topics:
         continue
 
@@ -332,11 +356,14 @@ proc validateAndRelay(g: GossipSub,
     toSendPeers.excl(peer)
     toSendPeers.excl(seenPeers)
 
+    if msg.stem.get(0) > 0:
+      toSendPeers.sampleDandelionPeers()
+
     # In theory, if topics are the same in all messages, we could batch - we'd
     # also have to be careful to only include validated messages
     g.broadcast(toSendPeers, RPCMsg(messages: @[msg]))
     trace "forwared message to peers", peers = toSendPeers.len, msgId, peer
-    for topic in msg.topicIDs:
+    for topic in msg.topicIds:
       if topic notin g.topics: continue
 
       if g.knownTopics.contains(topic):
@@ -390,7 +417,7 @@ method rpcHandler*(g: GossipSub,
 
       if not alreadyReceived:
         let delay = Moment.now() - g.firstSeen(msgId)
-        g.rewardDelivered(peer, msg.topicIDs, false, delay)
+        g.rewardDelivered(peer, msg.topicIds, false, delay)
 
       libp2p_gossipsub_duplicate.inc()
 
@@ -400,7 +427,7 @@ method rpcHandler*(g: GossipSub,
     libp2p_gossipsub_received.inc()
 
     # avoid processing messages we are not interested in
-    if msg.topicIDs.allIt(it notin g.topics):
+    if msg.topicIds.allIt(it notin g.topics):
       debug "Dropping message of topic without subscription", msgId = shortLog(msgId), peer
       continue
 
@@ -408,14 +435,14 @@ method rpcHandler*(g: GossipSub,
       # always validate if signature is present or required
       debug "Dropping message due to failed signature verification",
         msgId = shortLog(msgId), peer
-      g.punishInvalidMessage(peer, msg.topicIDs)
+      g.punishInvalidMessage(peer, msg.topicIds)
       continue
 
     if msg.seqno.len > 0 and msg.seqno.len != 8:
       # if we have seqno should be 8 bytes long
       debug "Dropping message due to invalid seqno length",
         msgId = shortLog(msgId), peer
-      g.punishInvalidMessage(peer, msg.topicIDs)
+      g.punishInvalidMessage(peer, msg.topicIds)
       continue
 
     # g.anonymize needs no evaluation when receiving messages
@@ -430,6 +457,13 @@ method rpcHandler*(g: GossipSub,
   if rpcMsg.control.isSome():
     g.handleControl(peer, rpcMsg.control.unsafeGet())
 
+  # Now, check subscription to update the meshes if required
+  for i in 0..<min(g.topicsHigh, rpcMsg.subscriptions.len):
+    let topic = rpcMsg.subscriptions[i].topic
+    if topic in g.topics and g.mesh.peers(topic) < g.parameters.dLow:
+      # rebalance but don't update metrics here, we do that only in the heartbeat
+      g.rebalanceMesh(topic, metrics = nil)
+
   g.updateMetrics(rpcMsg)
 
 method onTopicSubscription*(g: GossipSub, topic: string, subscribed: bool) =
@@ -506,7 +540,7 @@ method publish*(g: GossipSub,
     g.rng.shuffle(fanoutPeers)
     if fanoutPeers.len + peers.len > g.parameters.d:
       fanoutPeers.setLen(g.parameters.d - peers.len)
-    
+
     for fanPeer in fanoutPeers:
       peers.incl(fanPeer)
       if peers.len > g.parameters.d: break
@@ -520,29 +554,34 @@ method publish*(g: GossipSub,
 
   if peers.len == 0:
     let topicPeers = g.gossipsub.getOrDefault(topic).toSeq()
-    notice "No peers for topic, skipping publish",  peersOnTopic = topicPeers.len,
-                                                    connectedPeers = topicPeers.filterIt(it.connected).len,
-                                                    topic
+    debug "No peers for topic, skipping publish",  peersOnTopic = topicPeers.len,
+                                                   connectedPeers = topicPeers.filterIt(it.connected).len,
+                                                   topic
     # skipping topic as our metrics finds that heavy
     libp2p_gossipsub_failed_publish.inc()
     return 0
 
-  inc g.msgSeqno
+  peers.sampleDandelionPeers()
+
   let
     msg =
       if g.anonymize:
-        Message.init(none(PeerInfo), data, topic, none(uint64), false)
+        const
+          dandelionStemLo = 3.uint32
+          dandelionStemHi = 6.uint32
+        let stem = range[dandelionStemLo .. (dandelionStemHi - 1)].rand().uint32
+        Message.init(
+          none(PeerInfo), data, topic, none(uint64), false,
+          stem = some stem)
       else:
-        Message.init(some(g.peerInfo), data, topic, some(g.msgSeqno), g.sign)
-    msgIdResult = g.msgIdProvider(msg)
-
-  if msgIdResult.isErr:
-    trace "Error generating message id, skipping publish",
-      error = msgIdResult.error
-    libp2p_gossipsub_failed_publish.inc()
-    return 0
-
-  let msgId = msgIdResult.get
+        inc g.msgSeqno
+        Message.init(
+          some(g.peerInfo), data, topic, some(g.msgSeqno), g.sign)
+    msgId = g.msgIdProvider(msg).valueOr:
+      trace "Error generating message id, skipping publish",
+        error = error
+      libp2p_gossipsub_failed_publish.inc()
+      return 0
 
   logScope: msgId = shortLog(msgId)
 
@@ -566,22 +605,28 @@ method publish*(g: GossipSub,
 
   return peers.len
 
+proc maintainDirectPeer(g: GossipSub, id: PeerId, addrs: seq[MultiAddress]) {.async.} =
+  let peer = g.peers.getOrDefault(id)
+  if isNil(peer):
+    trace "Attempting to dial a direct peer", peer = id
+    try:
+      await g.switch.connect(id, addrs)
+      # populate the peer after it's connected
+      discard g.getOrCreatePeer(id, g.codecs)
+    except CancelledError as exc:
+      trace "Direct peer dial canceled"
+      raise exc
+    except CatchableError as exc:
+      debug "Direct peer error dialing", msg = exc.msg
+
+proc addDirectPeer*(g: GossipSub, id: PeerId, addrs: seq[MultiAddress]) {.async.} =
+  g.parameters.directPeers[id] = addrs
+  await g.maintainDirectPeer(id, addrs)
+
 proc maintainDirectPeers(g: GossipSub) {.async.} =
   heartbeat "GossipSub DirectPeers", 1.minutes:
     for id, addrs in g.parameters.directPeers:
-      let peer = g.peers.getOrDefault(id)
-      if isNil(peer):
-        trace "Attempting to dial a direct peer", peer = id
-        try:
-          # dial, internally connection will be stored
-          let _ = await g.switch.dial(id, addrs, g.codecs)
-          # populate the peer after it's connected
-          discard g.getOrCreatePeer(id, g.codecs)
-        except CancelledError as exc:
-          trace "Direct peer dial canceled"
-          raise exc
-        except CatchableError as exc:
-          debug "Direct peer error dialing", msg = exc.msg
+      await g.addDirectPeer(id, addrs)
 
 method start*(g: GossipSub) {.async.} =
   trace "gossipsub start"
@@ -593,9 +638,11 @@ method start*(g: GossipSub) {.async.} =
   g.heartbeatFut = g.heartbeat()
   g.scoringHeartbeatFut = g.scoringHeartbeat()
   g.directPeersLoop = g.maintainDirectPeers()
+  g.started = true
 
 method stop*(g: GossipSub) {.async.} =
   trace "gossipsub stop"
+  g.started = false
   if g.heartbeatFut.isNil:
     warn "Stopping gossipsub without starting it"
     return
@@ -618,7 +665,7 @@ method initPubSub*(g: GossipSub)
     raise newException(InitializationError, $validationRes.error)
 
   # init the floodsub stuff here, we customize timedcache in gossip!
-  g.seen = TimedCache[MessageID].init(g.parameters.seenTTL)
+  g.seen = TimedCache[MessageId].init(g.parameters.seenTTL)
 
   # init gossip stuff
   g.mcache = MCache.init(g.parameters.historyGossip, g.parameters.historyLength)

libp2p/protocols/pubsub/gossipsub/behavior.nim

@@ -1,13 +1,16 @@
-## Nim-LibP2P
-## Copyright (c) 2021 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 import std/[tables, sequtils, sets, algorithm]
 import chronos, chronicles, metrics
@@ -102,7 +105,7 @@ proc handleGraft*(g: GossipSub,
                  grafts: seq[ControlGraft]): seq[ControlPrune] = # {.raises: [Defect].} TODO chronicles exception on windows
   var prunes: seq[ControlPrune]
   for graft in grafts:
-    let topic = graft.topicID
+    let topic = graft.topicId
     trace "peer grafted topic", peer, topic
 
     # It is an error to GRAFT on a explicit peer
@@ -188,7 +191,7 @@ proc getPeers(prune: ControlPrune, peer: PubSubPeer): seq[(PeerId, Option[PeerRe
           trace "peer sent invalid SPR", peer, error=signedRecord.error
           none(PeerRecord)
         else:
-          if record.peerID != signedRecord.get().data.peerId:
+          if record.peerId != signedRecord.get().data.peerId:
             trace "peer sent envelope with wrong public key", peer
             none(PeerRecord)
           else:
@@ -201,7 +204,7 @@ proc getPeers(prune: ControlPrune, peer: PubSubPeer): seq[(PeerId, Option[PeerRe
 
 proc handlePrune*(g: GossipSub, peer: PubSubPeer, prunes: seq[ControlPrune]) {.raises: [Defect].} =
   for prune in prunes:
-    let topic = prune.topicID
+    let topic = prune.topicId
 
     trace "peer pruned topic", peer, topic
 
@@ -246,21 +249,21 @@ proc handleIHave*(g: GossipSub,
     let deIhaves = ihaves.deduplicate()
     for ihave in deIhaves:
       trace "peer sent ihave",
-        peer, topic = ihave.topicID, msgs = ihave.messageIDs
-      if ihave.topicID in g.mesh:
+        peer, topic = ihave.topicId, msgs = ihave.messageIds
+      if ihave.topicId in g.mesh:
         # also avoid duplicates here!
-        let deIhavesMsgs = ihave.messageIDs.deduplicate()
+        let deIhavesMsgs = ihave.messageIds.deduplicate()
         for msgId in deIhavesMsgs:
           if not g.hasSeen(msgId):
             if peer.iHaveBudget > 0:
-              res.messageIDs.add(msgId)
+              res.messageIds.add(msgId)
               dec peer.iHaveBudget
               trace "requested message via ihave", messageID=msgId
             else:
               break
     # shuffling res.messageIDs before sending it out to increase the likelihood
     # of getting an answer if the peer truncates the list due to internal size restrictions.
-    g.rng.shuffle(res.messageIDs)
+    g.rng.shuffle(res.messageIds)
     return res
 
 proc handleIWant*(g: GossipSub,
@@ -274,7 +277,7 @@ proc handleIWant*(g: GossipSub,
   else:
     let deIwants = iwants.deduplicate()
     for iwant in deIwants:
-      let deIwantsMsgs = iwant.messageIDs.deduplicate()
+      let deIwantsMsgs = iwant.messageIds.deduplicate()
       for mid in deIwantsMsgs:
         trace "peer sent iwant", peer, messageID = mid
         let msg = g.mcache.get(mid)
@@ -323,7 +326,7 @@ proc rebalanceMesh*(g: GossipSub, topic: string, metrics: ptr MeshMetrics = nil)
       candidates: seq[PubSubPeer]
       currentMesh = addr defaultMesh
     g.mesh.withValue(topic, v): currentMesh = v
-    g.gossipSub.withValue(topic, peerList):
+    g.gossipsub.withValue(topic, peerList):
       for it in peerList[]:
         if
             it.connected and
@@ -361,7 +364,7 @@ proc rebalanceMesh*(g: GossipSub, topic: string, metrics: ptr MeshMetrics = nil)
       candidates: seq[PubSubPeer]
       currentMesh = addr defaultMesh
     g.mesh.withValue(topic, v): currentMesh = v
-    g.gossipSub.withValue(topic, peerList):
+    g.gossipsub.withValue(topic, peerList):
       for it in peerList[]:
         if
             it.connected and
@@ -466,7 +469,7 @@ proc rebalanceMesh*(g: GossipSub, topic: string, metrics: ptr MeshMetrics = nil)
         avail: seq[PubSubPeer]
         currentMesh = addr defaultMesh
       g.mesh.withValue(topic, v): currentMesh = v
-      g.gossipSub.withValue(topic, peerList):
+      g.gossipsub.withValue(topic, peerList):
         for it in peerList[]:
           if
               # avoid negative score peers
@@ -611,7 +614,7 @@ proc getGossipPeers*(g: GossipSub): Table[PubSubPeer, ControlMessage] {.raises:
       allPeers.setLen(target)
 
     for peer in allPeers:
-      control.mGetOrPut(peer, ControlMessage()).ihave.add(ihave)
+      control.mgetOrPut(peer, ControlMessage()).ihave.add(ihave)
 
   libp2p_gossipsub_cache_window_size.set(cacheWindowSize.int64)
 
@@ -667,8 +670,8 @@ proc onHeartbeat(g: GossipSub) {.raises: [Defect].} =
     for peer, control in peers:
       # only ihave from here
       for ihave in control.ihave:
-        if g.knownTopics.contains(ihave.topicID):
-          libp2p_pubsub_broadcast_ihave.inc(labelValues = [ihave.topicID])
+        if g.knownTopics.contains(ihave.topicId):
+          libp2p_pubsub_broadcast_ihave.inc(labelValues = [ihave.topicId])
         else:
           libp2p_pubsub_broadcast_ihave.inc(labelValues = ["generic"])
       g.send(peer, RPCMsg(control: some(control)))

libp2p/protocols/pubsub/gossipsub/scoring.nim

@@ -1,13 +1,16 @@
-## Nim-LibP2P
-## Copyright (c) 2021 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 import std/[tables, sets, options]
 import chronos, chronicles, metrics
@@ -101,7 +104,6 @@ proc disconnectPeer(g: GossipSub, peer: PubSubPeer) {.async.} =
   except CatchableError as exc: # Never cancelled
     trace "Failed to close connection", peer, error = exc.name, msg = exc.msg
 
-{.push raises: [Defect].}
 
 proc updateScores*(g: GossipSub) = # avoid async
   ## https://github.com/libp2p/specs/blob/master/pubsub/gossipsub/gossipsub-v1.1.md#the-score-function

libp2p/protocols/pubsub/gossipsub/types.nim

@@ -1,13 +1,16 @@
-## Nim-LibP2P
-## Copyright (c) 2021 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 import chronos
 import std/[tables, sets]
@@ -141,7 +144,7 @@ type
     enablePX*: bool
 
   BackoffTable* = Table[string, Table[PeerId, Moment]]
-  ValidationSeenTable* = Table[MessageID, HashSet[PubSubPeer]]
+  ValidationSeenTable* = Table[MessageId, HashSet[PubSubPeer]]
 
   RoutingRecordsPair* = tuple[id: PeerId, record: Option[PeerRecord]]
   RoutingRecordsHandler* =

libp2p/protocols/pubsub/mcache.nim

@@ -1,13 +1,16 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 import std/[sets, tables, options]
 import rpc/[messages]
@@ -16,36 +19,36 @@ export sets, tables, messages, options
 
 type
   CacheEntry* = object
-    mid*: MessageID
-    topicIDs*: seq[string]
+    mid*: MessageId
+    topicIds*: seq[string]
 
   MCache* = object of RootObj
-    msgs*: Table[MessageID, Message]
+    msgs*: Table[MessageId, Message]
     history*: seq[seq[CacheEntry]]
     windowSize*: Natural
 
-func get*(c: MCache, mid: MessageID): Option[Message] =
+func get*(c: MCache, mid: MessageId): Option[Message] =
   if mid in c.msgs:
     try: some(c.msgs[mid])
     except KeyError: raiseAssert "checked"
   else:
     none(Message)
 
-func contains*(c: MCache, mid: MessageID): bool =
+func contains*(c: MCache, mid: MessageId): bool =
   mid in c.msgs
 
-func put*(c: var MCache, msgId: MessageID, msg: Message) =
+func put*(c: var MCache, msgId: MessageId, msg: Message) =
   if not c.msgs.hasKeyOrPut(msgId, msg):
     # Only add cache entry if the message was not already in the cache
-    c.history[0].add(CacheEntry(mid: msgId, topicIDs: msg.topicIDs))
+    c.history[0].add(CacheEntry(mid: msgId, topicIds: msg.topicIds))
 
-func window*(c: MCache, topic: string): HashSet[MessageID] =
+func window*(c: MCache, topic: string): HashSet[MessageId] =
   let
     len = min(c.windowSize, c.history.len)
 
   for i in 0..<len:
     for entry in c.history[i]:
-      for t in entry.topicIDs:
+      for t in entry.topicIds:
         if t == topic:
           result.incl(entry.mid)
           break

libp2p/protocols/pubsub/peertable.nim

@@ -1,13 +1,16 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 import std/[tables, sets]
 import ./pubsubpeer, ../../peerid

libp2p/protocols/pubsub/pubsub.nim

@@ -1,16 +1,25 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-{.push raises: [Defect].}
+## Base interface for pubsub protocols
+##
+## You can `subscribe<#subscribe%2CPubSub%2Cstring%2CTopicHandler>`_ to a topic,
+## `publish<#publish.e%2CPubSub%2Cstring%2Cseq%5Bbyte%5D>`_ something on it,
+## and eventually `unsubscribe<#unsubscribe%2CPubSub%2Cstring%2CTopicHandler>`_ from it.
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 import std/[tables, sequtils, sets, strutils]
-import chronos, chronicles, metrics, bearssl
+import chronos, chronicles, metrics
 import ./errors as pubsub_errors,
        ./pubsubpeer,
        ./rpc/[message, messages, protobuf],
@@ -75,37 +84,40 @@ declarePublicCounter(libp2p_pubsub_received_prune, "pubsub broadcast prune", lab
 type
   InitializationError* = object of LPError
 
-  TopicHandler* = proc(topic: string,
+  TopicHandler* {.public.} = proc(topic: string,
                        data: seq[byte]): Future[void] {.gcsafe, raises: [Defect].}
 
-  ValidatorHandler* = proc(topic: string,
+  ValidatorHandler* {.public.} = proc(topic: string,
                            message: Message): Future[ValidationResult] {.gcsafe, raises: [Defect].}
 
   TopicPair* = tuple[topic: string, handler: TopicHandler]
 
-  MsgIdProvider* =
-    proc(m: Message): Result[MessageID, ValidationResult] {.noSideEffect, raises: [Defect], gcsafe.}
+  MsgIdProvider* {.public.} =
+    proc(m: Message): Result[MessageId, ValidationResult] {.noSideEffect, raises: [Defect], gcsafe.}
 
-  SubscriptionValidator* =
+  SubscriptionValidator* {.public.} =
     proc(topic: string): bool {.raises: [Defect], gcsafe.}
+    ## Every time a peer send us a subscription (even to an unknown topic),
+    ## we have to store it, which may be an attack vector.
+    ## This callback can be used to reject topic we're not interested in
 
-  PubSub* = ref object of LPProtocol
+  PubSub* {.public.} = ref object of LPProtocol
     switch*: Switch                    # the switch used to dial/connect to peers
     peerInfo*: PeerInfo                # this peer's info
     topics*: Table[string, seq[TopicHandler]]      # the topics that _we_ are interested in
-    peers*: Table[PeerId, PubSubPeer]  ##\
-      ## Peers that we are interested to gossip with (but not necessarily
-      ## yet connected to)
-    triggerSelf*: bool                 # trigger own local handler on publish
-    verifySignature*: bool             # enable signature verification
-    sign*: bool                        # enable message signing
+    peers*: Table[PeerId, PubSubPeer]  #\
+      # Peers that we are interested to gossip with (but not necessarily
+      # yet connected to)
+    triggerSelf*: bool                 ## trigger own local handler on publish
+    verifySignature*: bool             ## enable signature verification
+    sign*: bool                        ## enable message signing
     validators*: Table[string, HashSet[ValidatorHandler]]
     observers: ref seq[PubSubObserver] # ref as in smart_ptr
-    msgIdProvider*: MsgIdProvider      # Turn message into message id (not nil)
+    msgIdProvider*: MsgIdProvider      ## Turn message into message id (not nil)
     msgSeqno*: uint64
-    anonymize*: bool                   # if we omit fromPeer and seqno from RPC messages we send
+    anonymize*: bool                   ## if we omit fromPeer and seqno from RPC messages we send
     subscriptionValidator*: SubscriptionValidator # callback used to validate subscriptions
-    topicsHigh*: int                  # the maximum number of topics a peer is allowed to subscribe to
+    topicsHigh*: int                  ## the maximum number of topics a peer is allowed to subscribe to
     maxMessageSize*: int          ##\ 
       ## the maximum raw message size we'll globally allow
       ## for finer tuning, check message size on topic validator
@@ -114,7 +126,7 @@ type
       ## lead to issues, from descoring to connection drops
       ##
       ## defaults to 1mB
-    rng*: ref BrHmacDrbgContext
+    rng*: ref HmacDrbgContext
 
     knownTopics*: HashSet[string]
 
@@ -154,7 +166,7 @@ proc broadcast*(
         libp2p_pubsub_broadcast_unsubscriptions.inc(npeers, labelValues = ["generic"])
 
   for smsg in msg.messages:
-    for topic in smsg.topicIDs:
+    for topic in smsg.topicIds:
       if p.knownTopics.contains(topic):
         libp2p_pubsub_broadcast_messages.inc(npeers, labelValues = [topic])
       else:
@@ -165,18 +177,18 @@ proc broadcast*(
 
     let control = msg.control.get()
     for ihave in control.ihave:
-      if p.knownTopics.contains(ihave.topicID):
-        libp2p_pubsub_broadcast_ihave.inc(npeers, labelValues = [ihave.topicID])
+      if p.knownTopics.contains(ihave.topicId):
+        libp2p_pubsub_broadcast_ihave.inc(npeers, labelValues = [ihave.topicId])
       else:
         libp2p_pubsub_broadcast_ihave.inc(npeers, labelValues = ["generic"])
     for graft in control.graft:
-      if p.knownTopics.contains(graft.topicID):
-        libp2p_pubsub_broadcast_graft.inc(npeers, labelValues = [graft.topicID])
+      if p.knownTopics.contains(graft.topicId):
+        libp2p_pubsub_broadcast_graft.inc(npeers, labelValues = [graft.topicId])
       else:
         libp2p_pubsub_broadcast_graft.inc(npeers, labelValues = ["generic"])
     for prune in control.prune:
-      if p.knownTopics.contains(prune.topicID):
-        libp2p_pubsub_broadcast_prune.inc(npeers, labelValues = [prune.topicID])
+      if p.knownTopics.contains(prune.topicId):
+        libp2p_pubsub_broadcast_prune.inc(npeers, labelValues = [prune.topicId])
       else:
         libp2p_pubsub_broadcast_prune.inc(npeers, labelValues = ["generic"])
 
@@ -227,8 +239,8 @@ proc updateMetrics*(p: PubSub, rpcMsg: RPCMsg) =
 
   for i in 0..<rpcMsg.messages.len():
     template smsg: untyped = rpcMsg.messages[i]
-    for j in 0..<smsg.topicIDs.len():
-      template topic: untyped = smsg.topicIDs[j]
+    for j in 0..<smsg.topicIds.len():
+      template topic: untyped = smsg.topicIds[j]
       if p.knownTopics.contains(topic):
         libp2p_pubsub_received_messages.inc(labelValues = [topic])
       else:
@@ -238,18 +250,18 @@ proc updateMetrics*(p: PubSub, rpcMsg: RPCMsg) =
     libp2p_pubsub_received_iwant.inc(rpcMsg.control.get().iwant.len.int64)
     template control: untyped = rpcMsg.control.unsafeGet()
     for ihave in control.ihave:
-      if p.knownTopics.contains(ihave.topicID):
-        libp2p_pubsub_received_ihave.inc(labelValues = [ihave.topicID])
+      if p.knownTopics.contains(ihave.topicId):
+        libp2p_pubsub_received_ihave.inc(labelValues = [ihave.topicId])
       else:
         libp2p_pubsub_received_ihave.inc(labelValues = ["generic"])
     for graft in control.graft:
-      if p.knownTopics.contains(graft.topicID):
-        libp2p_pubsub_received_graft.inc(labelValues = [graft.topicID])
+      if p.knownTopics.contains(graft.topicId):
+        libp2p_pubsub_received_graft.inc(labelValues = [graft.topicId])
       else:
         libp2p_pubsub_received_graft.inc(labelValues = ["generic"])
     for prune in control.prune:
-      if p.knownTopics.contains(prune.topicID):
-        libp2p_pubsub_received_prune.inc(labelValues = [prune.topicID])
+      if p.knownTopics.contains(prune.topicId):
+        libp2p_pubsub_received_prune.inc(labelValues = [prune.topicId])
       else:
         libp2p_pubsub_received_prune.inc(labelValues = ["generic"])
 
@@ -261,7 +273,7 @@ method rpcHandler*(p: PubSub,
 
 method onNewPeer(p: PubSub, peer: PubSubPeer) {.base.} = discard
 
-method onPubSubPeerEvent*(p: PubSub, peer: PubsubPeer, event: PubsubPeerEvent) {.base, gcsafe.} =
+method onPubSubPeerEvent*(p: PubSub, peer: PubSubPeer, event: PubSubPeerEvent) {.base, gcsafe.} =
   # Peer event is raised for the send connection in particular
   case event.kind
   of PubSubPeerEventKind.Connected:
@@ -277,18 +289,18 @@ proc getOrCreatePeer*(
   p.peers.withValue(peerId, peer):
     return peer[]
 
-  proc getConn(): Future[Connection] =
-    p.switch.dial(peerId, protos)
+  proc getConn(): Future[Connection] {.async.} =
+    return await p.switch.dial(peerId, protos)
 
   proc dropConn(peer: PubSubPeer) =
-    proc dropConnAsync(peer: PubsubPeer) {.async.} =
+    proc dropConnAsync(peer: PubSubPeer) {.async.} =
       try:
         await p.switch.disconnect(peer.peerId)
       except CatchableError as exc: # never cancelled
         trace "Failed to close connection", peer, error = exc.name, msg = exc.msg
     asyncSpawn dropConnAsync(peer)
 
-  proc onEvent(peer: PubsubPeer, event: PubsubPeerEvent) {.gcsafe.} =
+  proc onEvent(peer: PubSubPeer, event: PubSubPeerEvent) {.gcsafe.} =
     p.onPubSubPeerEvent(peer, event)
 
   # create new pubsub peer
@@ -303,7 +315,7 @@ proc getOrCreatePeer*(
   # metrics
   libp2p_pubsub_peers.set(p.peers.len.int64)
 
-  pubsubPeer.connect()
+  pubSubPeer.connect()
 
   return pubSubPeer
 
@@ -411,7 +423,7 @@ method onTopicSubscription*(p: PubSub, topic: string, subscribed: bool) {.base.}
 
 proc unsubscribe*(p: PubSub,
                   topic: string,
-                  handler: TopicHandler) =
+                  handler: TopicHandler) {.public.} =
   ## unsubscribe from a ``topic`` string
   ##
   p.topics.withValue(topic, handlers):
@@ -424,12 +436,13 @@ proc unsubscribe*(p: PubSub,
 
     p.updateTopicMetrics(topic)
 
-proc unsubscribe*(p: PubSub, topics: openArray[TopicPair]) =
+proc unsubscribe*(p: PubSub, topics: openArray[TopicPair]) {.public.} =
   ## unsubscribe from a list of ``topic`` handlers
   for t in topics:
     p.unsubscribe(t.topic, t.handler)
 
-proc unsubscribeAll*(p: PubSub, topic: string) =
+proc unsubscribeAll*(p: PubSub, topic: string) {.public.} =
+  ## unsubscribe every `handler` from `topic`
   if topic notin p.topics:
     debug "unsubscribeAll called for an unknown topic", topic
   else:
@@ -441,15 +454,14 @@ proc unsubscribeAll*(p: PubSub, topic: string) =
 
 proc subscribe*(p: PubSub,
                 topic: string,
-                handler: TopicHandler) =
+                handler: TopicHandler) {.public.} =
   ## subscribe to a topic
   ##
   ## ``topic``   - a string topic to subscribe to
   ##
-  ## ``handler`` - is a user provided proc
-  ##               that will be triggered
-  ##               on every received message
-  ##
+  ## ``handler`` - user provided proc that
+  ##               will be triggered on every
+  ##               received message
 
   # Check that this is an allowed topic
   if p.subscriptionValidator != nil and p.subscriptionValidator(topic) == false:
@@ -470,8 +482,9 @@ proc subscribe*(p: PubSub,
 
 method publish*(p: PubSub,
                 topic: string,
-                data: seq[byte]): Future[int] {.base, async.} =
+                data: seq[byte]): Future[int] {.base, async, public.} =
   ## publish to a ``topic``
+  ##
   ## The return value is the number of neighbours that we attempted to send the
   ## message to, excluding self. Note that this is an optimistic number of
   ## attempts - the number of peers that actually receive the message might
@@ -488,24 +501,19 @@ method initPubSub*(p: PubSub)
   if p.msgIdProvider == nil:
     p.msgIdProvider = defaultMsgIdProvider
 
-method start*(p: PubSub) {.async, base.} =
-  ## start pubsub
-  discard
-
-method stop*(p: PubSub) {.async, base.} =
-  ## stopt pubsub
-  discard
-
 method addValidator*(p: PubSub,
                      topic: varargs[string],
-                     hook: ValidatorHandler) {.base.} =
+                     hook: ValidatorHandler) {.base, public.} =
+  ## Add a validator to a `topic`. Each new message received in this
+  ## will be sent to `hook`. `hook` can return either `Accept`,
+  ## `Ignore` or `Reject` (which can descore the peer)
   for t in topic:
     trace "adding validator for topic", topicId = t
     p.validators.mgetOrPut(t, HashSet[ValidatorHandler]()).incl(hook)
 
 method removeValidator*(p: PubSub,
                         topic: varargs[string],
-                        hook: ValidatorHandler) {.base.} =
+                        hook: ValidatorHandler) {.base, public.} =
   for t in topic:
     p.validators.withValue(t, validators):
       validators[].excl(hook)
@@ -515,11 +523,11 @@ method removeValidator*(p: PubSub,
 method validate*(p: PubSub, message: Message): Future[ValidationResult] {.async, base.} =
   var pending: seq[Future[ValidationResult]]
   trace "about to validate message"
-  for topic in message.topicIDs:
-    trace "looking for validators on topic", topicID = topic,
+  for topic in message.topicIds:
+    trace "looking for validators on topic", topicId = topic,
                                              registered = toSeq(p.validators.keys)
     if topic in p.validators:
-      trace "running validators for topic", topicID = topic
+      trace "running validators for topic", topicId = topic
       for validator in p.validators[topic]:
         pending.add(validator(topic, message))
 
@@ -553,9 +561,9 @@ proc init*[PubParams: object | bool](
   msgIdProvider: MsgIdProvider = defaultMsgIdProvider,
   subscriptionValidator: SubscriptionValidator = nil,
   maxMessageSize: int = 1024 * 1024,
-  rng: ref BrHmacDrbgContext = newRng(),
+  rng: ref HmacDrbgContext = newRng(),
   parameters: PubParams = false): P
-  {.raises: [Defect, InitializationError].} =
+  {.raises: [Defect, InitializationError], public.} =
   let pubsub =
     when PubParams is bool:
       P(switch: switch,
@@ -598,9 +606,9 @@ proc init*[PubParams: object | bool](
 
   return pubsub
 
-proc addObserver*(p: PubSub; observer: PubSubObserver) = p.observers[] &= observer
+proc addObserver*(p: PubSub; observer: PubSubObserver) {.public.} = p.observers[] &= observer
 
-proc removeObserver*(p: PubSub; observer: PubSubObserver) =
+proc removeObserver*(p: PubSub; observer: PubSubObserver) {.public.} =
   let idx = p.observers[].find(observer)
   if idx != -1:
     p.observers[].del(idx)

libp2p/protocols/pubsub/pubsubpeer.nim

@@ -1,13 +1,16 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 import std/[sequtils, strutils, tables, hashes]
 import chronos, chronicles, nimcrypto/sha2, metrics
@@ -39,12 +42,12 @@ type
     Connected
     Disconnected
 
-  PubsubPeerEvent* = object
+  PubSubPeerEvent* = object
     kind*: PubSubPeerEventKind
 
   GetConn* = proc(): Future[Connection] {.gcsafe, raises: [Defect].}
-  DropConn* = proc(peer: PubsubPeer) {.gcsafe, raises: [Defect].} # have to pass peer as it's unknown during init
-  OnEvent* = proc(peer: PubSubPeer, event: PubsubPeerEvent) {.gcsafe, raises: [Defect].}
+  DropConn* = proc(peer: PubSubPeer) {.gcsafe, raises: [Defect].} # have to pass peer as it's unknown during init
+  OnEvent* = proc(peer: PubSubPeer, event: PubSubPeerEvent) {.gcsafe, raises: [Defect].}
 
   PubSubPeer* = ref object of RootObj
     getConn*: GetConn                   # callback to establish a new send connection
@@ -175,7 +178,7 @@ proc connectOnce(p: PubSubPeer): Future[void] {.async.} =
     p.address = some(p.sendConn.observedAddr)
 
     if p.onEvent != nil:
-      p.onEvent(p, PubsubPeerEvent(kind: PubSubPeerEventKind.Connected))
+      p.onEvent(p, PubSubPeerEvent(kind: PubSubPeerEventKind.Connected))
 
     await handle(p, newConn)
   finally:
@@ -186,7 +189,7 @@ proc connectOnce(p: PubSubPeer): Future[void] {.async.} =
 
     try:
       if p.onEvent != nil:
-        p.onEvent(p, PubsubPeerEvent(kind: PubSubPeerEventKind.Disconnected))
+        p.onEvent(p, PubSubPeerEvent(kind: PubSubPeerEventKind.Disconnected))
     except CancelledError as exc:
       raise exc
     except CatchableError as exc:

libp2p/protocols/pubsub/rpc/message.nim

@@ -1,13 +1,16 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 import hashes
 import chronicles, metrics, stew/[byteutils, endians2]
@@ -29,7 +32,7 @@ const PubSubPrefix = toBytes("libp2p-pubsub:")
 declareCounter(libp2p_pubsub_sig_verify_success, "pubsub successfully validated messages")
 declareCounter(libp2p_pubsub_sig_verify_failure, "pubsub failed validated messages")
 
-func defaultMsgIdProvider*(m: Message): Result[MessageID, ValidationResult] =
+func defaultMsgIdProvider*(m: Message): Result[MessageId, ValidationResult] =
   if m.seqno.len > 0 and m.fromPeer.data.len > 0:
     let mid = byteutils.toHex(m.seqno) & $m.fromPeer
     ok mid.toBytes()
@@ -62,7 +65,8 @@ proc init*(
     data: seq[byte],
     topic: string,
     seqno: Option[uint64],
-    sign: bool = true): Message
+    sign: bool = true,
+    stem = none(uint32)): Message
     {.gcsafe, raises: [Defect, LPError].} =
   var msg = Message(data: data, topicIDs: @[topic])
 
@@ -80,6 +84,8 @@ proc init*(
   elif sign:
     raise (ref LPError)(msg: "Cannot sign message without peer info")
 
+  msg.stem = stem
+
   msg
 
 proc init*(

libp2p/protocols/pubsub/rpc/messages.nim

@@ -1,13 +1,16 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 import options, sequtils
 import "../../.."/[
@@ -27,15 +30,16 @@ type
       subscribe*: bool
       topic*: string
 
-    MessageID* = seq[byte]
+    MessageId* = seq[byte]
 
     Message* = object
       fromPeer*: PeerId
       data*: seq[byte]
       seqno*: seq[byte]
-      topicIDs*: seq[string]
+      topicIds*: seq[string]
       signature*: seq[byte]
       key*: seq[byte]
+      stem*: Option[uint32]
 
     ControlMessage* = object
       ihave*: seq[ControlIHave]
@@ -44,17 +48,17 @@ type
       prune*: seq[ControlPrune]
 
     ControlIHave* = object
-      topicID*: string
-      messageIDs*: seq[MessageID]
+      topicId*: string
+      messageIds*: seq[MessageId]
 
     ControlIWant* = object
-      messageIDs*: seq[MessageID]
+      messageIds*: seq[MessageId]
 
     ControlGraft* = object
-      topicID*: string
+      topicId*: string
 
     ControlPrune* = object
-      topicID*: string
+      topicId*: string
       peers*: seq[PeerInfoMsg]
       backoff*: uint64
 
@@ -70,23 +74,23 @@ func withSubs*(
 
 func shortLog*(s: ControlIHave): auto =
   (
-    topicID: s.topicID.shortLog,
-    messageIDs: mapIt(s.messageIDs, it.shortLog)
+    topicId: s.topicId.shortLog,
+    messageIds: mapIt(s.messageIds, it.shortLog)
   )
 
 func shortLog*(s: ControlIWant): auto =
   (
-    messageIDs: mapIt(s.messageIDs, it.shortLog)
+    messageIds: mapIt(s.messageIds, it.shortLog)
   )
 
 func shortLog*(s: ControlGraft): auto =
   (
-    topicID: s.topicID.shortLog
+    topicId: s.topicId.shortLog
   )
 
 func shortLog*(s: ControlPrune): auto =
   (
-    topicID: s.topicID.shortLog
+    topicId: s.topicId.shortLog
   )
 
 func shortLog*(c: ControlMessage): auto =
@@ -97,14 +101,25 @@ func shortLog*(c: ControlMessage): auto =
     prune: mapIt(c.prune, it.shortLog)
   )
 
+func shortLog*(s: Option[uint32]): auto =
+  if s.isSome:
+    (
+      stem: $s.get
+    )
+  else:
+    (
+      stem: "(not set)"
+    )
+
 func shortLog*(msg: Message): auto =
   (
     fromPeer: msg.fromPeer.shortLog,
     data: msg.data.shortLog,
     seqno: msg.seqno.shortLog,
-    topicIDs: $msg.topicIDs,
+    topicIds: $msg.topicIds,
     signature: msg.signature.shortLog,
-    key: msg.key.shortLog
+    key: msg.key.shortLog,
+    stem: msg.stem.shortLog,
   )
 
 func shortLog*(m: RPCMsg): auto =

libp2p/protocols/pubsub/rpc/protobuf.nim

@@ -1,13 +1,16 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 import options
 import stew/assign2
@@ -17,7 +20,6 @@ import messages,
        ../../../utility,
        ../../../protobuf/minprotobuf
 
-{.push raises: [Defect].}
 
 logScope:
   topics = "pubsubprotobuf"
@@ -30,7 +32,7 @@ when defined(libp2p_protobuf_metrics):
 
 proc write*(pb: var ProtoBuffer, field: int, graft: ControlGraft) =
   var ipb = initProtoBuffer()
-  ipb.write(1, graft.topicID)
+  ipb.write(1, graft.topicId)
   ipb.finish()
   pb.write(field, ipb)
 
@@ -46,7 +48,7 @@ proc write*(pb: var ProtoBuffer, field: int, infoMsg: PeerInfoMsg) =
 
 proc write*(pb: var ProtoBuffer, field: int, prune: ControlPrune) =
   var ipb = initProtoBuffer()
-  ipb.write(1, prune.topicID)
+  ipb.write(1, prune.topicId)
   for peer in prune.peers:
     ipb.write(2, peer)
   ipb.write(3, prune.backoff)
@@ -58,8 +60,8 @@ proc write*(pb: var ProtoBuffer, field: int, prune: ControlPrune) =
 
 proc write*(pb: var ProtoBuffer, field: int, ihave: ControlIHave) =
   var ipb = initProtoBuffer()
-  ipb.write(1, ihave.topicID)
-  for mid in ihave.messageIDs:
+  ipb.write(1, ihave.topicId)
+  for mid in ihave.messageIds:
     ipb.write(2, mid)
   ipb.finish()
   pb.write(field, ipb)
@@ -69,7 +71,7 @@ proc write*(pb: var ProtoBuffer, field: int, ihave: ControlIHave) =
 
 proc write*(pb: var ProtoBuffer, field: int, iwant: ControlIWant) =
   var ipb = initProtoBuffer()
-  for mid in iwant.messageIDs:
+  for mid in iwant.messageIds:
     ipb.write(1, mid)
   if len(ipb.buffer) > 0:
     ipb.finish()
@@ -109,12 +111,14 @@ proc encodeMessage*(msg: Message, anonymize: bool): seq[byte] =
   pb.write(2, msg.data)
   if len(msg.seqno) > 0 and not anonymize:
     pb.write(3, msg.seqno)
-  for topic in msg.topicIDs:
+  for topic in msg.topicIds:
     pb.write(4, topic)
   if len(msg.signature) > 0 and not anonymize:
     pb.write(5, msg.signature)
   if len(msg.key) > 0 and not anonymize:
     pb.write(6, msg.key)
+  if msg.stem.isSome:
+    pb.write(7, msg.stem.get)
   pb.finish()
 
   when defined(libp2p_protobuf_metrics):
@@ -182,8 +186,8 @@ proc decodeIHave*(pb: ProtoBuffer): ProtoResult[ControlIHave] {.
     trace "decodeIHave: read topicId", topic_id = control.topicId
   else:
     trace "decodeIHave: topicId is missing"
-  if ? pb.getRepeatedField(2, control.messageIDs):
-    trace "decodeIHave: read messageIDs", message_ids = control.messageIDs
+  if ? pb.getRepeatedField(2, control.messageIds):
+    trace "decodeIHave: read messageIDs", message_ids = control.messageIds
   else:
     trace "decodeIHave: no messageIDs"
   ok(control)
@@ -194,8 +198,8 @@ proc decodeIWant*(pb: ProtoBuffer): ProtoResult[ControlIWant] {.inline.} =
 
   trace "decodeIWant: decoding message"
   var control = ControlIWant()
-  if ? pb.getRepeatedField(1, control.messageIDs):
-    trace "decodeIWant: read messageIDs", message_ids = control.messageIDs
+  if ? pb.getRepeatedField(1, control.messageIds):
+    trace "decodeIWant: read messageIDs", message_ids = control.messageIds
   else:
     trace "decodeIWant: no messageIDs"
   ok(control)
@@ -281,8 +285,8 @@ proc decodeMessage*(pb: ProtoBuffer): ProtoResult[Message] {.inline.} =
     trace "decodeMessage: read seqno", seqno = msg.seqno
   else:
     trace "decodeMessage: seqno is missing"
-  if ? pb.getRepeatedField(4, msg.topicIDs):
-    trace "decodeMessage: read topics", topic_ids = msg.topicIDs
+  if ? pb.getRepeatedField(4, msg.topicIds):
+    trace "decodeMessage: read topics", topic_ids = msg.topicIds
   else:
     trace "decodeMessage: topics are missing"
   if ? pb.getField(5, msg.signature):
@@ -293,6 +297,12 @@ proc decodeMessage*(pb: ProtoBuffer): ProtoResult[Message] {.inline.} =
     trace "decodeMessage: read public key", key = msg.key.shortLog()
   else:
     trace "decodeMessage: public key is missing"
+  var stem: uint32
+  if ? pb.getField(7, stem):
+    msg.stem = some stem
+    trace "decodeMessage: read stem", stem = msg.stem.shortLog()
+  else:
+    trace "decodeMessage: stem is missing"
   ok(msg)
 
 proc decodeMessages*(pb: ProtoBuffer): ProtoResult[seq[Message]] {.inline.} =

libp2p/protocols/pubsub/timedcache.nim

@@ -1,13 +1,16 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 import std/[tables]
 

libp2p/protocols/relay.nim

@@ -1,488 +0,0 @@
-## Nim-LibP2P
-## Copyright (c) 2022 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
-
-{.push raises: [Defect].}
-
-import options
-import sequtils, strutils, tables
-import chronos, chronicles
-
-import ../peerinfo,
-       ../switch,
-       ../multiaddress,
-       ../stream/connection,
-       ../protocols/protocol,
-       ../transports/transport,
-       ../utility,
-       ../errors
-
-const
-  RelayCodec* = "/libp2p/circuit/relay/0.1.0"
-  MsgSize* = 4096
-  MaxCircuit* = 1024
-  MaxCircuitPerPeer* = 64
-
-logScope:
-  topics = "libp2p relay"
-
-type
-  RelayType* = enum
-    Hop = 1
-    Stop = 2
-    Status = 3
-    CanHop = 4
-  RelayStatus* = enum
-    Success = 100
-    HopSrcAddrTooLong = 220
-    HopDstAddrTooLong = 221
-    HopSrcMultiaddrInvalid = 250
-    HopDstMultiaddrInvalid = 251
-    HopNoConnToDst = 260
-    HopCantDialDst = 261
-    HopCantOpenDstStream = 262
-    HopCantSpeakRelay = 270
-    HopCantRelayToSelf = 280
-    StopSrcAddrTooLong = 320
-    StopDstAddrTooLong = 321
-    StopSrcMultiaddrInvalid = 350
-    StopDstMultiaddrInvalid = 351
-    StopRelayRefused = 390
-    MalformedMessage = 400
-
-  RelayError* = object of LPError
-
-  RelayPeer* = object
-    peerId*: PeerID
-    addrs*: seq[MultiAddress]
-
-  AddConn* = proc(conn: Connection): Future[void] {.gcsafe, raises: [Defect].}
-
-  RelayMessage* = object
-    msgType*: Option[RelayType]
-    srcPeer*: Option[RelayPeer]
-    dstPeer*: Option[RelayPeer]
-    status*: Option[RelayStatus]
-
-  Relay* = ref object of LPProtocol
-    switch*: Switch
-    peerId: PeerID
-    dialer: Dial
-    canHop: bool
-    streamCount: int
-    hopCount: CountTable[PeerId]
-
-    addConn: AddConn
-
-    maxCircuit*: int
-    maxCircuitPerPeer*: int
-    msgSize*: int
-
-proc encodeMsg*(msg: RelayMessage): ProtoBuffer =
-  result = initProtoBuffer()
-
-  if isSome(msg.msgType):
-    result.write(1, msg.msgType.get().ord.uint)
-  if isSome(msg.srcPeer):
-    var peer = initProtoBuffer()
-    peer.write(1, msg.srcPeer.get().peerId)
-    for ma in msg.srcPeer.get().addrs:
-      peer.write(2, ma.data.buffer)
-    peer.finish()
-    result.write(2, peer.buffer)
-  if isSome(msg.dstPeer):
-    var peer = initProtoBuffer()
-    peer.write(1, msg.dstPeer.get().peerId)
-    for ma in msg.dstPeer.get().addrs:
-      peer.write(2, ma.data.buffer)
-    peer.finish()
-    result.write(3, peer.buffer)
-  if isSome(msg.status):
-    result.write(4, msg.status.get().ord.uint)
-
-  result.finish()
-
-proc decodeMsg*(buf: seq[byte]): Option[RelayMessage] =
-  var
-    rMsg: RelayMessage
-    msgTypeOrd: uint32
-    src: RelayPeer
-    dst: RelayPeer
-    statusOrd: uint32
-    pbSrc: ProtoBuffer
-    pbDst: ProtoBuffer
-
-  let
-    pb = initProtoBuffer(buf)
-    r1 = pb.getField(1, msgTypeOrd)
-    r2 = pb.getField(2, pbSrc)
-    r3 = pb.getField(3, pbDst)
-    r4 = pb.getField(4, statusOrd)
-
-  if r1.isErr() or r2.isErr() or r3.isErr() or r4.isErr():
-    return none(RelayMessage)
-
-  if r2.get() and
-     (pbSrc.getField(1, src.peerId).isErr() or
-      pbSrc.getRepeatedField(2, src.addrs).isErr()):
-    return none(RelayMessage)
-
-  if r3.get() and
-     (pbDst.getField(1, dst.peerId).isErr() or
-      pbDst.getRepeatedField(2, dst.addrs).isErr()):
-    return none(RelayMessage)
-
-  if r1.get(): rMsg.msgType = some(RelayType(msgTypeOrd))
-  if r2.get(): rMsg.srcPeer = some(src)
-  if r3.get(): rMsg.dstPeer = some(dst)
-  if r4.get(): rMsg.status = some(RelayStatus(statusOrd))
-  some(rMsg)
-
-proc sendStatus*(conn: Connection, code: RelayStatus) {.async, gcsafe.} =
-  trace "send status", status = $code & "(" & $ord(code) & ")"
-  let
-    msg = RelayMessage(
-      msgType: some(RelayType.Status),
-      status: some(code))
-    pb = encodeMsg(msg)
-
-  await conn.writeLp(pb.buffer)
-
-proc handleHopStream(r: Relay, conn: Connection, msg: RelayMessage) {.async, gcsafe.} =
-  r.streamCount.inc()
-  defer:
-    r.streamCount.dec()
-
-  if r.streamCount > r.maxCircuit:
-    trace "refusing connection; too many active circuit"
-    await sendStatus(conn, RelayStatus.HopCantSpeakRelay)
-    return
-
-  proc checkMsg(): Result[RelayMessage, RelayStatus] =
-    if not r.canHop:
-      return err(RelayStatus.HopCantSpeakRelay)
-    if msg.srcPeer.isNone:
-      return err(RelayStatus.HopSrcMultiaddrInvalid)
-    let src = msg.srcPeer.get()
-    if src.peerId != conn.peerId:
-      return err(RelayStatus.HopSrcMultiaddrInvalid)
-    if msg.dstPeer.isNone:
-      return err(RelayStatus.HopDstMultiaddrInvalid)
-    let dst = msg.dstPeer.get()
-    if dst.peerId == r.switch.peerInfo.peerId:
-      return err(RelayStatus.HopCantRelayToSelf)
-    if not r.switch.isConnected(dst.peerId):
-      trace "relay not connected to dst", dst
-      return err(RelayStatus.HopNoConnToDst)
-    ok(msg)
-
-  let check = checkMsg()
-  if check.isErr:
-    await sendStatus(conn, check.error())
-    return
-  let
-    src = msg.srcPeer.get()
-    dst = msg.dstPeer.get()
-
-  # TODO: if r.acl # access control list
-  #       and not r.acl.AllowHop(src.peerId, dst.peerId)
-  #         sendStatus(conn, RelayStatus.HopCantSpeakRelay)
-
-  r.hopCount.inc(src.peerId)
-  r.hopCount.inc(dst.peerId)
-  defer:
-    r.hopCount.inc(src.peerId, -1)
-    r.hopCount.inc(dst.peerId, -1)
-
-  if r.hopCount[src.peerId] > r.maxCircuitPerPeer:
-    trace "refusing connection; too many connection from src", src, dst
-    await sendStatus(conn, RelayStatus.HopCantSpeakRelay)
-    return
-
-  if r.hopCount[dst.peerId] > r.maxCircuitPerPeer:
-    trace "refusing connection; too many connection to dst", src, dst
-    await sendStatus(conn, RelayStatus.HopCantSpeakRelay)
-    return
-
-  let connDst = try:
-    await r.switch.dial(dst.peerId, @[RelayCodec])
-  except CancelledError as exc:
-    raise exc
-  except CatchableError as exc:
-    trace "error opening relay stream", dst, exc=exc.msg
-    await sendStatus(conn, RelayStatus.HopCantDialDst)
-    return
-  defer:
-    await connDst.close()
-
-  let msgToSend = RelayMessage(
-    msgType: some(RelayType.Stop),
-    srcPeer: some(src),
-    dstPeer: some(dst),
-    status: none(RelayStatus))
-
-  let msgRcvFromDstOpt = try:
-    await connDst.writeLp(encodeMsg(msgToSend).buffer)
-    decodeMsg(await connDst.readLp(r.msgSize))
-  except CancelledError as exc:
-    raise exc
-  except CatchableError as exc:
-    trace "error writing stop handshake or reading stop response", exc=exc.msg
-    await sendStatus(conn, RelayStatus.HopCantOpenDstStream)
-    return
-
-  if msgRcvFromDstOpt.isNone:
-    trace "error reading stop response", msg = msgRcvFromDstOpt
-    await sendStatus(conn, RelayStatus.HopCantOpenDstStream)
-    return
-
-  let msgRcvFromDst = msgRcvFromDstOpt.get()
-  if msgRcvFromDst.msgType.isNone or msgRcvFromDst.msgType.get() != RelayType.Status:
-    trace "unexcepted relay stop response", msgType = msgRcvFromDst.msgType
-    await sendStatus(conn, RelayStatus.HopCantOpenDstStream)
-    return
-
-  if msgRcvFromDst.status.isNone or msgRcvFromDst.status.get() != RelayStatus.Success:
-    trace "relay stop failure", status=msgRcvFromDst.status
-    await sendStatus(conn, RelayStatus.HopCantOpenDstStream)
-    return
-
-  await sendStatus(conn, RelayStatus.Success)
-
-  trace "relaying connection", src, dst
-
-  proc bridge(conn: Connection, connDst: Connection) {.async.} =
-    const bufferSize = 4096
-    var
-      bufSrcToDst: array[bufferSize, byte]
-      bufDstToSrc: array[bufferSize, byte]
-      futSrc = conn.readOnce(addr bufSrcToDst[0], bufSrcToDst.high + 1)
-      futDst = connDst.readOnce(addr bufDstToSrc[0], bufDstToSrc.high + 1)
-      bytesSendFromSrcToDst = 0
-      bytesSendFromDstToSrc = 0
-      bufRead: int
-
-    while not conn.closed() and not connDst.closed():
-      try:
-        await futSrc or futDst
-        if futSrc.finished():
-          bufRead = await futSrc
-          bytesSendFromSrcToDst.inc(bufRead)
-          await connDst.write(@bufSrcToDst[0..<bufRead])
-          zeroMem(addr(bufSrcToDst), bufSrcToDst.high + 1)
-          futSrc = conn.readOnce(addr bufSrcToDst[0], bufSrcToDst.high + 1)
-        if futDst.finished():
-          bufRead = await futDst
-          bytesSendFromDstToSrc += bufRead
-          await conn.write(bufDstToSrc[0..<bufRead])
-          zeroMem(addr(bufDstToSrc), bufDstToSrc.high + 1)
-          futDst = connDst.readOnce(addr bufDstToSrc[0], bufDstToSrc.high + 1)
-      except CancelledError as exc:
-        raise exc
-      except CatchableError as exc:
-        if conn.closed() or conn.atEof():
-          trace "relay src closed connection", src
-        if connDst.closed() or connDst.atEof():
-          trace "relay dst closed connection", dst
-        trace "relay error", exc=exc.msg
-        break
-
-    trace "end relaying", bytesSendFromSrcToDst, bytesSendFromDstToSrc
-
-    await futSrc.cancelAndWait()
-    await futDst.cancelAndWait()
-  await bridge(conn, connDst)
-
-proc handleStopStream(r: Relay, conn: Connection, msg: RelayMessage) {.async, gcsafe.} =
-  if msg.srcPeer.isNone:
-    await sendStatus(conn, RelayStatus.StopSrcMultiaddrInvalid)
-    return
-  let src = msg.srcPeer.get()
-
-  if msg.dstPeer.isNone:
-    await sendStatus(conn, RelayStatus.StopDstMultiaddrInvalid)
-    return
-
-  let dst = msg.dstPeer.get()
-  if dst.peerId != r.switch.peerInfo.peerId:
-    await sendStatus(conn, RelayStatus.StopDstMultiaddrInvalid)
-    return
-
-  trace "get a relay connection", src, conn
-
-  if r.addConn == nil:
-    await sendStatus(conn, RelayStatus.StopRelayRefused)
-    await conn.close()
-    return
-  await sendStatus(conn, RelayStatus.Success)
-  # This sound redundant but the callback could, in theory, be set to nil during
-  # sendStatus(Success) so it's safer to double check
-  if r.addConn != nil: await r.addConn(conn)
-  else: await conn.close()
-
-proc handleCanHop(r: Relay, conn: Connection, msg: RelayMessage) {.async, gcsafe.} =
-  await sendStatus(conn,
-    if r.canHop:
-      RelayStatus.Success
-    else:
-      RelayStatus.HopCantSpeakRelay
-  )
-
-proc new*(T: typedesc[Relay], switch: Switch, canHop: bool): T =
-  let relay = T(switch: switch, canHop: canHop)
-  relay.init()
-  relay
-
-method init*(r: Relay) =
-  proc handleStream(conn: Connection, proto: string) {.async, gcsafe.} =
-    try:
-      let msgOpt = decodeMsg(await conn.readLp(r.msgSize))
-
-      if msgOpt.isNone:
-        await sendStatus(conn, RelayStatus.MalformedMessage)
-        return
-      else:
-        trace "relay handle stream", msg = msgOpt.get()
-      let msg = msgOpt.get()
-
-      case msg.msgType.get:
-        of RelayType.Hop: await r.handleHopStream(conn, msg)
-        of RelayType.Stop: await r.handleStopStream(conn, msg)
-        of RelayType.CanHop: await r.handleCanHop(conn, msg)
-        else:
-          trace "Unexpected relay handshake", msgType=msg.msgType
-          await sendStatus(conn, RelayStatus.MalformedMessage)
-    except CancelledError as exc:
-      raise exc
-    except CatchableError as exc:
-      trace "exception in relay handler", exc = exc.msg, conn
-    finally:
-      trace "exiting relay handler", conn
-      await conn.close()
-
-  r.handler = handleStream
-  r.codecs = @[RelayCodec]
-
-  r.maxCircuit = MaxCircuit
-  r.maxCircuitPerPeer = MaxCircuitPerPeer
-  r.msgSize = MsgSize
-
-proc dialPeer(
-    r: Relay,
-    conn: Connection,
-    dstPeerId: PeerId,
-    dstAddrs: seq[MultiAddress]): Future[Connection] {.async.} =
-  var
-    msg = RelayMessage(
-      msgType: some(RelayType.Hop),
-      srcPeer: some(RelayPeer(peerId: r.switch.peerInfo.peerId, addrs: r.switch.peerInfo.addrs)),
-      dstPeer: some(RelayPeer(peerId: dstPeerId, addrs: dstAddrs)),
-      status: none(RelayStatus))
-    pb = encodeMsg(msg)
-
-  trace "Dial peer", msgSend=msg
-
-  try:
-    await conn.writeLp(pb.buffer)
-  except CancelledError as exc:
-    raise exc
-  except CatchableError as exc:
-    trace "error writing hop request", exc=exc.msg
-    raise exc
-
-  let msgRcvFromRelayOpt = try:
-    decodeMsg(await conn.readLp(r.msgSize))
-  except CancelledError as exc:
-    raise exc
-  except CatchableError as exc:
-    trace "error reading stop response", exc=exc.msg
-    await sendStatus(conn, RelayStatus.HopCantOpenDstStream)
-    raise exc
-
-  if msgRcvFromRelayOpt.isNone:
-    trace "error reading stop response", msg = msgRcvFromRelayOpt
-    await sendStatus(conn, RelayStatus.HopCantOpenDstStream)
-    raise newException(RelayError, "Hop can't open destination stream")
-
-  let msgRcvFromRelay = msgRcvFromRelayOpt.get()
-  if msgRcvFromRelay.msgType.isNone or msgRcvFromRelay.msgType.get() != RelayType.Status:
-    trace "unexcepted relay stop response", msgType = msgRcvFromRelay.msgType
-    await sendStatus(conn, RelayStatus.HopCantOpenDstStream)
-    raise newException(RelayError, "Hop can't open destination stream")
-
-  if msgRcvFromRelay.status.isNone or msgRcvFromRelay.status.get() != RelayStatus.Success:
-    trace "relay stop failure", status=msgRcvFromRelay.status
-    await sendStatus(conn, RelayStatus.HopCantOpenDstStream)
-    raise newException(RelayError, "Hop can't open destination stream")
-  result = conn
-
-#
-# Relay Transport
-#
-
-type
-  RelayTransport* = ref object of Transport
-    relay*: Relay
-    queue: AsyncQueue[Connection]
-    relayRunning: bool
-
-method start*(self: RelayTransport, ma: seq[MultiAddress]) {.async.} =
-  if self.relayRunning:
-    trace "Relay transport already running"
-    return
-  await procCall Transport(self).start(ma)
-  self.relayRunning = true
-  self.relay.addConn = proc(conn: Connection) {.async, gcsafe, raises: [Defect].} =
-    await self.queue.addLast(conn)
-    await conn.join()
-  trace "Starting Relay transport"
-
-method stop*(self: RelayTransport) {.async, gcsafe.} =
-  self.running = false
-  self.relayRunning = false
-  self.relay.addConn = nil
-  while not self.queue.empty():
-    await self.queue.popFirstNoWait().close()
-
-method accept*(self: RelayTransport): Future[Connection] {.async, gcsafe.} =
-  result = await self.queue.popFirst()
-
-proc dial*(self: RelayTransport, ma: MultiAddress): Future[Connection] {.async, gcsafe.} =
-  let
-    sma = toSeq(ma.items())
-    relayAddrs = sma[0..sma.len-4].mapIt(it.tryGet()).foldl(a & b)
-  var
-    relayPeerId: PeerId
-    dstPeerId: PeerId
-  if not relayPeerId.init(($(sma[^3].get())).split('/')[2]):
-    raise newException(RelayError, "Relay doesn't exist")
-  if not dstPeerId.init(($(sma[^1].get())).split('/')[2]):
-    raise newException(RelayError, "Destination doesn't exist")
-  trace "Dial", relayPeerId, relayAddrs, dstPeerId
-
-  let conn = await self.relay.switch.dial(relayPeerId, @[ relayAddrs ], RelayCodec)
-  result = await self.relay.dialPeer(conn, dstPeerId, @[])
-
-method dial*(
-  self: RelayTransport,
-  hostname: string,
-  address: MultiAddress): Future[Connection] {.async, gcsafe.} =
-  result = await self.dial(address)
-
-method handles*(self: RelayTransport, ma: MultiAddress): bool {.gcsafe} =
-  if ma.protocols.isOk:
-    let sma = toSeq(ma.items())
-    if sma.len >= 3:
-      result = CircuitRelay.match(sma[^2].get()) and
-               P2PPattern.match(sma[^1].get())
-  trace "Handles return", ma, result
-
-proc new*(T: typedesc[RelayTransport], relay: Relay, upgrader: Upgrade): T =
-  result = T(relay: relay, upgrader: upgrader)
-  result.running = true
-  result.queue = newAsyncQueue[Connection](0)

libp2p/protocols/relay/client.nim

@@ -0,0 +1,294 @@
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import times, options
+
+import chronos, chronicles
+
+import ./relay,
+       ./messages,
+       ./rconn,
+       ./utils,
+       ../../peerinfo,
+       ../../switch,
+       ../../multiaddress,
+       ../../stream/connection
+
+
+logScope:
+  topics = "libp2p relay relay-client"
+
+const RelayClientMsgSize = 4096
+
+type
+  RelayClientError* = object of LPError
+  ReservationError* = object of RelayClientError
+  RelayV1DialError* = object of RelayClientError
+  RelayV2DialError* = object of RelayClientError
+  RelayClientAddConn* = proc(conn: Connection,
+                        duration: uint32,
+                        data: uint64): Future[void] {.gcsafe, raises: [Defect].}
+  RelayClient* = ref object of Relay
+    onNewConnection*: RelayClientAddConn
+    canHop: bool
+
+  Rsvp* = object
+    expire*: uint64           # required, Unix expiration time (UTC)
+    addrs*: seq[MultiAddress] # relay address for reserving peer
+    voucher*: Option[Voucher] # optional, reservation voucher
+    limitDuration*: uint32    # seconds
+    limitData*: uint64        # bytes
+
+proc sendStopError(conn: Connection, code: StatusV2) {.async.} =
+  trace "send stop status", status = $code & " (" & $ord(code) & ")"
+  let msg = StopMessage(msgType: StopMessageType.Status, status: some(code))
+  await conn.writeLp(encode(msg).buffer)
+
+proc handleRelayedConnect(cl: RelayClient, conn: Connection, msg: StopMessage) {.async.} =
+  if msg.peer.isNone():
+    await sendStopError(conn, MalformedMessage)
+    return
+  let
+    # TODO: check the go version to see in which way this could fail
+    # it's unclear in the spec
+    src = msg.peer.get()
+    limitDuration = msg.limit.duration
+    limitData = msg.limit.data
+    msg = StopMessage(
+      msgType: StopMessageType.Status,
+      status: some(Ok))
+    pb = encode(msg)
+
+  trace "incoming relay connection", src
+
+  if cl.onNewConnection == nil:
+    await sendStopError(conn, StatusV2.ConnectionFailed)
+    await conn.close()
+    return
+  await conn.writeLp(pb.buffer)
+  # This sound redundant but the callback could, in theory, be set to nil during
+  # conn.writeLp so it's safer to double check
+  if cl.onNewConnection != nil: await cl.onNewConnection(conn, limitDuration, limitData)
+  else: await conn.close()
+
+proc reserve*(cl: RelayClient,
+              peerId: PeerId,
+              addrs: seq[MultiAddress] = @[]): Future[Rsvp] {.async.} =
+  let conn = await cl.switch.dial(peerId, addrs, RelayV2HopCodec)
+  defer: await conn.close()
+  let
+    pb = encode(HopMessage(msgType: HopMessageType.Reserve))
+    msg = try:
+      await conn.writeLp(pb.buffer)
+      HopMessage.decode(await conn.readLp(RelayClientMsgSize)).get()
+    except CancelledError as exc:
+      raise exc
+    except CatchableError as exc:
+      trace "error writing or reading reservation message", exc=exc.msg
+      raise newException(ReservationError, exc.msg)
+
+  if msg.msgType != HopMessageType.Status:
+    raise newException(ReservationError, "Unexpected relay response type")
+  if msg.status.get(UnexpectedMessage) != Ok:
+    raise newException(ReservationError, "Reservation failed")
+  if msg.reservation.isNone():
+    raise newException(ReservationError, "Missing reservation information")
+
+  let reservation = msg.reservation.get()
+  if reservation.expire > int64.high().uint64 or
+     now().utc > reservation.expire.int64.fromUnix.utc:
+    raise newException(ReservationError, "Bad expiration date")
+  result.expire = reservation.expire
+  result.addrs = reservation.addrs
+
+  if reservation.svoucher.isSome():
+    let svoucher = SignedVoucher.decode(reservation.svoucher.get())
+    if svoucher.isErr() or svoucher.get().data.relayPeerId != peerId:
+      raise newException(ReservationError, "Invalid voucher")
+    result.voucher = some(svoucher.get().data)
+
+  result.limitDuration = msg.limit.duration
+  result.limitData = msg.limit.data
+
+proc dialPeerV1*(
+    cl: RelayClient,
+    conn: Connection,
+    dstPeerId: PeerId,
+    dstAddrs: seq[MultiAddress]): Future[Connection] {.async.} =
+  var
+    msg = RelayMessage(
+      msgType: some(RelayType.Hop),
+      srcPeer: some(RelayPeer(peerId: cl.switch.peerInfo.peerId, addrs: cl.switch.peerInfo.addrs)),
+      dstPeer: some(RelayPeer(peerId: dstPeerId, addrs: dstAddrs)))
+    pb = encode(msg)
+
+  trace "Dial peer", msgSend=msg
+
+  try:
+    await conn.writeLp(pb.buffer)
+  except CancelledError as exc:
+    raise exc
+  except CatchableError as exc:
+    trace "error writing hop request", exc=exc.msg
+    raise exc
+
+  let msgRcvFromRelayOpt = try:
+    RelayMessage.decode(await conn.readLp(RelayClientMsgSize))
+  except CancelledError as exc:
+    raise exc
+  except CatchableError as exc:
+    trace "error reading stop response", exc=exc.msg
+    await sendStatus(conn, StatusV1.HopCantOpenDstStream)
+    raise exc
+
+  try:
+    if msgRcvFromRelayOpt.isNone:
+      raise newException(RelayV1DialError, "Hop can't open destination stream")
+    let msgRcvFromRelay = msgRcvFromRelayOpt.get()
+    if msgRcvFromRelay.msgType.isNone or msgRcvFromRelay.msgType.get() != RelayType.Status:
+      raise newException(RelayV1DialError, "Hop can't open destination stream: wrong message type")
+    if msgRcvFromRelay.status.isNone or msgRcvFromRelay.status.get() != StatusV1.Success:
+      raise newException(RelayV1DialError, "Hop can't open destination stream: status failed")
+  except RelayV1DialError as exc:
+    await sendStatus(conn, StatusV1.HopCantOpenDstStream)
+    raise exc
+  result = conn
+
+proc dialPeerV2*(
+    cl: RelayClient,
+    conn: RelayConnection,
+    dstPeerId: PeerId,
+    dstAddrs: seq[MultiAddress]): Future[Connection] {.async.} =
+  let
+    p = Peer(peerId: dstPeerId, addrs: dstAddrs)
+    pb = encode(HopMessage(msgType: HopMessageType.Connect, peer: some(p)))
+
+  trace "Dial peer", p
+
+  let msgRcvFromRelay = try:
+    await conn.writeLp(pb.buffer)
+    HopMessage.decode(await conn.readLp(RelayClientMsgSize)).get()
+  except CancelledError as exc:
+    raise exc
+  except CatchableError as exc:
+    trace "error reading stop response", exc=exc.msg
+    raise newException(RelayV2DialError, exc.msg)
+
+  if msgRcvFromRelay.msgType != HopMessageType.Status:
+    raise newException(RelayV2DialError, "Unexpected stop response")
+  if msgRcvFromRelay.status.get(UnexpectedMessage) != Ok:
+    trace "Relay stop failed", msg = msgRcvFromRelay.status.get()
+    raise newException(RelayV2DialError, "Relay stop failure")
+  conn.limitDuration = msgRcvFromRelay.limit.duration
+  conn.limitData = msgRcvFromRelay.limit.data
+  return conn
+
+proc handleStopStreamV2(cl: RelayClient, conn: Connection) {.async, gcsafe.} =
+  let msgOpt = StopMessage.decode(await conn.readLp(RelayClientMsgSize))
+  if msgOpt.isNone():
+    await sendHopStatus(conn, MalformedMessage)
+    return
+  trace "client circuit relay v2 handle stream", msg = msgOpt.get()
+  let msg = msgOpt.get()
+
+  if msg.msgType == StopMessageType.Connect:
+    await cl.handleRelayedConnect(conn, msg)
+  else:
+    trace "Unexpected client / relayv2 handshake", msgType=msg.msgType
+    await sendStopError(conn, MalformedMessage)
+
+proc handleStop(cl: RelayClient, conn: Connection, msg: RelayMessage) {.async, gcsafe.} =
+  if msg.srcPeer.isNone:
+    await sendStatus(conn, StatusV1.StopSrcMultiaddrInvalid)
+    return
+  let src = msg.srcPeer.get()
+
+  if msg.dstPeer.isNone:
+    await sendStatus(conn, StatusV1.StopDstMultiaddrInvalid)
+    return
+
+  let dst = msg.dstPeer.get()
+  if dst.peerId != cl.switch.peerInfo.peerId:
+    await sendStatus(conn, StatusV1.StopDstMultiaddrInvalid)
+    return
+
+  trace "get a relay connection", src, conn
+
+  if cl.onNewConnection == nil:
+    await sendStatus(conn, StatusV1.StopRelayRefused)
+    await conn.close()
+    return
+  await sendStatus(conn, StatusV1.Success)
+  # This sound redundant but the callback could, in theory, be set to nil during
+  # sendStatus(Success) so it's safer to double check
+  if cl.onNewConnection != nil: await cl.onNewConnection(conn, 0, 0)
+  else: await conn.close()
+
+proc handleStreamV1(cl: RelayClient, conn: Connection) {.async, gcsafe.} =
+  let msgOpt = RelayMessage.decode(await conn.readLp(RelayClientMsgSize))
+  if msgOpt.isNone:
+    await sendStatus(conn, StatusV1.MalformedMessage)
+    return
+  trace "client circuit relay v1 handle stream", msg = msgOpt.get()
+  let msg = msgOpt.get()
+  case msg.msgType.get:
+    of RelayType.Hop:
+      if cl.canHop: await cl.handleHop(conn, msg)
+      else: await sendStatus(conn, StatusV1.HopCantSpeakRelay)
+    of RelayType.Stop: await cl.handleStop(conn, msg)
+    of RelayType.CanHop:
+      if cl.canHop: await sendStatus(conn, StatusV1.Success)
+      else: await sendStatus(conn, StatusV1.HopCantSpeakRelay)
+    else:
+      trace "Unexpected relay handshake", msgType=msg.msgType
+      await sendStatus(conn, StatusV1.MalformedMessage)
+
+proc new*(T: typedesc[RelayClient], canHop: bool = false,
+     reservationTTL: times.Duration = DefaultReservationTTL,
+     limitDuration: uint32 = DefaultLimitDuration,
+     limitData: uint64 = DefaultLimitData,
+     heartbeatSleepTime: uint32 = DefaultHeartbeatSleepTime,
+     maxCircuit: int = MaxCircuit,
+     maxCircuitPerPeer: int = MaxCircuitPerPeer,
+     msgSize: int = RelayClientMsgSize,
+     circuitRelayV1: bool = false): T =
+
+  let cl = T(canHop: canHop,
+             reservationTTL: reservationTTL,
+             limit: Limit(duration: limitDuration, data: limitData),
+             heartbeatSleepTime: heartbeatSleepTime,
+             maxCircuit: maxCircuit,
+             maxCircuitPerPeer: maxCircuitPerPeer,
+             msgSize: msgSize,
+             isCircuitRelayV1: circuitRelayV1)
+  proc handleStream(conn: Connection, proto: string) {.async, gcsafe.} =
+    try:
+      case proto:
+      of RelayV1Codec: await cl.handleStreamV1(conn)
+      of RelayV2StopCodec: await cl.handleStopStreamV2(conn)
+      of RelayV2HopCodec: await cl.handleHopStreamV2(conn)
+    except CancelledError as exc:
+      raise exc
+    except CatchableError as exc:
+      trace "exception in client handler", exc = exc.msg, conn
+    finally:
+      trace "exiting client handler", conn
+      await conn.close()
+
+  cl.handler = handleStream
+  cl.codecs = if cl.canHop:
+                @[RelayV1Codec, RelayV2HopCodec, RelayV2StopCodec]
+              else:
+                @[RelayV1Codec, RelayV2StopCodec]
+  cl

libp2p/protocols/relay/messages.nim

@@ -0,0 +1,370 @@
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import options, macros, sequtils
+import stew/objects
+import ../../peerinfo,
+       ../../signed_envelope
+
+# Circuit Relay V1 Message
+
+type
+  RelayType* {.pure.} = enum
+    Hop = 1
+    Stop = 2
+    Status = 3
+    CanHop = 4
+
+  StatusV1* {.pure.} = enum
+    Success = 100
+    HopSrcAddrTooLong = 220
+    HopDstAddrTooLong = 221
+    HopSrcMultiaddrInvalid = 250
+    HopDstMultiaddrInvalid = 251
+    HopNoConnToDst = 260
+    HopCantDialDst = 261
+    HopCantOpenDstStream = 262
+    HopCantSpeakRelay = 270
+    HopCantRelayToSelf = 280
+    StopSrcAddrTooLong = 320
+    StopDstAddrTooLong = 321
+    StopSrcMultiaddrInvalid = 350
+    StopDstMultiaddrInvalid = 351
+    StopRelayRefused = 390
+    MalformedMessage = 400
+
+  RelayPeer* = object
+    peerId*: PeerId
+    addrs*: seq[MultiAddress]
+
+  RelayMessage* = object
+    msgType*: Option[RelayType]
+    srcPeer*: Option[RelayPeer]
+    dstPeer*: Option[RelayPeer]
+    status*: Option[StatusV1]
+
+proc encode*(msg: RelayMessage): ProtoBuffer =
+  result = initProtoBuffer()
+
+  if isSome(msg.msgType):
+    result.write(1, msg.msgType.get().ord.uint)
+  if isSome(msg.srcPeer):
+    var peer = initProtoBuffer()
+    peer.write(1, msg.srcPeer.get().peerId)
+    for ma in msg.srcPeer.get().addrs:
+      peer.write(2, ma.data.buffer)
+    peer.finish()
+    result.write(2, peer.buffer)
+  if isSome(msg.dstPeer):
+    var peer = initProtoBuffer()
+    peer.write(1, msg.dstPeer.get().peerId)
+    for ma in msg.dstPeer.get().addrs:
+      peer.write(2, ma.data.buffer)
+    peer.finish()
+    result.write(3, peer.buffer)
+  if isSome(msg.status):
+    result.write(4, msg.status.get().ord.uint)
+
+  result.finish()
+
+proc decode*(_: typedesc[RelayMessage], buf: seq[byte]): Option[RelayMessage] =
+  var
+    rMsg: RelayMessage
+    msgTypeOrd: uint32
+    src: RelayPeer
+    dst: RelayPeer
+    statusOrd: uint32
+    pbSrc: ProtoBuffer
+    pbDst: ProtoBuffer
+
+  let
+    pb = initProtoBuffer(buf)
+    r1 = pb.getField(1, msgTypeOrd)
+    r2 = pb.getField(2, pbSrc)
+    r3 = pb.getField(3, pbDst)
+    r4 = pb.getField(4, statusOrd)
+
+  if r1.isErr() or r2.isErr() or r3.isErr() or r4.isErr():
+    return none(RelayMessage)
+
+  if r2.get() and
+     (pbSrc.getField(1, src.peerId).isErr() or
+      pbSrc.getRepeatedField(2, src.addrs).isErr()):
+    return none(RelayMessage)
+
+  if r3.get() and
+     (pbDst.getField(1, dst.peerId).isErr() or
+      pbDst.getRepeatedField(2, dst.addrs).isErr()):
+    return none(RelayMessage)
+
+  if r1.get():
+    if msgTypeOrd.int notin RelayType:
+      return none(RelayMessage)
+    rMsg.msgType = some(RelayType(msgTypeOrd))
+  if r2.get(): rMsg.srcPeer = some(src)
+  if r3.get(): rMsg.dstPeer = some(dst)
+  if r4.get():
+    if statusOrd.int notin StatusV1:
+      return none(RelayMessage)
+    rMsg.status = some(StatusV1(statusOrd))
+  some(rMsg)
+
+# Voucher
+
+type
+  Voucher* = object
+    relayPeerId*: PeerId     # peer ID of the relay
+    reservingPeerId*: PeerId # peer ID of the reserving peer
+    expiration*: uint64      # UNIX UTC expiration time for the reservation
+
+proc decode*(_: typedesc[Voucher], buf: seq[byte]): Result[Voucher, ProtoError] =
+  let pb = initProtoBuffer(buf)
+  var v = Voucher()
+
+  ? pb.getRequiredField(1, v.relayPeerId)
+  ? pb.getRequiredField(2, v.reservingPeerId)
+  ? pb.getRequiredField(3, v.expiration)
+
+  ok(v)
+
+proc encode*(v: Voucher): seq[byte] =
+  var pb = initProtoBuffer()
+
+  pb.write(1, v.relayPeerId)
+  pb.write(2, v.reservingPeerId)
+  pb.write(3, v.expiration)
+
+  pb.finish()
+  pb.buffer
+
+proc init*(T: typedesc[Voucher],
+          relayPeerId: PeerId,
+          reservingPeerId: PeerId,
+          expiration: uint64): T =
+  T(
+    relayPeerId = relayPeerId,
+    reservingPeerId = reservingPeerId,
+    expiration: expiration
+  )
+
+type SignedVoucher* = SignedPayload[Voucher]
+
+proc payloadDomain*(_: typedesc[Voucher]): string = "libp2p-relay-rsvp"
+proc payloadType*(_: typedesc[Voucher]): seq[byte] = @[ (byte)0x03, (byte)0x02 ]
+
+proc checkValid*(spr: SignedVoucher): Result[void, EnvelopeError] =
+  if not spr.data.relayPeerId.match(spr.envelope.publicKey):
+    err(EnvelopeInvalidSignature)
+  else:
+    ok()
+
+# Circuit Relay V2 Hop Message
+
+type
+  Peer* = object
+    peerId*: PeerId
+    addrs*: seq[MultiAddress]
+  Reservation* = object
+    expire*: uint64              # required, Unix expiration time (UTC)
+    addrs*: seq[MultiAddress]    # relay address for reserving peer
+    svoucher*: Option[seq[byte]] # optional, reservation voucher
+  Limit* = object
+    duration*: uint32            # seconds
+    data*: uint64                # bytes
+
+  StatusV2* = enum
+    Ok = 100
+    ReservationRefused = 200
+    ResourceLimitExceeded = 201
+    PermissionDenied = 202
+    ConnectionFailed = 203
+    NoReservation = 204
+    MalformedMessage = 400
+    UnexpectedMessage = 401
+  HopMessageType* {.pure.} = enum
+    Reserve = 0
+    Connect = 1
+    Status = 2
+  HopMessage* = object
+    msgType*: HopMessageType
+    peer*: Option[Peer]
+    reservation*: Option[Reservation]
+    limit*: Limit
+    status*: Option[StatusV2]
+
+proc encode*(msg: HopMessage): ProtoBuffer =
+  var pb = initProtoBuffer()
+
+  pb.write(1, msg.msgType.ord.uint)
+  if msg.peer.isSome():
+    var ppb = initProtoBuffer()
+    ppb.write(1, msg.peer.get().peerId)
+    for ma in msg.peer.get().addrs:
+      ppb.write(2, ma.data.buffer)
+    ppb.finish()
+    pb.write(2, ppb.buffer)
+  if msg.reservation.isSome():
+    let rsrv = msg.reservation.get()
+    var rpb = initProtoBuffer()
+    rpb.write(1, rsrv.expire)
+    for ma in rsrv.addrs:
+      rpb.write(2, ma.data.buffer)
+    if rsrv.svoucher.isSome():
+      rpb.write(3, rsrv.svoucher.get())
+    rpb.finish()
+    pb.write(3, rpb.buffer)
+  if msg.limit.duration > 0 or msg.limit.data > 0:
+    var lpb = initProtoBuffer()
+    if msg.limit.duration > 0: lpb.write(1, msg.limit.duration)
+    if msg.limit.data > 0: lpb.write(2, msg.limit.data)
+    lpb.finish()
+    pb.write(4, lpb.buffer)
+  if msg.status.isSome():
+    pb.write(5, msg.status.get().ord.uint)
+
+  pb.finish()
+  pb
+
+proc decode*(_: typedesc[HopMessage], buf: seq[byte]): Option[HopMessage] =
+  var
+    msg: HopMessage
+    msgTypeOrd: uint32
+    pbPeer: ProtoBuffer
+    pbReservation: ProtoBuffer
+    pbLimit: ProtoBuffer
+    statusOrd: uint32
+    peer: Peer
+    reservation: Reservation
+    limit: Limit
+    res: bool
+
+  let
+    pb = initProtoBuffer(buf)
+    r1 = pb.getRequiredField(1, msgTypeOrd)
+    r2 = pb.getField(2, pbPeer)
+    r3 = pb.getField(3, pbReservation)
+    r4 = pb.getField(4, pbLimit)
+    r5 = pb.getField(5, statusOrd)
+
+  if r1.isErr() or r2.isErr() or r3.isErr() or r4.isErr() or r5.isErr():
+    return none(HopMessage)
+
+  if r2.get() and
+     (pbPeer.getRequiredField(1, peer.peerId).isErr() or
+      pbPeer.getRepeatedField(2, peer.addrs).isErr()):
+    return none(HopMessage)
+
+  if r3.get():
+    var svoucher: seq[byte]
+    let rSVoucher = pbReservation.getField(3, svoucher)
+    if pbReservation.getRequiredField(1, reservation.expire).isErr() or
+       pbReservation.getRepeatedField(2, reservation.addrs).isErr() or
+       rSVoucher.isErr():
+      return none(HopMessage)
+    if rSVoucher.get(): reservation.svoucher = some(svoucher)
+
+  if r4.get() and
+     (pbLimit.getField(1, limit.duration).isErr() or
+      pbLimit.getField(2, limit.data).isErr()):
+    return none(HopMessage)
+
+  if not checkedEnumAssign(msg.msgType, msgTypeOrd):
+    return none(HopMessage)
+  if r2.get(): msg.peer = some(peer)
+  if r3.get(): msg.reservation = some(reservation)
+  if r4.get(): msg.limit = limit
+  if r5.get():
+    if statusOrd.int notin StatusV2:
+      return none(HopMessage)
+    msg.status = some(StatusV2(statusOrd))
+  some(msg)
+
+# Circuit Relay V2 Stop Message
+
+type
+  StopMessageType* {.pure.} = enum
+    Connect = 0
+    Status = 1
+  StopMessage* = object
+    msgType*: StopMessageType
+    peer*: Option[Peer]
+    limit*: Limit
+    status*: Option[StatusV2]
+
+
+proc encode*(msg: StopMessage): ProtoBuffer =
+  var pb = initProtoBuffer()
+
+  pb.write(1, msg.msgType.ord.uint)
+  if msg.peer.isSome():
+    var ppb = initProtoBuffer()
+    ppb.write(1, msg.peer.get().peerId)
+    for ma in msg.peer.get().addrs:
+      ppb.write(2, ma.data.buffer)
+    ppb.finish()
+    pb.write(2, ppb.buffer)
+  if msg.limit.duration > 0 or msg.limit.data > 0:
+    var lpb = initProtoBuffer()
+    if msg.limit.duration > 0: lpb.write(1, msg.limit.duration)
+    if msg.limit.data > 0: lpb.write(2, msg.limit.data)
+    lpb.finish()
+    pb.write(3, lpb.buffer)
+  if msg.status.isSome():
+    pb.write(4, msg.status.get().ord.uint)
+
+  pb.finish()
+  pb
+
+proc decode*(_: typedesc[StopMessage], buf: seq[byte]): Option[StopMessage] =
+  var
+    msg: StopMessage
+    msgTypeOrd: uint32
+    pbPeer: ProtoBuffer
+    pbLimit: ProtoBuffer
+    statusOrd: uint32
+    peer: Peer
+    limit: Limit
+    rVoucher: ProtoResult[bool]
+    res: bool
+
+  let
+    pb = initProtoBuffer(buf)
+    r1 = pb.getRequiredField(1, msgTypeOrd)
+    r2 = pb.getField(2, pbPeer)
+    r3 = pb.getField(3, pbLimit)
+    r4 = pb.getField(4, statusOrd)
+
+  if r1.isErr() or r2.isErr() or r3.isErr() or r4.isErr():
+    return none(StopMessage)
+
+  if r2.get() and
+     (pbPeer.getRequiredField(1, peer.peerId).isErr() or
+      pbPeer.getRepeatedField(2, peer.addrs).isErr()):
+    return none(StopMessage)
+
+  if r3.get() and
+     (pbLimit.getField(1, limit.duration).isErr() or
+      pbLimit.getField(2, limit.data).isErr()):
+    return none(StopMessage)
+
+  if msgTypeOrd.int notin StopMessageType.low.ord .. StopMessageType.high.ord:
+    return none(StopMessage)
+  msg.msgType = StopMessageType(msgTypeOrd)
+  if r2.get(): msg.peer = some(peer)
+  if r3.get(): msg.limit = limit
+  if r4.get():
+    if statusOrd.int notin StatusV2:
+      return none(StopMessage)
+    msg.status = some(StatusV2(statusOrd))
+  some(msg)

libp2p/protocols/relay/rconn.nim

@@ -0,0 +1,61 @@
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import chronos
+
+import ../../stream/connection
+
+type
+  RelayConnection* = ref object of Connection
+    conn*: Connection
+    limitDuration*: uint32
+    limitData*: uint64
+    dataSent*: uint64
+
+method readOnce*(
+    self: RelayConnection,
+    pbytes: pointer,
+    nbytes: int): Future[int] {.async.} =
+  self.activity = true
+  return await self.conn.readOnce(pbytes, nbytes)
+
+method write*(self: RelayConnection, msg: seq[byte]): Future[void] {.async.} =
+  self.dataSent.inc(msg.len)
+  if self.limitData != 0 and self.dataSent > self.limitData:
+    await self.close()
+    return
+  self.activity = true
+  await self.conn.write(msg)
+
+method closeImpl*(self: RelayConnection): Future[void] {.async.} =
+  await self.conn.close()
+  await procCall Connection(self).closeImpl()
+
+method getWrapped*(self: RelayConnection): Connection = self.conn
+
+proc new*(
+  T: typedesc[RelayConnection],
+  conn: Connection,
+  limitDuration: uint32,
+  limitData: uint64): T =
+  let rc = T(conn: conn, limitDuration: limitDuration, limitData: limitData)
+  rc.initStream()
+  if limitDuration > 0:
+    proc checkDurationConnection() {.async.} =
+      let sleep = sleepAsync(limitDuration.seconds())
+      await sleep or conn.join()
+      if sleep.finished: await conn.close()
+      else: sleep.cancel()
+    asyncSpawn checkDurationConnection()
+  return rc

libp2p/protocols/relay/relay.nim

@@ -0,0 +1,386 @@
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import options, sequtils, tables, sugar
+
+import chronos, chronicles
+
+import ./messages,
+       ./rconn,
+       ./utils,
+       ../../peerinfo,
+       ../../switch,
+       ../../multiaddress,
+       ../../multicodec,
+       ../../stream/connection,
+       ../../protocols/protocol,
+       ../../transports/transport,
+       ../../errors,
+       ../../utils/heartbeat,
+       ../../signed_envelope
+
+# TODO:
+# * Eventually replace std/times by chronos/timer. Currently chronos/timer
+#   doesn't offer the possibility to get a datetime in UNIX UTC
+# * Eventually add an access control list in the handleReserve, handleConnect
+#   and handleHop
+# * Better reservation management ie find a way to re-reserve when the end is nigh
+
+import std/times
+export chronicles
+
+const
+  RelayMsgSize* = 4096
+  DefaultReservationTTL* = initDuration(hours = 1)
+  DefaultLimitDuration* = 120
+  DefaultLimitData* = 1 shl 17
+  DefaultHeartbeatSleepTime* = 1
+  MaxCircuit* = 128
+  MaxCircuitPerPeer* = 16
+
+logScope:
+  topics = "libp2p relay"
+
+type
+  RelayV2Error* = object of LPError
+  SendStopError = object of RelayV2Error
+
+# Relay Side
+
+type
+  Relay* = ref object of LPProtocol
+    switch*: Switch
+    peerCount: CountTable[PeerId]
+
+    # number of reservation (relayv2) + number of connection (relayv1)
+    maxCircuit*: int
+
+    maxCircuitPerPeer*: int
+    msgSize*: int
+    # RelayV1
+    isCircuitRelayV1*: bool
+    streamCount: int
+    # RelayV2
+    rsvp: Table[PeerId, DateTime]
+    reservationLoop: Future[void]
+    reservationTTL*: times.Duration
+    heartbeatSleepTime*: uint32
+    limit*: Limit
+
+# Relay V2
+
+proc createReserveResponse(
+    r: Relay,
+    pid: PeerId,
+    expire: DateTime): Result[HopMessage, CryptoError] =
+  let
+    expireUnix = expire.toTime.toUnix.uint64
+    v = Voucher(relayPeerId: r.switch.peerInfo.peerId,
+                  reservingPeerId: pid,
+                  expiration: expireUnix)
+    sv = ? SignedVoucher.init(r.switch.peerInfo.privateKey, v)
+    ma = ? MultiAddress.init("/p2p/" & $r.switch.peerInfo.peerId).orErr(CryptoError.KeyError)
+    rsrv = Reservation(expire: expireUnix,
+                       addrs: r.switch.peerInfo.addrs.mapIt(
+                         ? it.concat(ma).orErr(CryptoError.KeyError)),
+                       svoucher: some(? sv.encode))
+    msg = HopMessage(msgType: HopMessageType.Status,
+                     reservation: some(rsrv),
+                     limit: r.limit,
+                     status: some(Ok))
+  return ok(msg)
+
+proc isRelayed(conn: Connection): bool =
+  var wrappedConn = conn
+  while not isNil(wrappedConn):
+    if wrappedConn of RelayConnection:
+      return true
+    wrappedConn = wrappedConn.getWrapped()
+  return false
+
+proc handleReserve(r: Relay, conn: Connection) {.async, gcsafe.} =
+  if conn.isRelayed():
+    trace "reservation attempt over relay connection", pid = conn.peerId
+    await sendHopStatus(conn, PermissionDenied)
+    return
+
+  if r.peerCount[conn.peerId] + r.rsvp.len() >= r.maxCircuit:
+    trace "Too many reservations", pid = conn.peerId
+    await sendHopStatus(conn, ReservationRefused)
+    return
+  let
+    pid = conn.peerId
+    expire = now().utc + r.reservationTTL
+    msg = r.createReserveResponse(pid, expire)
+
+  trace "reserving relay slot for", pid
+  if msg.isErr():
+    trace "error signing the voucher", error = error(msg), pid
+    return
+  r.rsvp[pid] = expire
+  await conn.writeLp(encode(msg.get()).buffer)
+
+proc handleConnect(r: Relay,
+                   connSrc: Connection,
+                   msg: HopMessage) {.async, gcsafe.} =
+  if connSrc.isRelayed():
+    trace "connection attempt over relay connection"
+    await sendHopStatus(connSrc, PermissionDenied)
+    return
+  if msg.peer.isNone():
+    await sendHopStatus(connSrc, MalformedMessage)
+    return
+
+  let
+    src = connSrc.peerId
+    dst = msg.peer.get().peerId
+  if dst notin r.rsvp:
+    trace "refusing connection, no reservation", src, dst
+    await sendHopStatus(connSrc, NoReservation)
+    return
+
+  r.peerCount.inc(src)
+  r.peerCount.inc(dst)
+  defer:
+    r.peerCount.inc(src, -1)
+    r.peerCount.inc(dst, -1)
+
+  if r.peerCount[src] > r.maxCircuitPerPeer or
+     r.peerCount[dst] > r.maxCircuitPerPeer:
+    trace "too many connections", src = r.peerCount[src],
+                                  dst = r.peerCount[dst],
+                                  max = r.maxCircuitPerPeer
+    await sendHopStatus(connSrc, ResourceLimitExceeded)
+    return
+
+  let connDst = try:
+    await r.switch.dial(dst, RelayV2StopCodec)
+  except CancelledError as exc:
+    raise exc
+  except CatchableError as exc:
+    trace "error opening relay stream", dst, exc=exc.msg
+    await sendHopStatus(connSrc, ConnectionFailed)
+    return
+  defer:
+    await connDst.close()
+
+  proc sendStopMsg() {.async.} =
+    let stopMsg = StopMessage(msgType: StopMessageType.Connect,
+                            peer: some(Peer(peerId: src, addrs: @[])),
+                            limit: r.limit)
+    await connDst.writeLp(encode(stopMsg).buffer)
+    let msg = StopMessage.decode(await connDst.readLp(r.msgSize)).get()
+    if msg.msgType != StopMessageType.Status:
+      raise newException(SendStopError, "Unexpected stop response, not a status message")
+    if msg.status.get(UnexpectedMessage) != Ok:
+      raise newException(SendStopError, "Relay stop failure")
+    await connSrc.writeLp(encode(HopMessage(msgType: HopMessageType.Status,
+                                          status: some(Ok))).buffer)
+  try:
+    await sendStopMsg()
+  except CancelledError as exc:
+    raise exc
+  except CatchableError as exc:
+    trace "error sending stop message", msg = exc.msg
+    await sendHopStatus(connSrc, ConnectionFailed)
+    return
+
+  trace "relaying connection", src, dst
+  let
+    rconnSrc = RelayConnection.new(connSrc, r.limit.duration, r.limit.data)
+    rconnDst = RelayConnection.new(connDst, r.limit.duration, r.limit.data)
+  defer:
+    await rconnSrc.close()
+    await rconnDst.close()
+  await bridge(rconnSrc, rconnDst)
+
+proc handleHopStreamV2*(r: Relay, conn: Connection) {.async, gcsafe.} =
+  let msgOpt = HopMessage.decode(await conn.readLp(r.msgSize))
+  if msgOpt.isNone():
+    await sendHopStatus(conn, MalformedMessage)
+    return
+  trace "relayv2 handle stream", msg = msgOpt.get()
+  let msg = msgOpt.get()
+  case msg.msgType:
+  of HopMessageType.Reserve: await r.handleReserve(conn)
+  of HopMessageType.Connect: await r.handleConnect(conn, msg)
+  else:
+    trace "Unexpected relayv2 handshake", msgType=msg.msgType
+    await sendHopStatus(conn, MalformedMessage)
+
+# Relay V1
+
+proc handleHop*(r: Relay, connSrc: Connection, msg: RelayMessage) {.async, gcsafe.} =
+  r.streamCount.inc()
+  defer: r.streamCount.dec()
+  if r.streamCount + r.rsvp.len() >= r.maxCircuit:
+    trace "refusing connection; too many active circuit", streamCount = r.streamCount, rsvp = r.rsvp.len()
+    await sendStatus(connSrc, StatusV1.HopCantSpeakRelay)
+    return
+
+  proc checkMsg(): Result[RelayMessage, StatusV1] =
+    if msg.srcPeer.isNone:
+      return err(StatusV1.HopSrcMultiaddrInvalid)
+    let src = msg.srcPeer.get()
+    if src.peerId != connSrc.peerId:
+      return err(StatusV1.HopSrcMultiaddrInvalid)
+    if msg.dstPeer.isNone:
+      return err(StatusV1.HopDstMultiaddrInvalid)
+    let dst = msg.dstPeer.get()
+    if dst.peerId == r.switch.peerInfo.peerId:
+      return err(StatusV1.HopCantRelayToSelf)
+    if not r.switch.isConnected(dst.peerId):
+      trace "relay not connected to dst", dst
+      return err(StatusV1.HopNoConnToDst)
+    ok(msg)
+  let check = checkMsg()
+  if check.isErr:
+    await sendStatus(connSrc, check.error())
+    return
+
+  let
+    src = msg.srcPeer.get()
+    dst = msg.dstPeer.get()
+  if r.peerCount[src.peerId] >= r.maxCircuitPerPeer or
+     r.peerCount[dst.peerId] >= r.maxCircuitPerPeer:
+    trace "refusing connection; too many connection from src or to dst", src, dst
+    await sendStatus(connSrc, StatusV1.HopCantSpeakRelay)
+    return
+  r.peerCount.inc(src.peerId)
+  r.peerCount.inc(dst.peerId)
+  defer:
+    r.peerCount.inc(src.peerId, -1)
+    r.peerCount.inc(dst.peerId, -1)
+
+  let connDst = try:
+    await r.switch.dial(dst.peerId, RelayV1Codec)
+  except CancelledError as exc:
+    raise exc
+  except CatchableError as exc:
+    trace "error opening relay stream", dst, exc=exc.msg
+    await sendStatus(connSrc, StatusV1.HopCantDialDst)
+    return
+  defer:
+    await connDst.close()
+
+  let msgToSend = RelayMessage(
+    msgType: some(RelayType.Stop),
+    srcPeer: some(src),
+    dstPeer: some(dst))
+
+  let msgRcvFromDstOpt = try:
+    await connDst.writeLp(encode(msgToSend).buffer)
+    RelayMessage.decode(await connDst.readLp(r.msgSize))
+  except CancelledError as exc:
+    raise exc
+  except CatchableError as exc:
+    trace "error writing stop handshake or reading stop response", exc=exc.msg
+    await sendStatus(connSrc, StatusV1.HopCantOpenDstStream)
+    return
+
+  if msgRcvFromDstOpt.isNone:
+    trace "error reading stop response", msg = msgRcvFromDstOpt
+    await sendStatus(connSrc, StatusV1.HopCantOpenDstStream)
+    return
+
+  let msgRcvFromDst = msgRcvFromDstOpt.get()
+  if msgRcvFromDst.msgType.get(RelayType.Stop) != RelayType.Status or
+     msgRcvFromDst.status.get(StatusV1.StopRelayRefused) != StatusV1.Success:
+    trace "unexcepted relay stop response", msgRcvFromDst
+    await sendStatus(connSrc, StatusV1.HopCantOpenDstStream)
+    return
+
+  await sendStatus(connSrc, StatusV1.Success)
+  trace "relaying connection", src, dst
+  await bridge(connSrc, connDst)
+
+proc handleStreamV1(r: Relay, conn: Connection) {.async, gcsafe.} =
+  let msgOpt = RelayMessage.decode(await conn.readLp(r.msgSize))
+  if msgOpt.isNone:
+    await sendStatus(conn, StatusV1.MalformedMessage)
+    return
+  trace "relay handle stream", msg = msgOpt.get()
+  let msg = msgOpt.get()
+  case msg.msgType.get:
+    of RelayType.Hop: await r.handleHop(conn, msg)
+    of RelayType.Stop: await sendStatus(conn, StatusV1.StopRelayRefused)
+    of RelayType.CanHop: await sendStatus(conn, StatusV1.Success)
+    else:
+      trace "Unexpected relay handshake", msgType=msg.msgType
+      await sendStatus(conn, StatusV1.MalformedMessage)
+
+proc setup*(r: Relay, switch: Switch) =
+  r.switch = switch
+  r.switch.addPeerEventHandler(
+    proc (peerId: PeerId, event: PeerEvent) {.async.} =
+      r.rsvp.del(peerId),
+    Left)
+
+proc new*(T: typedesc[Relay],
+     reservationTTL: times.Duration = DefaultReservationTTL,
+     limitDuration: uint32 = DefaultLimitDuration,
+     limitData: uint64 = DefaultLimitData,
+     heartbeatSleepTime: uint32 = DefaultHeartbeatSleepTime,
+     maxCircuit: int = MaxCircuit,
+     maxCircuitPerPeer: int = MaxCircuitPerPeer,
+     msgSize: int = RelayMsgSize,
+     circuitRelayV1: bool = false): T =
+
+  let r = T(reservationTTL: reservationTTL,
+            limit: Limit(duration: limitDuration, data: limitData),
+            heartbeatSleepTime: heartbeatSleepTime,
+            maxCircuit: maxCircuit,
+            maxCircuitPerPeer: maxCircuitPerPeer,
+            msgSize: msgSize,
+            isCircuitRelayV1: circuitRelayV1)
+
+  proc handleStream(conn: Connection, proto: string) {.async, gcsafe.} =
+    try:
+      case proto:
+      of RelayV2HopCodec: await r.handleHopStreamV2(conn)
+      of RelayV1Codec: await r.handleStreamV1(conn)
+    except CancelledError as exc:
+      raise exc
+    except CatchableError as exc:
+      debug "exception in relayv2 handler", exc = exc.msg, conn
+    finally:
+      trace "exiting relayv2 handler", conn
+      await conn.close()
+
+  r.codecs = if r.isCircuitRelayV1: @[RelayV1Codec]
+             else: @[RelayV2HopCodec, RelayV1Codec]
+  r.handler = handleStream
+  r
+
+proc deletesReservation(r: Relay) {.async.} =
+  heartbeat "Reservation timeout", r.heartbeatSleepTime.seconds():
+    let n = now().utc
+    for k in toSeq(r.rsvp.keys):
+      if n > r.rsvp[k]:
+        r.rsvp.del(k)
+
+method start*(r: Relay) {.async.} =
+  if not r.reservationLoop.isNil:
+    warn "Starting relay twice"
+    return
+  r.reservationLoop = r.deletesReservation()
+  r.started = true
+
+method stop*(r: Relay) {.async.} =
+  if r.reservationLoop.isNil:
+    warn "Stopping relay without starting it"
+    return
+  r.started = false
+  r.reservationLoop.cancel()
+  r.reservationLoop = nil

libp2p/protocols/relay/rtransport.nim

@@ -0,0 +1,109 @@
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import sequtils, strutils
+
+import chronos, chronicles
+
+import ./client,
+       ./rconn,
+       ./utils,
+       ../../switch,
+       ../../stream/connection,
+       ../../transports/transport
+
+logScope:
+  topics = "libp2p relay relay-transport"
+
+type
+  RelayTransport* = ref object of Transport
+    client*: RelayClient
+    queue: AsyncQueue[Connection]
+    selfRunning: bool
+
+method start*(self: RelayTransport, ma: seq[MultiAddress]) {.async.} =
+  if self.selfRunning:
+    trace "Relay transport already running"
+    return
+
+  self.client.onNewConnection = proc(
+    conn: Connection,
+    duration: uint32 = 0,
+    data: uint64 = 0) {.async, gcsafe, raises: [Defect].} =
+      await self.queue.addLast(RelayConnection.new(conn, duration, data))
+      await conn.join()
+  self.selfRunning = true
+  await procCall Transport(self).start(ma)
+  trace "Starting Relay transport"
+
+method stop*(self: RelayTransport) {.async, gcsafe.} =
+  self.running = false
+  self.selfRunning = false
+  self.client.onNewConnection = nil
+  while not self.queue.empty():
+    await self.queue.popFirstNoWait().close()
+
+method accept*(self: RelayTransport): Future[Connection] {.async, gcsafe.} =
+  result = await self.queue.popFirst()
+
+proc dial*(self: RelayTransport, ma: MultiAddress): Future[Connection] {.async, gcsafe.} =
+  let
+    sma = toSeq(ma.items())
+    relayAddrs = sma[0..sma.len-4].mapIt(it.tryGet()).foldl(a & b)
+  var
+    relayPeerId: PeerId
+    dstPeerId: PeerId
+  if not relayPeerId.init(($(sma[^3].get())).split('/')[2]):
+    raise newException(RelayV2DialError, "Relay doesn't exist")
+  if not dstPeerId.init(($(sma[^1].get())).split('/')[2]):
+    raise newException(RelayV2DialError, "Destination doesn't exist")
+  trace "Dial", relayPeerId, dstPeerId
+
+  let conn = await self.client.switch.dial(
+    relayPeerId,
+    @[ relayAddrs ],
+    @[ RelayV2HopCodec, RelayV1Codec ])
+  conn.dir = Direction.Out
+  var rc: RelayConnection
+  try:
+    case conn.protocol:
+    of RelayV1Codec:
+      return await self.client.dialPeerV1(conn, dstPeerId, @[])
+    of RelayV2HopCodec:
+      rc = RelayConnection.new(conn, 0, 0)
+      return await self.client.dialPeerV2(rc, dstPeerId, @[])
+  except CancelledError as exc:
+    raise exc
+  except CatchableError as exc:
+    if not rc.isNil: await rc.close()
+    raise exc
+
+method dial*(
+  self: RelayTransport,
+  hostname: string,
+  address: MultiAddress): Future[Connection] {.async, gcsafe.} =
+  result = await self.dial(address)
+
+method handles*(self: RelayTransport, ma: MultiAddress): bool {.gcsafe} =
+  if ma.protocols.isOk():
+    let sma = toSeq(ma.items())
+    if sma.len >= 3:
+      result = CircuitRelay.match(sma[^2].get()) and
+               P2PPattern.match(sma[^1].get())
+  trace "Handles return", ma, result
+
+proc new*(T: typedesc[RelayTransport], cl: RelayClient, upgrader: Upgrade): T =
+  result = T(client: cl, upgrader: upgrader)
+  result.running = true
+  result.queue = newAsyncQueue[Connection](0)

libp2p/protocols/relay/utils.nim

@@ -0,0 +1,87 @@
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import options
+
+import chronos, chronicles
+
+import ./messages,
+       ../../stream/connection
+
+logScope:
+  topics = "libp2p relay relay-utils"
+
+const
+  RelayV1Codec* = "/libp2p/circuit/relay/0.1.0"
+  RelayV2HopCodec* = "/libp2p/circuit/relay/0.2.0/hop"
+  RelayV2StopCodec* = "/libp2p/circuit/relay/0.2.0/stop"
+
+proc sendStatus*(conn: Connection, code: StatusV1) {.async, gcsafe.} =
+  trace "send relay/v1 status", status = $code & "(" & $ord(code) & ")"
+  let
+    msg = RelayMessage(msgType: some(RelayType.Status), status: some(code))
+    pb = encode(msg)
+  await conn.writeLp(pb.buffer)
+
+proc sendHopStatus*(conn: Connection, code: StatusV2) {.async, gcsafe.} =
+  trace "send hop relay/v2 status", status = $code & "(" & $ord(code) & ")"
+  let
+    msg = HopMessage(msgType: HopMessageType.Status, status: some(code))
+    pb = encode(msg)
+  await conn.writeLp(pb.buffer)
+
+proc sendStopStatus*(conn: Connection, code: StatusV2) {.async.} =
+  trace "send stop relay/v2 status", status = $code & " (" & $ord(code) & ")"
+  let
+    msg = StopMessage(msgType: StopMessageType.Status, status: some(code))
+    pb = encode(msg)
+  await conn.writeLp(pb.buffer)
+
+proc bridge*(connSrc: Connection, connDst: Connection) {.async.} =
+  const bufferSize = 4096
+  var
+    bufSrcToDst: array[bufferSize, byte]
+    bufDstToSrc: array[bufferSize, byte]
+    futSrc = connSrc.readOnce(addr bufSrcToDst[0], bufSrcToDst.high + 1)
+    futDst = connDst.readOnce(addr bufDstToSrc[0], bufDstToSrc.high + 1)
+    bytesSendFromSrcToDst = 0
+    bytesSendFromDstToSrc = 0
+    bufRead: int
+
+  try:
+    while not connSrc.closed() and not connDst.closed():
+      await futSrc or futDst
+      if futSrc.finished():
+        bufRead = await futSrc
+        bytesSendFromSrcToDst.inc(bufRead)
+        await connDst.write(@bufSrcToDst[0..<bufRead])
+        zeroMem(addr(bufSrcToDst), bufSrcToDst.high + 1)
+        futSrc = connSrc.readOnce(addr bufSrcToDst[0], bufSrcToDst.high + 1)
+      if futDst.finished():
+        bufRead = await futDst
+        bytesSendFromDstToSrc += bufRead
+        await connSrc.write(bufDstToSrc[0..<bufRead])
+        zeroMem(addr(bufDstToSrc), bufDstToSrc.high + 1)
+        futDst = connDst.readOnce(addr bufDstToSrc[0], bufDstToSrc.high + 1)
+  except CancelledError as exc:
+    raise exc
+  except CatchableError as exc:
+    if connSrc.closed() or connSrc.atEof():
+      trace "relay src closed connection", src = connSrc.peerId
+    if connDst.closed() or connDst.atEof():
+      trace "relay dst closed connection", dst = connDst.peerId
+    trace "relay error", exc=exc.msg
+  trace "end relaying", bytesSendFromSrcToDst, bytesSendFromDstToSrc
+  await futSrc.cancelAndWait()
+  await futDst.cancelAndWait()

libp2p/protocols/secure/noise.nim

@@ -1,18 +1,21 @@
-## Nim-LibP2P
-## Copyright (c) 2020 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 import std/[oids, strformat]
 import chronos
 import chronicles
-import bearssl
+import bearssl/[rand, hash]
 import stew/[endians2, byteutils]
 import nimcrypto/[utils, sha2, hmac]
 import ../../stream/[connection, streamseq]
@@ -78,7 +81,7 @@ type
     rs: Curve25519Key
 
   Noise* = ref object of Secure
-    rng: ref BrHmacDrbgContext
+    rng: ref HmacDrbgContext
     localPrivateKey: PrivateKey
     localPublicKey: seq[byte]
     noiseKeys: KeyPair
@@ -106,7 +109,7 @@ func shortLog*(conn: NoiseConnection): auto =
 
 chronicles.formatIt(NoiseConnection): shortLog(it)
 
-proc genKeyPair(rng: var BrHmacDrbgContext): KeyPair =
+proc genKeyPair(rng: var HmacDrbgContext): KeyPair =
   result.privateKey = Curve25519Key.random(rng)
   result.publicKey = result.privateKey.public()
 
@@ -602,7 +605,7 @@ method init*(p: Noise) {.gcsafe.} =
 
 proc new*(
   T: typedesc[Noise],
-  rng: ref BrHmacDrbgContext,
+  rng: ref HmacDrbgContext,
   privateKey: PrivateKey,
   outgoing: bool = true,
   commonPrologue: seq[byte] = @[]): T =