[skip ci] Debug new discovery tuto

Co-authored-by: Tanguy <tanguy@status.im>

.appveyor.yml

@@ -39,8 +39,7 @@ install:
   - CD ..
 
   # install and build go-libp2p-daemon
-  - curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_p2pd.sh
-  - bash build_p2pd.sh p2pdCache v0.2.4
+  - bash scripts/build_p2pd.sh p2pdCache v0.3.0
 
 build_script:
   - nimble install -y --depsOnly

.assets/small-logo.svg

@@ -0,0 +1,96 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:v="https://vecta.io/nano" xmlns:xlink="http://www.w3.org/1999/xlink" width="172.071" height="196.414" viewBox="0 0 45.527 51.968">
+   <g transform="matrix(.2822 0 0 .2822 -212.833275 -150.656248)">
+      <path d="M835.432 533.821l-12.483 9.783c-6.482-.207-19.197 1.251-26.086 3.769-6.346-4.04-11.923-8.5-11.923-8.5l-7.762 13.071c-4.444 2.375-8.906 5.046-12.883 8.58l-10.162-4.17c6.125 12.414 10.243 24.844 21.445 32.316 17.834-28.299 100.705-25.691 118.907-.16 11.764-6.165 16.339-19.429 20.965-31.674-.507.168-6.802 2.285-10.882 3.849-2.436-2.665-8.179-6.763-11.443-8.741-3.096-5.696-7.602-13.391-7.602-13.391s-5.337 3.988-11.523 8.34c-8.357-1.55-18.465-3.433-26.966-2.967-5.787-4.779-11.603-10.104-11.603-10.104z" fill="#f3d400" />
+      <g opacity=".9" transform="matrix(.9375 0 0 .9375 765.1166 550.13225)">
+         <path d="M99.952 106.898l.215-.107 24.755-14.248-24.97-14.535-24.97 14.374z" fill="#cc2a65" />
+         <use xlink:href="#B" fill="#a21d4c" />
+         <path d="M124.922 92.542l-24.755 14.248-.215.107v28.89l24.97-14.356z" fill="#b62454" />
+         <path d="M50.012 106.737l.215-.125 24.755-14.248-24.97-14.517-24.97 14.356z" fill="#c8d92b" />
+         <path d="M50.012 135.609v-28.872l-24.97-14.535v28.89h.018z" fill="#c2d02f" />
+         <path d="M74.982 92.381l-24.755 14.23-.215.125v28.872.018h.018l24.952-14.356v-.018z" fill="#b9be33" />
+         <path d="M74.982 121.253l.215-.107 24.755-14.248-24.97-14.535-24.97 14.374z" fill="#cc2a65" />
+         <use xlink:href="#B" x="-24.97" y="14.356" fill="#a21d4c" />
+         <path d="M99.952 106.898l-24.755 14.248-.215.107v28.89H75l24.952-14.356z" fill="#b62454" />
+         <path d="M124.905 121.415l.215-.125 24.737-14.23-24.952-14.535-24.97 14.356z" fill="#a159a2" />
+         <path d="M124.905 150.305v-28.89l-24.97-14.535v28.89h.018z" fill="#772a86" />
+         <path d="M149.875 107.059l-24.755 14.23-.215.125v28.89l24.97-14.356z" fill="#8e3b95" />
+         <path d="M74.982 92.345l.215-.125 24.737-14.248-24.952-14.517-24.97 14.356z" fill="#bec831" />
+         <path d="M74.982 121.217V92.345L50.012 77.81v28.89h.018z" fill="#a1a938" />
+         <path d="M99.952 77.989l-24.755 14.23-.215.125v28.872.018l24.97-14.356v-.018z" fill="#999b37" />
+         <path d="M75 60.645l.197-.125 24.755-14.23L75 31.755 50.029 46.11l24.952 14.535z" fill="#bec831" />
+         <path d="M74.982 89.535L75 60.645 50.029 46.11 50.012 75h.018z" fill="#a1a938" />
+         <path d="M99.97 46.307L75.197 60.52l-.197.125h-.018v28.89l24.97-14.338v-.018z" fill="#999b37" />
+         <path d="M99.952 75.179l.215-.107 24.755-14.23L99.97 46.306 75 60.644z" fill="#ee539a" />
+         <path d="M99.952 104.069v-28.89L75 60.644l-.018 28.89H75z" fill="#d01b68" />
+         <path d="M124.922 60.841l-24.755 14.23-.215.107v28.89l24.97-14.338.018-28.89z" fill="#ec0f68" />
+         <path d="M124.923 89.731l.215-.125 24.755-14.23-24.952-14.535h-.018l-24.97 14.338z" fill="#a159a2" />
+         <path d="M124.905 118.622l.018-28.89-24.97-14.535v28.872.018z" fill="#772a86" />
+         <path d="M149.893 75.376l-24.755 14.23-.215.125-.018 28.89h.018l24.97-14.356z" fill="#8e3b95" />
+         <path d="M50.03 75l.197-.125 24.755-14.23L50.03 46.109 25.06 60.447l24.952 14.535z" fill="#c8d92b" />
+         <path d="M50.012 103.872L50.03 75 25.06 60.447l-.018 28.89h.018z" fill="#c2d02f" />
+         <path d="M75 60.644l-24.773 14.23-.197.125-.018 28.872 24.97-14.338z" fill="#b9be33" />
+         <path d="M74.982 89.534l.215-.125 24.755-14.23L75 60.644l-24.97 14.338z" fill="#f7af19" />
+         <path d="M74.982 118.425v-.018.018-28.89L50.029 75l-.018 28.872.018.018z" fill="#f2901f" />
+         <path d="M99.952 75.179l-24.755 14.23-.215.125v28.89l24.97-14.356.018-28.89z" fill="#f9a120" />
+         <path d="M99.934 135.769l.215-.125 24.684-14.356-25.042-14.409L74.91 121.36z" fill="#833593" />
+         <path d="M100.077 164.66l-.143-28.89-25.042-14.409.143 28.89h.018z" fill="#652977" />
+         <path d="M124.833 121.288l-24.684 14.356-.215.125.143 28.89 24.899-14.481z" fill="#4d1f5b" />
+         <path d="M99.952 104.069l.215-.107 24.755-14.23L99.97 75.179h-.018l-24.97 14.356z" fill="#a159a2" />
+         <path d="M99.934 132.959l.018-28.89-24.97-14.535v28.89z" fill="#772a86" />
+         <path d="M124.922 89.732l-24.755 14.23-.215.107-.018 28.89h.018l24.97-14.338z" fill="#8e3b95" />
+         <path d="M25.042 121.074l.197-.125 24.755-14.248-24.952-14.517h-.018L.071 106.54l24.952 14.535z" fill="#f6dd03" />
+         <path d="M25.024 149.947h.018v-28.872L.071 106.54v28.89z" fill="#f9bb1d" />
+         <path d="M49.994 106.719l-24.755 14.23-.197.125v28.872.018l24.952-14.356h.018v-.018-28.872z" fill="#e9ae20" />
+         <path d="M25.06 89.338l.197-.125 24.755-14.23L25.06 60.447.089 74.803l24.952 14.535z" fill="#f6dd03" />
+         <path d="M25.042 118.228l.018-28.89L.089 74.803.072 103.675l.018.018z" fill="#f9bb1d" />
+         <path d="M50.03 75L25.257 89.212l-.197.125-.018 28.89 24.97-14.356v.018-.018z" fill="#e9ae20" />
+         <path d="M50.012 135.59l.215-.107 24.737-14.248L50.012 106.7l-24.97 14.374z" fill="#f7af19" />
+         <path d="M50.012 164.481v-28.89l-24.97-14.517v28.872.018z" fill="#f2901f" />
+         <path d="M74.964 121.235l-24.755 14.248-.197.107v28.89l24.97-14.356v-28.89z" fill="#f9a120" />
+         <path d="M50.012 103.872l.215-.107 24.755-14.23L50.03 74.982 25.06 89.338z" fill="#f7af19" />
+         <path d="M50.012 132.763v-28.89L25.06 89.338l-.018 28.89h.018z" fill="#f2901f" />
+         <path d="M74.982 89.535l-24.755 14.23-.215.107v28.89l24.97-14.338.018-28.89z" fill="#f9a120" />
+         <path d="M74.982 150.125l.197-.125 24.755-14.23-24.952-14.535h-.018l-24.952 14.356 24.952 14.535z" fill="#f7af19" />
+         <path d="M74.964 179.015h.018v-28.89l-24.97-14.517v28.872.018z" fill="#f2901f" />
+         <path d="M99.934 135.77L75.179 150l-.197.125v28.89l24.97-14.356v-28.89z" fill="#f9a120" />
+         <path d="M74.982 118.425l.215-.125 24.755-14.23L75 89.535h-.018l-24.97 14.338z" fill="#31838b" />
+         <path d="M74.964 147.297l.018-28.872-24.97-14.535v-.018 28.89.018z" fill="#22626c" />
+         <path d="M99.952 104.069L75.197 118.3l-.215.125-.018 28.872v.018h.018l24.97-14.356z" fill="#1b4b56" />
+         <path d="M74.982 28.962l.215-.125 24.737-14.248L74.982.072l-24.97 14.356 24.97 14.517z" fill="#bec831" />
+         <path d="M74.982 57.834V28.962l-24.97-14.535v28.89h.018z" fill="#a1a938" />
+         <path d="M99.952 14.606l-24.755 14.23-.215.125v28.872l24.97-14.356z" fill="#999b37" />
+         <path d="M74.964 28.944l.215-.125 24.755-14.23L74.982.054h-.018l-24.97 14.338z" fill="#a159a2" />
+         <path d="M74.946 57.835l.018-28.89-24.97-14.535v28.872.018z" fill="#772a86" />
+         <path d="M99.934 14.589l-24.755 14.23-.215.125-.018 28.89h.018l24.97-14.356z" fill="#8e3b95" />
+         <path d="M99.952 43.479l.215-.107 24.755-14.248-24.97-14.535-24.97 14.356z" fill="#ee539a" />
+         <use xlink:href="#B" y="-63.419" fill="#d01b68" />
+         <path d="M124.922 29.123l-24.755 14.248-.215.107v28.89l24.97-14.356z" fill="#ec0f68" />
+         <path d="M50.03 43.317l.215-.125L75 28.961 50.048 14.427h-.018L25.06 28.765z" fill="#31838b" />
+         <path d="M50.012 72.189l.018-28.872-24.97-14.535v-.018 28.89.018z" fill="#22626c" />
+         <path d="M75 28.961l-24.755 14.23-.215.125-.018 28.872v.018h.018L75 57.852z" fill="#1b4b56" />
+         <path d="M124.923 58.013l.215-.125 24.737-14.23-24.952-14.535-24.97 14.356z" fill="#cc2a65" />
+         <use xlink:href="#B" x="24.971" y="-48.884" fill="#a21d4c" />
+         <path d="M149.893 43.658l-24.755 14.23-.215.125v28.89l24.97-14.356z" fill="#b62454" />
+         <path d="M74.982 57.835l.215-.107 24.755-14.248-24.97-14.535L50.012 43.3z" fill="#c8d92b" />
+         <path d="M74.982 86.725v-28.89l-24.97-14.517V72.19l.018.018z" fill="#c2d02f" />
+         <path d="M99.952 43.479L75.197 57.727l-.215.107v28.89H75l24.952-14.356z" fill="#b9be33" />
+         <path d="M99.952 72.369l.215-.125 24.755-14.23-24.97-14.535-24.97 14.356z" fill="#33b4d7" />
+         <use xlink:href="#B" y="-34.529" fill="#209ac5" />
+         <path d="M124.922 58.014l-24.755 14.23-.215.125v28.89h.018l24.952-14.356z" fill="#0f8cae" />
+         <path d="M25.06 57.673l.197-.125L50.012 43.3 25.06 28.783h-.018L.089 43.139l24.952 14.535z" fill="#94d6e3" />
+         <path d="M25.042 86.546h.018V57.673L.089 43.139v28.89z" fill="#73ccdd" />
+         <path d="M50.012 43.318l-24.755 14.23-.197.125v28.872.018l24.952-14.356h.018v-.018-28.872z" fill="#3bafbb" />
+         <path d="M50.03 72.19l.215-.107 24.737-14.248L50.03 43.318 25.06 57.674z" fill="#94d6e3" />
+         <path d="M50.03 101.08V72.208v-.018L25.06 57.674v28.872.018z" fill="#73ccdd" />
+         <path d="M74.982 57.835L50.227 72.083l-.197.107v28.89L75 86.725v-28.89z" fill="#3bafbb" />
+         <path d="M75 86.724l.197-.107 24.755-14.248L75 57.834h-.018L50.029 72.189l24.952 14.535z" fill="#33b4d7" />
+         <path d="M74.982 115.614H75v-28.89l-24.97-14.517v28.872.018z" fill="#209ac5" />
+         <path d="M99.952 72.368L75.197 86.617l-.197.107v28.89l24.97-14.356v-28.89z" fill="#0f8cae" />
+      </g>
+      <path d="M759.126 567.007s10.273 21.02 16.364 35.698c25.549 33.869 90.792 36.224 119.235.656 9.484-17.619 16.733-36.357 16.733-36.357-7.297 10.862-20.094 18.056-27.408 22.095-5.197 2.861-17.189 4.59-17.189 4.59l-31.482-16.393-31.663 16.065s-11.832-1.91-17.189-4.426c-10.811-5.799-19.735-12.549-27.401-21.928z" fill="#ffe953" />
+   </g>
+   <defs>
+      <path id="B" d="M99.952 135.788v-28.89l-24.97-14.517v28.872l.018.018z" />
+   </defs>
+</svg>

.github/workflows/bumper.yml

@@ -0,0 +1,43 @@
+name: Bumper
+on:
+  push:
+    branches:
+      - unstable
+      - bumper
+  workflow_dispatch:
+
+jobs:
+  bumpProjects:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        target: [
+          { repo: status-im/nimbus-eth2, branch: unstable },
+          { repo: status-im/nwaku, branch: master },
+          { repo: status-im/nim-codex, branch: main }
+        ]
+    steps:
+      - name: Clone repo
+        uses: actions/checkout@v2
+        with:
+          repository: ${{ matrix.target.repo }}
+          ref: ${{ matrix.target.branch }}
+          path: nbc
+          submodules: true
+          fetch-depth: 0
+          token:  ${{ secrets.ACTIONS_GITHUB_TOKEN }}
+
+      - name: Checkout this ref
+        run: |
+          cd nbc/vendor/nim-libp2p
+          git checkout $GITHUB_SHA
+
+      - name: Commit this bump
+        run: |
+          cd nbc
+          git config --global user.email "${{ github.actor }}@users.noreply.github.com"
+          git config --global user.name = "${{ github.actor }}"
+          git commit -a -m "auto-bump nim-libp2p"
+          git branch -D nim-libp2p-auto-bump-${GITHUB_REF##*/} || true
+          git switch -c nim-libp2p-auto-bump-${GITHUB_REF##*/}
+          git push -f origin nim-libp2p-auto-bump-${GITHUB_REF##*/}

.github/workflows/ci.yml

@@ -0,0 +1,168 @@
+name: CI
+on:
+  push:
+    branches:
+      - master
+      - unstable
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  build:
+    timeout-minutes: 90
+    strategy:
+      fail-fast: false
+      matrix:
+        target:
+          - os: linux
+            cpu: amd64
+          - os: linux
+            cpu: i386
+          - os: macos
+            cpu: amd64
+          - os: windows
+            cpu: amd64
+          #- os: windows
+            #cpu: i386
+        branch: [version-1-2, version-1-6]
+        include:
+          - target:
+              os: linux
+            builder: ubuntu-20.04
+            shell: bash
+          - target:
+              os: macos
+            builder: macos-12
+            shell: bash
+          - target:
+              os: windows
+            builder: windows-2019
+            shell: msys2 {0}
+
+    defaults:
+      run:
+        shell: ${{ matrix.shell }}
+
+    name: '${{ matrix.target.os }}-${{ matrix.target.cpu }} (Nim ${{ matrix.branch }})'
+    runs-on: ${{ matrix.builder }}
+    continue-on-error: ${{ matrix.branch == 'version-1-6' || matrix.branch == 'devel' }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          submodules: true
+
+      - name: Install build dependencies (Linux i386)
+        if: runner.os == 'Linux' && matrix.target.cpu == 'i386'
+        run: |
+          sudo dpkg --add-architecture i386
+          sudo apt-get update -qq
+          sudo DEBIAN_FRONTEND='noninteractive' apt-get install \
+            --no-install-recommends -yq gcc-multilib g++-multilib \
+            libssl-dev:i386
+          mkdir -p external/bin
+          cat << EOF > external/bin/gcc
+          #!/bin/bash
+          exec $(which gcc) -m32 "\$@"
+          EOF
+          cat << EOF > external/bin/g++
+          #!/bin/bash
+          exec $(which g++) -m32 "\$@"
+          EOF
+          chmod 755 external/bin/gcc external/bin/g++
+          echo '${{ github.workspace }}/external/bin' >> $GITHUB_PATH
+
+      - name: MSYS2 (Windows i386)
+        if: runner.os == 'Windows' && matrix.target.cpu == 'i386'
+        uses: msys2/setup-msys2@v2
+        with:
+          path-type: inherit
+          msystem: MINGW32
+          install: >-
+            base-devel
+            git
+            mingw-w64-i686-toolchain
+
+      - name: MSYS2 (Windows amd64)
+        if: runner.os == 'Windows' && matrix.target.cpu == 'amd64'
+        uses: msys2/setup-msys2@v2
+        with:
+          path-type: inherit
+          install: >-
+            base-devel
+            git
+            mingw-w64-x86_64-toolchain
+
+      - name: Restore Nim DLLs dependencies (Windows) from cache
+        if: runner.os == 'Windows'
+        id: windows-dlls-cache
+        uses: actions/cache@v2
+        with:
+          path: external/dlls
+          key: 'dlls'
+
+      - name: Install DLL dependencies (Windows)
+        if: >
+          steps.windows-dlls-cache.outputs.cache-hit != 'true' &&
+          runner.os == 'Windows'
+        run: |
+          mkdir external
+          curl -L "https://nim-lang.org/download/windeps.zip" -o external/windeps.zip
+          7z x external/windeps.zip -oexternal/dlls
+
+      - name: Path to cached dependencies (Windows)
+        if: >
+          runner.os == 'Windows'
+        run: |
+          echo '${{ github.workspace }}'"/external/dlls" >> $GITHUB_PATH
+
+      - name: Derive environment variables
+        run: |
+          if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then
+            PLATFORM=x64
+          else
+            PLATFORM=x86
+          fi
+          echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV
+
+          ncpu=
+          MAKE_CMD="make"
+          case '${{ runner.os }}' in
+          'Linux')
+            ncpu=$(nproc)
+            ;;
+          'macOS')
+            ncpu=$(sysctl -n hw.ncpu)
+            ;;
+          'Windows')
+            ncpu=$NUMBER_OF_PROCESSORS
+            MAKE_CMD="mingw32-make"
+            ;;
+          esac
+          [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
+          echo "ncpu=$ncpu" >> $GITHUB_ENV
+          echo "MAKE_CMD=${MAKE_CMD}" >> $GITHUB_ENV
+
+      - name: Build Nim and Nimble
+        run: |
+          curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh
+          env MAKE="${MAKE_CMD} -j${ncpu}" ARCH_OVERRIDE=${PLATFORM} NIM_COMMIT=${{ matrix.branch }} \
+            QUICK_AND_DIRTY_COMPILER=1 QUICK_AND_DIRTY_NIMBLE=1 CC=gcc \
+            bash build_nim.sh nim csources dist/nimble NimBinaries
+          echo '${{ github.workspace }}/nim/bin' >> $GITHUB_PATH
+
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: '~1.15.5'
+
+      - name: Install p2pd
+        run: |
+          V=1 bash scripts/build_p2pd.sh p2pdCache 124530a3
+
+      - name: Run tests
+        run: |
+          nim --version
+          nimble --version
+          nimble install_pinned
+          nimble test

.github/workflows/codecov.yml

@@ -0,0 +1,140 @@
+name: nim-libp2p codecov builds
+
+on:
+  #On push to common branches, this computes the "bases stats" for PRs
+  push:
+    branches:
+      - master
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  GossipSub:
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        nim-options: [
+          "",
+          "-d:libp2p_pubsub_anonymize=true -d:libp2p_pubsub_sign=false -d:libp2p_pubsub_verify=false",
+          "-d:libp2p_pubsub_sign=true -d:libp2p_pubsub_verify=true"
+        ]
+        test-program: [
+          "tests/pubsub/testpubsub",
+          "tests/pubsub/testfloodsub",
+          "tests/pubsub/testgossipinternal"
+        ]
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Run
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y lcov build-essential git curl
+          mkdir coverage
+          curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh
+          env MAKE="make -j${NPROC}" bash build_nim.sh Nim csources dist/nimble NimBinaries
+          export PATH="$PATH:$PWD/Nim/bin"
+          nimble install_pinned
+          export NIM_OPTIONS="--opt:speed -d:debug --verbosity:0 --hints:off --lineDir:on -d:chronicles_log_level=INFO --warning[CaseTransition]:off --warning[ObservableStores]:off --warning[LockLevel]:off --nimcache:nimcache --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage ${{ matrix.nim-options }}"
+          nim c $NIM_OPTIONS -r ${{ matrix.test-program }}
+          cd nimcache; rm *.c; cd ..
+          lcov --capture --directory nimcache --output-file coverage/coverage.info
+          shopt -s globstar
+          ls `pwd`/libp2p/{*,**/*}.nim
+          lcov --extract coverage/coverage.info  `pwd`/libp2p/{*,**/*}.nim --output-file coverage/coverage.f.info
+          export COV_UUID=`cksum <<< "${{ matrix.test-program }} $NIM_OPTIONS" | cut -f 1 -d ' '`
+          genhtml coverage/coverage.f.info --output-directory coverage/$COV_UUID-output
+          echo ${{ matrix.test-program }} > coverage/$COV_UUID-nim_options.txt
+          echo $NIM_OPTIONS >> coverage/$COV_UUID-nim_options.txt
+          bash <(curl -s https://codecov.io/bash) -f coverage/coverage.f.info || echo "Codecov did not collect coverage reports"
+      - uses: actions/upload-artifact@master
+        with:
+          name: coverage
+          path: coverage
+
+  Tests:
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        nim-options: [
+          ""
+        ]
+        test-program: [
+          "tests/testnative",
+        ]
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Run
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y lcov build-essential git curl
+          mkdir coverage
+          curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh
+          env MAKE="make -j${NPROC}" bash build_nim.sh Nim csources dist/nimble NimBinaries
+          export PATH="$PATH:$PWD/Nim/bin"
+          nimble install_pinned
+          export NIM_OPTIONS="--opt:speed -d:debug --verbosity:0 --hints:off --lineDir:on -d:chronicles_log_level=INFO --warning[CaseTransition]:off --warning[ObservableStores]:off --warning[LockLevel]:off --nimcache:nimcache --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage ${{ matrix.nim-options }} --clearNimblePath --NimblePath:nimbledeps/pkgs"
+          nim c $NIM_OPTIONS -r ${{ matrix.test-program }}
+          cd nimcache; rm *.c; cd ..
+          lcov --capture --directory nimcache --output-file coverage/coverage.info
+          shopt -s globstar
+          ls `pwd`/libp2p/{*,**/*}.nim
+          lcov --extract coverage/coverage.info  `pwd`/libp2p/{*,**/*}.nim --output-file coverage/coverage.f.info
+          export COV_UUID=`cksum <<< "${{ matrix.test-program }} $NIM_OPTIONS" | cut -f 1 -d ' '`
+          genhtml coverage/coverage.f.info --output-directory coverage/$COV_UUID-output
+          echo ${{ matrix.test-program }} > coverage/$COV_UUID-nim_options.txt
+          echo $NIM_OPTIONS >> coverage/$COV_UUID-nim_options.txt
+          bash <(curl -s https://codecov.io/bash) -f coverage/coverage.f.info || echo "Codecov did not collect coverage reports"
+      - uses: actions/upload-artifact@master
+        with:
+          name: coverage
+          path: coverage
+
+  Filter:
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        nim-options: [
+          "",
+          "-d:libp2p_pki_schemes=secp256k1",
+          "-d:libp2p_pki_schemes=secp256k1;ed25519",
+          "-d:libp2p_pki_schemes=secp256k1;ed25519;ecnist",
+        ]
+        test-program: [
+          "tests/testpkifilter",
+        ]
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Run
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y lcov build-essential git curl
+          mkdir coverage
+          curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh
+          env MAKE="make -j${NPROC}" bash build_nim.sh Nim csources dist/nimble NimBinaries
+          export PATH="$PATH:$PWD/Nim/bin"
+          nimble install_pinned
+          export NIM_OPTIONS="--opt:speed -d:debug --verbosity:0 --hints:off --lineDir:on -d:chronicles_log_level=INFO --warning[CaseTransition]:off --warning[ObservableStores]:off --warning[LockLevel]:off --nimcache:nimcache --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage ${{ matrix.nim-options }}"
+          nim c $NIM_OPTIONS -r ${{ matrix.test-program }}
+          cd nimcache; rm *.c; cd ..
+          lcov --capture --directory nimcache --output-file coverage/coverage.info
+          shopt -s globstar
+          ls `pwd`/libp2p/{*,**/*}.nim
+          lcov --extract coverage/coverage.info  `pwd`/libp2p/{*,**/*}.nim --output-file coverage/coverage.f.info
+          export COV_UUID=`cksum <<< "${{ matrix.test-program }} $NIM_OPTIONS" | cut -f 1 -d ' '`
+          genhtml coverage/coverage.f.info --output-directory coverage/$COV_UUID-output
+          echo ${{ matrix.test-program }} > coverage/$COV_UUID-nim_options.txt
+          echo $NIM_OPTIONS >> coverage/$COV_UUID-nim_options.txt
+          bash <(curl -s https://codecov.io/bash) -f coverage/coverage.f.info || echo "Codecov did not collect coverage reports"
+      - uses: actions/upload-artifact@master
+        with:
+          name: coverage
+          path: coverage
+
+
+

.github/workflows/doc.yml

@@ -0,0 +1,103 @@
+name: Docgen
+on:
+  push:
+  workflow_dispatch:
+
+
+jobs:
+  build:
+    timeout-minutes: 20
+
+    name: 'Generate & upload documentation'
+    runs-on: 'ubuntu-20.04'
+    continue-on-error: true
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          submodules: true
+
+      - uses: jiro4989/setup-nim-action@v1
+        with:
+          nim-version: 'stable'
+
+      - name: Generate doc
+        run: |
+          nim --version
+          nimble --version
+          nimble install_pinned
+          # nim doc can "fail", but the doc is still generated
+          nim doc --git.url:https://github.com/status-im/nim-libp2p --git.commit:${GITHUB_REF##*/} --outdir:${GITHUB_REF##*/} --project libp2p || true
+
+          # check that the folder exists
+          ls ${GITHUB_REF##*/}
+
+      - name: Clone the gh-pages branch
+        uses: actions/checkout@v2
+        with:
+          repository: status-im/nim-libp2p
+          ref: gh-pages
+          path: subdoc
+          submodules: true
+          fetch-depth: 0
+
+      - name: Commit & push
+        run: |
+          cd subdoc
+
+          # Delete merged branches doc's
+          for branch in $(git branch -vv | grep ': gone]' | awk '{print $1}'); do rm -rf $branch; done
+
+          # Update / create this branch doc
+          rm -rf ${GITHUB_REF##*/}
+          mv ../${GITHUB_REF##*/} .
+
+          # Remove .idx files
+          # NOTE: git also uses idx files in his
+          # internal folder, hence the `*` instead of `.`
+          find * -name "*.idx" -delete
+          git add .
+          git config --global user.email "${{ github.actor }}@users.noreply.github.com"
+          git config --global user.name = "${{ github.actor }}"
+          git commit -a -m "update docs for ${GITHUB_REF##*/}"
+          git push origin gh-pages
+
+  update_site:
+    if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/docs'
+    name: 'Rebuild website'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.x
+
+      - uses: jiro4989/setup-nim-action@v1
+        with:
+          nim-version: 'stable'
+
+      - name: Generate website
+        run: pip install mkdocs-material && nimble website
+
+      - name: Clone the gh-pages branch
+        uses: actions/checkout@v2
+        with:
+          repository: status-im/nim-libp2p
+          ref: gh-pages
+          path: subdoc
+          fetch-depth: 0
+
+      - name: Commit & push
+        run: |
+          cd subdoc
+
+          rm -rf docs
+          mv ../site docs
+
+          git add .
+          git config --global user.email "${{ github.actor }}@users.noreply.github.com"
+          git config --global user.name = "${{ github.actor }}"
+          git commit -a -m "update website"
+          git push origin gh-pages

.github/workflows/multi_nim.yml

@@ -0,0 +1,171 @@
+name: Daily
+on:
+  schedule:
+    - cron: "30 6 * * *"
+  workflow_dispatch:
+
+jobs:
+  build:
+    timeout-minutes: 120
+    strategy:
+      fail-fast: false
+      matrix:
+        target:
+          - os: linux
+            cpu: amd64
+          - os: linux
+            cpu: i386
+          - os: macos
+            cpu: amd64
+          - os: windows
+            cpu: amd64
+          #- os: windows
+            #cpu: i386
+        branch: [version-1-2, version-1-4, version-1-6, devel]
+        include:
+          - target:
+              os: linux
+            builder: ubuntu-20.04
+            shell: bash
+          - target:
+              os: macos
+            builder: macos-12
+            shell: bash
+          - target:
+              os: windows
+            builder: windows-2019
+            shell: msys2 {0}
+
+    defaults:
+      run:
+        shell: ${{ matrix.shell }}
+
+    name: '${{ matrix.target.os }}-${{ matrix.target.cpu }} (Nim ${{ matrix.branch }})'
+    runs-on: ${{ matrix.builder }}
+    continue-on-error: ${{ matrix.branch == 'version-1-6' || matrix.branch == 'devel' }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          ref: unstable
+          submodules: true
+
+      - name: Install build dependencies (Linux i386)
+        if: runner.os == 'Linux' && matrix.target.cpu == 'i386'
+        run: |
+          sudo dpkg --add-architecture i386
+          sudo apt-get update -qq
+          sudo DEBIAN_FRONTEND='noninteractive' apt-get install \
+            --no-install-recommends -yq gcc-multilib g++-multilib \
+            libssl-dev:i386
+          mkdir -p external/bin
+          cat << EOF > external/bin/gcc
+          #!/bin/bash
+          exec $(which gcc) -m32 "\$@"
+          EOF
+          cat << EOF > external/bin/g++
+          #!/bin/bash
+          exec $(which g++) -m32 "\$@"
+          EOF
+          chmod 755 external/bin/gcc external/bin/g++
+          echo '${{ github.workspace }}/external/bin' >> $GITHUB_PATH
+
+      - name: MSYS2 (Windows i386)
+        if: runner.os == 'Windows' && matrix.target.cpu == 'i386'
+        uses: msys2/setup-msys2@v2
+        with:
+          path-type: inherit
+          msystem: MINGW32
+          install: >-
+            base-devel
+            git
+            mingw-w64-i686-toolchain
+
+      - name: MSYS2 (Windows amd64)
+        if: runner.os == 'Windows' && matrix.target.cpu == 'amd64'
+        uses: msys2/setup-msys2@v2
+        with:
+          path-type: inherit
+          install: >-
+            base-devel
+            git
+            mingw-w64-x86_64-toolchain
+
+      - name: Restore Nim DLLs dependencies (Windows) from cache
+        if: runner.os == 'Windows'
+        id: windows-dlls-cache
+        uses: actions/cache@v2
+        with:
+          path: external/dlls
+          key: 'dlls'
+
+      - name: Install DLL dependencies (Windows)
+        if: >
+          steps.windows-dlls-cache.outputs.cache-hit != 'true' &&
+          runner.os == 'Windows'
+        run: |
+          mkdir external
+          curl -L "https://nim-lang.org/download/windeps.zip" -o external/windeps.zip
+          7z x external/windeps.zip -oexternal/dlls
+
+      - name: Path to cached dependencies (Windows)
+        if: >
+          runner.os == 'Windows'
+        run: |
+          echo '${{ github.workspace }}'"/external/dlls" >> $GITHUB_PATH
+
+      - name: Derive environment variables
+        run: |
+          if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then
+            PLATFORM=x64
+          else
+            PLATFORM=x86
+          fi
+          echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV
+
+          ncpu=
+          MAKE_CMD="make"
+          case '${{ runner.os }}' in
+          'Linux')
+            ncpu=$(nproc)
+            ;;
+          'macOS')
+            ncpu=$(sysctl -n hw.ncpu)
+            ;;
+          'Windows')
+            ncpu=$NUMBER_OF_PROCESSORS
+            MAKE_CMD="mingw32-make"
+            ;;
+          esac
+          [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
+          echo "ncpu=$ncpu" >> $GITHUB_ENV
+          echo "MAKE_CMD=${MAKE_CMD}" >> $GITHUB_ENV
+
+      - name: Build Nim and Nimble
+        run: |
+          curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh
+          env MAKE="${MAKE_CMD} -j${ncpu}" ARCH_OVERRIDE=${PLATFORM} NIM_COMMIT=${{ matrix.branch }} \
+            QUICK_AND_DIRTY_COMPILER=1 QUICK_AND_DIRTY_NIMBLE=1 CC=gcc \
+            bash build_nim.sh nim csources dist/nimble NimBinaries
+          echo '${{ github.workspace }}/nim/bin' >> $GITHUB_PATH
+
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: '~1.15.5'
+
+      - name: Install p2pd
+        run: |
+          V=1 bash scripts/build_p2pd.sh p2pdCache 124530a3
+
+      - name: Run tests
+        run: |
+          nim --version
+          nimble --version
+          nimble install -y --depsOnly
+          nimble test
+          if [[ "${{ matrix.branch }}" == "version-1-6" || "${{ matrix.branch }}" == "devel" ]]; then
+            echo -e "\nTesting with '--gc:orc':\n"
+            export NIMFLAGS="${NIMFLAGS} --gc:orc"
+            nimble test
+          fi

.gitignore

@@ -1,4 +1,5 @@
 nimcache/
+nimbledeps/
 
 # Executables shall be put in an ignored build/ directory
 # Ignore dynamic, static libs and libtool archive files
@@ -10,3 +11,8 @@ build/
 *.exe
 *.dll
 .vscode/
+.DS_Store
+tests/pubsub/testgossipsub
+examples/*.md
+nimble.develop
+nimble.paths

.pinned

@@ -0,0 +1,16 @@
+bearssl;https://github.com/status-im/nim-bearssl@#f4c4233de453cb7eac0ce3f3ffad6496295f83ab
+chronicles;https://github.com/status-im/nim-chronicles@#32ac8679680ea699f7dbc046e8e0131cac97d41a
+chronos;https://github.com/status-im/nim-chronos@#9df76c39df254c7ff0cec6dec5c9f345f2819c91
+dnsclient;https://github.com/ba0f3/dnsclient.nim@#6647ca8bd9ffcc13adaecb9cb6453032063967db
+faststreams;https://github.com/status-im/nim-faststreams@#6112432b3a81d9db116cd5d64c39648881cfff29
+httputils;https://github.com/status-im/nim-http-utils@#e88e231dfcef4585fe3b2fbd9b664dbd28a88040
+json_serialization;https://github.com/status-im/nim-json-serialization@#e5b18fb710c3d0167ec79f3b892f5a7a1bc6d1a4
+metrics;https://github.com/status-im/nim-metrics@#0a6477268e850d7bc98347b3875301524871765f
+nimcrypto;https://github.com/cheatfate/nimcrypto@#24e006df85927f64916e60511620583b11403178
+secp256k1;https://github.com/status-im/nim-secp256k1@#c7f1a37d9b0f17292649bfed8bf6cef83cf4221f
+serialization;https://github.com/status-im/nim-serialization@#493d18b8292fc03aa4f835fd825dea1183f97466
+stew;https://github.com/status-im/nim-stew@#f2e58ba4c8da65548c824e4fa8732db9739f6505
+testutils;https://github.com/status-im/nim-testutils@#dfc4c1b39f9ded9baf6365014de2b4bfb4dafc34
+unittest2;https://github.com/status-im/nim-unittest2@#f180f596c88dfd266f746ed6f8dbebce39c824db
+websock;https://github.com/status-im/nim-websock@#2424f2b215c0546f97d8b147e21544521c7545b0
+zlib;https://github.com/status-im/nim-zlib@#6a6670afba6b97b29b920340e2641978c05ab4d8

.travis.yml

@@ -1,55 +0,0 @@
-language: go
-
-dist: bionic
-
-# https://docs.travis-ci.com/user/caching/
-cache:
-  directories:
-    - NimBinaries
-    - p2pdCache
-
-git:
-  # when multiple CI builds are queued, the tested commit needs to be in the last X commits cloned with "--depth X"
-  depth: 10
-
-go: "1.14.x"
-
-matrix:
-  include:
-    - os: linux
-      arch: amd64
-      env:
-        - NPROC=2
-      before_install:
-        - export GOPATH=$HOME/go
-    ## arm64 is very unreliable and slow, disabled for now
-    # - os: linux
-    #   arch: arm64
-    #   env:
-    #     - NPROC=6 # Worth trying more than 2 parallel jobs: https://travis-ci.community/t/no-cache-support-on-arm64/5416/8
-    #     # (also used to get a different cache key than the amd64 one)
-    #   before_install:
-    #     - export GOPATH=$HOME/go
-    - os: osx
-      env:
-        - NPROC=2
-      before_install:
-        - export GOPATH=$HOME/go
-
-install:
-  # build nim from our own branch - this to avoid the day-to-day churn and
-  # regressions of the fast-paced Nim development while maintaining the
-  # flexibility to apply patches
-  - curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh
-  - env MAKE="make -j${NPROC}" bash build_nim.sh Nim csources dist/nimble NimBinaries
-  - export PATH="$PWD/Nim/bin:$GOPATH/bin:$PATH"
-
-  # install and build go-libp2p-daemon
-  - curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_p2pd.sh
-  - bash build_p2pd.sh p2pdCache v0.2.4
-
-script:
-  - nimble install -y --depsOnly
-  - nimble test
-  - nimble examples_build
-

README.md

@@ -1,83 +1,102 @@
 <h1 align="center">
-  <a href="libp2p.io"><img width="250" src="https://github.com/libp2p/libp2p/blob/master/logo/black-bg-2.png?raw=true" alt="libp2p hex logo" /></a>
+  <a href="https://libp2p.io"><img width="250" src="./.assets/full-logo.svg?raw=true" alt="nim-libp2p logo" /></a>
 </h1>
 
 <h3 align="center">The Nim implementation of the libp2p Networking Stack.</h3>
 
 <p align="center">
-<a href="https://travis-ci.org/status-im/nim-libp2p"><img src="https://travis-ci.org/status-im/nim-libp2p.svg?branch=master" /></a>
-<a href="https://ci.appveyor.com/project/nimbus/nim-libp2p/branch/master"><img src="https://ci.appveyor.com/api/projects/status/pqgif5bcie6cp3wi/branch/master?svg=true" /></a>
-<a href="https://dev.azure.com/nimbus-dev/nim-libp2p/_build?definitionId=5&branchName=master"><img src="https://img.shields.io/azure-devops/build/nimbus-dev/dc5eed24-3f6c-4c06-8466-3d060abd6c8b/5/master?label=Azure%20%28Linux%2064-bit%2C%20Windows%2032-bit%2F64-bit%2C%20MacOS%2064-bit%29" /></a>
+<a href="https://github.com/status-im/nim-libp2p/actions"><img src="https://github.com/status-im/nim-libp2p/actions/workflows/ci.yml/badge.svg" /></a>
+<a href="https://codecov.io/gh/status-im/nim-libp2p"><img src="https://codecov.io/gh/status-im/nim-libp2p/branch/master/graph/badge.svg?token=UR5JRQ249W"/></a>
+
 </p>
 
 <p align="center">
 <a href="https://opensource.org/licenses/Apache-2.0"><img src="https://img.shields.io/badge/License-Apache%202.0-blue.svg" /></a>
 <a href="https://opensource.org/licenses/MIT"><img src="https://img.shields.io/badge/License-MIT-blue.svg" /></a>
-<a href=""><img src="https://img.shields.io/badge/nim-%3E%3D1.0.6-orange.svg?style=flat-square" /></a>
+<img src="https://img.shields.io/badge/nim-%3E%3D1.2.0-orange.svg?style=flat-square" />
 </p>
 
 ## Introduction
 
-An implementation of [libp2p](https://libp2p.io/) in Nim. Also provides a Nim wrapper of the [Libp2p Go daemon](https://github.com/libp2p/go-libp2p).
-
-## Project Status
-The current native Nim libp2p implementation support is experimental and shouldn't be relied on for production use. It is under active development and contributions are highly welcomed. :)
-
-Check our [examples folder](/examples) to get started! 
+An implementation of [libp2p](https://libp2p.io/) in [Nim](https://nim-lang.org/).
 
 # Table of Contents
 - [Background](#background)
 - [Install](#install)
-  - [Prerequisite](#prerequisite)
-- [Usage](#usage)
-  - [API](#api)
-  - [Getting Started](#getting-started)
-  - [Tutorials and Examples](#tutorials-and-examples)
-  - [Using the Go Daemon](#using-the-go-daemon)
+- [Getting Started](#getting-started)
+- [Modules](#modules)
+- [Users](#users)
 - [Development](#development)
-  - [Tests](#tests)
-  - [Packages](#packages)
-- [Contribute](#contribute)
-  - [Core Developers](#core-developers) 
+  - [Contribute](#contribute)
+  - [Core Developers](#core-developers)
 - [License](#license)
 
-## Background   
+## Background
 libp2p is a networking stack and library modularized out of [The IPFS Project](https://github.com/ipfs/ipfs), and bundled separately for other tools to use.
 
-libp2p is the product of a long and arduous quest of understanding; a deep dive into the internet's network stack and the peer-to-peer protocols from the past. Building large scale peer-to-peer systems has been complex and difficult in the last 15 years and libp2p is a way to fix that. It is a "network stack", a suite of networking protocols that cleanly separates concerns and enables sophisticated applications to only use the protocols they absolutely need, without giving up interoperability and upgradeability. 
+libp2p is the product of a long and arduous quest of understanding; a deep dive into the internet's network stack and the peer-to-peer protocols from the past. Building large scale peer-to-peer systems has been complex and difficult in the last 15 years and libp2p is a way to fix that. It is a "network stack", a suite of networking protocols that cleanly separates concerns and enables sophisticated applications to only use the protocols they absolutely need, without giving up interoperability and upgradeability.
 
 libp2p grew out of IPFS, but it is built so that lots of people can use it, for lots of different projects.
 
-- Learn more about libp2p at [**libp2p.io**](https://libp2p.io) and follow our evolving documentation efforts at [**docs.libp2p.io**](https://docs.libp2p.io). 
-- [Here](https://github.com/libp2p/libp2p#description) is an overview of libp2p and its implementations in other programming languages. 
+- Learn more about libp2p at [**libp2p.io**](https://libp2p.io) and follow our evolving documentation efforts at [**docs.libp2p.io**](https://docs.libp2p.io).
+- [Here](https://github.com/libp2p/libp2p#description) is an overview of libp2p and its implementations in other programming languages.
 
 ## Install
+**Prerequisite**
+- [Nim](https://nim-lang.org/install.html)
 ```
 nimble install libp2p
 ```
-### Prerequisite 
-- [Nim](https://nim-lang.org/install.html)
 
-## Usage
+## Getting Started
+You'll find the documentation [here](https://status-im.github.io/nim-libp2p/docs/).
 
-### API
-The specification is available in the [docs/api](docs/api) folder.
+**Go Daemon:**
+Please find the installation and usage intructions in [daemonapi.md](examples/go-daemon/daemonapi.md).
 
-### Getting Started
-Please read the [GETTING_STARTED.md](docs/GETTING_STARTED.md) guide. 
+## Modules
 
-### Tutorials and Examples 
-Example code can be found in the [examples folder](/examples).
-#### Direct Chat Tutorial
-- [Part I](https://our.status.im/nim-libp2p-tutorial-a-peer-to-peer-chat-example-1/): Set up the main function and use multi-thread for processing IO.
-- [Part II](https://our.status.im/nim-libp2p-tutorial-a-peer-to-peer-chat-example-2/): Dial remote peer and allow customized user input commands.
-- [Part III](https://our.status.im/nim-libp2p-tutorial-a-peer-to-peer-chat-example-3/): Configure and establish a libp2p node. 
+List of packages modules implemented in nim-libp2p:
 
+| Name                                                       | Description                                                                                                      |
+| ---------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------- |
+| **Libp2p**                                                 |                                                                                                                  |
+| [libp2p](libp2p/switch.nim)                                | The core of the project                                                                                          |
+| [connmanager](libp2p/connmanager.nim)                      | Connection manager                                                                                               |
+| [identify / push identify](libp2p/protocols/identify.nim)  | [Identify](https://docs.libp2p.io/concepts/protocols/#identify) protocol                                         |
+| [ping](libp2p/protocols/ping.nim)                          | [Ping](https://docs.libp2p.io/concepts/protocols/#ping) protocol                                                 |
+| [libp2p-daemon-client](libp2p/daemon/daemonapi.nim)        | [go-daemon](https://github.com/libp2p/go-libp2p-daemon) nim wrapper                                              |
+| [interop-libp2p](tests/testinterop.nim)                    | Interop tests                                                                                                    |
+| **Transports**                                             |                                                                                                                  |
+| [libp2p-tcp](libp2p/transports/tcptransport.nim)           | TCP transport                                                                                                    |
+| [libp2p-ws](libp2p/transports/wstransport.nim)             | WebSocket & WebSocket Secure transport                                                                           |
+| **Secure Channels**                                        |                                                                                                                  |
+| [libp2p-secio](libp2p/protocols/secure/secio.nim)          | [Secio](https://docs.libp2p.io/concepts/protocols/#secio) secure channel                                         |
+| [libp2p-noise](libp2p/protocols/secure/noise.nim)          | [Noise](https://github.com/libp2p/specs/tree/master/noise) secure channel                                        |
+| [libp2p-plaintext](libp2p/protocols/secure/plaintext.nim)  | [Plain Text](https://github.com/libp2p/specs/tree/master/plaintext) for development purposes                     |
+| **Stream Multiplexers**                                    |                                                                                                                  |
+| [libp2p-mplex](libp2p/muxers/mplex/mplex.nim)              | [MPlex](https://github.com/libp2p/specs/tree/master/mplex) multiplexer                                           |
+| **Data Types**                                             |                                                                                                                  |
+| [peer-id](libp2p/peerid.nim)                               | [Cryptographic identifiers](https://docs.libp2p.io/concepts/peer-id/)                                            |
+| [peer-store](libp2p/peerstore.nim)                         | ["Phone book" of known peers](https://docs.libp2p.io/concepts/peer-id/#peerinfo)                                 |
+| [multiaddress](libp2p/multiaddress.nim)                    | [Composable network addresses](https://github.com/multiformats/multiaddr)                                        |
+| [signed envelope](libp2p/signed_envelope.nim)              | [Signed generic data container](https://github.com/libp2p/specs/blob/master/RFC/0002-signed-envelopes.md)        |
+| [routing record](libp2p/routing_record.nim)                | [Signed peer dialing informations](https://github.com/libp2p/specs/blob/master/RFC/0003-routing-records.md)      |
+| **Utilities**                                              |                                                                                                                  |
+| [libp2p-crypto](libp2p/crypto)                             | Cryptographic backend                                                                                            |
+| [libp2p-crypto-secp256k1](libp2p/crypto/secp.nim)          |                                                                                                                  |
+| **Pubsub**                                                 |                                                                                                                  |
+| [libp2p-pubsub](libp2p/protocols/pubsub/pubsub.nim)        | Pub-Sub generic interface                                                                                        |
+| [libp2p-floodsub](libp2p/protocols/pubsub/floodsub.nim)    | FloodSub implementation                                                                                          |
+| [libp2p-gossipsub](libp2p/protocols/pubsub/gossipsub.nim)  | [GossipSub](https://docs.libp2p.io/concepts/publish-subscribe/) implementation                                   |
 
-### Using the Go Daemon
-Please find the installation and usage intructions in [daemonapi.md](docs/api/libp2p/daemonapi.md). 
+## Users
 
-Examples can be found in the [examples/go-daemon folder](https://github.com/status-im/nim-libp2p/tree/readme/examples/go-daemon);
+nim-libp2p is used by:
+- [Nimbus](https://github.com/status-im/nimbus-eth2), an Ethereum client
+- [nwaku](https://github.com/status-im/nwaku), a decentralized messaging application
+- [nim-codex](https://github.com/status-im/nim-codex), a decentralized storage application
+- (open a pull request if you want to be included here)
 
 ## Development
 **Clone and Install dependencies:**
@@ -88,75 +107,43 @@ cd nim-libp2p
 nimble install
 ```
 
-### Tests 
-
-#### Prerequisite
-- [Go 1.12+](https://golang.org/dl/) 
-
-#### Run unit tests
+**Run unit tests**
 ```sh
 # run all the unit tests
 nimble test
 ```
 
-### Packages 
+### Contribute
 
-List of packages currently in existence for nim-libp2p:
-
-#### Libp2p
-- [libp2p](https://github.com/status-im/nim-libp2p)
-- [libp2p-daemon-client](https://github.com/status-im/nim-libp2p/blob/master/libp2p/daemon/daemonapi.nim)
-- [interop-libp2p](https://github.com/status-im/nim-libp2p/blob/master/tests/testinterop.nim)
-
-#### Transports
-- [libp2p-tcp](https://github.com/status-im/nim-libp2p/blob/master/libp2p/transports/tcptransport.nim)
-
-#### Secure Channels
-- [libp2p-secio](https://github.com/status-im/nim-libp2p/blob/master/libp2p/protocols/secure/secio.nim)
-
-#### Stream Multiplexers
-- [libp2p-mplex](https://github.com/status-im/nim-libp2p/blob/master/libp2p/muxers/mplex/mplex.nim)
-
-#### Utilities
-- [libp2p-crypto](https://github.com/status-im/nim-libp2p/tree/master/libp2p/crypto)
-- [libp2p-crypto-secp256k1](https://github.com/status-im/nim-libp2p/blob/master/libp2p/crypto/secp.nim)
-
-#### Data Types
-- [peer-id](https://github.com/status-im/nim-libp2p/blob/master/libp2p/peer.nim)
-- [peer-info](https://github.com/status-im/nim-libp2p/blob/master/libp2p/peerinfo.nim)
-
-#### Pubsub
-- [libp2p-pubsub](https://github.com/status-im/nim-libp2p/blob/master/libp2p/protocols/pubsub/pubsub.nim)
-- [libp2p-floodsub](https://github.com/status-im/nim-libp2p/blob/master/libp2p/protocols/pubsub/floodsub.nim)
-- [libp2p-gossipsub](https://github.com/status-im/nim-libp2p/blob/master/libp2p/protocols/pubsub/gossipsub.nim)
-
-
-Packages that exist in the original libp2p specs and are under active development: 
-- libp2p-daemon
-- libp2p-webrtc-direct
-- libp2p-webrtc-star
-- libp2p-websockets
-- libp2p-spdy
-- libp2p-bootstrap
-- libp2p-kad-dht
-- libp2p-mdns
-- libp2p-webrtc-star
-- libp2p-delegated-content-routing
-- libp2p-delegated-peer-routing
-- libp2p-nat-mgnr
-- libp2p-utils
-
-** Note that the current stack reflects the minimal requirements for the upcoming Eth2 implementation. 
-
-
-## Contribute
 The libp2p implementation in Nim is a work in progress. We welcome contributors to help out! Specifically, you can:
-- Go through the modules and **check out existing issues**. This would be especially useful for modules in active development. Some knowledge of IPFS/libp2p may be required, as well as the infrastructure behind it. 
-- **Perform code reviews**. Feel free to let us know if you found anything that can a) speed up the project development b) ensure better quality and c) reduce possible future bugs. 
-- **Add tests**. Help nim-libp2p to be more robust by adding more tests to the [tests folder](https://github.com/status-im/nim-libp2p/tree/master/tests).
+- Go through the modules and **check out existing issues**. This would be especially useful for modules in active development. Some knowledge of IPFS/libp2p may be required, as well as the infrastructure behind it.
+- **Perform code reviews**. Feel free to let us know if you found anything that can a) speed up the project development b) ensure better quality and c) reduce possible future bugs.
+- **Add tests**. Help nim-libp2p to be more robust by adding more tests to the [tests folder](tests/).
 
-### Core Developers 
-[@cheatfate](https://github.com/cheatfate), [Dmitriy Ryajov](https://github.com/dryajov), [Giovanni Petrantoni](https://github.com/sinkingsugar), [Zahary Karadjov](https://github.com/zah)
+The code follows the [Status Nim Style Guide](https://status-im.github.io/nim-style-guide/).
+
+### Core Developers
+[@cheatfate](https://github.com/cheatfate), [Dmitriy Ryajov](https://github.com/dryajov), [Tanguy](https://github.com/Menduist), [Zahary Karadjov](https://github.com/zah)
+
+### Tips and tricks
+
+**enable expensive metrics:**
+
+```bash
+nim c -d:libp2p_expensive_metrics some_file.nim
+```
+
+**use identify metrics**
+
+```bash
+nim c -d:libp2p_agents_metrics -d:KnownLibP2PAgents=nimbus,lighthouse,prysm,teku some_file.nim
+```
+
+**specify gossipsub specific topics to measure**
+
+```bash
+nim c -d:KnownLibP2PTopics=topic1,topic2,topic3 some_file.nim
+```
 
 ## License
 
@@ -168,4 +155,4 @@ or
 
 * Apache License, Version 2.0, ([LICENSE-APACHEv2](LICENSE-APACHEv2) or http://www.apache.org/licenses/LICENSE-2.0)
 
-at your option. This file may not be copied, modified, or distributed except according to those terms.
+at your option. These files may not be copied, modified, or distributed except according to those terms.

azure-pipelines.yml

@@ -1,119 +0,0 @@
-strategy:
-  maxParallel: 10
-  matrix:
-    # Nim requires enforcing ARCH="x86" and UCPU
-    # for 32-bit targets as it seems like Azure machines are 64-bit
-    # TEST_LANG env variable support TODO
-    Windows_32bit:
-      VM: 'windows-latest'
-      ARCH: x86
-      PLATFORM: x86
-      TEST_LANG: c
-    Windows_64bit:
-      VM: 'windows-latest'
-      PLATFORM: x64
-      TEST_LANG: c
-
-pool:
-  vmImage: $(VM)
-
-variables:
-  V: 0 # Scripts verbosity, 1 for debugging build scripts
-
-steps:
-  - task: CacheBeta@1
-    displayName: 'cache Nim binaries'
-    inputs:
-      key: NimBinaries | $(Agent.OS) | $(PLATFORM) | "$(Build.SourceBranchName)" | "v4"
-      path: NimBinaries
-
-  - task: CacheBeta@1
-    displayName: 'cache Go libp2p daemon'
-    inputs:
-      key: p2pdCache | $(Agent.OS) | $(PLATFORM) | "v3"
-      path: p2pdCache
-
-  - task: CacheBeta@1
-    displayName: 'cache MinGW-w64'
-    inputs:
-      key: mingwCache | 8_1_0 | $(PLATFORM) | "v2"
-      path: mingwCache
-
-  - powershell: |
-      Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem' -Name 'LongPathsEnabled' -Value 1
-    displayName: 'long path support'
-
-  - bash: |
-      set -e
-
-      # custom MinGW-w64 versions for both 32-bit and 64-bit, since we need a 64-bit build of p2pd
-      echo "Installing MinGW-w64"
-
-      install_mingw() {
-        mkdir -p mingwCache
-        cd mingwCache
-        if [[ ! -e "$MINGW_FILE" ]]; then
-          curl -OLsS "$MINGW_URL"
-        fi
-        7z x -y -bd "$MINGW_FILE" >/dev/null
-        mkdir -p /c/custom
-        mv "$MINGW_DIR" /c/custom/
-        cd ..
-      }
-
-      # 32-bit
-      MINGW_FILE="i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z"
-      MINGW_URL="https://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win32/Personal%20Builds/mingw-builds/8.1.0/threads-posix/dwarf/${MINGW_FILE}"
-      MINGW_DIR="mingw32"
-      install_mingw
-
-      # 64-bit
-      MINGW_FILE="x86_64-8.1.0-release-posix-seh-rt_v6-rev0.7z"
-      MINGW_URL="https://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win64/Personal%20Builds/mingw-builds/8.1.0/threads-posix/seh/${MINGW_FILE}"
-      MINGW_DIR="mingw64"
-      install_mingw
-
-      if [[ $PLATFORM == "x86" ]]; then
-        MINGW_DIR="mingw32"
-      else
-        MINGW_DIR="mingw64"
-      fi
-      export PATH="/c/custom/${MINGW_DIR}/bin:${PATH}"
-      echo "PATH=${PATH}"
-      which gcc
-      gcc -v
-
-      # detect number of cores
-      export ncpu="$NUMBER_OF_PROCESSORS"
-      [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=2
-      echo "Found ${ncpu} cores"
-
-      # build nim from our own branch - this to avoid the day-to-day churn and
-      # regressions of the fast-paced Nim development while maintaining the
-      # flexibility to apply patches
-      curl -OLsS https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh
-      env MAKE="mingw32-make -j${ncpu}" ARCH_OVERRIDE=$(PLATFORM) bash build_nim.sh Nim csources dist/nimble NimBinaries
-
-      export PATH="${PWD}/Nim/bin:${PATH}"
-      echo "PATH=${PATH}"
-
-      # install and build go-libp2p-daemon
-      go version
-
-      export GOPATH="${PWD}/go"
-      export PATH="${GOPATH}/bin:${PATH}"
-      echo "PATH=${PATH}"
-
-      curl -OLsS https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_p2pd.sh
-      # we can't seem to be able to build a 32-bit p2pd
-      env PATH="/c/custom/mingw64/bin:${PATH}" bash build_p2pd.sh p2pdCache v0.2.4
-
-      # install dependencies
-      nimble refresh
-      nimble install -y --depsOnly
-
-      # run tests
-      nimble test
-      nimble examples_build
-    displayName: 'build and test'
-

codecov.yml

@@ -0,0 +1,19 @@
+codecov:
+  notify:
+    require_ci_to_pass: true
+    # must be the number of coverage report builds
+    # notice that this number is for PRs;
+    # like this we disabled notify on pure branches report
+    # which is fine I guess
+    after_n_builds: 28
+comment:
+  layout: "reach, diff, flags, files"
+  after_n_builds: 28 # must be the number of coverage report builds
+coverage:
+  status:
+    project:
+      default: # This can be anything, but it needs to exist as the name
+        # basic settings
+        target: auto
+        threshold: 5%
+        base: auto

config.nims

@@ -0,0 +1,23 @@
+# to allow locking
+if dirExists("nimbledeps/pkgs"):
+  switch("NimblePath", "nimbledeps/pkgs")
+
+switch("warning", "CaseTransition:off")
+switch("warning", "ObservableStores:off")
+switch("warning", "LockLevel:off")
+--define:chronosStrictException
+--styleCheck:usages
+if (NimMajor, NimMinor) < (1, 6):
+  --styleCheck:hint
+else:
+  --styleCheck:error
+
+# Avoid some rare stack corruption while using exceptions with a SEH-enabled
+# toolchain: https://github.com/status-im/nimbus-eth2/issues/3121
+if defined(windows) and not defined(vcc):
+  --define:nimRawSetjmp
+
+# begin Nimble config (version 1)
+when fileExists("nimble.paths"):
+  include "nimble.paths"
+# end Nimble config

docs/API.md

@@ -1,3 +0,0 @@
-# API
-
-Coming Soon... 

docs/GETTING_STARTED.md

@@ -1,99 +0,0 @@
-# Getting Started
-Welcome to nim-libp2p! This guide will walk you through a peer to peer chat example. <br>
-The full code can be found in [directchat.nim](examples/directchat.nim) under the examples folder.
-
-
-### Direct Chat Example
-To run nim-libp2p, add it to your project's nimble file and spawn a node as follows:
-
-```nim
-import tables
-import chronos
-import ../libp2p/[switch,
-                  multistream,
-                  protocols/identify,
-                  connection,
-                  transports/transport,
-                  transports/tcptransport,
-                  multiaddress,
-                  peerinfo,
-                  crypto/crypto,
-                  peerid,
-                  protocols/protocol,
-                  muxers/muxer,
-                  muxers/mplex/mplex,
-                  muxers/mplex/types,
-                  protocols/secure/secio,
-                  protocols/secure/secure]
-
-const TestCodec = "/test/proto/1.0.0" # custom protocol string
-
-type
-  TestProto = ref object of LPProtocol # declare a custom protocol
-
-method init(p: TestProto) {.gcsafe.} =
-  # handle incoming connections in closure
-  proc handle(conn: Connection, proto: string) {.async, gcsafe.} =
-    echo "Got from remote - ", cast[string](await conn.readLp(1024))
-    await conn.writeLp("Hello!")
-    await conn.close()
-
-  p.codec = TestCodec # init proto with the correct string id
-  p.handler = handle # set proto handler
-
-proc createSwitch(ma: MultiAddress): (Switch, PeerInfo) =
-  ## Helper to create a swith
-
-  let seckey = PrivateKey.random(RSA) # use a random key for peer id
-  var peerInfo = PeerInfo.init(seckey) # create a peer id and assign
-  peerInfo.addrs.add(ma) # set this peer's multiaddresses (can be any number)
-
-  let identify = newIdentify(peerInfo) # create the identify proto
-
-  proc createMplex(conn: Connection): Muxer =
-    # helper proc to create multiplexers,
-    # use this to perform any custom setup up,
-    # such as adjusting timeout or anything else
-    # that the muxer requires
-    result = newMplex(conn)
-
-  let mplexProvider = newMuxerProvider(createMplex, MplexCodec) # create multiplexer
-  let transports = @[Transport(newTransport(TcpTransport))] # add all transports (tcp only for now, but can be anything in the future)
-  let muxers = {MplexCodec: mplexProvider}.toTable() # add all muxers
-  let secureManagers = {SecioCodec: Secure(newSecio(seckey))}.toTable() # setup the secio and any other secure provider
-
-  # create the switch
-  let switch = newSwitch(peerInfo,
-                         transports,
-                         identify,
-                         muxers,
-                         secureManagers)
-  result = (switch, peerInfo)
-
-proc main() {.async, gcsafe.} =
-  let ma1: MultiAddress = Multiaddress.init("/ip4/0.0.0.0/tcp/0")
-  let ma2: MultiAddress = Multiaddress.init("/ip4/0.0.0.0/tcp/0")
-
-  var peerInfo1, peerInfo2: PeerInfo
-  var switch1, switch2: Switch
-  (switch1, peerInfo1) = createSwitch(ma1) # create node 1
-
-  # setup the custom proto
-  let testProto = new TestProto
-  testProto.init() # run it's init method to perform any required initialization
-  switch1.mount(testProto) # mount the proto
-  var switch1Fut = await switch1.start() # start the node
-
-  (switch2, peerInfo2) = createSwitch(ma2) # create node 2
-  var switch2Fut = await switch2.start() # start second node
-  let conn = await switch2.dial(switch1.peerInfo, TestCodec) # dial the first node
-
-  await conn.writeLp("Hello!") # writeLp send a length prefixed buffer over the wire
-  # readLp reads length prefixed bytes and returns a buffer without the prefix
-  echo "Remote responded with - ", cast[string](await conn.readLp(1024))
-
-  await allFutures(switch1.stop(), switch2.stop()) # close connections and shutdown all transports
-  await allFutures(switch1Fut & switch2Fut) # wait for all transports to shutdown
-
-waitFor(main())
-```

docs/GO_DAEMON.md

@@ -1,56 +0,0 @@
-# Table of Contents
-- [Introduction](#introduction)
-- [Installation](#installation)
-- [Usage](#usage)
-  - [Example](#example)
-  - [Getting Started](#getting-started)
-
-# Introduction
-This is a libp2p-backed daemon wrapping the functionalities of go-libp2p for use in Nim. <br>
-For more information about the go daemon, check out [this repository](https://github.com/libp2p/go-libp2p-daemon).
-
-# Installation
-```sh
-# clone and install dependencies
-git clone https://github.com/status-im/nim-libp2p
-cd nim-libp2p
-nimble install
-
-# perform unit tests
-nimble test
-
-# update the git submodule to install the go daemon 
-git submodule update --init --recursive
-go version
-git clone https://github.com/libp2p/go-libp2p-daemon
-cd go-libp2p-daemon
-git checkout v0.0.1
-go install ./...
-cd ..
-```
-
-# Usage 
-
-## Example
-Examples can be found in the [examples folder](https://github.com/status-im/nim-libp2p/tree/readme/examples/go-daemon)
-
-## Getting Started
-Try out the chat example. Full code can be found [here](https://github.com/status-im/nim-libp2p/blob/master/examples/chat.nim):
-
-```bash
-nim c -r --threads:on examples\chat.nim
-```
-
-This will output a peer ID such as `QmbmHfVvouKammmQDJck4hz33WvVktNEe7pasxz2HgseRu` which you can use in another instance to connect to it.
-
-```bash
-./example/chat
-/connect QmbmHfVvouKammmQDJck4hz33WvVktNEe7pasxz2HgseRu
-```
-
-You can now chat between the instances!
-
-![Chat example](https://imgur.com/caYRu8K.gif)
-
-
-

docs/README.md

@@ -1,29 +0,0 @@
-# Introduction
-This folder contains the documentation for each nim-libp2p module and the sample code for the tutorials. 
-
-# Table of Contents 
-### [Getting Started](GETTING_STARTED.md)
-### Tutorials
-- P2P Chat Example
-    - [part I](tutorial/directchat/start.nim)
-    - [part II](tutorial/directchat/second.nim)
-### API Specifications
-- libp2p
-    - [libp2p-daemon-client](api/libp2p/daemonapi.md)
-    - [interop-libp2p](api/libp2p/interop.md)
-- transports
-    - [libp2p-tcp](api/transports/tcptransport.md)
-- secure channels
-    - [libp2p-secio](api/secure_channels/secio.md)
-- stream multiplexers
-    - [libp2p-mplex](api/stream_multiplexers/mplex.md)
-- utilities
-    - [libp2p-crypto](api/utilities/crypto.md)
-    - [libp2p-crypto-secp256k1](api/utilities/secp256k1.md)
-- data types
-    - [peer-id](api/data_types/peer.md)
-    - [peer-info](api/data_types/peerinfo.md)
-- pubsub
-    - [libp2p-pubsub](api/pubsub/pubsub.md)
-    - [libp2p-floodsub](api/pubsub/floodsub.md)
-    - [libp2p-gossipsub](api/pubsub/gossipsub.md)

docs/tutorial/directchat/second.nim

@@ -1,150 +0,0 @@
-when not(compileOption("threads")):
-  {.fatal: "Please, compile this program with the --threads:on option!".}
-
-import tables, strformat, strutils
-import chronos
-import ../libp2p/[switch,
-                  multistream,
-                  crypto/crypto,
-                  protocols/identify,
-                  connection,
-                  transports/transport,
-                  transports/tcptransport,
-                  multiaddress,
-                  peerinfo,
-                  peerid,
-                  protocols/protocol,
-                  protocols/secure/secure,
-                  protocols/secure/secio,
-                  muxers/muxer,
-                  muxers/mplex/mplex,
-                  muxers/mplex/types]
-
-const ChatCodec = "/nim-libp2p/chat/1.0.0"
-const DefaultAddr = "/ip4/127.0.0.1/tcp/55505"
-
-const Help = """
-  Commands: /[?|hep|connect|disconnect|exit]
-  help: Prints this help
-  connect: dials a remote peer
-  disconnect: ends current session
-  exit: closes the chat
-"""
-
-type ChatProto = ref object of LPProtocol
-  switch: Switch          # a single entry point for dialing and listening to peer
-  transp: StreamTransport # transport streams between read & write file descriptor
-  conn: Connection        # create and close read & write stream
-  connected: bool         # if the node is connected to another peer
-  started: bool           # if the node has started
-
-# copied from https://github.com/status-im/nim-beacon-chain/blob/0ed657e953740a92458f23033d47483ffa17ccb0/beacon_chain/eth2_network.nim#L109-L115
-proc initAddress(T: type MultiAddress, str: string): T =
-  let address = MultiAddress.init(str)
-  if IPFS.match(address) and matchPartial(multiaddress.TCP, address):
-    result = address
-  else:
-    raise newException(MultiAddressError,
-                         "Invalid bootstrap node multi-address")
-
-proc dialPeer(p: ChatProto, address: string) {.async.} =
-  let multiAddr = MultiAddress.initAddress(address);
-  let parts = address.split("/")
-  let remotePeer = PeerInfo.init(parts[^1],
-                                 [multiAddr])
-
-  echo &"dialing peer: {multiAddr}"
-  p.conn = await p.switch.dial(remotePeer, ChatCodec)
-  p.connected = true
-
-proc readAndPrint(p: ChatProto) {.async.} =
-  while true:
-    while p.connected:
-      echo cast[string](await p.conn.readLp(1024))
-    await sleepAsync(100.millis)
-
-proc writeAndPrint(p: ChatProto) {.async.} =
-  while true:
-    if not p.connected:
-      echo "type an address or wait for a connection:"
-      echo "type /[help|?] for help"
-
-    let line = await p.transp.readLine()
-    if line.startsWith("/help") or line.startsWith("/?") or not p.started:
-      echo Help
-      continue
-
-    if line.startsWith("/disconnect"):
-      echo "Ending current session"
-      if p.connected and p.conn.closed.not:
-        await p.conn.close()
-      p.connected = false
-    elif line.startsWith("/connect"):
-      if p.connected:
-        var yesno = "N"
-        echo "a session is already in progress, do you want end it [y/N]?"
-        yesno = await p.transp.readLine()
-        if yesno.cmpIgnoreCase("y") == 0:
-          await p.conn.close()
-          p.connected = false
-        elif yesno.cmpIgnoreCase("n") == 0:
-          continue
-        else:
-          echo "unrecognized response"
-          continue
-
-      echo "enter address of remote peer"
-      let address = await p.transp.readLine()
-      if address.len > 0:
-        await p.dialPeer(address)
-
-    elif line.startsWith("/exit"):
-      if p.connected and p.conn.closed.not:
-        await p.conn.close()
-        p.connected = false
-
-      await p.switch.stop()
-      echo "quitting..."
-      quit(0)
-    else:
-      if p.connected:
-        await p.conn.writeLp(line)
-      else:
-        try:
-          if line.startsWith("/") and "ipfs" in line:
-            await p.dialPeer(line)
-        except:
-          echo &"unable to dial remote peer {line}"
-          echo getCurrentExceptionMsg()
-
-proc readWriteLoop(p: ChatProto) {.async.} =
-  asyncCheck p.writeAndPrint() # execute the async function but does not block
-  asyncCheck p.readAndPrint()
-
-proc processInput(rfd: AsyncFD) {.async.} =
-  let transp = fromPipe(rfd)
-  while true:
-    let a = await transp.readLine()
-    echo "You just entered: " & a
-
-proc readInput(wfd: AsyncFD) {.thread.} =
-  ## This procedure performs reading from `stdin` and sends data over
-  ## pipe to main thread.
-  let transp = fromPipe(wfd)
-
-  while true:
-    let line = stdin.readLine()
-    discard waitFor transp.write(line & "\r\n")
-
-proc main() {.async.} =
-  let (rfd, wfd) = createAsyncPipe()
-  if rfd == asyncInvalidPipe or wfd == asyncInvalidPipe:
-    raise newException(ValueError, "Could not initialize pipe!")
-
-  var thread: Thread[AsyncFD]
-  thread.createThread(readInput, wfd)
-
-  await processInput(rfd)
-
-when isMainModule:      # isMainModule = true when the module is compiled as the main file
-  waitFor(main())

docs/tutorial/directchat/start.nim

@@ -1,39 +0,0 @@
-when not(compileOption("threads")):
-  {.fatal: "Please, compile this program with the --threads:on option!".}
-  
-import chronos          # an efficient library for async
-
-proc processInput(rfd: AsyncFD) {.async.} =
-  echo "Type something below to see if the multithread IO works:\nType 'exit' to exit."
-
-  let transp = fromPipe(rfd)
-  while true:
-    let a = await transp.readLine()
-
-    if a == "exit":
-      quit(0);
-
-    echo "You just entered: " & a
-
-proc readInput(wfd: AsyncFD) {.thread.} =
-  ## This procedure performs reading from `stdin` and sends data over
-  ## pipe to main thread.
-  let transp = fromPipe(wfd)
-
-  while true:
-    let line = stdin.readLine()
-    discard waitFor transp.write(line & "\r\n")
-
-proc main() {.async.} =
-  let (rfd, wfd) = createAsyncPipe()
-  if rfd == asyncInvalidPipe or wfd == asyncInvalidPipe:
-    raise newException(ValueError, "Could not initialize pipe!")
-  
-  var thread: Thread[AsyncFD]
-  thread.createThread(readInput, wfd)
-  
-  await processInput(rfd)
-
-when isMainModule:      # isMainModule = true when the module is compiled as the main file
-  waitFor(main())
- 

docs/tutorial/second.nim

@@ -1,150 +0,0 @@
-when not(compileOption("threads")):
-  {.fatal: "Please, compile this program with the --threads:on option!".}
-
-import tables, strformat, strutils
-import chronos
-import ../libp2p/[switch,
-                  multistream,
-                  crypto/crypto,
-                  protocols/identify,
-                  connection,
-                  transports/transport,
-                  transports/tcptransport,
-                  multiaddress,
-                  peerinfo,
-                  peerid,
-                  protocols/protocol,
-                  protocols/secure/secure,
-                  protocols/secure/secio,
-                  muxers/muxer,
-                  muxers/mplex/mplex,
-                  muxers/mplex/types]
-
-const ChatCodec = "/nim-libp2p/chat/1.0.0"
-const DefaultAddr = "/ip4/127.0.0.1/tcp/55505"
-
-const Help = """
-  Commands: /[?|hep|connect|disconnect|exit]
-  help: Prints this help
-  connect: dials a remote peer
-  disconnect: ends current session
-  exit: closes the chat
-"""
-
-type ChatProto = ref object of LPProtocol
-  switch: Switch          # a single entry point for dialing and listening to peer
-  transp: StreamTransport # transport streams between read & write file descriptor
-  conn: Connection        # create and close read & write stream
-  connected: bool         # if the node is connected to another peer
-  started: bool           # if the node has started
-
-# copied from https://github.com/status-im/nim-beacon-chain/blob/0ed657e953740a92458f23033d47483ffa17ccb0/beacon_chain/eth2_network.nim#L109-L115
-proc initAddress(T: type MultiAddress, str: string): T =
-  let address = MultiAddress.init(str)
-  if IPFS.match(address) and matchPartial(multiaddress.TCP, address):
-    result = address
-  else:
-    raise newException(MultiAddressError,
-                         "Invalid bootstrap node multi-address")
-
-proc dialPeer(p: ChatProto, address: string) {.async.} =
-  let multiAddr = MultiAddress.initAddress(address);
-  let parts = address.split("/")
-  let remotePeer = PeerInfo.init(parts[^1],
-                                 [multiAddr])
-
-  echo &"dialing peer: {multiAddr}"
-  p.conn = await p.switch.dial(remotePeer, ChatCodec)
-  p.connected = true
-
-proc readAndPrint(p: ChatProto) {.async.} =
-  while true:
-    while p.connected:
-      echo cast[string](await p.conn.readLp(1024))
-    await sleepAsync(100.millis)
-
-proc writeAndPrint(p: ChatProto) {.async.} =
-  while true:
-    if not p.connected:
-      echo "type an address or wait for a connection:"
-      echo "type /[help|?] for help"
-
-    let line = await p.transp.readLine()
-    if line.startsWith("/help") or line.startsWith("/?") or not p.started:
-      echo Help
-      continue
-
-    if line.startsWith("/disconnect"):
-      echo "Ending current session"
-      if p.connected and p.conn.closed.not:
-        await p.conn.close()
-      p.connected = false
-    elif line.startsWith("/connect"):
-      if p.connected:
-        var yesno = "N"
-        echo "a session is already in progress, do you want end it [y/N]?"
-        yesno = await p.transp.readLine()
-        if yesno.cmpIgnoreCase("y") == 0:
-          await p.conn.close()
-          p.connected = false
-        elif yesno.cmpIgnoreCase("n") == 0:
-          continue
-        else:
-          echo "unrecognized response"
-          continue
-
-      echo "enter address of remote peer"
-      let address = await p.transp.readLine()
-      if address.len > 0:
-        await p.dialPeer(address)
-
-    elif line.startsWith("/exit"):
-      if p.connected and p.conn.closed.not:
-        await p.conn.close()
-        p.connected = false
-
-      await p.switch.stop()
-      echo "quitting..."
-      quit(0)
-    else:
-      if p.connected:
-        await p.conn.writeLp(line)
-      else:
-        try:
-          if line.startsWith("/") and "ipfs" in line:
-            await p.dialPeer(line)
-        except:
-          echo &"unable to dial remote peer {line}"
-          echo getCurrentExceptionMsg()
-
-proc readWriteLoop(p: ChatProto) {.async.} =
-  asyncCheck p.writeAndPrint() # execute the async function but does not block
-  asyncCheck p.readAndPrint()
-
-proc processInput(rfd: AsyncFD) {.async.} =
-  let transp = fromPipe(rfd)
-  while true:
-    let a = await transp.readLine()
-    echo "You just entered: " & a
-
-proc readInput(wfd: AsyncFD) {.thread.} =
-  ## This procedure performs reading from `stdin` and sends data over
-  ## pipe to main thread.
-  let transp = fromPipe(wfd)
-
-  while true:
-    let line = stdin.readLine()
-    discard waitFor transp.write(line & "\r\n")
-
-proc main() {.async.} =
-  let (rfd, wfd) = createAsyncPipe()
-  if rfd == asyncInvalidPipe or wfd == asyncInvalidPipe:
-    raise newException(ValueError, "Could not initialize pipe!")
-
-  var thread: Thread[AsyncFD]
-  thread.createThread(readInput, wfd)
-
-  await processInput(rfd)
-
-when isMainModule:      # isMainModule = true when the module is compiled as the main file
-  waitFor(main())

docs/tutorial/start.nim

@@ -1,39 +0,0 @@
-when not(compileOption("threads")):
-  {.fatal: "Please, compile this program with the --threads:on option!".}
-  
-import chronos          # an efficient library for async
-
-proc processInput(rfd: AsyncFD) {.async.} =
-  echo "Type something below to see if the multithread IO works:\nType 'exit' to exit."
-
-  let transp = fromPipe(rfd)
-  while true:
-    let a = await transp.readLine()
-
-    if a == "exit":
-      quit(0);
-
-    echo "You just entered: " & a
-
-proc readInput(wfd: AsyncFD) {.thread.} =
-  ## This procedure performs reading from `stdin` and sends data over
-  ## pipe to main thread.
-  let transp = fromPipe(wfd)
-
-  while true:
-    let line = stdin.readLine()
-    discard waitFor transp.write(line & "\r\n")
-
-proc main() {.async.} =
-  let (rfd, wfd) = createAsyncPipe()
-  if rfd == asyncInvalidPipe or wfd == asyncInvalidPipe:
-    raise newException(ValueError, "Could not initialize pipe!")
-  
-  var thread: Thread[AsyncFD]
-  thread.createThread(readInput, wfd)
-  
-  await processInput(rfd)
-
-when isMainModule:      # isMainModule = true when the module is compiled as the main file
-  waitFor(main())
- 

examples/README.md

@@ -0,0 +1,6 @@
+# nim-libp2p documentation
+
+Welcome to the nim-libp2p documentation!
+
+Here, you'll find [tutorials](tutorial_1_connect.md) to help you get started, as well as 
+the [full reference](https://status-im.github.io/nim-libp2p/master/libp2p.html).

examples/circuitrelay.nim

@@ -0,0 +1,84 @@
+## # Circuit Relay example
+##
+## Circuit Relay can be used when a node cannot reach another node
+## directly, but can reach it through a another node (the Relay).
+##
+## That may happen because of NAT, Firewalls, or incompatible transports.
+##
+## More informations [here](https://docs.libp2p.io/concepts/circuit-relay/).
+import chronos, stew/byteutils
+import libp2p,
+       libp2p/protocols/connectivity/relay/[relay, client]
+
+# Helper to create a circuit relay node
+proc createCircuitRelaySwitch(r: Relay): Switch =
+  SwitchBuilder.new()
+    .withRng(newRng())
+    .withAddresses(@[ MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet() ])
+    .withTcpTransport()
+    .withMplex()
+    .withNoise()
+    .withCircuitRelay(r)
+    .build()
+
+proc main() {.async.} =
+  # Create a custom protocol
+  let customProtoCodec = "/test"
+  var proto = new LPProtocol
+  proto.codec = customProtoCodec
+  proto.handler = proc(conn: Connection, proto: string) {.async.} =
+    var msg = string.fromBytes(await conn.readLp(1024))
+    echo "1 - Dst Received: ", msg
+    assert "test1" == msg
+    await conn.writeLp("test2")
+    msg = string.fromBytes(await conn.readLp(1024))
+    echo "2 - Dst Received: ", msg
+    assert "test3" == msg
+    await conn.writeLp("test4")
+
+  let
+    relay = Relay.new()
+    clSrc = RelayClient.new()
+    clDst = RelayClient.new()
+
+    # Create three hosts, enable relay client on two of them.
+    # The third one can relay connections for other peers.
+    # RelayClient can use a relay, Relay is a relay.
+    swRel = createCircuitRelaySwitch(relay)
+    swSrc = createCircuitRelaySwitch(clSrc)
+    swDst = createCircuitRelaySwitch(clDst)
+
+    # Create a relay address to swDst using swRel as the relay
+    addrs = MultiAddress.init($swRel.peerInfo.addrs[0] & "/p2p/" &
+                              $swRel.peerInfo.peerId & "/p2p-circuit/p2p/" &
+                              $swDst.peerInfo.peerId).get()
+
+  swDst.mount(proto)
+
+  await swRel.start()
+  await swSrc.start()
+  await swDst.start()
+
+  # Connect both Src and Dst to the relay, but not to each other.
+  await swSrc.connect(swRel.peerInfo.peerId, swRel.peerInfo.addrs)
+  await swDst.connect(swRel.peerInfo.peerId, swRel.peerInfo.addrs)
+
+  # Dst reserve a slot on the relay.
+  let rsvp = await clDst.reserve(swRel.peerInfo.peerId, swRel.peerInfo.addrs)
+
+  # Src dial Dst using the relay
+  let conn = await swSrc.dial(swDst.peerInfo.peerId, @[ addrs ], customProtoCodec)
+
+  await conn.writeLp("test1")
+  var msg = string.fromBytes(await conn.readLp(1024))
+  echo "1 - Src Received: ", msg
+  assert "test2" == msg
+  await conn.writeLp("test3")
+  msg = string.fromBytes(await conn.readLp(1024))
+  echo "2 - Src Received: ", msg
+  assert "test4" == msg
+
+  await relay.stop()
+  await allFutures(swSrc.stop(), swDst.stop(), swRel.stop())
+
+waitFor(main())

examples/directchat.nim

@@ -1,147 +1,151 @@
 when not(compileOption("threads")):
   {.fatal: "Please, compile this program with the --threads:on option!".}
 
-import tables, strformat, strutils, bearssl
-import chronos                              # an efficient library for async
-import ../libp2p/[switch,                   # manage transports, a single entry point for dialing and listening
-                  multistream,              # tag stream with short header to identify it
-                  crypto/crypto,            # cryptographic functions
-                  errors,                   # error handling utilities
-                  protocols/identify,       # identify the peer info of a peer
-                  stream/connection,        # create and close stream read / write connections
-                  transports/transport,     # listen and dial to other peers using p2p protocol
-                  transports/tcptransport,  # listen and dial to other peers using client-server protocol
-                  multiaddress,             # encode different addressing schemes. For example, /ip4/7.7.7.7/tcp/6543 means it is using IPv4 protocol and TCP
-                  peerinfo,                 # manage the information of a peer, such as peer ID and public / private key
-                  peerid,                   # Implement how peers interact
-                  protocols/protocol,       # define the protocol base type
-                  protocols/secure/secure,  # define the protocol of secure connection
-                  protocols/secure/secio,   # define the protocol of secure input / output, allows encrypted communication that uses public keys to validate signed messages instead of a certificate authority like in TLS
-                  muxers/muxer,             # define an interface for stream multiplexing, allowing peers to offer many protocols over a single connection
-                  muxers/mplex/mplex,       # implement stream multiplexing
-                  muxers/mplex/types]       # define some contants and message types for stream multiplexing
+import
+  strformat, strutils,
+  stew/byteutils,
+  chronos,
+  libp2p
 
-const ChatCodec = "/nim-libp2p/chat/1.0.0"
-const DefaultAddr = "/ip4/127.0.0.1/tcp/55505"
+const DefaultAddr = "/ip4/127.0.0.1/tcp/0"
 
 const Help = """
-  Commands: /[?|hep|connect|disconnect|exit]
+  Commands: /[?|help|connect|disconnect|exit]
   help: Prints this help
   connect: dials a remote peer
   disconnect: ends current session
   exit: closes the chat
 """
 
-type ChatProto = ref object of LPProtocol
-  switch: Switch          # a single entry point for dialing and listening to peer
-  transp: StreamTransport # transport streams between read & write file descriptor
-  conn: Connection        # create and close read & write stream
-  connected: bool         # if the node is connected to another peer
-  started: bool           # if the node has started
+type
+  Chat = ref object
+    switch: Switch          # a single entry point for dialing and listening to peer
+    stdinReader: StreamTransport # transport streams between read & write file descriptor
+    conn: Connection        # connection to the other peer
+    connected: bool         # if the node is connected to another peer
 
+##
+# Stdout helpers, to write the prompt
+##
+proc writePrompt(c: Chat) =
+  if c.connected:
+    stdout.write '\r' & $c.switch.peerInfo.peerId & ": "
+    stdout.flushFile()
 
-proc initAddress(T: type MultiAddress, str: string): T =
-  let address = MultiAddress.init(str).tryGet()
-  if IPFS.match(address) and matchPartial(multiaddress.TCP, address):
-    result = address
-  else:
-    raise newException(ValueError,
-                         "Invalid bootstrap node multi-address")
+proc writeStdout(c: Chat, str: string) =
+  echo '\r' & str
+  c.writePrompt()
 
-proc dialPeer(p: ChatProto, address: string) {.async.} =
-  let multiAddr = MultiAddress.initAddress(address);
-  let parts = address.split("/")
-  let remotePeer = PeerInfo.init(parts[^1],
-                                 [multiAddr])
+##
+# Chat Protocol
+##
+const ChatCodec = "/nim-libp2p/chat/1.0.0"
+
+type
+  ChatProto = ref object of LPProtocol
+
+proc new(T: typedesc[ChatProto], c: Chat): T =
+  let chatproto = T()
+
+  # create handler for incoming connection
+  proc handle(stream: Connection, proto: string) {.async.} =
+    if c.connected and not c.conn.closed:
+      c.writeStdout "a chat session is already in progress - refusing incoming peer!"
+      await stream.close()
+    else:
+      await c.handlePeer(stream)
+      await stream.close()
+
+  # assign the new handler
+  chatproto.handler = handle
+  chatproto.codec = ChatCodec
+  return chatproto
+
+##
+# Chat application
+##
+proc handlePeer(c: Chat, conn: Connection) {.async.} =
+  # Handle a peer (incoming or outgoing)
+  try:
+    c.conn = conn
+    c.connected = true
+    c.writeStdout $conn.peerId & " connected"
+
+    # Read loop
+    while true:
+      let
+        strData = await conn.readLp(1024)
+        str = string.fromBytes(strData)
+      c.writeStdout $conn.peerId & ": " & $str
+
+  except LPStreamEOFError:
+    defer: c.writeStdout $conn.peerId & " disconnected"
+    await c.conn.close()
+    c.connected = false
+
+proc dialPeer(c: Chat, address: string) {.async.} =
+  # Parse and dial address
+  let
+    multiAddr = MultiAddress.init(address).tryGet()
+    # split the peerId part /p2p/...
+    peerIdBytes = multiAddr[multiCodec("p2p")]
+      .tryGet()
+      .protoAddress()
+      .tryGet()
+    remotePeer = PeerId.init(peerIdBytes).tryGet()
+    # split the wire address
+    ip4Addr = multiAddr[multiCodec("ip4")].tryGet()
+    tcpAddr = multiAddr[multiCodec("tcp")].tryGet()
+    wireAddr = ip4Addr & tcpAddr
 
   echo &"dialing peer: {multiAddr}"
-  p.conn = await p.switch.dial(remotePeer, ChatCodec)
-  p.connected = true
+  asyncSpawn c.handlePeer(await c.switch.dial(remotePeer, @[wireAddr], ChatCodec))
 
-proc readAndPrint(p: ChatProto) {.async.} =
+proc readLoop(c: Chat) {.async.} =
   while true:
-    while p.connected:
-      # TODO: echo &"{p.id} -> "
-
-      echo cast[string](await p.conn.readLp(1024))
-    await sleepAsync(100.millis)
-
-proc writeAndPrint(p: ChatProto) {.async.} =
-  while true:
-    if not p.connected:
+    if not c.connected:
       echo "type an address or wait for a connection:"
       echo "type /[help|?] for help"
 
-    let line = await p.transp.readLine()
-    if line.startsWith("/help") or line.startsWith("/?") or not p.started:
+    c.writePrompt()
+
+    let line = await c.stdinReader.readLine()
+    if line.startsWith("/help") or line.startsWith("/?"):
       echo Help
       continue
 
     if line.startsWith("/disconnect"):
-      echo "Ending current session"
-      if p.connected and p.conn.closed.not:
-        await p.conn.close()
-      p.connected = false
+      c.writeStdout "Ending current session"
+      if c.connected and c.conn.closed.not:
+        await c.conn.close()
+      c.connected = false
     elif line.startsWith("/connect"):
-      if p.connected:
-        var yesno = "N"
-        echo "a session is already in progress, do you want end it [y/N]?"
-        yesno = await p.transp.readLine()
-        if yesno.cmpIgnoreCase("y") == 0:
-          await p.conn.close()
-          p.connected = false
-        elif yesno.cmpIgnoreCase("n") == 0:
-          continue
-        else:
-          echo "unrecognized response"
-          continue
-
-      echo "enter address of remote peer"
-      let address = await p.transp.readLine()
+      c.writeStdout "enter address of remote peer"
+      let address = await c.stdinReader.readLine()
       if address.len > 0:
-        await p.dialPeer(address)
+        await c.dialPeer(address)
 
     elif line.startsWith("/exit"):
-      if p.connected and p.conn.closed.not:
-        await p.conn.close()
-        p.connected = false
+      if c.connected and c.conn.closed.not:
+        await c.conn.close()
+        c.connected = false
 
-      await p.switch.stop()
-      echo "quitting..."
-      quit(0)
+      await c.switch.stop()
+      c.writeStdout "quitting..."
+      return
     else:
-      if p.connected:
-        await p.conn.writeLp(line)
+      if c.connected:
+        await c.conn.writeLp(line)
       else:
         try:
-          if line.startsWith("/") and "ipfs" in line:
-            await p.dialPeer(line)
-        except:
+          if line.startsWith("/") and "p2p" in line:
+            await c.dialPeer(line)
+        except CatchableError as exc:
           echo &"unable to dial remote peer {line}"
-          echo getCurrentExceptionMsg()
-
-proc readWriteLoop(p: ChatProto) {.async.} =
-  asyncCheck p.writeAndPrint() # execute the async function but does not block
-  asyncCheck p.readAndPrint()
-
-proc newChatProto(switch: Switch, transp: StreamTransport): ChatProto =
-  var chatproto = ChatProto(switch: switch, transp: transp, codec: ChatCodec)
-
-  # create handler for incoming connection
-  proc handle(stream: Connection, proto: string) {.async.} =
-    if chatproto.connected and not chatproto.conn.closed:
-      echo "a chat session is already in progress - disconnecting!"
-      await stream.close()
-    else:
-      chatproto.conn = stream
-      chatproto.connected = true
-
-  # assign the new handler
-  chatproto.handler = handle
-  return chatproto
+          echo exc.msg
 
 proc readInput(wfd: AsyncFD) {.thread.} =
-  ## This procedure performs reading from `stdin` and sends data over
+  ## This thread performs reading from `stdin` and sends data over
   ## pipe to main thread.
   let transp = fromPipe(wfd)
 
@@ -149,66 +153,46 @@ proc readInput(wfd: AsyncFD) {.thread.} =
     let line = stdin.readLine()
     discard waitFor transp.write(line & "\r\n")
 
-proc processInput(rfd: AsyncFD, rng: ref BrHmacDrbgContext) {.async.} =
-  let transp = fromPipe(rfd)
-
-  let seckey = PrivateKey.random(RSA, rng[]).get()
-  let peerInfo = PeerInfo.init(seckey)
-  var localAddress = DefaultAddr
-  while true:
-    echo &"Type an address to bind to or Enter to use the default {DefaultAddr}"
-    let a = await transp.readLine()
-    try:
-      if a.len > 0:
-        peerInfo.addrs.add(Multiaddress.init(a).tryGet())
-        break
-
-      peerInfo.addrs.add(Multiaddress.init(localAddress).tryGet())
-      break
-    except:
-      echo "invalid address"
-      localAddress = DefaultAddr
-      continue
-
-  # a constructor for building different multiplexers under various connections
-  proc createMplex(conn: Connection): Muxer =
-    result = newMplex(conn)
-
-  let mplexProvider = newMuxerProvider(createMplex, MplexCodec)
-  let transports = @[Transport(TcpTransport.init())]
-  let muxers = [(MplexCodec, mplexProvider)].toTable()
-  let identify = newIdentify(peerInfo)
-  let secureManagers = [Secure(newSecio(rng, seckey))]
-  let switch = newSwitch(peerInfo,
-                         transports,
-                         identify,
-                         muxers,
-                         secureManagers)
-
-  let chatProto = newChatProto(switch, transp)
-  switch.mount(chatProto)
-  let libp2pFuts = await switch.start()
-  chatProto.started = true
-
-  let id = peerInfo.peerId.pretty
-  echo "PeerID: " & id
-  echo "listening on: "
-  for a in peerInfo.addrs:
-    echo &"{a}/ipfs/{id}"
-
-  await chatProto.readWriteLoop()
-  await allFuturesThrowing(libp2pFuts)
-
 proc main() {.async.} =
-  let rng = newRng() # Singe random number source for the whole application
-  let (rfd, wfd) = createAsyncPipe()
-  if rfd == asyncInvalidPipe or wfd == asyncInvalidPipe:
-    raise newException(ValueError, "Could not initialize pipe!")
+  let
+    rng = newRng() # Single random number source for the whole application
+
+    # Pipe to read stdin from main thread
+    (rfd, wfd) = createAsyncPipe()
+    stdinReader = fromPipe(rfd)
 
   var thread: Thread[AsyncFD]
-  thread.createThread(readInput, wfd)
+  try:
+    thread.createThread(readInput, wfd)
+  except Exception as exc:
+    quit("Failed to create thread: " & exc.msg)
 
-  await processInput(rfd, rng)
+  var localAddress = MultiAddress.init(DefaultAddr).tryGet()
 
-when isMainModule: # isMainModule = true when the module is compiled as the main file
-  waitFor(main())
+  var switch = SwitchBuilder
+    .new()
+    .withRng(rng)       # Give the application RNG
+    .withAddress(localAddress)
+    .withTcpTransport() # Use TCP as transport
+    .withMplex()        # Use Mplex as muxer
+    .withNoise()        # Use Noise as secure manager
+    .build()
+
+  let chat = Chat(
+    switch: switch,
+    stdinReader: stdinReader)
+
+  switch.mount(ChatProto.new(chat))
+
+  await switch.start()
+
+  let id = $switch.peerInfo.peerId
+  echo "PeerId: " & id
+  echo "listening on: "
+  for a in switch.peerInfo.addrs:
+    echo &"{a}/p2p/{id}"
+
+  await chat.readLoop()
+  quit(0)
+
+waitFor(main())

examples/go-daemon/bootstrap.nim

@@ -35,7 +35,7 @@ proc main() {.async.} =
     if item.protoCode() == mcip4 or item.protoCode() == mcip6:
       echo $item & "/ipfs/" & id.peer.pretty()
 
-  asyncCheck monitor(api)
+  asyncSpawn monitor(api)
 
   proc pubsubLogger(api: DaemonAPI,
                     ticket: PubsubTicket,

examples/go-daemon/chat.nim

@@ -19,7 +19,7 @@ proc threadMain(wfd: AsyncFD) {.thread.} =
   ## This procedure performs reading from `stdin` and sends data over
   ## pipe to main thread.
   var transp = fromPipe(wfd)
- 
+
   while true:
     var line = stdin.readLine()
     let res = waitFor transp.write(line & "\r\n")
@@ -41,7 +41,7 @@ proc serveThread(udata: CustomData) {.async.} =
       if line.startsWith("/connect"):
         var parts = line.split(" ")
         if len(parts) == 2:
-          var peerId = PeerID.init(parts[1])
+          var peerId = PeerId.init(parts[1])
           var address = MultiAddress.init(multiCodec("p2p-circuit"))
           address &= MultiAddress.init(multiCodec("p2p"), peerId)
           echo "= Searching for peer ", peerId.pretty()
@@ -55,11 +55,11 @@ proc serveThread(udata: CustomData) {.async.} =
           var stream = await udata.api.openStream(peerId, ServerProtocols)
           udata.remotes.add(stream.transp)
           echo "= Connected to peer chat ", parts[1]
-          asyncCheck remoteReader(stream.transp)
+          asyncSpawn remoteReader(stream.transp)
       elif line.startsWith("/search"):
         var parts = line.split(" ")
         if len(parts) == 2:
-          var peerId = PeerID.init(parts[1])
+          var peerId = PeerId.init(parts[1])
           echo "= Searching for peer ", peerId.pretty()
           var id = await udata.api.dhtFindPeer(peerId)
           echo "= Peer " & parts[1] & " found at addresses:"
@@ -68,7 +68,7 @@ proc serveThread(udata: CustomData) {.async.} =
       elif line.startsWith("/consearch"):
         var parts = line.split(" ")
         if len(parts) == 2:
-          var peerId = PeerID.init(parts[1])
+          var peerId = PeerId.init(parts[1])
           echo "= Searching for peers connected to peer ", parts[1]
           var peers = await udata.api.dhtFindPeersConnectedToPeer(peerId)
           echo "= Found ", len(peers), " connected to peer ", parts[1]
@@ -127,7 +127,7 @@ proc main() {.async.} =
       echo ">> ", line
 
   await data.api.addHandler(ServerProtocols, streamHandler)
-  echo "= Your PeerID is ", id.peer.pretty()
+  echo "= Your PeerId is ", id.peer.pretty()
   await data.serveFut
 
 when isMainModule:

examples/go-daemon/daemonapi.md

@@ -38,13 +38,13 @@ Examples can be found in the [examples folder](https://github.com/status-im/nim-
 Try out the chat example. Full code can be found [here](https://github.com/status-im/nim-libp2p/blob/master/examples/chat.nim):
 
 ```bash
-nim c -r --threads:on examples\chat.nim
+nim c -r --threads:on examples/directchat.nim
 ```
 
 This will output a peer ID such as `QmbmHfVvouKammmQDJck4hz33WvVktNEe7pasxz2HgseRu` which you can use in another instance to connect to it.
 
 ```bash
-./example/chat
+./examples/directchat
 /connect QmbmHfVvouKammmQDJck4hz33WvVktNEe7pasxz2HgseRu
 ```
 

examples/helloworld.nim

@@ -0,0 +1,89 @@
+import chronos         # an efficient library for async
+import stew/byteutils  # various utils
+import libp2p
+
+##
+# Create our custom protocol
+##
+const TestCodec = "/test/proto/1.0.0" # custom protocol string identifier
+
+type
+  TestProto = ref object of LPProtocol # declare a custom protocol
+
+proc new(T: typedesc[TestProto]): T =
+
+  # every incoming connections will be in handled in this closure
+  proc handle(conn: Connection, proto: string) {.async, gcsafe.} =
+    echo "Got from remote - ", string.fromBytes(await conn.readLp(1024))
+    await conn.writeLp("Roger p2p!")
+
+    # We must close the connections ourselves when we're done with it
+    await conn.close()
+
+  return T(codecs: @[TestCodec], handler: handle)
+
+##
+# Helper to create a switch/node
+##
+proc createSwitch(ma: MultiAddress, rng: ref HmacDrbgContext): Switch =
+  var switch = SwitchBuilder
+    .new()
+    .withRng(rng)       # Give the application RNG
+    .withAddress(ma)    # Our local address(es)
+    .withTcpTransport() # Use TCP as transport
+    .withMplex()        # Use Mplex as muxer
+    .withNoise()        # Use Noise as secure manager
+    .build()
+
+  result = switch
+
+##
+# The actual application
+##
+proc main() {.async, gcsafe.} =
+  let
+    rng = newRng() # Single random number source for the whole application
+    # port 0 will take a random available port
+    # `tryGet` will throw an exception if the MultiAddress failed
+    # (for instance, if the address is not well formatted)
+    ma1 = MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet()
+    ma2 = MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet()
+
+  # setup the custom proto
+  let testProto = TestProto.new()
+
+  # setup the two nodes
+  let
+    switch1 = createSwitch(ma1, rng) #Create the two switches
+    switch2 = createSwitch(ma2, rng)
+
+  # mount the proto on switch1
+  # the node will now listen for this proto
+  # and call the handler everytime a client request it
+  switch1.mount(testProto)
+
+  # Start the nodes. This will start the transports
+  # and listen on each local addresses
+  await switch1.start()
+  await switch2.start()
+
+  # the node addrs is populated with it's
+  # actual port during the start
+
+  # use the second node to dial the first node
+  # using the first node peerid and address
+  # and specify our custom protocol codec
+  let conn = await switch2.dial(switch1.peerInfo.peerId, switch1.peerInfo.addrs, TestCodec)
+
+  # conn is now a fully setup connection, we talk directly to the node1 custom protocol handler
+  await conn.writeLp("Hello p2p!") # writeLp send a length prefixed buffer over the wire
+
+  # readLp reads length prefixed bytes and returns a buffer without the prefix
+  echo "Remote responded with - ", string.fromBytes(await conn.readLp(1024))
+
+  # We must close the connection ourselves when we're done with it
+  await conn.close()
+
+  await allFutures(switch1.stop(), switch2.stop()) # close connections and shutdown all transports
+
+waitFor(main())

examples/hexdump.nim

@@ -71,8 +71,8 @@ proc dumpHex*(pbytes: pointer, nbytes: int, items = 1, ascii = true): string =
     result = result & asciiText
   result = result & "\n"
 
-proc dumpHex*[T](v: openarray[T], items: int = 0, ascii = true): string =
-  ## Return hexadecimal memory dump representation of openarray[T] ``v``.
+proc dumpHex*[T](v: openArray[T], items: int = 0, ascii = true): string =
+  ## Return hexadecimal memory dump representation of openArray[T] ``v``.
   ## ``items``  - number of bytes in group (supported ``items`` count is
   ##  0, 1, 2, 4, 8). If ``items`` is ``0`` group size will depend on
   ## ``sizeof(T)``.

examples/tutorial_1_connect.nim

@@ -0,0 +1,95 @@
+## # Simple ping tutorial
+##
+## Hi all, welcome to the first nim-libp2p tutorial!
+##
+## !!! tips ""
+##     This tutorial is for everyone who is interested in building peer-to-peer applications. No Nim programming experience is needed.
+##
+## To give you a quick overview, **Nim** is the programming language we are using and **nim-libp2p** is the Nim implementation of [libp2p](https://libp2p.io/), a modular library that enables the development of peer-to-peer network applications.
+##
+## Hope you'll find it helpful in your journey of learning. Happy coding! ;)
+##
+## ## Before you start
+## The only prerequisite here is [Nim](https://nim-lang.org/), the programming language with a Python-like syntax and a performance similar to C. Detailed information can be found [here](https://nim-lang.org/docs/tut1.html).
+##
+## Install Nim via their [official website](https://nim-lang.org/install.html).
+## Check Nim's installation via `nim --version` and its package manager Nimble via `nimble --version`.
+##
+## You can now install the latest version of `nim-libp2p`:
+## ```bash
+## nimble install libp2p@#master
+## ```
+##
+## ## A simple ping application
+## We'll start by creating a simple application, which is starting two libp2p [switch](https://docs.libp2p.io/concepts/stream-multiplexing/#switch-swarm), and pinging each other using the [Ping](https://docs.libp2p.io/concepts/protocols/#ping) protocol.
+##
+## !!! tips ""
+##     You can find the source of this tutorial (and other tutorials) in the [libp2p/examples](https://github.com/status-im/nim-libp2p/tree/master/examples) folder!
+##
+## Let's create a `part1.nim`, and import our dependencies:
+import chronos
+
+import libp2p
+import libp2p/protocols/ping
+
+## [chronos](https://github.com/status-im/nim-chronos) the asynchronous framework used by `nim-libp2p`
+##
+## Next, we'll create an helper procedure to create our switches. A switch needs a bit of configuration, and it will be easier to do this configuration only once:
+proc createSwitch(ma: MultiAddress, rng: ref HmacDrbgContext): Switch =
+  var switch = SwitchBuilder
+    .new()
+    .withRng(rng)       # Give the application RNG
+    .withAddress(ma)    # Our local address(es)
+    .withTcpTransport() # Use TCP as transport
+    .withMplex()        # Use Mplex as muxer
+    .withNoise()        # Use Noise as secure manager
+    .build()
+
+  return switch
+
+## This will create a switch using [Mplex](https://docs.libp2p.io/concepts/stream-multiplexing/) as a multiplexer, Noise to secure the communication, and TCP as an underlying transport.
+##
+## You can of course tweak this, to use a different or multiple transport, or tweak the configuration of Mplex and Noise, but this is some sane defaults that we'll use going forward.
+##
+##
+## Let's now start to create our main procedure:
+proc main() {.async, gcsafe.} =
+  let
+    rng = newRng()
+    localAddress = MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet()
+    pingProtocol = Ping.new(rng=rng)
+  ## We created some variables that we'll need for the rest of the application: the global `rng` instance, our `localAddress`, and an instance of the `Ping` protocol.
+  ## The address is in the [MultiAddress](https://github.com/multiformats/multiaddr) format. The port `0` means "take any port available".
+  ##
+  ## `tryGet` is procedure which is part of [nim-result](https://github.com/arnetheduck/nim-result/), that will throw an exception if the supplied MultiAddress is invalid.
+  ##
+  ## We can now create our two switches:
+  let
+    switch1 = createSwitch(localAddress, rng)
+    switch2 = createSwitch(localAddress, rng)
+
+  switch1.mount(pingProtocol)
+
+  await switch1.start()
+  await switch2.start()
+  ## We've **mounted** the `pingProtocol` on our first switch. This means that the first switch will actually listen for any ping requests coming in, and handle them accordingly.
+  ##
+  ## Now that we've started the nodes, they are listening for incoming peers.
+  ## We can find out which port was attributed, and the resulting local addresses, by using `switch1.peerInfo.addrs`.
+  ##
+  ## We'll **dial** the first switch from the second one, by specifying it's **Peer ID**, it's **MultiAddress** and the **`Ping` protocol codec**:
+  let conn = await switch2.dial(switch1.peerInfo.peerId, switch1.peerInfo.addrs, PingCodec)
+  ## We now have a `Ping` connection setup between the second and the first switch, we can use it to actually ping the node:
+  # ping the other node and echo the ping duration
+  echo "ping: ", await pingProtocol.ping(conn)
+
+  # We must close the connection ourselves when we're done with it
+  await conn.close()
+  ## And that's it! Just a little bit of cleanup: shutting down the switches, waiting for them to stop, and we'll call our `main` procedure:
+  await allFutures(switch1.stop(), switch2.stop()) # close connections and shutdown all transports
+
+waitFor(main())
+
+## You can now run this program using `nim c -r part1.nim`, and you should see the dialing sequence, ending with a ping output.
+##
+## In the [next tutorial](tutorial_2_customproto.md), we'll look at how to create our own custom protocol.

examples/tutorial_2_customproto.nim

@@ -0,0 +1,74 @@
+## # Custom protocol in libp2p
+##
+## In the [previous tutorial](tutorial_1_connect.md), we've looked at how to create a simple ping program using the `nim-libp2p`.
+##
+## We'll now look at how to create a custom protocol inside the libp2p
+##
+## Let's create a `part2.nim`, and import our dependencies:
+import chronos
+import stew/byteutils
+
+import libp2p
+## This is similar to the first tutorial, except we don't need to import the `Ping` protocol.
+##
+## Next, we'll declare our custom protocol
+const TestCodec = "/test/proto/1.0.0"
+
+type TestProto = ref object of LPProtocol
+
+## We've set a [protocol ID](https://docs.libp2p.io/concepts/protocols/#protocol-ids), and created a custom `LPProtocol`. In a more complex protocol, we could use this structure to store interesting variables.
+##
+## A protocol generally has two part: and handling/server part, and a dialing/client part.
+## Theses two parts can be identical, but in our trivial protocol, the server will wait for a message from the client, and the client will send a message, so we have to handle the two cases separately.
+##
+## Let's start with the server part:
+
+proc new(T: typedesc[TestProto]): T =
+  # every incoming connections will in be handled in this closure
+  proc handle(conn: Connection, proto: string) {.async, gcsafe.} =
+    # Read up to 1024 bytes from this connection, and transform them into
+    # a string
+    echo "Got from remote - ", string.fromBytes(await conn.readLp(1024))
+    # We must close the connections ourselves when we're done with it
+    await conn.close()
+
+  return T(codecs: @[TestCodec], handler: handle)
+
+## This is a constructor for our `TestProto`, that will specify our `codecs` and a `handler`, which will be called for each incoming peer asking for this protocol.
+## In our handle, we simply read a message from the connection and `echo` it.
+##
+## We can now create our client part:
+proc hello(p: TestProto, conn: Connection) {.async.} =
+  await conn.writeLp("Hello p2p!")
+
+## Again, pretty straight-forward, we just send a message on the connection.
+##
+## We can now create our main procedure:
+proc main() {.async, gcsafe.} =
+  let
+    rng = newRng()
+    testProto = TestProto.new()
+    switch1 = newStandardSwitch(rng=rng)
+    switch2 = newStandardSwitch(rng=rng)
+
+  switch1.mount(testProto)
+
+  await switch1.start()
+  await switch2.start()
+
+  let conn = await switch2.dial(switch1.peerInfo.peerId, switch1.peerInfo.addrs, TestCodec)
+
+  await testProto.hello(conn)
+
+  # We must close the connection ourselves when we're done with it
+  await conn.close()
+
+  await allFutures(switch1.stop(), switch2.stop()) # close connections and shutdown all transports
+
+## This is very similar to the first tutorial's `main`, the only noteworthy difference is that we use `newStandardSwitch`, which is similar to the `createSwitch` of the first tutorial, but is bundled directly in libp2p
+##
+## We can now wrap our program by calling our main proc:
+waitFor(main())
+
+## And that's it!
+## In the [next tutorial](tutorial_3_protobuf.md), we'll create a more complex protocol using Protobuf.

examples/tutorial_3_protobuf.nim

@@ -0,0 +1,162 @@
+## # Protobuf usage
+##
+## In the [previous tutorial](tutorial_2_customproto.md), we created a simple "ping" protocol.
+## Most real protocol want their messages to be structured and extensible, which is why
+## most real protocols use [protobuf](https://developers.google.com/protocol-buffers) to
+## define their message structures.
+##
+## Here, we'll create a slightly more complex protocol, which parses & generate protobuf
+## messages. Let's start by importing our dependencies, as usual:
+import chronos
+import stew/results # for Opt[T]
+
+import libp2p
+
+## ## Protobuf encoding & decoding
+## This will be the structure of our messages:
+## ```protobuf
+## message MetricList {
+##   message Metric {
+##     string name = 1;
+##     float value = 2;
+##   }
+##
+##   repeated Metric metrics = 2;
+## }
+## ```
+## We'll create our protobuf types, encoders & decoders, according to this format.
+## To create the encoders & decoders, we are going to use minprotobuf
+## (included in libp2p).
+##
+## While more modern technics
+## (such as [nim-protobuf-serialization](https://github.com/status-im/nim-protobuf-serialization))
+## exists, minprotobuf is currently the recommended method to handle protobuf, since it has
+## been used in production extensively, and audited.
+type
+  Metric = object
+    name: string
+    value: float
+
+  MetricList = object
+    metrics: seq[Metric]
+
+{.push raises: [].}
+
+proc encode(m: Metric): ProtoBuffer =
+  result = initProtoBuffer()
+  result.write(1, m.name)
+  result.write(2, m.value)
+  result.finish()
+
+proc decode(_: type Metric, buf: seq[byte]): Result[Metric, ProtoError] =
+  var res: Metric
+  let pb = initProtoBuffer(buf)
+  # "getField" will return a Result[bool, ProtoError].
+  # The Result will hold an error if the protobuf is invalid.
+  # The Result will hold "false" if the field is missing
+  #
+  # We are just checking the error, and ignoring whether the value
+  # is present or not (default values are valid).
+  discard ? pb.getField(1, res.name)
+  discard ? pb.getField(2, res.value)
+  ok(res)
+
+proc encode(m: MetricList): ProtoBuffer =
+  result = initProtoBuffer()
+  for metric in m.metrics:
+    result.write(1, metric.encode())
+  result.finish()
+
+proc decode(_: type MetricList, buf: seq[byte]): Result[MetricList, ProtoError] =
+  var
+    res: MetricList
+    metrics: seq[seq[byte]]
+  let pb = initProtoBuffer(buf)
+  discard ? pb.getRepeatedField(1, metrics)
+
+  for metric in metrics:
+    res.metrics &= ? Metric.decode(metric)
+  ok(res)
+
+## ## Results instead of exceptions
+## As you can see, this part of the program also uses Results instead of exceptions for error handling.
+## We start by `{.push raises: [].}`, which will prevent every non-async function from raising
+## exceptions.
+##
+## Then, we use [nim-result](https://github.com/arnetheduck/nim-result) to convey
+## errors to function callers. A `Result[T, E]` will either hold a valid result of type
+## T, or an error of type E.
+##
+## You can check if the call succeeded by using `res.isOk`, and then get the
+## value using `res.value` or the error by using `res.error`.
+##
+## Another useful tool is `?`, which will unpack a Result if it succeeded,
+## or if it failed, exit the current procedure returning the error.
+##
+## nim-result is packed with other functionalities that you'll find in the
+## nim-result repository.
+##
+## Results and exception are generally interchangeable, but have different semantics
+## that you may or may not prefer.
+##
+## ## Creating the protocol
+## We'll next create a protocol, like in the last tutorial, to request these metrics from our host
+type
+  MetricCallback = proc: Future[MetricList] {.raises: [], gcsafe.}
+  MetricProto = ref object of LPProtocol
+    metricGetter: MetricCallback
+
+proc new(_: typedesc[MetricProto], cb: MetricCallback): MetricProto =
+  let res = MetricProto(metricGetter: cb)
+  proc handle(conn: Connection, proto: string) {.async, gcsafe.} =
+    let
+      metrics = await res.metricGetter()
+      asProtobuf = metrics.encode()
+    await conn.writeLp(asProtobuf.buffer)
+    await conn.close()
+
+  res.codecs = @["/metric-getter/1.0.0"]
+  res.handler = handle
+  return res
+
+proc fetch(p: MetricProto, conn: Connection): Future[MetricList] {.async.} =
+  let protobuf = await conn.readLp(2048)
+  # tryGet will raise an exception if the Result contains an error.
+  # It's useful to bridge between exception-world and result-world
+  return MetricList.decode(protobuf).tryGet()
+
+## We can now create our main procedure:
+proc main() {.async, gcsafe.} =
+  let rng = newRng()
+  proc randomMetricGenerator: Future[MetricList] {.async.} =
+    let metricCount = rng[].generate(uint32) mod 16
+    for i in 0 ..< metricCount + 1:
+      result.metrics.add(Metric(
+        name: "metric_" & $i,
+        value: float(rng[].generate(uint16)) / 1000.0
+      ))
+    return result
+  let
+    metricProto1 = MetricProto.new(randomMetricGenerator)
+    metricProto2 = MetricProto.new(randomMetricGenerator)
+    switch1 = newStandardSwitch(rng=rng)
+    switch2 = newStandardSwitch(rng=rng)
+
+  switch1.mount(metricProto1)
+
+  await switch1.start()
+  await switch2.start()
+
+  let
+    conn = await switch2.dial(switch1.peerInfo.peerId, switch1.peerInfo.addrs, metricProto2.codecs)
+    metrics = await metricProto2.fetch(conn)
+  await conn.close()
+
+  for metric in metrics.metrics:
+    echo metric.name, " = ", metric.value
+
+  await allFutures(switch1.stop(), switch2.stop()) # close connections and shutdown all transports
+
+waitFor(main())
+
+## If you run this program, you should see random metrics being sent from the switch1 to the switch2.

examples/tutorial_5_discovery.nim

@@ -0,0 +1,167 @@
+## # Discovery method
+##
+## In the [previous tutorial](tutorial_4_gossipsub.md), we built a custom protocol using [protobuf](https://developers.google.com/protocol-buffers) and
+## spread informations (some metrics) on the network using gossipsub.
+## For this tutorial, on the other hand, we'll go back on a simple example
+## we'll try to discover a specific peer to ping on the network.
+##
+## First, as usual, we import the dependencies:
+import chronos
+import stew/byteutils
+
+import libp2p
+import libp2p/protocols/rendezvous
+import libp2p/discovery/rendezvousinterface
+import libp2p/discovery/discoverymngr
+
+## We'll not use newStandardSwitch this time as we need the discovery protocol
+## [RendezVous](https://github.com/libp2p/specs/blob/master/rendezvous/README.md) to be mounted on the switch using withRendezVous.
+##
+## Note that other discovery methods such as [Kademlia](https://github.com/libp2p/specs/blob/master/kad-dht/README.md) or [discv5](https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md) exist (not everyone
+## of those are implemented yet though), but to make it simple for this tutorial,
+## we'll stick to RendezVous.
+proc createSwitch(rdv: RendezVous = RendezVous.new()): Switch =
+  SwitchBuilder.new()
+    .withRng(newRng())
+    .withAddresses(@[ MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet() ])
+    .withTcpTransport()
+    .withMplex()
+    .withNoise()
+    .withRendezVous(rdv)
+    .build()
+
+const DumbCodec = "/dumb/proto/1.0.0"
+type DumbProto = ref object of LPProtocol
+proc new(T: typedesc[DumbProto], nodeNumber: int): T =
+  proc handle(conn: Connection, proto: string) {.async, gcsafe.} =
+    echo "Node", nodeNumber, " received: ", string.fromBytes(await conn.readLp(1024))
+    await conn.close()
+
+  return T(codecs: @[DumbCodec], handler: handle)
+
+## We can create our main procedure:
+proc main() {.async, gcsafe.} =
+  # First of all, we'll create and start a boot node that'll be
+  # used as an information hub on the network
+  let bootNode = createSwitch()
+  await bootNode.start()
+
+  var
+    switches: seq[Switch] = @[]
+    discManagers: seq[DiscoveryManager] = @[]
+
+  for i in 0..5:
+    # Create a RendezVous Protocol
+    let rdv = RendezVous.new()
+    # Create a switch, mount the RendezVous protocol to it, start it and
+    # eventually connect it to the bootNode
+    let switch = createSwitch(rdv)
+    switch.mount(DumbProto.new(i))
+    await switch.start()
+    await switch.connect(bootNode.peerInfo.peerId, bootNode.peerInfo.addrs)
+    switches.add(switch)
+
+    # A discovery manager is a simple tool, you set it up by adding discovery
+    # interfaces (such as RendezVousInterface) then you can use it to advertise
+    # something on the network or to request something from it.
+    let dm = DiscoveryManager()
+    # A RendezVousInterface is a RendezVous protocol wrapped to be usable by the
+    # DiscoveryManager. You can initialize a time to advertise/request (tta/ttr)
+    # for advertise/request every x seconds.
+    dm.add(RendezVousInterface.new(rdv))
+    # Each nodes of the network'll advertise on some topics (even gang or odd club)
+    if i mod 2 == 0:
+      dm.advertise(RdvNamespace("Even Gang"))
+    else:
+      dm.advertise(RdvNamespace("Odd Club"))
+    discManagers.add(dm)
+
+  let
+    rdv = RendezVous.new()
+    newcomer = createSwitch(rdv)
+    dm = DiscoveryManager()
+  await newcomer.start()
+  await newcomer.connect(bootNode.peerInfo.peerId, bootNode.peerInfo.addrs)
+  dm.add(RendezVousInterface.new(rdv, ttr = 250.milliseconds))
+
+  let query = dm.request(RdvNamespace("Odd Club"))
+  for _ in 0..2:
+    echo "start"
+    let
+      res = await query.getPeer()
+    echo "0 ", res[PeerId], " ", res.getAll(MultiAddress)
+    let
+      conn = await newcomer.dial(res[PeerId], res.getAll(MultiAddress), DumbCodec)
+    echo "1"
+    await conn.writeLp("Odd Club suuuucks! Even Gang is better!")
+    echo "2"
+    await sleepAsync(300.milliseconds)
+    await conn.close()
+    echo "stop"
+  query.stop()
+
+
+
+#  # Create a "network" of six nodes looking like that:
+#  # node0 <-> node1 <-> node2 <-> node3 <-> node4 <-> node5
+#  var
+#    switches: seq[Switch] = @[]
+#    discManagers: seq[DiscoveryManager] = @[]
+#  for _ in 0..5:
+#    # Create a RendezVous Protocol
+#    let rdv = RendezVous.new()
+#    # Create a switch, mount the RendezVous protocol to it, start it and
+#    # enventually connect it to the previous node in the sequence
+#    let switch = createSwitch(rdv)
+#    if switches.len == 0:
+#      
+#    await switch.start()
+#    if switches.len > 0:
+#      await switches[^1].connect(switch.peerInfo.peerId, switch.peerInfo.addrs)
+#    switches.add(switch)
+#
+#    # A discovery manager is a simple tool, you setup it by adding discovery
+#    # interfaces (such as RendezVousInterface) then you can use it to advertise
+#    # something on the network or to request something from it.
+#    let dm = DiscoveryManager()
+#    # A RendezVousInterface is a RendezVous protocol wrapped to be usable by the
+#    # DiscoveryManager. You can initialize a time to advertise/request (tta/ttr)
+#    # for advertise/request every x seconds.
+#    dm.add(RendezVousInterface.new(rdv, ttr = 250.milliseconds))
+#    discManagers.add(dm)
+#  
+#  # The first node of the network advertise on someTopic
+#  discManagers[0].advertise(RdvNamespace("someTopic"))
+#  for i in 1..4:
+#    # All the four "middle" nodes request informations on someTopic
+#    # to make the message spread. With a time to request of 250ms, it should
+#    # spread on the four nodes in a little over 1 second.
+#    discard discManagers[i].request(RdvNamespace("someTopic"))
+#  let
+#    # The ending node request on someTopic aswell and await for information.
+#    query = discManagers[5].request(RdvNamespace("someTopic"))
+#    # getPeer give you a PeerAttribute containing informations about the peer.
+#    # The most useful are the PeerId and the MultiAddresses
+#    res = await query.getPeer()
+#
+#  # Now that a peer have been found, we can stop the query
+#  query.stop()
+#
+#  doAssert(res[PeerId] == switches[0].peerInfo.peerId)
+#  let resMa = res.getAll(MultiAddress)
+#  for ma in resMa:
+#    doAssert(ma in switches[0].peerInfo.addrs)
+#  echo "Peer Found: ", res[PeerId], " ", resMa
+#  # The peer Id and the address of the node0 is discovered by the node5 after
+#  # being pass by all the intermediate node.
+#
+#  # The node5 can connect the node0 to talk about someTopic directly.
+#
+#  # Stop all the DiscoveryManagers
+#  for dm in discManagers: dm.stop()
+#
+#  # Stop all the switches
+#  let futStop = switches.mapIt(it.stop())
+#  await allFutures(futStop)
+
+waitFor(main())

libp2p.nim

@@ -1,17 +1,72 @@
-## Nim-LibP2P
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-import
-  libp2p/daemon/[daemonapi, transpool],
-  libp2p/protobuf/minprotobuf,
-  libp2p/varint
+when defined(nimdoc):
+  ## Welcome to the nim-libp2p reference!
+  ##
+  ## On the left, you'll find a switch that allows you to see private
+  ## procedures. By default, you'll only see the public one (marked with `{.public.}`)
+  ##
+  ## The difference between public and private procedures is that public procedure
+  ## stay backward compatible during the Major version, whereas private ones can
+  ## change at each new Minor version.
+  ##
+  ## If you're new to nim-libp2p, you can find a tutorial `here<https://github.com/status-im/nim-libp2p/blob/master/examples/tutorial_1_connect.md>`_
+  ## that can help you get started.
 
-export
-  daemonapi, transpool, minprotobuf, varint
+  # Import stuff for doc
+  import libp2p/[
+      protobuf/minprotobuf,
+      switch,
+      stream/lpstream,
+      builders,
+      transports/tcptransport,
+      transports/wstransport,
+      protocols/ping,
+      protocols/pubsub,
+      peerid,
+      peerinfo,
+      peerstore,
+      multiaddress]
 
+  proc dummyPrivateProc*() =
+    ## A private proc example
+    discard
+else:
+  import
+    libp2p/[protobuf/minprotobuf,
+            muxers/muxer,
+            muxers/mplex/mplex,
+            stream/lpstream,
+            stream/bufferstream,
+            stream/connection,
+            transports/transport,
+            transports/tcptransport,
+            transports/wstransport,
+            protocols/secure/noise,
+            protocols/ping,
+            cid,
+            multihash,
+            multibase,
+            multicodec,
+            errors,
+            switch,
+            peerid,
+            peerinfo,
+            multiaddress,
+            builders,
+            crypto/crypto,
+            protocols/pubsub]
+
+  export
+    minprotobuf, switch, peerid, peerinfo,
+    connection, multiaddress, crypto, lpstream,
+    bufferstream, muxer, mplex, transport,
+    tcptransport, noise, errors, cid, multihash,
+    multicodec, builders, pubsub

libp2p.nimble

@@ -5,32 +5,43 @@ version       = "0.0.2"
 author        = "Status Research & Development GmbH"
 description   = "LibP2P implementation"
 license       = "MIT"
-skipDirs      = @["tests", "examples", "Nim"]
+skipDirs      = @["tests", "examples", "Nim", "tools", "scripts", "docs"]
 
 requires "nim >= 1.2.0",
          "nimcrypto >= 0.4.1",
+         "dnsclient >= 0.3.0 & < 0.4.0",
          "bearssl >= 0.1.4",
-         "chronicles >= 0.7.2",
-         "chronos >= 2.3.8",
+         "chronicles >= 0.10.2",
+         "chronos >= 3.0.6",
          "metrics",
          "secp256k1",
-         "stew >= 0.1.0"
+         "stew#head",
+         "websock"
 
-proc runTest(filename: string, verify: bool = true, sign: bool = true) =
-  var excstr = "nim c -r --opt:speed -d:debug --verbosity:0 --hints:off -d:chronicles_log_level=info"
-  excstr.add(" --warning[CaseTransition]:off --warning[ObservableStores]:off --warning[LockLevel]:off")
+proc runTest(filename: string, verify: bool = true, sign: bool = true,
+             moreoptions: string = "") =
+  var excstr = "nim c --skipParentCfg --opt:speed -d:debug -d:libp2p_agents_metrics -d:libp2p_protobuf_metrics -d:libp2p_network_protocols_metrics -d:libp2p_mplex_metrics "
+  excstr.add(" -d:chronicles_sinks=textlines[stdout],json[dynamic] -d:chronicles_log_level=TRACE ")
+  excstr.add(" -d:chronicles_runtime_filtering=TRUE ")
+  excstr.add(" " & getEnv("NIMFLAGS") & " ")
+  excstr.add(" --verbosity:0 --hints:off ")
   excstr.add(" -d:libp2p_pubsub_sign=" & $sign)
   excstr.add(" -d:libp2p_pubsub_verify=" & $verify)
-  excstr.add(" tests/" & filename)
-  exec excstr
+  excstr.add(" " & moreoptions & " ")
+  exec excstr & " -r " & " tests/" & filename
   rmFile "tests/" & filename.toExe
 
-proc buildSample(filename: string) =
-  var excstr = "nim c --opt:speed --threads:on -d:debug --verbosity:0 --hints:off"
-  excstr.add(" --warning[CaseTransition]:off --warning[ObservableStores]:off --warning[LockLevel]:off")
+proc buildSample(filename: string, run = false) =
+  var excstr = "nim c --opt:speed --threads:on -d:debug --verbosity:0 --hints:off -p:. "
   excstr.add(" examples/" & filename)
   exec excstr
-  rmFile "examples" & filename.toExe
+  if run:
+    exec "./examples/" & filename.toExe
+  rmFile "examples/" & filename.toExe
+
+proc tutorialToMd(filename: string) =
+  let markdown = gorge "cat " & filename & " | nim c -r --verbosity:0 --hints:off tools/markdown_builder.nim "
+  writeFile(filename.replace(".nim", ".md"), markdown)
 
 task testnative, "Runs libp2p native tests":
   runTest("testnative")
@@ -42,14 +53,85 @@ task testinterop, "Runs interop tests":
   runTest("testinterop")
 
 task testpubsub, "Runs pubsub tests":
+  runTest("pubsub/testgossipinternal", sign = false, verify = false, moreoptions = "-d:pubsub_internal_testing")
   runTest("pubsub/testpubsub")
   runTest("pubsub/testpubsub", sign = false, verify = false)
+  runTest("pubsub/testpubsub", sign = false, verify = false, moreoptions = "-d:libp2p_pubsub_anonymize=true")
+
+task testpubsub_slim, "Runs pubsub tests":
+  runTest("pubsub/testgossipinternal", sign = false, verify = false, moreoptions = "-d:pubsub_internal_testing")
+  runTest("pubsub/testpubsub")
+
+task testfilter, "Run PKI filter test":
+  runTest("testpkifilter",
+           moreoptions = "-d:libp2p_pki_schemes=\"secp256k1\"")
+  runTest("testpkifilter",
+           moreoptions = "-d:libp2p_pki_schemes=\"secp256k1;ed25519\"")
+  runTest("testpkifilter",
+           moreoptions = "-d:libp2p_pki_schemes=\"secp256k1;ed25519;ecnist\"")
+  runTest("testpkifilter",
+           moreoptions = "-d:libp2p_pki_schemes=")
 
 task test, "Runs the test suite":
   exec "nimble testnative"
   exec "nimble testpubsub"
   exec "nimble testdaemon"
   exec "nimble testinterop"
+  exec "nimble testfilter"
+  exec "nimble examples_build"
+
+task test_slim, "Runs the (slimmed down) test suite":
+  exec "nimble testnative"
+  exec "nimble testpubsub_slim"
+  exec "nimble testfilter"
+  exec "nimble examples_build"
+
+task website, "Build the website":
+  tutorialToMd("examples/tutorial_1_connect.nim")
+  tutorialToMd("examples/tutorial_2_customproto.nim")
+  tutorialToMd("examples/tutorial_3_protobuf.nim")
+  tutorialToMd("examples/tutorial_5_discovery.nim")
+  tutorialToMd("examples/circuitrelay.nim")
+  exec "mkdocs build"
 
 task examples_build, "Build the samples":
   buildSample("directchat")
+  buildSample("helloworld", true)
+  buildSample("circuitrelay", true)
+  buildSample("tutorial_1_connect", true)
+  buildSample("tutorial_2_customproto", true)
+  if (NimMajor, NimMinor) > (1, 2):
+    # This tutorial relies on post 1.4 exception tracking
+    buildSample("tutorial_3_protobuf", true)
+    buildSample("tutorial_5_discovery", true)
+
+# pin system
+# while nimble lockfile
+# isn't available
+
+const PinFile = ".pinned"
+task pin, "Create a lockfile":
+  # pinner.nim was originally here
+  # but you can't read output from
+  # a command in a nimscript
+  exec "nim c -r tools/pinner.nim"
+
+import sequtils
+import os
+task install_pinned, "Reads the lockfile":
+  let toInstall = readFile(PinFile).splitWhitespace().mapIt((it.split(";", 1)[0], it.split(";", 1)[1]))
+  # [('packageName', 'packageFullUri')]
+
+  rmDir("nimbledeps")
+  mkDir("nimbledeps")
+  exec "nimble install -y " & toInstall.mapIt(it[1]).join(" ")
+
+  # Remove the automatically installed deps
+  # (inefficient you say?)
+  let allowedDirectories = toInstall.mapIt(it[0] & "-" & it[1].split('@')[1])
+  for dependency in listDirs("nimbledeps/pkgs"):
+    if dependency.extractFilename notin allowedDirectories:
+      rmDir(dependency)
+
+task unpin, "Restore global package use":
+  rmDir("nimbledeps")

libp2p/builders.nim

@@ -0,0 +1,317 @@
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+## This module contains a Switch Building helper.
+runnableExamples:
+  let switch =
+   SwitchBuilder.new()
+   .withRng(rng)
+   .withAddresses(multiaddress)
+   # etc
+   .build()
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import
+  options, tables, chronos, chronicles, sequtils,
+  switch, peerid, peerinfo, stream/connection, multiaddress,
+  crypto/crypto, transports/[transport, tcptransport],
+  muxers/[muxer, mplex/mplex, yamux/yamux],
+  protocols/[identify, secure/secure, secure/noise, rendezvous],
+  protocols/connectivity/[autonat, relay/relay, relay/client, relay/rtransport],
+  connmanager, upgrademngrs/muxedupgrade,
+  nameresolving/nameresolver,
+  errors, utility
+
+export
+  switch, peerid, peerinfo, connection, multiaddress, crypto, errors
+
+type
+  TransportProvider* {.public.} = proc(upgr: Upgrade): Transport {.gcsafe, raises: [Defect].}
+
+  SecureProtocol* {.pure.} = enum
+    Noise,
+    Secio {.deprecated.}
+
+  SwitchBuilder* = ref object
+    privKey: Option[PrivateKey]
+    addresses: seq[MultiAddress]
+    secureManagers: seq[SecureProtocol]
+    muxers: seq[MuxerProvider]
+    transports: seq[TransportProvider]
+    rng: ref HmacDrbgContext
+    maxConnections: int
+    maxIn: int
+    sendSignedPeerRecord: bool
+    maxOut: int
+    maxConnsPerPeer: int
+    protoVersion: string
+    agentVersion: string
+    nameResolver: NameResolver
+    peerStoreCapacity: Option[int]
+    autonat: bool
+    circuitRelay: Relay
+    rdv: RendezVous
+
+proc new*(T: type[SwitchBuilder]): T {.public.} =
+  ## Creates a SwitchBuilder
+
+  let address = MultiAddress
+  .init("/ip4/127.0.0.1/tcp/0")
+  .expect("Should initialize to default")
+
+  SwitchBuilder(
+    privKey: none(PrivateKey),
+    addresses: @[address],
+    secureManagers: @[],
+    maxConnections: MaxConnections,
+    maxIn: -1,
+    maxOut: -1,
+    maxConnsPerPeer: MaxConnectionsPerPeer,
+    protoVersion: ProtoVersion,
+    agentVersion: AgentVersion)
+
+proc withPrivateKey*(b: SwitchBuilder, privateKey: PrivateKey): SwitchBuilder {.public.} =
+  ## Set the private key of the switch. Will be used to
+  ## generate a PeerId
+
+  b.privKey = some(privateKey)
+  b
+
+proc withAddress*(b: SwitchBuilder, address: MultiAddress): SwitchBuilder {.public.} =
+  ## | Set the listening address of the switch
+  ## | Calling it multiple time will override the value
+
+  b.addresses = @[address]
+  b
+
+proc withAddresses*(b: SwitchBuilder, addresses: seq[MultiAddress]): SwitchBuilder {.public.} =
+  ## | Set the listening addresses of the switch
+  ## | Calling it multiple time will override the value
+
+  b.addresses = addresses
+  b
+
+proc withSignedPeerRecord*(b: SwitchBuilder, sendIt = true): SwitchBuilder {.public.} =
+  b.sendSignedPeerRecord = sendIt
+  b
+
+proc withMplex*(
+    b: SwitchBuilder,
+    inTimeout = 5.minutes,
+    outTimeout = 5.minutes,
+    maxChannCount = 200): SwitchBuilder {.public.} =
+  ## | Uses `Mplex <https://docs.libp2p.io/concepts/stream-multiplexing/#mplex>`_ as a multiplexer
+  ## | `Timeout` is the duration after which a inactive connection will be closed
+  proc newMuxer(conn: Connection): Muxer =
+    Mplex.new(
+      conn,
+      inTimeout,
+      outTimeout,
+      maxChannCount)
+
+  assert b.muxers.countIt(it.codec == MplexCodec) == 0, "Mplex build multiple times"
+  b.muxers.add(MuxerProvider.new(newMuxer, MplexCodec))
+  b
+
+proc withYamux*(b: SwitchBuilder): SwitchBuilder =
+  proc newMuxer(conn: Connection): Muxer = Yamux.new(conn)
+
+  assert b.muxers.countIt(it.codec == YamuxCodec) == 0, "Yamux build multiple times"
+  b.muxers.add(MuxerProvider.new(newMuxer, YamuxCodec))
+  b
+
+proc withNoise*(b: SwitchBuilder): SwitchBuilder {.public.} =
+  b.secureManagers.add(SecureProtocol.Noise)
+  b
+
+proc withTransport*(b: SwitchBuilder, prov: TransportProvider): SwitchBuilder {.public.} =
+  ## Use a custom transport
+  runnableExamples:
+    let switch =
+      SwitchBuilder.new()
+      .withTransport(proc(upgr: Upgrade): Transport = TcpTransport.new(flags, upgr))
+      .build()
+  b.transports.add(prov)
+  b
+
+proc withTcpTransport*(b: SwitchBuilder, flags: set[ServerFlags] = {}): SwitchBuilder {.public.} =
+  b.withTransport(proc(upgr: Upgrade): Transport = TcpTransport.new(flags, upgr))
+
+proc withRng*(b: SwitchBuilder, rng: ref HmacDrbgContext): SwitchBuilder {.public.} =
+  b.rng = rng
+  b
+
+proc withMaxConnections*(b: SwitchBuilder, maxConnections: int): SwitchBuilder {.public.} =
+  ## Maximum concurrent connections of the switch. You should either use this, or
+  ## `withMaxIn <#withMaxIn,SwitchBuilder,int>`_ & `withMaxOut<#withMaxOut,SwitchBuilder,int>`_
+  b.maxConnections = maxConnections
+  b
+
+proc withMaxIn*(b: SwitchBuilder, maxIn: int): SwitchBuilder {.public.} =
+  ## Maximum concurrent incoming connections. Should be used with `withMaxOut<#withMaxOut,SwitchBuilder,int>`_
+  b.maxIn = maxIn
+  b
+
+proc withMaxOut*(b: SwitchBuilder, maxOut: int): SwitchBuilder {.public.} =
+  ## Maximum concurrent outgoing connections. Should be used with `withMaxIn<#withMaxIn,SwitchBuilder,int>`_
+  b.maxOut = maxOut
+  b
+
+proc withMaxConnsPerPeer*(b: SwitchBuilder, maxConnsPerPeer: int): SwitchBuilder {.public.} =
+  b.maxConnsPerPeer = maxConnsPerPeer
+  b
+
+proc withPeerStore*(b: SwitchBuilder, capacity: int): SwitchBuilder {.public.} =
+  b.peerStoreCapacity = some(capacity)
+  b
+
+proc withProtoVersion*(b: SwitchBuilder, protoVersion: string): SwitchBuilder {.public.} =
+  b.protoVersion = protoVersion
+  b
+
+proc withAgentVersion*(b: SwitchBuilder, agentVersion: string): SwitchBuilder {.public.} =
+  b.agentVersion = agentVersion
+  b
+
+proc withNameResolver*(b: SwitchBuilder, nameResolver: NameResolver): SwitchBuilder {.public.} =
+  b.nameResolver = nameResolver
+  b
+
+proc withAutonat*(b: SwitchBuilder): SwitchBuilder =
+  b.autonat = true
+  b
+
+proc withCircuitRelay*(b: SwitchBuilder, r: Relay = Relay.new()): SwitchBuilder =
+  b.circuitRelay = r
+  b
+
+proc withRendezVous*(b: SwitchBuilder, rdv: RendezVous = RendezVous.new()): SwitchBuilder =
+  b.rdv = rdv
+  b
+
+proc build*(b: SwitchBuilder): Switch
+  {.raises: [Defect, LPError], public.} =
+
+  if b.rng == nil: # newRng could fail
+    raise newException(Defect, "Cannot initialize RNG")
+
+  let pkRes = PrivateKey.random(b.rng[])
+  let
+    seckey = b.privKey.get(otherwise = pkRes.expect("Expected default Private Key"))
+
+  var
+    secureManagerInstances: seq[Secure]
+  if SecureProtocol.Noise in b.secureManagers:
+    secureManagerInstances.add(Noise.new(b.rng, seckey).Secure)
+
+  let
+    peerInfo = PeerInfo.new(
+      seckey,
+      b.addresses,
+      protoVersion = b.protoVersion,
+      agentVersion = b.agentVersion)
+
+  let
+    identify = Identify.new(peerInfo, b.sendSignedPeerRecord)
+    connManager = ConnManager.new(b.maxConnsPerPeer, b.maxConnections, b.maxIn, b.maxOut)
+    ms = MultistreamSelect.new()
+    muxedUpgrade = MuxedUpgrade.new(identify, b.muxers, secureManagerInstances, connManager, ms)
+
+  let
+    transports = block:
+      var transports: seq[Transport]
+      for tProvider in b.transports:
+        transports.add(tProvider(muxedUpgrade))
+      transports
+
+  if b.secureManagers.len == 0:
+    b.secureManagers &= SecureProtocol.Noise
+
+  if isNil(b.rng):
+    b.rng = newRng()
+
+  let peerStore =
+    if isSome(b.peerStoreCapacity):
+      PeerStore.new(b.peerStoreCapacity.get())
+    else:
+      PeerStore.new()
+
+  let switch = newSwitch(
+    peerInfo = peerInfo,
+    transports = transports,
+    identity = identify,
+    secureManagers = secureManagerInstances,
+    connManager = connManager,
+    ms = ms,
+    nameResolver = b.nameResolver,
+    peerStore = peerStore)
+
+  if b.autonat:
+    let autonat = Autonat.new(switch)
+    switch.mount(autonat)
+
+  if not isNil(b.circuitRelay):
+    if b.circuitRelay of RelayClient:
+      switch.addTransport(RelayTransport.new(RelayClient(b.circuitRelay), muxedUpgrade))
+    b.circuitRelay.setup(switch)
+    switch.mount(b.circuitRelay)
+
+  if not isNil(b.rdv):
+    b.rdv.setup(switch)
+    switch.mount(b.rdv)
+
+  return switch
+
+proc newStandardSwitch*(
+  privKey = none(PrivateKey),
+  addrs: MultiAddress | seq[MultiAddress] = MultiAddress.init("/ip4/127.0.0.1/tcp/0").tryGet(),
+  secureManagers: openArray[SecureProtocol] = [
+      SecureProtocol.Noise,
+    ],
+  transportFlags: set[ServerFlags] = {},
+  rng = newRng(),
+  inTimeout: Duration = 5.minutes,
+  outTimeout: Duration = 5.minutes,
+  maxConnections = MaxConnections,
+  maxIn = -1,
+  maxOut = -1,
+  maxConnsPerPeer = MaxConnectionsPerPeer,
+  nameResolver: NameResolver = nil,
+  sendSignedPeerRecord = false,
+  peerStoreCapacity = 1000): Switch
+  {.raises: [Defect, LPError], public.} =
+  ## Helper for common switch configurations.
+
+  if SecureProtocol.Secio in secureManagers:
+      quit("Secio is deprecated!") # use of secio is unsafe
+
+  let addrs = when addrs is MultiAddress: @[addrs] else: addrs
+  var b = SwitchBuilder
+    .new()
+    .withAddresses(addrs)
+    .withRng(rng)
+    .withSignedPeerRecord(sendSignedPeerRecord)
+    .withMaxConnections(maxConnections)
+    .withMaxIn(maxIn)
+    .withMaxOut(maxOut)
+    .withMaxConnsPerPeer(maxConnsPerPeer)
+    .withPeerStore(capacity=peerStoreCapacity)
+    .withMplex(inTimeout, outTimeout)
+    .withTcpTransport(transportFlags)
+    .withNameResolver(nameResolver)
+    .withNoise()
+
+  if privKey.isSome():
+    b = b.withPrivateKey(privKey.get())
+
+  b.build()

libp2p/cid.nim

@@ -1,20 +1,28 @@
-## Nim-LibP2P
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module implementes CID (Content IDentifier).
-import tables
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import tables, hashes
 import multibase, multicodec, multihash, vbuffer, varint
-import stew/base58
+import stew/[base58, results]
+
+export results
 
 type
-  CidStatus* {.pure.} = enum
-    Error, Success, Incorrect, Overrun
+  CidError* {.pure.} = enum
+    Error, Incorrect, Unsupported, Overrun
 
   CidVersion* = enum
     CIDvIncorrect, CIDv0, CIDv1, CIDvReserved
@@ -25,8 +33,6 @@ type
     hpos*: int
     data*: VBuffer
 
-  CidError* = object of CatchableError
-
 const
   ContentIdsList = [
     multiCodec("raw"),
@@ -59,75 +65,82 @@ const
   ]
 
 proc initCidCodeTable(): Table[int, MultiCodec] {.compileTime.} =
-  result = initTable[int, MultiCodec]()
   for item in ContentIdsList:
     result[int(item)] = item
 
 const
   CodeContentIds = initCidCodeTable()
 
-proc decode(data: openarray[byte], cid: var Cid): CidStatus =
-  if len(data) == 34:
-    if data[0] == 0x12'u8 and data[1] == 0x20'u8:
-      cid.cidver = CIDv0
-      cid.mcodec = multiCodec("dag-pb")
-      cid.hpos = 0
-      cid.data = initVBuffer(data)
-      result = CidStatus.Success
-  if cid.cidver == CIDvIncorrect:
+template orError*(exp: untyped, err: untyped): untyped =
+  (exp.mapErr do (_: auto) -> auto: err)
+
+proc decode(data: openArray[byte]): Result[Cid, CidError] =
+  if len(data) == 34 and data[0] == 0x12'u8 and data[1] == 0x20'u8:
+    ok(Cid(
+        cidver: CIDv0,
+        mcodec: multiCodec("dag-pb"),
+        hpos: 0,
+        data: initVBuffer(data)))
+  else:
     var version, codec: uint64
     var res, offset: int
     var vb = initVBuffer(data)
     if vb.isEmpty():
-      return CidStatus.Incorrect
-    res = vb.readVarint(version)
-    if res == -1:
-      return CidStatus.Incorrect
-    offset += res
-    if version != 1'u64:
-      return CidStatus.Incorrect
-    res = vb.readVarint(codec)
-    if res == -1:
-      return CidStatus.Incorrect
-    offset += res
-    var mcodec = CodeContentIds.getOrDefault(cast[int](codec),
-                                             InvalidMultiCodec)
-    if mcodec == InvalidMultiCodec:
-      return CidStatus.Incorrect
-    if not MultiHash.validate(vb.buffer.toOpenArray(vb.offset,
-                                                    vb.buffer.high)):
-      return CidStatus.Incorrect
-    vb.finish()
-    cid.cidver = CIDv1
-    cid.mcodec = mcodec
-    cid.hpos = offset
-    cid.data = vb
-    result = CidStatus.Success
+      err(CidError.Incorrect)
+    else:
+      res = vb.readVarint(version)
+      if res == -1:
+        err(CidError.Incorrect)
+      else:
+        offset += res
+        if version != 1'u64:
+          err(CidError.Incorrect)
+        else:
+          res = vb.readVarint(codec)
+          if res == -1:
+            err(CidError.Incorrect)
+          else:
+            offset += res
+            var mcodec = CodeContentIds.getOrDefault(cast[int](codec),
+                                                    InvalidMultiCodec)
+            if mcodec == InvalidMultiCodec:
+              err(CidError.Incorrect)
+            else:
+              if not MultiHash.validate(vb.buffer.toOpenArray(vb.offset,
+                                                              vb.buffer.high)):
+                err(CidError.Incorrect)
+              else:
+                vb.finish()
+                ok(Cid(
+                    cidver: CIDv1,
+                    mcodec: mcodec,
+                    hpos: offset,
+                    data: vb))
 
-proc decode(data: openarray[char], cid: var Cid): CidStatus =
+proc decode(data: openArray[char]): Result[Cid, CidError] =
   var buffer: seq[byte]
   var plen = 0
   if len(data) < 2:
-    return CidStatus.Incorrect
+    return err(CidError.Incorrect)
   if len(data) == 46:
     if data[0] == 'Q' and data[1] == 'm':
       buffer = newSeq[byte](BTCBase58.decodedLength(len(data)))
       if BTCBase58.decode(data, buffer, plen) != Base58Status.Success:
-        return CidStatus.Incorrect
+        return err(CidError.Incorrect)
       buffer.setLen(plen)
   if len(buffer) == 0:
     let length = MultiBase.decodedLength(data[0], len(data))
     if length == -1:
-      return CidStatus.Incorrect
+      return err(CidError.Incorrect)
     buffer = newSeq[byte](length)
     if MultiBase.decode(data, buffer, plen) != MultiBaseStatus.Success:
-      return CidStatus.Incorrect
+      return err(CidError.Incorrect)
     buffer.setLen(plen)
     if buffer[0] == 0x12'u8:
-      return CidStatus.Incorrect
-  result = decode(buffer, cid)
+      return err(CidError.Incorrect)
+  decode(buffer)
 
-proc validate*(ctype: typedesc[Cid], data: openarray[byte]): bool =
+proc validate*(ctype: typedesc[Cid], data: openArray[byte]): bool =
   ## Returns ``true`` is data has valid binary CID representation.
   var version, codec: uint64
   var res: VarintResult[void]
@@ -157,30 +170,30 @@ proc validate*(ctype: typedesc[Cid], data: openarray[byte]): bool =
     return false
   result = true
 
-proc mhash*(cid: Cid): MultiHash =
+proc mhash*(cid: Cid): Result[MultiHash, CidError] =
   ## Returns MultiHash part of CID.
   if cid.cidver notin {CIDv0, CIDv1}:
-    raise newException(CidError, "Incorrect CID!")
-  result = MultiHash.init(
-    cid.data.buffer.toOpenArray(cid.hpos, cid.data.high)).tryGet()
+    err(CidError.Incorrect)
+  else:
+    MultiHash.init(cid.data.buffer.toOpenArray(cid.hpos, cid.data.high)).orError(CidError.Incorrect)
 
-proc contentType*(cid: Cid): MultiCodec =
+proc contentType*(cid: Cid): Result[MultiCodec, CidError] =
   ## Returns content type part of CID
   if cid.cidver notin {CIDv0, CIDv1}:
-    raise newException(CidError, "Incorrect CID!")
-  result = cid.mcodec
+    err(CidError.Incorrect)
+  else:
+    ok(cid.mcodec)
 
 proc version*(cid: Cid): CidVersion =
   ## Returns CID version
   result = cid.cidver
 
-proc init*[T: char|byte](ctype: typedesc[Cid], data: openarray[T]): Cid =
+proc init*[T: char|byte](ctype: typedesc[Cid], data: openArray[T]): Result[Cid, CidError] =
   ## Create new content identifier using array of bytes or string ``data``.
-  if decode(data, result) != CidStatus.Success:
-    raise newException(CidError, "Incorrect CID!")
+  decode(data)
 
 proc init*(ctype: typedesc[Cid], version: CidVersion, content: MultiCodec,
-           hash: MultiHash): Cid =
+           hash: MultiHash): Result[Cid, CidError] =
   ## Create new content identifier using content type ``content`` and
   ## MultiHash ``hash`` using version ``version``.
   ##
@@ -188,33 +201,35 @@ proc init*(ctype: typedesc[Cid], version: CidVersion, content: MultiCodec,
   ## Cid.init(CIDv0, multiCodec("dag-pb"), MultiHash.digest("sha2-256", data))
   ##
   ## All other encodings and hashes are not supported by CIDv0.
-  result.cidver = version
-  
+
+  var res: Cid
+  res.cidver = version
+
   if version == CIDv0:
     if content != multiCodec("dag-pb"):
-      raise newException(CidError,
-                         "CIDv0 supports only `dag-pb` content type!")
-    result.data = initVBuffer()
+      return err(CidError.Unsupported)
+    res.data = initVBuffer()
     if hash.mcodec != multiCodec("sha2-256"):
-      raise newException(CidError,
-                         "CIDv0 supports only `sha2-256` hash digest!")
-    result.mcodec = content
-    result.data.write(hash)
-    result.data.finish()
+      return err(CidError.Unsupported)
+    res.mcodec = content
+    res.data.write(hash)
+    res.data.finish()
+    return ok(res)
   elif version == CIDv1:
     let mcodec = CodeContentIds.getOrDefault(cast[int](content),
                                              InvalidMultiCodec)
     if mcodec == InvalidMultiCodec:
-      raise newException(CidError, "Incorrect content type")
-    result.mcodec = mcodec
-    result.data = initVBuffer()
-    result.data.writeVarint(cast[uint64](1))
-    result.data.write(mcodec)
-    result.hpos = len(result.data.buffer)
-    result.data.write(hash)
-    result.data.finish()
+      return err(CidError.Incorrect)
+    res.mcodec = mcodec
+    res.data = initVBuffer()
+    res.data.writeVarint(cast[uint64](1))
+    res.data.write(mcodec)
+    res.hpos = len(res.data.buffer)
+    res.data.write(hash)
+    res.data.finish()
+    return ok(res)
   else:
-    raise newException(CidError, "CID version is not supported" & $version)
+    return err(CidError.Unsupported)
 
 proc `==`*(a: Cid, b: Cid): bool =
   ## Compares content identifiers ``a`` and ``b``, returns ``true`` if hashes
@@ -255,9 +270,18 @@ proc encode*(mbtype: typedesc[MultiBase], encoding: string,
   ## ``encoding``.
   result = MultiBase.encode(encoding, cid.data.buffer).tryGet()
 
+proc hash*(cid: Cid): Hash {.inline.} =
+  hash(cid.data.buffer)
+
 proc `$`*(cid: Cid): string =
   ## Return official string representation of content identifier ``cid``.
   if cid.cidver == CIDv0:
-    result = BTCBase58.encode(cid.data.buffer)
+    BTCBase58.encode(cid.data.buffer)
   elif cid.cidver == CIDv1:
-    result = Multibase.encode("base58btc", cid.data.buffer).tryGet()
+    let res = MultiBase.encode("base58btc", cid.data.buffer)
+    if res.isOk():
+      res.get()
+    else:
+      ""
+  else:
+    ""

libp2p/connmanager.nim

@@ -0,0 +1,539 @@
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import std/[options, tables, sequtils, sets]
+import pkg/[chronos, chronicles, metrics]
+import peerinfo,
+       peerstore,
+       stream/connection,
+       muxers/muxer,
+       utils/semaphore,
+       errors
+
+logScope:
+  topics = "libp2p connmanager"
+
+declareGauge(libp2p_peers, "total connected peers")
+
+const
+  MaxConnections* = 50
+  MaxConnectionsPerPeer* = 1
+
+type
+  TooManyConnectionsError* = object of LPError
+
+  ConnEventKind* {.pure.} = enum
+    Connected,    # A connection was made and securely upgraded - there may be
+                  # more than one concurrent connection thus more than one upgrade
+                  # event per peer.
+
+    Disconnected  # Peer disconnected - this event is fired once per upgrade
+                  # when the associated connection is terminated.
+
+  ConnEvent* = object
+    case kind*: ConnEventKind
+    of ConnEventKind.Connected:
+      incoming*: bool
+    else:
+      discard
+
+  ConnEventHandler* =
+    proc(peerId: PeerId, event: ConnEvent): Future[void]
+      {.gcsafe, raises: [Defect].}
+
+  PeerEventKind* {.pure.} = enum
+    Left,
+    Identified,
+    Joined
+
+  PeerEvent* = object
+    case kind*: PeerEventKind
+      of PeerEventKind.Joined:
+        initiator*: bool
+      else:
+        discard
+
+  PeerEventHandler* =
+    proc(peerId: PeerId, event: PeerEvent): Future[void] {.gcsafe, raises: [Defect].}
+
+  MuxerHolder = object
+    muxer: Muxer
+    handle: Future[void]
+
+  ConnManager* = ref object of RootObj
+    maxConnsPerPeer: int
+    inSema*: AsyncSemaphore
+    outSema*: AsyncSemaphore
+    conns: Table[PeerId, HashSet[Connection]]
+    muxed: Table[Connection, MuxerHolder]
+    connEvents: array[ConnEventKind, OrderedSet[ConnEventHandler]]
+    peerEvents: array[PeerEventKind, OrderedSet[PeerEventHandler]]
+    peerStore*: PeerStore
+
+  ConnectionSlot* = object
+    connManager: ConnManager
+    direction: Direction
+
+proc newTooManyConnectionsError(): ref TooManyConnectionsError {.inline.} =
+  result = newException(TooManyConnectionsError, "Too many connections")
+
+proc new*(C: type ConnManager,
+           maxConnsPerPeer = MaxConnectionsPerPeer,
+           maxConnections = MaxConnections,
+           maxIn = -1,
+           maxOut = -1): ConnManager =
+  var inSema, outSema: AsyncSemaphore
+  if maxIn > 0 or maxOut > 0:
+    inSema = newAsyncSemaphore(maxIn)
+    outSema = newAsyncSemaphore(maxOut)
+  elif maxConnections > 0:
+    inSema = newAsyncSemaphore(maxConnections)
+    outSema = inSema
+  else:
+    raiseAssert "Invalid connection counts!"
+
+  C(maxConnsPerPeer: maxConnsPerPeer,
+    inSema: inSema,
+    outSema: outSema)
+
+proc connCount*(c: ConnManager, peerId: PeerId): int =
+  c.conns.getOrDefault(peerId).len
+
+proc addConnEventHandler*(c: ConnManager,
+                          handler: ConnEventHandler,
+                          kind: ConnEventKind) =
+  ## Add peer event handler - handlers must not raise exceptions!
+  ##
+
+  try:
+    if isNil(handler): return
+    c.connEvents[kind].incl(handler)
+  except Exception as exc:
+    # TODO: there is an Exception being raised
+    # somewhere in the depths of the std.
+    # Might be related to https://github.com/nim-lang/Nim/issues/17382
+
+    raiseAssert exc.msg
+
+proc removeConnEventHandler*(c: ConnManager,
+                             handler: ConnEventHandler,
+                             kind: ConnEventKind) =
+  try:
+    c.connEvents[kind].excl(handler)
+  except Exception as exc:
+    # TODO: there is an Exception being raised
+    # somewhere in the depths of the std.
+    # Might be related to https://github.com/nim-lang/Nim/issues/17382
+
+    raiseAssert exc.msg
+
+proc triggerConnEvent*(c: ConnManager,
+                       peerId: PeerId,
+                       event: ConnEvent) {.async, gcsafe.} =
+  try:
+    trace "About to trigger connection events", peer = peerId
+    if c.connEvents[event.kind].len() > 0:
+      trace "triggering connection events", peer = peerId, event = $event.kind
+      var connEvents: seq[Future[void]]
+      for h in c.connEvents[event.kind]:
+        connEvents.add(h(peerId, event))
+
+      checkFutures(await allFinished(connEvents))
+  except CancelledError as exc:
+    raise exc
+  except CatchableError as exc:
+    warn "Exception in triggerConnEvents",
+      msg = exc.msg, peer = peerId, event = $event
+
+proc addPeerEventHandler*(c: ConnManager,
+                          handler: PeerEventHandler,
+                          kind: PeerEventKind) =
+  ## Add peer event handler - handlers must not raise exceptions!
+  ##
+
+  if isNil(handler): return
+  try:
+    c.peerEvents[kind].incl(handler)
+  except Exception as exc:
+    # TODO: there is an Exception being raised
+    # somewhere in the depths of the std.
+    # Might be related to https://github.com/nim-lang/Nim/issues/17382
+
+    raiseAssert exc.msg
+
+proc removePeerEventHandler*(c: ConnManager,
+                             handler: PeerEventHandler,
+                             kind: PeerEventKind) =
+  try:
+    c.peerEvents[kind].excl(handler)
+  except Exception as exc:
+    # TODO: there is an Exception being raised
+    # somewhere in the depths of the std.
+    # Might be related to https://github.com/nim-lang/Nim/issues/17382
+
+    raiseAssert exc.msg
+
+proc triggerPeerEvents*(c: ConnManager,
+                        peerId: PeerId,
+                        event: PeerEvent) {.async, gcsafe.} =
+
+  trace "About to trigger peer events", peer = peerId
+  if c.peerEvents[event.kind].len == 0:
+    return
+
+  try:
+    let count = c.connCount(peerId)
+    if event.kind == PeerEventKind.Joined and count != 1:
+      trace "peer already joined", peer = peerId, event = $event
+      return
+    elif event.kind == PeerEventKind.Left and count != 0:
+      trace "peer still connected or already left", peer = peerId, event = $event
+      return
+
+    trace "triggering peer events", peer = peerId, event = $event
+
+    var peerEvents: seq[Future[void]]
+    for h in c.peerEvents[event.kind]:
+      peerEvents.add(h(peerId, event))
+
+    checkFutures(await allFinished(peerEvents))
+  except CancelledError as exc:
+    raise exc
+  except CatchableError as exc: # handlers should not raise!
+    warn "Exception in triggerPeerEvents", exc = exc.msg, peer = peerId
+
+proc contains*(c: ConnManager, conn: Connection): bool =
+  ## checks if a connection is being tracked by the
+  ## connection manager
+  ##
+
+  if isNil(conn):
+    return
+
+  return conn in c.conns.getOrDefault(conn.peerId)
+
+proc contains*(c: ConnManager, peerId: PeerId): bool =
+  peerId in c.conns
+
+proc contains*(c: ConnManager, muxer: Muxer): bool =
+  ## checks if a muxer is being tracked by the connection
+  ## manager
+  ##
+
+  if isNil(muxer):
+    return
+
+  let conn = muxer.connection
+  if conn notin c:
+    return
+
+  if conn notin c.muxed:
+    return
+
+  return muxer == c.muxed.getOrDefault(conn).muxer
+
+proc closeMuxerHolder(muxerHolder: MuxerHolder) {.async.} =
+  trace "Cleaning up muxer", m = muxerHolder.muxer
+
+  await muxerHolder.muxer.close()
+  if not(isNil(muxerHolder.handle)):
+    try:
+      await muxerHolder.handle # TODO noraises?
+    except CatchableError as exc:
+      trace "Exception in close muxer handler", exc = exc.msg
+  trace "Cleaned up muxer", m = muxerHolder.muxer
+
+proc delConn(c: ConnManager, conn: Connection) =
+  let peerId = conn.peerId
+  c.conns.withValue(peerId, peerConns):
+    peerConns[].excl(conn)
+
+    if peerConns[].len == 0:
+      c.conns.del(peerId) # invalidates `peerConns`
+
+    libp2p_peers.set(c.conns.len.int64)
+    trace "Removed connection", conn
+
+proc cleanupConn(c: ConnManager, conn: Connection) {.async.} =
+  ## clean connection's resources such as muxers and streams
+
+  if isNil(conn):
+    trace "Wont cleanup a nil connection"
+    return
+
+  # Remove connection from all tables without async breaks
+  var muxer = some(MuxerHolder())
+  if not c.muxed.pop(conn, muxer.get()):
+    muxer = none(MuxerHolder)
+
+  delConn(c, conn)
+
+  try:
+    if muxer.isSome:
+      await closeMuxerHolder(muxer.get())
+  finally:
+    await conn.close()
+
+  trace "Connection cleaned up", conn
+
+proc onConnUpgraded(c: ConnManager, conn: Connection) {.async.} =
+  try:
+    trace "Triggering connect events", conn
+    conn.upgrade()
+
+    let peerId = conn.peerId
+    await c.triggerPeerEvents(
+      peerId, PeerEvent(kind: PeerEventKind.Joined, initiator: conn.dir == Direction.Out))
+
+    await c.triggerConnEvent(
+      peerId, ConnEvent(kind: ConnEventKind.Connected, incoming: conn.dir == Direction.In))
+  except CatchableError as exc:
+    # This is top-level procedure which will work as separate task, so it
+    # do not need to propagate CancelledError and should handle other errors
+    warn "Unexpected exception in switch peer connection cleanup",
+      conn, msg = exc.msg
+
+proc peerCleanup(c: ConnManager, conn: Connection) {.async.} =
+  try:
+    trace "Triggering disconnect events", conn
+    let peerId = conn.peerId
+    await c.triggerConnEvent(
+      peerId, ConnEvent(kind: ConnEventKind.Disconnected))
+    await c.triggerPeerEvents(peerId, PeerEvent(kind: PeerEventKind.Left))
+
+    if not(c.peerStore.isNil):
+      c.peerStore.cleanup(peerId)
+  except CatchableError as exc:
+    # This is top-level procedure which will work as separate task, so it
+    # do not need to propagate CancelledError and should handle other errors
+    warn "Unexpected exception peer cleanup handler",
+      conn, msg = exc.msg
+
+proc onClose(c: ConnManager, conn: Connection) {.async.} =
+  ## connection close even handler
+  ##
+  ## triggers the connections resource cleanup
+  ##
+  try:
+    await conn.join()
+    trace "Connection closed, cleaning up", conn
+    await c.cleanupConn(conn)
+  except CancelledError:
+    # This is top-level procedure which will work as separate task, so it
+    # do not need to propagate CancelledError.
+    debug "Unexpected cancellation in connection manager's cleanup", conn
+  except CatchableError as exc:
+    debug "Unexpected exception in connection manager's cleanup",
+          errMsg = exc.msg, conn
+  finally:
+    trace "Triggering peerCleanup", conn
+    asyncSpawn c.peerCleanup(conn)
+
+proc selectConn*(c: ConnManager,
+                peerId: PeerId,
+                dir: Direction): Connection =
+  ## Select a connection for the provided peer and direction
+  ##
+  let conns = toSeq(
+    c.conns.getOrDefault(peerId))
+    .filterIt( it.dir == dir )
+
+  if conns.len > 0:
+    return conns[0]
+
+proc selectConn*(c: ConnManager, peerId: PeerId): Connection =
+  ## Select a connection for the provided giving priority
+  ## to outgoing connections
+  ##
+
+  var conn = c.selectConn(peerId, Direction.Out)
+  if isNil(conn):
+    conn = c.selectConn(peerId, Direction.In)
+  if isNil(conn):
+    trace "connection not found", peerId
+
+  return conn
+
+proc selectMuxer*(c: ConnManager, conn: Connection): Muxer =
+  ## select the muxer for the provided connection
+  ##
+
+  if isNil(conn):
+    return
+
+  if conn in c.muxed:
+    return c.muxed.getOrDefault(conn).muxer
+  else:
+    debug "no muxer for connection", conn
+
+proc storeConn*(c: ConnManager, conn: Connection)
+  {.raises: [Defect, LPError].} =
+  ## store a connection
+  ##
+
+  if isNil(conn):
+    raise newException(LPError, "Connection cannot be nil")
+
+  if conn.closed or conn.atEof:
+    raise newException(LPError, "Connection closed or EOF")
+
+  let peerId = conn.peerId
+  if c.conns.getOrDefault(peerId).len > c.maxConnsPerPeer:
+    debug "Too many connections for peer",
+      conn, conns = c.conns.getOrDefault(peerId).len
+
+    raise newTooManyConnectionsError()
+
+  c.conns.mgetOrPut(peerId, HashSet[Connection]()).incl(conn)
+  libp2p_peers.set(c.conns.len.int64)
+
+  # Launch on close listener
+  # All the errors are handled inside `onClose()` procedure.
+  asyncSpawn c.onClose(conn)
+
+  trace "Stored connection",
+    conn, direction = $conn.dir, connections = c.conns.len
+
+proc getIncomingSlot*(c: ConnManager): Future[ConnectionSlot] {.async.} =
+  await c.inSema.acquire()
+  return ConnectionSlot(connManager: c, direction: In)
+
+proc getOutgoingSlot*(c: ConnManager, forceDial = false): Future[ConnectionSlot] {.async.} =
+  if forceDial:
+    c.outSema.forceAcquire()
+  elif not c.outSema.tryAcquire():
+    trace "Too many outgoing connections!", count = c.outSema.count,
+                                            max = c.outSema.size
+    raise newTooManyConnectionsError()
+  return ConnectionSlot(connManager: c, direction: Out)
+
+proc release*(cs: ConnectionSlot) =
+  if cs.direction == In:
+    cs.connManager.inSema.release()
+  else:
+    cs.connManager.outSema.release()
+
+proc trackConnection*(cs: ConnectionSlot, conn: Connection) =
+  if isNil(conn):
+    cs.release()
+    return
+
+  proc semaphoreMonitor() {.async.} =
+    try:
+      await conn.join()
+    except CatchableError as exc:
+      trace "Exception in semaphore monitor, ignoring", exc = exc.msg
+
+    cs.release()
+
+  asyncSpawn semaphoreMonitor()
+
+proc storeMuxer*(c: ConnManager,
+                 muxer: Muxer,
+                 handle: Future[void] = nil)
+                 {.raises: [Defect, CatchableError].} =
+  ## store the connection and muxer
+  ##
+
+  if isNil(muxer):
+    raise newException(CatchableError, "muxer cannot be nil")
+
+  if isNil(muxer.connection):
+    raise newException(CatchableError, "muxer's connection cannot be nil")
+
+  if muxer.connection notin c:
+    raise newException(CatchableError, "cant add muxer for untracked connection")
+
+  c.muxed[muxer.connection] = MuxerHolder(
+    muxer: muxer,
+    handle: handle)
+
+  trace "Stored muxer",
+    muxer, handle = not handle.isNil, connections = c.conns.len
+
+  asyncSpawn c.onConnUpgraded(muxer.connection)
+
+proc getStream*(c: ConnManager,
+                peerId: PeerId,
+                dir: Direction): Future[Connection] {.async, gcsafe.} =
+  ## get a muxed stream for the provided peer
+  ## with the given direction
+  ##
+
+  let muxer = c.selectMuxer(c.selectConn(peerId, dir))
+  if not(isNil(muxer)):
+    return await muxer.newStream()
+
+proc getStream*(c: ConnManager,
+                peerId: PeerId): Future[Connection] {.async, gcsafe.} =
+  ## get a muxed stream for the passed peer from any connection
+  ##
+
+  let muxer = c.selectMuxer(c.selectConn(peerId))
+  if not(isNil(muxer)):
+    return await muxer.newStream()
+
+proc getStream*(c: ConnManager,
+                conn: Connection): Future[Connection] {.async, gcsafe.} =
+  ## get a muxed stream for the passed connection
+  ##
+
+  let muxer = c.selectMuxer(conn)
+  if not(isNil(muxer)):
+    return await muxer.newStream()
+
+proc dropPeer*(c: ConnManager, peerId: PeerId) {.async.} =
+  ## drop connections and cleanup resources for peer
+  ##
+  trace "Dropping peer", peerId
+  let conns = c.conns.getOrDefault(peerId)
+  for conn in conns:
+    trace  "Removing connection", conn
+    delConn(c, conn)
+
+  var muxers: seq[MuxerHolder]
+  for conn in conns:
+    if conn in c.muxed:
+      muxers.add c.muxed[conn]
+      c.muxed.del(conn)
+
+  for muxer in muxers:
+    await closeMuxerHolder(muxer)
+
+  for conn in conns:
+    await conn.close()
+    trace "Dropped peer", peerId
+
+  trace "Peer dropped", peerId
+
+proc close*(c: ConnManager) {.async.} =
+  ## cleanup resources for the connection
+  ## manager
+  ##
+
+  trace "Closing ConnManager"
+  let conns = c.conns
+  c.conns.clear()
+
+  let muxed = c.muxed
+  c.muxed.clear()
+
+  for _, muxer in muxed:
+    await closeMuxerHolder(muxer)
+
+  for _, conns2 in conns:
+    for conn in conns2:
+      await conn.close()
+
+  trace "Closed ConnManager"

libp2p/crypto/chacha20poly1305.nim

@@ -1,11 +1,11 @@
-## Nim-Libp2p
-## Copyright (c) 2020 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module integrates BearSSL ChaCha20+Poly1305
 ##
@@ -15,16 +15,14 @@
 
 # RFC @ https://tools.ietf.org/html/rfc7539
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
-import bearssl
-
-# have to do this due to a nim bug and raises[] on callbacks
-# https://github.com/nim-lang/Nim/issues/13905
-proc ourPoly1305CtmulRun*(key: pointer; iv: pointer; data: pointer; len: int;
-                      aad: pointer; aadLen: int; tag: pointer; ichacha: pointer;
-                      encrypt: cint) {.cdecl, importc: "br_poly1305_ctmul_run",
-                                     header: "bearssl_block.h".}
+import bearssl/blockx
+from stew/assign2 import assign
+from stew/ranges/ptr_arith import baseAddr
 
 const
   ChaChaPolyKeySize = 32
@@ -37,17 +35,17 @@ type
   ChaChaPolyNonce* = array[ChaChaPolyNonceSize, byte]
   ChaChaPolyTag* = array[ChaChaPolyTagSize, byte]
 
-proc intoChaChaPolyKey*(s: openarray[byte]): ChaChaPolyKey =
+proc intoChaChaPolyKey*(s: openArray[byte]): ChaChaPolyKey =
   assert s.len == ChaChaPolyKeySize
-  copyMem(addr result[0], unsafeaddr s[0], ChaChaPolyKeySize)
+  assign(result, s)
 
-proc intoChaChaPolyNonce*(s: openarray[byte]): ChaChaPolyNonce =
+proc intoChaChaPolyNonce*(s: openArray[byte]): ChaChaPolyNonce =
   assert s.len == ChaChaPolyNonceSize
-  copyMem(addr result[0], unsafeaddr s[0], ChaChaPolyNonceSize)
+  assign(result, s)
 
-proc intoChaChaPolyTag*(s: openarray[byte]): ChaChaPolyTag =
+proc intoChaChaPolyTag*(s: openArray[byte]): ChaChaPolyTag =
   assert s.len == ChaChaPolyTagSize
-  copyMem(addr result[0], unsafeaddr s[0], ChaChaPolyTagSize)
+  assign(result, s)
 
 # bearssl allows us to use optimized versions
 # this is reconciled at runtime
@@ -57,44 +55,46 @@ proc encrypt*(_: type[ChaChaPoly],
                  key: ChaChaPolyKey,
                  nonce: ChaChaPolyNonce,
                  tag: var ChaChaPolyTag,
-                 data: var openarray[byte],
-                 aad: openarray[byte]) =
+                 data: var openArray[byte],
+                 aad: openArray[byte]) =
   let
     ad = if aad.len > 0:
-           unsafeaddr aad[0]
+           unsafeAddr aad[0]
          else:
            nil
 
-  ourPoly1305CtmulRun(
-    unsafeaddr key[0],
-    unsafeaddr nonce[0],
-    addr data[0],
-    data.len,
+  poly1305CtmulRun(
+    unsafeAddr key[0],
+    unsafeAddr nonce[0],
+    baseAddr(data),
+    uint(data.len),
     ad,
-    aad.len,
-    addr tag[0],
-    chacha20CtRun,
+    uint(aad.len),
+    baseAddr(tag),
+    # cast is required to workaround https://github.com/nim-lang/Nim/issues/13905
+    cast[Chacha20Run](chacha20CtRun),
     #[encrypt]# 1.cint)
 
 proc decrypt*(_: type[ChaChaPoly],
                  key: ChaChaPolyKey,
                  nonce: ChaChaPolyNonce,
                  tag: var ChaChaPolyTag,
-                 data: var openarray[byte],
-                 aad: openarray[byte]) =
+                 data: var openArray[byte],
+                 aad: openArray[byte]) =
   let
     ad = if aad.len > 0:
-          unsafeaddr aad[0]
+          unsafeAddr aad[0]
          else:
            nil
 
-  ourPoly1305CtmulRun(
-    unsafeaddr key[0],
-    unsafeaddr nonce[0],
-    addr data[0],
-    data.len,
+  poly1305CtmulRun(
+    unsafeAddr key[0],
+    unsafeAddr nonce[0],
+    baseAddr(data),
+    uint(data.len),
     ad,
-    aad.len,
-    addr tag[0],
-    chacha20CtRun,
+    uint(aad.len),
+    baseAddr(tag),
+    # cast is required to workaround https://github.com/nim-lang/Nim/issues/13905
+    cast[Chacha20Run](chacha20CtRun),
     #[decrypt]# 0.cint)

libp2p/crypto/curve25519.nim

@@ -1,11 +1,11 @@
-## Nim-Libp2p
-## Copyright (c) 2020 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022-2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module integrates BearSSL Cyrve25519 mul and mulgen
 ##
@@ -15,10 +15,14 @@
 
 # RFC @ https://tools.ietf.org/html/rfc7748
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
-import bearssl
+import bearssl/[ec, rand, hash]
 import stew/results
+from stew/assign2 import assign
 export results
 
 const
@@ -27,32 +31,15 @@ const
 type
   Curve25519* = object
   Curve25519Key* = array[Curve25519KeySize, byte]
-  pcuchar = ptr cuchar
   Curve25519Error* = enum
     Curver25519GenError
 
-proc intoCurve25519Key*(s: openarray[byte]): Curve25519Key =
+proc intoCurve25519Key*(s: openArray[byte]): Curve25519Key =
   assert s.len == Curve25519KeySize
-  copyMem(addr result[0], unsafeaddr s[0], Curve25519KeySize)
+  assign(result, s)
 
 proc getBytes*(key: Curve25519Key): seq[byte] = @key
 
-const
-  ForbiddenCurveValues: array[12, Curve25519Key] = [
-                [0.byte, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-                [1.byte, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
-                [224.byte, 235, 122, 124, 59, 65, 184, 174, 22, 86, 227, 250, 241, 159, 196, 106, 218, 9, 141, 235, 156, 50, 177, 253, 134, 98, 5, 22, 95, 73, 184, 0],
-                [95.byte, 156, 149, 188, 163, 80, 140, 36, 177, 208, 177, 85, 156, 131, 239, 91, 4, 68, 92, 196, 88, 28, 142, 134, 216, 34, 78, 221, 208, 159, 17, 87],
-                [236.byte, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 127],
-                [237.byte, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 127],
-                [238.byte, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 127],
-                [205.byte, 235, 122, 124, 59, 65, 184, 174, 22, 86, 227, 250, 241, 159, 196, 106, 218, 9, 141, 235, 156, 50, 177, 253, 134, 98, 5, 22, 95, 73, 184, 128],
-                [76.byte, 156, 149, 188, 163, 80, 140, 36, 177, 208, 177, 85, 156, 131, 239, 91, 4, 68, 92, 196, 88, 28, 142, 134, 216, 34, 78, 221, 208, 159, 17, 215],
-                [217.byte, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255],
-                [218.byte, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255],
-                [219.byte, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 25],
-        ]
-
 proc byteswap(buf: var Curve25519Key) {.inline.} =
   for i in 0..<16:
     let
@@ -60,55 +47,45 @@ proc byteswap(buf: var Curve25519Key) {.inline.} =
     buf[i] = buf[31 - i]
     buf[31 - i] = x
 
-proc mul*(_: type[Curve25519], dst: var Curve25519Key, scalar: Curve25519Key, point: Curve25519Key) =
-  let defaultBrEc = brEcGetDefault()
+proc mul*(_: type[Curve25519], point: var Curve25519Key, multiplier: Curve25519Key) =
+  let defaultBrEc = ecGetDefault()
 
-  # The source point is provided in array G (of size Glen bytes);
-  # the multiplication result is written over it.
-  dst = scalar
-
-  # point needs to be big-endian
+  # multiplier needs to be big-endian
   var
-    rpoint = point
-  rpoint.byteswap()
+    multiplierBs = multiplier
+  multiplierBs.byteswap()
   let
     res = defaultBrEc.mul(
-      cast[pcuchar](addr dst[0]),
+      addr point[0],
       Curve25519KeySize,
-      cast[pcuchar](addr rpoint[0]),
+      addr multiplierBs[0],
       Curve25519KeySize,
       EC_curve25519)
   assert res == 1
 
-proc mulgen*(_: type[Curve25519], dst: var Curve25519Key, point: Curve25519Key) =
-  let defaultBrEc = brEcGetDefault()
+proc mulgen(_: type[Curve25519], dst: var Curve25519Key, point: Curve25519Key) =
+  let defaultBrEc = ecGetDefault()
 
   var
     rpoint = point
   rpoint.byteswap()
 
-  block iterate:
-    while true:
-      block derive:
-        let
-          size = defaultBrEc.mulgen(
-            cast[pcuchar](addr dst[0]),
-            cast[pcuchar](addr rpoint[0]),
-            Curve25519KeySize,
-            EC_curve25519)
-        assert size == Curve25519KeySize
-        for forbid in ForbiddenCurveValues:
-          if dst == forbid:
-            break derive
-        break iterate
+  let
+    size = defaultBrEc.mulgen(
+      addr dst[0],
+      addr rpoint[0],
+      Curve25519KeySize,
+      EC_curve25519)
+
+  assert size == Curve25519KeySize
 
 proc public*(private: Curve25519Key): Curve25519Key =
   Curve25519.mulgen(result, private)
 
-proc random*(_: type[Curve25519Key], rng: var BrHmacDrbgContext): Curve25519Key =
+proc random*(_: type[Curve25519Key], rng: var HmacDrbgContext): Curve25519Key =
   var res: Curve25519Key
-  let defaultBrEc = brEcGetDefault()
-  let len = brEcKeygen(
+  let defaultBrEc = ecGetDefault()
+  let len = ecKeygen(
     addr rng.vtable, defaultBrEc, nil, addr res[0], EC_curve25519)
   # Per bearssl documentation, the keygen only fails if the curve is
   # unrecognised -

libp2p/crypto/ecnist.nim

@@ -1,11 +1,11 @@
-## Nim-Libp2p
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module implements constant-time ECDSA and ECDHE for NIST elliptic
 ## curves secp256r1, secp384r1 and secp521r1.
@@ -14,13 +14,17 @@
 ## BearSSL library <https://bearssl.org/>
 ## Copyright(C) 2018 Thomas Pornin <pornin@bolet.org>.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
-import bearssl
-import nimcrypto/utils
+import bearssl/[ec, rand, hash]
+# We use `ncrutils` for constant-time hexadecimal encoding/decoding procedures.
+import nimcrypto/utils as ncrutils
 import minasn1
 export minasn1.Asn1Error
-import stew/results
+import stew/[results, ctops]
 export results
 
 const
@@ -39,12 +43,12 @@ const
 
 type
   EcPrivateKey* = ref object
-    buffer*: array[BR_EC_KBUF_PRIV_MAX_SIZE, byte]
-    key*: BrEcPrivateKey
+    buffer*: array[EC_KBUF_PRIV_MAX_SIZE, byte]
+    key*: ec.EcPrivateKey
 
   EcPublicKey* = ref object
-    buffer*: array[BR_EC_KBUF_PUB_MAX_SIZE, byte]
-    key*: BrEcPublicKey
+    buffer*: array[EC_KBUF_PUB_MAX_SIZE, byte]
+    key*: ec.EcPublicKey
 
   EcKeyPair* = object
     seckey*: EcPrivateKey
@@ -54,9 +58,9 @@ type
     buffer*: seq[byte]
 
   EcCurveKind* = enum
-    Secp256r1 = BR_EC_SECP256R1,
-    Secp384r1 = BR_EC_SECP384R1,
-    Secp521r1 = BR_EC_SECP521R1
+    Secp256r1 = EC_secp256r1,
+    Secp384r1 = EC_secp384r1,
+    Secp521r1 = EC_secp521r1
 
   EcPKI* = EcPrivateKey | EcPublicKey | EcSignature
 
@@ -93,37 +97,37 @@ proc NEQ(x, y: uint32): uint32 {.inline.} =
 proc LT0(x: int32): uint32 {.inline.} =
   result = cast[uint32](x) shr 31
 
-proc checkScalar(scalar: openarray[byte], curve: cint): uint32 =
+proc checkScalar(scalar: openArray[byte], curve: cint): uint32 =
   ## Return ``1`` if all of the following hold:
   ##   - len(``scalar``) <= ``orderlen``
   ##   - ``scalar`` != 0
   ##   - ``scalar`` is lower than the curve ``order``.
   ##
   ## Otherwise, return ``0``.
-  var impl = brEcGetDefault()
-  var orderlen = 0
-  var order = cast[ptr UncheckedArray[byte]](impl.order(curve, addr orderlen))
+  var impl = ecGetDefault()
+  var orderlen: uint = 0
+  var order = cast[ptr UncheckedArray[byte]](impl.order(curve, orderlen))
 
   var z = 0'u32
   var c = 0'i32
   for u in scalar:
     z = z or u
-  if len(scalar) == orderlen:
+  if len(scalar) == int(orderlen):
     for i in 0..<len(scalar):
       c = c or (-(cast[int32](EQ0(c))) and CMP(scalar[i], order[i]))
   else:
     c = -1
   result = NEQ(z, 0'u32) and LT0(c)
 
-proc checkPublic(key: openarray[byte], curve: cint): uint32 =
+proc checkPublic(key: openArray[byte], curve: cint): uint32 =
   ## Return ``1`` if public key ``key`` is on curve.
   var ckey = @key
-  var x = [0x00'u8, 0x01'u8]
-  var impl = brEcGetDefault()
-  var orderlen = 0
-  discard impl.order(curve, addr orderlen)
-  result = impl.mul(cast[ptr cuchar](unsafeAddr ckey[0]), len(ckey),
-                    cast[ptr cuchar](addr x[0]), len(x), curve)
+  var x = [byte 0x00, 0x01]
+  var impl = ecGetDefault()
+  var orderlen: uint = 0
+  discard impl.order(curve, orderlen)
+  result = impl.mul(unsafeAddr ckey[0], uint(len(ckey)),
+                    addr x[0], uint(len(x)), curve)
 
 proc getOffset(pubkey: EcPublicKey): int {.inline.} =
   let o = cast[uint](pubkey.key.q) - cast[uint](unsafeAddr pubkey.buffer[0])
@@ -173,7 +177,7 @@ proc copy*[T: EcPKI](dst: var T, src: T): bool =
           dst.buffer = src.buffer
           dst.key.curve = src.key.curve
           dst.key.xlen = length
-          dst.key.x = cast[ptr cuchar](addr dst.buffer[offset])
+          dst.key.x = addr dst.buffer[offset]
           result = true
     elif T is EcPublicKey:
       let length = src.key.qlen
@@ -183,7 +187,7 @@ proc copy*[T: EcPKI](dst: var T, src: T): bool =
           dst.buffer = src.buffer
           dst.key.curve = src.key.curve
           dst.key.qlen = length
-          dst.key.q = cast[ptr cuchar](addr dst.buffer[offset])
+          dst.key.q = addr dst.buffer[offset]
           result = true
     else:
       let length = len(src.buffer)
@@ -229,32 +233,32 @@ proc clear*[T: EcPKI|EcKeyPair](pki: var T) =
 
 proc random*(
     T: typedesc[EcPrivateKey], kind: EcCurveKind,
-    rng: var BrHmacDrbgContext): EcResult[EcPrivateKey] =
+    rng: var HmacDrbgContext): EcResult[EcPrivateKey] =
   ## Generate new random EC private key using BearSSL's HMAC-SHA256-DRBG
   ## algorithm.
   ##
   ## ``kind`` elliptic curve kind of your choice (secp256r1, secp384r1 or
   ## secp521r1).
-  var ecimp = brEcGetDefault()
+  var ecimp = ecGetDefault()
   var res = new EcPrivateKey
-  if brEcKeygen(addr rng.vtable, ecimp,
+  if ecKeygen(addr rng.vtable, ecimp,
                 addr res.key, addr res.buffer[0],
                 cast[cint](kind)) == 0:
     err(EcKeyGenError)
   else:
     ok(res)
 
-proc getKey*(seckey: EcPrivateKey): EcResult[EcPublicKey] =
+proc getPublicKey*(seckey: EcPrivateKey): EcResult[EcPublicKey] =
   ## Calculate and return EC public key from private key ``seckey``.
   if isNil(seckey):
     return err(EcKeyIncorrectError)
 
-  var ecimp = brEcGetDefault()
+  var ecimp = ecGetDefault()
   if seckey.key.curve in EcSupportedCurvesCint:
-    var length = getPublicKeyLength(cast[EcCurveKind](seckey.key.curve))
     var res = new EcPublicKey
-    if brEcComputePublicKey(ecimp, addr res.key,
-                            addr res.buffer[0], unsafeAddr seckey.key) == 0:
+    assert res.buffer.len > getPublicKeyLength(cast[EcCurveKind](seckey.key.curve))
+    if ecComputePub(ecimp, addr res.key,
+                    addr res.buffer[0], unsafeAddr seckey.key) == 0:
       err(EcKeyIncorrectError)
     else:
       ok(res)
@@ -263,7 +267,7 @@ proc getKey*(seckey: EcPrivateKey): EcResult[EcPublicKey] =
 
 proc random*(
     T: typedesc[EcKeyPair], kind: EcCurveKind,
-    rng: var BrHmacDrbgContext): EcResult[T] =
+    rng: var HmacDrbgContext): EcResult[T] =
   ## Generate new random EC private and public keypair using BearSSL's
   ## HMAC-SHA256-DRBG algorithm.
   ##
@@ -271,7 +275,7 @@ proc random*(
   ## secp521r1).
   let
     seckey = ? EcPrivateKey.random(kind, rng)
-    pubkey = ? seckey.getKey()
+    pubkey = ? seckey.getPublicKey()
     key = EcKeyPair(seckey: seckey, pubkey: pubkey)
   ok(key)
 
@@ -289,7 +293,7 @@ proc `$`*(seckey: EcPrivateKey): string =
         result = "Corrupted key"
       else:
         let e = offset + cast[int](seckey.key.xlen) - 1
-        result = toHex(seckey.buffer.toOpenArray(offset, e))
+        result = ncrutils.toHex(seckey.buffer.toOpenArray(offset, e))
 
 proc `$`*(pubkey: EcPublicKey): string =
   ## Return string representation of EC public key.
@@ -305,16 +309,16 @@ proc `$`*(pubkey: EcPublicKey): string =
         result = "Corrupted key"
       else:
         let e = offset + cast[int](pubkey.key.qlen) - 1
-        result = toHex(pubkey.buffer.toOpenArray(offset, e))
+        result = ncrutils.toHex(pubkey.buffer.toOpenArray(offset, e))
 
 proc `$`*(sig: EcSignature): string =
   ## Return hexadecimal string representation of EC signature.
   if isNil(sig) or len(sig.buffer) == 0:
     result = "Empty or uninitialized ECNIST signature"
   else:
-    result = toHex(sig.buffer)
+    result = ncrutils.toHex(sig.buffer)
 
-proc toRawBytes*(seckey: EcPrivateKey, data: var openarray[byte]): EcResult[int] =
+proc toRawBytes*(seckey: EcPrivateKey, data: var openArray[byte]): EcResult[int] =
   ## Serialize EC private key ``seckey`` to raw binary form and store it
   ## to ``data``.
   ##
@@ -330,7 +334,7 @@ proc toRawBytes*(seckey: EcPrivateKey, data: var openarray[byte]): EcResult[int]
   else:
     err(EcKeyIncorrectError)
 
-proc toRawBytes*(pubkey: EcPublicKey, data: var openarray[byte]): EcResult[int] =
+proc toRawBytes*(pubkey: EcPublicKey, data: var openArray[byte]): EcResult[int] =
   ## Serialize EC public key ``pubkey`` to uncompressed form specified in
   ## section 4.3.6 of ANSI X9.62.
   ##
@@ -346,7 +350,7 @@ proc toRawBytes*(pubkey: EcPublicKey, data: var openarray[byte]): EcResult[int]
   else:
     err(EcKeyIncorrectError)
 
-proc toRawBytes*(sig: EcSignature, data: var openarray[byte]): int =
+proc toRawBytes*(sig: EcSignature, data: var openArray[byte]): int =
   ## Serialize EC signature ``sig`` to raw binary form and store it to ``data``.
   ##
   ## Returns number of bytes (octets) needed to store EC signature, or `0`
@@ -357,7 +361,7 @@ proc toRawBytes*(sig: EcSignature, data: var openarray[byte]): int =
     if len(sig.buffer) > 0:
       copyMem(addr data[0], unsafeAddr sig.buffer[0], len(sig.buffer))
 
-proc toBytes*(seckey: EcPrivateKey, data: var openarray[byte]): EcResult[int] =
+proc toBytes*(seckey: EcPrivateKey, data: var openArray[byte]): EcResult[int] =
   ## Serialize EC private key ``seckey`` to ASN.1 DER binary form and store it
   ## to ``data``.
   ##
@@ -367,25 +371,29 @@ proc toBytes*(seckey: EcPrivateKey, data: var openarray[byte]): EcResult[int] =
     return err(EcKeyIncorrectError)
   if seckey.key.curve in EcSupportedCurvesCint:
     var offset, length: int
-    var pubkey = ? seckey.getKey()
+    var pubkey = ? seckey.getPublicKey()
     var b = Asn1Buffer.init()
     var p = Asn1Composite.init(Asn1Tag.Sequence)
     var c0 = Asn1Composite.init(0)
     var c1 = Asn1Composite.init(1)
-    if seckey.key.curve == BR_EC_SECP256R1:
+    if seckey.key.curve == EC_secp256r1:
       c0.write(Asn1Tag.Oid, Asn1OidSecp256r1)
-    elif seckey.key.curve == BR_EC_SECP384R1:
+    elif seckey.key.curve == EC_secp384r1:
       c0.write(Asn1Tag.Oid, Asn1OidSecp384r1)
-    elif seckey.key.curve == BR_EC_SECP521R1:
+    elif seckey.key.curve == EC_secp521r1:
       c0.write(Asn1Tag.Oid, Asn1OidSecp521r1)
     c0.finish()
     offset = pubkey.getOffset()
-    length = pubkey.key.qlen
+    if offset < 0:
+      return err(EcKeyIncorrectError)
+    length = int(pubkey.key.qlen)
     c1.write(Asn1Tag.BitString,
              pubkey.buffer.toOpenArray(offset, offset + length - 1))
     c1.finish()
     offset = seckey.getOffset()
-    length = seckey.key.xlen
+    if offset < 0:
+      return err(EcKeyIncorrectError)
+    length = int(seckey.key.xlen)
     p.write(1'u64)
     p.write(Asn1Tag.OctetString,
             seckey.buffer.toOpenArray(offset, offset + length - 1))
@@ -403,7 +411,7 @@ proc toBytes*(seckey: EcPrivateKey, data: var openarray[byte]): EcResult[int] =
     err(EcKeyIncorrectError)
 
 
-proc toBytes*(pubkey: EcPublicKey, data: var openarray[byte]): EcResult[int] =
+proc toBytes*(pubkey: EcPublicKey, data: var openArray[byte]): EcResult[int] =
   ## Serialize EC public key ``pubkey`` to ASN.1 DER binary form and store it
   ## to ``data``.
   ##
@@ -416,16 +424,18 @@ proc toBytes*(pubkey: EcPublicKey, data: var openarray[byte]): EcResult[int] =
     var p = Asn1Composite.init(Asn1Tag.Sequence)
     var c = Asn1Composite.init(Asn1Tag.Sequence)
     c.write(Asn1Tag.Oid, Asn1OidEcPublicKey)
-    if pubkey.key.curve == BR_EC_SECP256R1:
+    if pubkey.key.curve == EC_secp256r1:
       c.write(Asn1Tag.Oid, Asn1OidSecp256r1)
-    elif pubkey.key.curve == BR_EC_SECP384R1:
+    elif pubkey.key.curve == EC_secp384r1:
       c.write(Asn1Tag.Oid, Asn1OidSecp384r1)
-    elif pubkey.key.curve == BR_EC_SECP521R1:
+    elif pubkey.key.curve == EC_secp521r1:
       c.write(Asn1Tag.Oid, Asn1OidSecp521r1)
     c.finish()
     p.write(c)
     let offset = getOffset(pubkey)
-    let length = pubkey.key.qlen
+    if offset < 0:
+      return err(EcKeyIncorrectError)
+    let length = int(pubkey.key.qlen)
     p.write(Asn1Tag.BitString,
             pubkey.buffer.toOpenArray(offset, offset + length - 1))
     p.finish()
@@ -438,7 +448,7 @@ proc toBytes*(pubkey: EcPublicKey, data: var openarray[byte]): EcResult[int] =
   else:
     err(EcKeyIncorrectError)
 
-proc toBytes*(sig: EcSignature, data: var openarray[byte]): EcResult[int] =
+proc toBytes*(sig: EcSignature, data: var openArray[byte]): EcResult[int] =
   ## Serialize EC signature ``sig`` to ASN.1 DER binary form and store it
   ## to ``data``.
   ##
@@ -509,7 +519,7 @@ proc getRawBytes*(pubkey: EcPublicKey): EcResult[seq[byte]] =
     let length = ? pubkey.toRawBytes(res)
     res.setLen(length)
     discard ? pubkey.toRawBytes(res)
-    ok(res)
+    return ok(res)
   else:
     return err(EcKeyIncorrectError)
 
@@ -540,8 +550,8 @@ proc `==`*(pubkey1, pubkey2: EcPublicKey): bool =
     let op2 = pubkey2.getOffset()
     if op1 == -1 or op2 == -1:
       return false
-    result = equalMem(unsafeAddr pubkey1.buffer[op1],
-                      unsafeAddr pubkey2.buffer[op2], pubkey1.key.qlen)
+    return CT.isEqual(pubkey1.buffer.toOpenArray(op1, pubkey1.key.qlen - 1),
+                      pubkey2.buffer.toOpenArray(op2, pubkey2.key.qlen - 1))
 
 proc `==`*(seckey1, seckey2: EcPrivateKey): bool =
   ## Returns ``true`` if both keys ``seckey1`` and ``seckey2`` are equal.
@@ -560,21 +570,32 @@ proc `==`*(seckey1, seckey2: EcPrivateKey): bool =
     let op2 = seckey2.getOffset()
     if op1 == -1 or op2 == -1:
       return false
-    result = equalMem(unsafeAddr seckey1.buffer[op1],
-                      unsafeAddr seckey2.buffer[op2], seckey1.key.xlen)
+    return CT.isEqual(seckey1.buffer.toOpenArray(op1, seckey1.key.xlen - 1),
+                      seckey2.buffer.toOpenArray(op2, seckey2.key.xlen - 1))
 
-proc `==`*(sig1, sig2: EcSignature): bool =
+proc `==`*(a, b: EcSignature): bool =
   ## Return ``true`` if both signatures ``sig1`` and ``sig2`` are equal.
-  if isNil(sig1) and isNil(sig2):
-    result = true
-  elif isNil(sig1) and (not isNil(sig2)):
-    result = false
-  elif isNil(sig2) and (not isNil(sig1)):
-    result = false
+  if isNil(a) and isNil(b):
+    true
+  elif isNil(a) and (not isNil(b)):
+    false
+  elif isNil(b) and (not isNil(a)):
+    false
   else:
-    result = (sig1.buffer == sig2.buffer)
+    # We need to cover all the cases because Signature initialization procedure
+    # do not perform any checks.
+    if len(a.buffer) == 0 and len(b.buffer) == 0:
+      true
+    elif len(a.buffer) == 0 and len(b.buffer) != 0:
+      false
+    elif len(b.buffer) == 0 and len(a.buffer) != 0:
+      false
+    elif len(a.buffer) != len(b.buffer):
+      false
+    else:
+      CT.isEqual(a.buffer, b.buffer)
 
-proc init*(key: var EcPrivateKey, data: openarray[byte]): Result[void, Asn1Error] =
+proc init*(key: var EcPrivateKey, data: openArray[byte]): Result[void, Asn1Error] =
   ## Initialize EC `private key` or `signature` ``key`` from ASN.1 DER binary
   ## representation ``data``.
   ##
@@ -620,14 +641,14 @@ proc init*(key: var EcPrivateKey, data: openarray[byte]): Result[void, Asn1Error
   if checkScalar(raw.toOpenArray(), curve) == 1'u32:
     key = new EcPrivateKey
     copyMem(addr key.buffer[0], addr raw.buffer[raw.offset], raw.length)
-    key.key.x = cast[ptr cuchar](addr key.buffer[0])
-    key.key.xlen = raw.length
+    key.key.x = addr key.buffer[0]
+    key.key.xlen = uint(raw.length)
     key.key.curve = curve
     ok()
   else:
     err(Asn1Error.Incorrect)
 
-proc init*(pubkey: var EcPublicKey, data: openarray[byte]): Result[void, Asn1Error] =
+proc init*(pubkey: var EcPublicKey, data: openArray[byte]): Result[void, Asn1Error] =
   ## Initialize EC public key ``pubkey`` from ASN.1 DER binary representation
   ## ``data``.
   ##
@@ -679,14 +700,14 @@ proc init*(pubkey: var EcPublicKey, data: openarray[byte]): Result[void, Asn1Err
   if checkPublic(raw.toOpenArray(), curve) != 0:
     pubkey = new EcPublicKey
     copyMem(addr pubkey.buffer[0], addr raw.buffer[raw.offset], raw.length)
-    pubkey.key.q = cast[ptr cuchar](addr pubkey.buffer[0])
-    pubkey.key.qlen = raw.length
+    pubkey.key.q = addr pubkey.buffer[0]
+    pubkey.key.qlen = uint(raw.length)
     pubkey.key.curve = curve
     ok()
   else:
     err(Asn1Error.Incorrect)
 
-proc init*(sig: var EcSignature, data: openarray[byte]): Result[void, Asn1Error] =
+proc init*(sig: var EcSignature, data: openArray[byte]): Result[void, Asn1Error] =
   ## Initialize EC signature ``sig`` from raw binary representation ``data``.
   ##
   ## Procedure returns ``Result[void, Asn1Error]``.
@@ -697,14 +718,16 @@ proc init*(sig: var EcSignature, data: openarray[byte]): Result[void, Asn1Error]
   else:
     err(Asn1Error.Incorrect)
 
-proc init*[T: EcPKI](sospk: var T, data: string): Result[void, Asn1Error] {.inline.} =
+proc init*[T: EcPKI](sospk: var T,
+                     data: string): Result[void, Asn1Error] {.inline.} =
   ## Initialize EC `private key`, `public key` or `signature` ``sospk`` from
   ## ASN.1 DER hexadecimal string representation ``data``.
   ##
   ## Procedure returns ``Asn1Status``.
-  sospk.init(fromHex(data))
+  sospk.init(ncrutils.fromHex(data))
 
-proc init*(t: typedesc[EcPrivateKey], data: openarray[byte]): EcResult[EcPrivateKey] =
+proc init*(t: typedesc[EcPrivateKey],
+           data: openArray[byte]): EcResult[EcPrivateKey] =
   ## Initialize EC private key from ASN.1 DER binary representation ``data`` and
   ## return constructed object.
   var key: EcPrivateKey
@@ -714,7 +737,8 @@ proc init*(t: typedesc[EcPrivateKey], data: openarray[byte]): EcResult[EcPrivate
   else:
     ok(key)
 
-proc init*(t: typedesc[EcPublicKey], data: openarray[byte]): EcResult[EcPublicKey] =
+proc init*(t: typedesc[EcPublicKey],
+           data: openArray[byte]): EcResult[EcPublicKey] =
   ## Initialize EC public key from ASN.1 DER binary representation ``data`` and
   ## return constructed object.
   var key: EcPublicKey
@@ -724,7 +748,8 @@ proc init*(t: typedesc[EcPublicKey], data: openarray[byte]): EcResult[EcPublicKe
   else:
     ok(key)
 
-proc init*(t: typedesc[EcSignature], data: openarray[byte]): EcResult[EcSignature] =
+proc init*(t: typedesc[EcSignature],
+           data: openArray[byte]): EcResult[EcSignature] =
   ## Initialize EC signature from raw binary representation ``data`` and
   ## return constructed object.
   var sig: EcSignature
@@ -737,12 +762,9 @@ proc init*(t: typedesc[EcSignature], data: openarray[byte]): EcResult[EcSignatur
 proc init*[T: EcPKI](t: typedesc[T], data: string): EcResult[T] =
   ## Initialize EC `private key`, `public key` or `signature` from hexadecimal
   ## string representation ``data`` and return constructed object.
-  try:
-    t.init(fromHex(data))
-  except ValueError:
-    err(EcKeyIncorrectError)
+  t.init(ncrutils.fromHex(data))
 
-proc initRaw*(key: var EcPrivateKey, data: openarray[byte]): bool =
+proc initRaw*(key: var EcPrivateKey, data: openArray[byte]): bool =
   ## Initialize EC `private key` or `scalar` ``key`` from raw binary
   ## representation ``data``.
   ##
@@ -766,12 +788,12 @@ proc initRaw*(key: var EcPrivateKey, data: openarray[byte]): bool =
       let length = len(data)
       key = new EcPrivateKey
       copyMem(addr key.buffer[0], unsafeAddr data[0], length)
-      key.key.x = cast[ptr cuchar](addr key.buffer[0])
-      key.key.xlen = length
+      key.key.x = addr key.buffer[0]
+      key.key.xlen = uint(length)
       key.key.curve = curve
       result = true
 
-proc initRaw*(pubkey: var EcPublicKey, data: openarray[byte]): bool =
+proc initRaw*(pubkey: var EcPublicKey, data: openArray[byte]): bool =
   ## Initialize EC public key ``pubkey`` from raw binary representation
   ## ``data``.
   ##
@@ -797,12 +819,12 @@ proc initRaw*(pubkey: var EcPublicKey, data: openarray[byte]): bool =
       let length = len(data)
       pubkey = new EcPublicKey
       copyMem(addr pubkey.buffer[0], unsafeAddr data[0], length)
-      pubkey.key.q = cast[ptr cuchar](addr pubkey.buffer[0])
-      pubkey.key.qlen = length
+      pubkey.key.q = addr pubkey.buffer[0]
+      pubkey.key.qlen = uint(length)
       pubkey.key.curve = curve
       result = true
 
-proc initRaw*(sig: var EcSignature, data: openarray[byte]): bool =
+proc initRaw*(sig: var EcSignature, data: openArray[byte]): bool =
   ## Initialize EC signature ``sig`` from raw binary representation ``data``.
   ##
   ## Length of ``data`` array must be ``Sig256Length``, ``Sig384Length``
@@ -822,9 +844,10 @@ proc initRaw*[T: EcPKI](sospk: var T, data: string): bool {.inline.} =
   ## raw hexadecimal string representation ``data``.
   ##
   ## Procedure returns ``true`` on success, ``false`` otherwise.
-  result = sospk.initRaw(fromHex(data))
+  result = sospk.initRaw(ncrutils.fromHex(data))
 
-proc initRaw*(t: typedesc[EcPrivateKey], data: openarray[byte]): EcResult[EcPrivateKey] =
+proc initRaw*(t: typedesc[EcPrivateKey],
+              data: openArray[byte]): EcResult[EcPrivateKey] =
   ## Initialize EC private key from raw binary representation ``data`` and
   ## return constructed object.
   var res: EcPrivateKey
@@ -833,7 +856,8 @@ proc initRaw*(t: typedesc[EcPrivateKey], data: openarray[byte]): EcResult[EcPriv
   else:
     ok(res)
 
-proc initRaw*(t: typedesc[EcPublicKey], data: openarray[byte]): EcResult[EcPublicKey] =
+proc initRaw*(t: typedesc[EcPublicKey],
+              data: openArray[byte]): EcResult[EcPublicKey] =
   ## Initialize EC public key from raw binary representation ``data`` and
   ## return constructed object.
   var res: EcPublicKey
@@ -842,7 +866,8 @@ proc initRaw*(t: typedesc[EcPublicKey], data: openarray[byte]): EcResult[EcPubli
   else:
     ok(res)
 
-proc initRaw*(t: typedesc[EcSignature], data: openarray[byte]): EcResult[EcSignature] =
+proc initRaw*(t: typedesc[EcSignature],
+              data: openArray[byte]): EcResult[EcSignature] =
   ## Initialize EC signature from raw binary representation ``data`` and
   ## return constructed object.
   var res: EcSignature
@@ -854,14 +879,14 @@ proc initRaw*(t: typedesc[EcSignature], data: openarray[byte]): EcResult[EcSigna
 proc initRaw*[T: EcPKI](t: typedesc[T], data: string): T {.inline.} =
   ## Initialize EC `private key`, `public key` or `signature` from raw
   ## hexadecimal string representation ``data`` and return constructed object.
-  result = t.initRaw(fromHex(data))
+  result = t.initRaw(ncrutils.fromHex(data))
 
 proc scalarMul*(pub: EcPublicKey, sec: EcPrivateKey): EcPublicKey =
   ## Return scalar multiplication of ``pub`` and ``sec``.
   ##
   ## Returns point in curve as ``pub * sec`` or ``nil`` otherwise.
   doAssert((not isNil(pub)) and (not isNil(sec)))
-  var impl = brEcGetDefault()
+  var impl = ecGetDefault()
   if sec.key.curve in EcSupportedCurvesCint:
     if pub.key.curve == sec.key.curve:
       var key = new EcPublicKey
@@ -869,16 +894,16 @@ proc scalarMul*(pub: EcPublicKey, sec: EcPrivateKey): EcPublicKey =
         let poffset = key.getOffset()
         let soffset = sec.getOffset()
         if poffset >= 0 and soffset >= 0:
-          let res = impl.mul(cast[ptr cuchar](addr key.buffer[poffset]),
+          let res = impl.mul(addr key.buffer[poffset],
                              key.key.qlen,
-                             cast[ptr cuchar](unsafeAddr sec.buffer[soffset]),
+                             unsafeAddr sec.buffer[soffset],
                              sec.key.xlen,
                              key.key.curve)
           if res != 0:
             result = key
 
 proc toSecret*(pubkey: EcPublicKey, seckey: EcPrivateKey,
-               data: var openarray[byte]): int =
+               data: var openArray[byte]): int =
   ## Calculate ECDHE shared secret using Go's elliptic/curve approach, using
   ## remote public key ``pubkey`` and local private key ``seckey`` and store
   ## shared secret to ``data``.
@@ -891,11 +916,11 @@ proc toSecret*(pubkey: EcPublicKey, seckey: EcPrivateKey,
   doAssert((not isNil(pubkey)) and (not isNil(seckey)))
   var mult = scalarMul(pubkey, seckey)
   if not isNil(mult):
-    if seckey.key.curve == BR_EC_SECP256R1:
+    if seckey.key.curve == EC_secp256r1:
       result = Secret256Length
-    elif seckey.key.curve == BR_EC_SECP384R1:
+    elif seckey.key.curve == EC_secp384r1:
       result = Secret384Length
-    elif seckey.key.curve == BR_EC_SECP521R1:
+    elif seckey.key.curve == EC_secp521r1:
       result = Secret521Length
     if len(data) >= result:
       var qplus1 = cast[pointer](cast[uint](mult.key.q) + 1'u)
@@ -915,24 +940,24 @@ proc getSecret*(pubkey: EcPublicKey, seckey: EcPrivateKey): seq[byte] =
     copyMem(addr result[0], addr data[0], res)
 
 proc sign*[T: byte|char](seckey: EcPrivateKey,
-                         message: openarray[T]): EcResult[EcSignature] {.gcsafe.} =
+                      message: openArray[T]): EcResult[EcSignature] {.gcsafe.} =
   ## Get ECDSA signature of data ``message`` using private key ``seckey``.
   if isNil(seckey):
     return err(EcKeyIncorrectError)
-  var hc: BrHashCompatContext
+  var hc: HashCompatContext
   var hash: array[32, byte]
-  var impl = brEcGetDefault()
+  var impl = ecGetDefault()
   if seckey.key.curve in EcSupportedCurvesCint:
     var sig = new EcSignature
     sig.buffer = newSeq[byte](256)
     var kv = addr sha256Vtable
     kv.init(addr hc.vtable)
     if len(message) > 0:
-      kv.update(addr hc.vtable, unsafeAddr message[0], len(message))
+      kv.update(addr hc.vtable, unsafeAddr message[0], uint(len(message)))
     else:
       kv.update(addr hc.vtable, nil, 0)
-    kv.output(addr hc.vtable, addr hash[0])
-    let res = brEcdsaSignAsn1(impl, kv, addr hash[0], addr seckey.key,
+    kv.out(addr hc.vtable, addr hash[0])
+    let res = ecdsaI31SignAsn1(impl, kv, addr hash[0], addr seckey.key,
                               addr sig.buffer[0])
     # Clear context with initial value
     kv.init(addr hc.vtable)
@@ -944,7 +969,7 @@ proc sign*[T: byte|char](seckey: EcPrivateKey,
   else:
     err(EcKeyIncorrectError)
 
-proc verify*[T: byte|char](sig: EcSignature, message: openarray[T],
+proc verify*[T: byte|char](sig: EcSignature, message: openArray[T],
                            pubkey: EcPublicKey): bool {.inline.} =
   ## Verify ECDSA signature ``sig`` using public key ``pubkey`` and data
   ## ``message``.
@@ -952,20 +977,20 @@ proc verify*[T: byte|char](sig: EcSignature, message: openarray[T],
   ## Return ``true`` if message verification succeeded, ``false`` if
   ## verification failed.
   doAssert((not isNil(sig)) and (not isNil(pubkey)))
-  var hc: BrHashCompatContext
+  var hc: HashCompatContext
   var hash: array[32, byte]
-  var impl = brEcGetDefault()
+  var impl = ecGetDefault()
   if pubkey.key.curve in EcSupportedCurvesCint:
     var kv = addr sha256Vtable
     kv.init(addr hc.vtable)
     if len(message) > 0:
-      kv.update(addr hc.vtable, unsafeAddr message[0], len(message))
+      kv.update(addr hc.vtable, unsafeAddr message[0], uint(len(message)))
     else:
       kv.update(addr hc.vtable, nil, 0)
-    kv.output(addr hc.vtable, addr hash[0])
-    let res = brEcdsaVerifyAsn1(impl, addr hash[0], len(hash),
-                                unsafeAddr pubkey.key,
-                                addr sig.buffer[0], len(sig.buffer))
+    kv.out(addr hc.vtable, addr hash[0])
+    let res = ecdsaI31VrfyAsn1(impl, addr hash[0], uint(len(hash)),
+                               unsafeAddr pubkey.key,
+                               addr sig.buffer[0], uint(len(sig.buffer)))
     # Clear context with initial value
     kv.init(addr hc.vtable)
     result = (res == 1)

libp2p/crypto/ed25519/constants.nim

@@ -1,11 +1,11 @@
-## Nim-Libp2p
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module implements ED25519.
 ## This is pure nim implementation of ED25519 ref10.

libp2p/crypto/ed25519/ed25519.nim

@@ -1,25 +1,31 @@
-## Nim-Libp2p
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module implements ED25519.
 ## This code is a port of the public domain, "ref10" implementation of ed25519
 ## from SUPERCOP.
 
-{.push raises: Defect.}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
-import constants, bearssl
-import nimcrypto/[hash, sha2, utils]
-import stew/results
+import bearssl/rand
+import constants
+import nimcrypto/[hash, sha2]
+# We use `ncrutils` for constant-time hexadecimal encoding/decoding procedures.
+import nimcrypto/utils as ncrutils
+import stew/[results, ctops]
 export results
 
 # This workaround needed because of some bugs in Nim Static[T].
-export hash, sha2
+export hash, sha2, rand
 
 const
   EdPrivateKeySize* = 64
@@ -163,30 +169,30 @@ proc feCopy(h: var Fe, f: Fe) =
   h[8] = f8
   h[9] = f9
 
-proc load3(inp: openarray[byte]): uint64 =
+proc load_3(inp: openArray[byte]): uint64 =
   result = cast[uint64](inp[0])
   result = result or (cast[uint64](inp[1]) shl 8)
   result = result or (cast[uint64](inp[2]) shl 16)
 
-proc load4(inp: openarray[byte]): uint64 =
+proc load_4(inp: openArray[byte]): uint64 =
   result = cast[uint64](inp[0])
   result = result or (cast[uint64](inp[1]) shl 8)
   result = result or (cast[uint64](inp[2]) shl 16)
   result = result or (cast[uint64](inp[3]) shl 24)
 
-proc feFromBytes(h: var Fe, s: openarray[byte]) =
+proc feFromBytes(h: var Fe, s: openArray[byte]) =
   var c0, c1, c2, c3, c4, c5, c6, c7, c8, c9: int64
 
-  var h0 = cast[int64](load4(s.toOpenArray(0, 3)))
-  var h1 = cast[int64](load3(s.toOpenArray(4, 6))) shl 6
-  var h2 = cast[int64](load3(s.toOpenArray(7, 9))) shl 5
-  var h3 = cast[int64](load3(s.toOpenArray(10, 12))) shl 3
-  var h4 = cast[int64](load3(s.toOpenArray(13, 15))) shl 2
-  var h5 = cast[int64](load4(s.toOpenArray(16, 19)))
-  var h6 = cast[int64](load3(s.toOpenArray(20, 22))) shl 7
-  var h7 = cast[int64](load3(s.toOpenArray(23, 25))) shl 5
-  var h8 = cast[int64](load3(s.toOpenArray(26, 28))) shl 4
-  var h9 = (cast[int64](load3(s.toOpenArray(29, 31))) and 8388607'i32) shl 2
+  var h0 = cast[int64](load_4(s.toOpenArray(0, 3)))
+  var h1 = cast[int64](load_3(s.toOpenArray(4, 6))) shl 6
+  var h2 = cast[int64](load_3(s.toOpenArray(7, 9))) shl 5
+  var h3 = cast[int64](load_3(s.toOpenArray(10, 12))) shl 3
+  var h4 = cast[int64](load_3(s.toOpenArray(13, 15))) shl 2
+  var h5 = cast[int64](load_4(s.toOpenArray(16, 19)))
+  var h6 = cast[int64](load_3(s.toOpenArray(20, 22))) shl 7
+  var h7 = cast[int64](load_3(s.toOpenArray(23, 25))) shl 5
+  var h8 = cast[int64](load_3(s.toOpenArray(26, 28))) shl 4
+  var h9 = (cast[int64](load_3(s.toOpenArray(29, 31))) and 8388607'i32) shl 2
 
   c9 = ashr((h9 + (1'i64 shl 24)), 25); h0 = h0 + (c9 * 19); h9 -= (c9 shl 25)
   c1 = ashr((h1 + (1'i64 shl 24)), 25); h2 = h2 + c1; h1 -= (c1 shl 25)
@@ -211,7 +217,7 @@ proc feFromBytes(h: var Fe, s: openarray[byte]) =
   h[8] = cast[int32](h8)
   h[9] = cast[int32](h9)
 
-proc feToBytes(s: var openarray[byte], h: Fe) =
+proc feToBytes(s: var openArray[byte], h: Fe) =
   var h0 = h[0]; var h1 = h[1]; var h2 = h[2]; var h3 = h[3]; var h4 = h[4]
   var h5 = h[5]; var h6 = h[6]; var h7 = h[7]; var h8 = h[8]; var h9 = h[9]
   var q, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9: int32
@@ -448,7 +454,7 @@ proc feNeg(h: var Fe, f: Fe) =
   h[0] = h0; h[1] = h1; h[2] = h2; h[3] = h3; h[4] = h4
   h[5] = h5; h[6] = h6; h[7] = h7; h[8] = h8; h[9] = h9
 
-proc verify32(x: openarray[byte], y: openarray[byte]): int32 =
+proc verify32(x: openArray[byte], y: openArray[byte]): int32 =
   var d = 0'u32
   d = d or (x[0] xor y[0])
   d = d or (x[1] xor y[1])
@@ -798,7 +804,7 @@ proc geAdd(r: var GeP1P1, p: GeP3, q: GeCached) =
   feAdd(r.z, t0, r.t)
   feSub(r.t, t0, r.t)
 
-proc geFromBytesNegateVartime(h: var GeP3, s: openarray[byte]): int32 =
+proc geFromBytesNegateVartime(h: var GeP3, s: openArray[byte]): int32 =
   var u, v, v3, vxx, check: Fe
 
   feFromBytes(h.y, s)
@@ -874,12 +880,12 @@ proc geSub(r: var GeP1P1, p: GeP3, q: GeCached) =
   feSub(r.z, t0, r.t)
   feAdd(r.t, t0, r.t)
 
-proc geToBytes(s: var openarray[byte], h: GeP2) =
+proc geToBytes(s: var openArray[byte], h: GeP2) =
   var recip, x, y: Fe
   feInvert(recip, h.z)
   feMul(x, h.x, recip)
   feMul(y, h.y, recip)
-  feTobytes(s, y)
+  feToBytes(s, y)
   s[31] = s[31] xor cast[byte](feIsNegative(x) shl 7)
 
 proc geP1P1toP2(r: var GeP2, p: GeP1P1) =
@@ -923,10 +929,10 @@ proc geP3toP2(r: var GeP2, p: GeP3) =
 
 proc geP3dbl(r: var GeP1P1, p: GeP3) =
   var q: GeP2
-  geP3ToP2(q, p)
+  geP3toP2(q, p)
   geP2dbl(r, q)
 
-proc geP3ToBytes(s: var openarray[byte], h: GeP3) =
+proc geP3ToBytes(s: var openArray[byte], h: GeP3) =
   var recip, x, y: Fe
 
   feInvert(recip, h.z);
@@ -983,7 +989,7 @@ proc select(t: var GePrecomp, pos: int, b: int8) =
   feNeg(minust.xy2d, t.xy2d)
   cmov(t, minust, bnegative)
 
-proc geScalarMultBase(h: var GeP3, a: openarray[byte]) =
+proc geScalarMultBase(h: var GeP3, a: openArray[byte]) =
   var e: array[64, int8]
   var carry: int8
   var r: GeP1P1
@@ -1008,8 +1014,8 @@ proc geScalarMultBase(h: var GeP3, a: openarray[byte]) =
     geMadd(r, h, t)
     geP1P1toP3(h, r)
 
-  geP3dbl(r, h); geP1P1ToP2(s, r)
-  geP2dbl(r, s); geP1P1ToP2(s, r)
+  geP3dbl(r, h); geP1P1toP2(s, r)
+  geP2dbl(r, s); geP1P1toP2(s, r)
   geP2dbl(r, s); geP1P1toP2(s, r)
   geP2dbl(r, s); geP1P1toP3(h, r)
 
@@ -1018,7 +1024,7 @@ proc geScalarMultBase(h: var GeP3, a: openarray[byte]) =
     geMadd(r, h, t)
     geP1P1toP3(h, r)
 
-proc scMulAdd(s: var openarray[byte], a, b, c: openarray[byte]) =
+proc scMulAdd(s: var openArray[byte], a, b, c: openArray[byte]) =
   var a0 = 2097151'i64 and cast[int64](load_3(a.toOpenArray(0, 2)))
   var a1 = 2097151'i64 and cast[int64](load_4(a.toOpenArray(2, 5)) shr 5)
   var a2 = 2097151'i64 and cast[int64](load_3(a.toOpenArray(5, 7)) shr 2)
@@ -1318,7 +1324,7 @@ proc scMulAdd(s: var openarray[byte], a, b, c: openarray[byte]) =
   s[30] = cast[uint8](ashr(s11, 9))
   s[31] = cast[uint8](ashr(s11, 17))
 
-proc scReduce(s: var openarray[byte]) =
+proc scReduce(s: var openArray[byte]) =
   var s0 = 2097151'i64 and cast[int64](load_3(s.toOpenArray(0, 2)));
   var s1 = 2097151'i64 and cast[int64](load_4(s.toOpenArray(2, 5)) shr 5)
   var s2 = 2097151'i64 and cast[int64](load_3(s.toOpenArray(5, 7)) shr 2)
@@ -1544,7 +1550,7 @@ proc scReduce(s: var openarray[byte]) =
   s[30] = cast[byte](ashr(s11, 9))
   s[31] = cast[byte](ashr(s11, 17))
 
-proc slide(r: var openarray[int8], a: openarray[byte]) =
+proc slide(r: var openArray[int8], a: openArray[byte]) =
   for i in 0..<256:
     r[i] = cast[int8](1'u8 and (a[i shr 3] shr (i and 7)))
   for i in 0..<256:
@@ -1565,8 +1571,8 @@ proc slide(r: var openarray[int8], a: openarray[byte]) =
             break
         inc(b)
 
-proc geDoubleScalarMultVartime(r: var GeP2, a: openarray[byte], A: GeP3,
-                               b: openarray[byte]) =
+proc geDoubleScalarMultVartime(r: var GeP2, a: openArray[byte], A: GeP3,
+                               b: openArray[byte]) =
   var
     aslide: array[256, int8]
     bslide: array[256, int8]
@@ -1630,7 +1636,7 @@ proc NEQ(x, y: uint32): uint32 {.inline.} =
 proc LT0(x: int32): uint32 {.inline.} =
   result = cast[uint32](x) shr 31
 
-proc checkScalar*(scalar: openarray[byte]): uint32 =
+proc checkScalar*(scalar: openArray[byte]): uint32 =
   var z = 0'u32
   var c = 0'i32
   for u in scalar:
@@ -1642,14 +1648,14 @@ proc checkScalar*(scalar: openarray[byte]): uint32 =
     c = -1
   result = NEQ(z, 0'u32) and LT0(c)
 
-proc random*(t: typedesc[EdPrivateKey], rng: var BrHmacDrbgContext): EdPrivateKey =
+proc random*(t: typedesc[EdPrivateKey], rng: var HmacDrbgContext): EdPrivateKey =
   ## Generate new random ED25519 private key using the given random number generator
   var
     point: GeP3
     pk: array[EdPublicKeySize, byte]
     res: EdPrivateKey
 
-  brHmacDrbgGenerate(addr rng, addr res.data[0], 32)
+  hmacDrbgGenerate(rng, res.data.toOpenArray(0, 31))
 
   var hh = sha512.digest(res.data.toOpenArray(0, 31))
   hh.data[0] = hh.data[0] and 0xF8'u8
@@ -1661,14 +1667,14 @@ proc random*(t: typedesc[EdPrivateKey], rng: var BrHmacDrbgContext): EdPrivateKe
 
   res
 
-proc random*(t: typedesc[EdKeyPair], rng: var BrHmacDrbgContext): EdKeyPair =
+proc random*(t: typedesc[EdKeyPair], rng: var HmacDrbgContext): EdKeyPair =
   ## Generate new random ED25519 private and public keypair using OS specific
   ## CSPRNG.
   var
     point: GeP3
     res: EdKeyPair
 
-  brHmacDrbgGenerate(addr rng, addr res.seckey.data[0], 32)
+  hmacDrbgGenerate(rng, res.seckey.data.toOpenArray(0, 31))
 
   var hh = sha512.digest(res.seckey.data.toOpenArray(0, 31))
   hh.data[0] = hh.data[0] and 0xF8'u8
@@ -1680,11 +1686,11 @@ proc random*(t: typedesc[EdKeyPair], rng: var BrHmacDrbgContext): EdKeyPair =
 
   res
 
-proc getKey*(key: EdPrivateKey): EdPublicKey =
+proc getPublicKey*(key: EdPrivateKey): EdPublicKey =
   ## Calculate and return ED25519 public key from private key ``key``.
   copyMem(addr result.data[0], unsafeAddr key.data[32], 32)
 
-proc toBytes*(key: EdPrivateKey, data: var openarray[byte]): int =
+proc toBytes*(key: EdPrivateKey, data: var openArray[byte]): int =
   ## Serialize ED25519 `private key` ``key`` to raw binary form and store it
   ## to ``data``.
   ##
@@ -1694,7 +1700,7 @@ proc toBytes*(key: EdPrivateKey, data: var openarray[byte]): int =
   if len(data) >= result:
     copyMem(addr data[0], unsafeAddr key.data[0], len(key.data))
 
-proc toBytes*(key: EdPublicKey, data: var openarray[byte]): int =
+proc toBytes*(key: EdPublicKey, data: var openArray[byte]): int =
   ## Serialize ED25519 `public key` ``key`` to raw binary form and store it
   ## to ``data``.
   ##
@@ -1704,7 +1710,7 @@ proc toBytes*(key: EdPublicKey, data: var openarray[byte]): int =
   if len(data) >= result:
     copyMem(addr data[0], unsafeAddr key.data[0], len(key.data))
 
-proc toBytes*(sig: EdSignature, data: var openarray[byte]): int =
+proc toBytes*(sig: EdSignature, data: var openArray[byte]): int =
   ## Serialize ED25519 `signature` ``sig`` to raw binary form and store it
   ## to ``data``.
   ##
@@ -1725,26 +1731,29 @@ proc getBytes*(sig: EdSignature): seq[byte] = @(sig.data)
 
 proc `==`*(eda, edb: EdPrivateKey): bool =
   ## Compare ED25519 `private key` objects for equality.
-  result = (eda.data == edb.data)
+  result = CT.isEqual(eda.data, edb.data)
 
 proc `==`*(eda, edb: EdPublicKey): bool =
   ## Compare ED25519 `public key` objects for equality.
-  result = (eda.data == edb.data)
+  result = CT.isEqual(eda.data, edb.data)
 
 proc `==`*(eda, edb: EdSignature): bool =
   ## Compare ED25519 `signature` objects for equality.
-  result = (eda.data == edb.data)
+  result = CT.isEqual(eda.data, edb.data)
 
-proc `$`*(key: EdPrivateKey): string = toHex(key.data)
+proc `$`*(key: EdPrivateKey): string =
   ## Return string representation of ED25519 `private key`.
+  ncrutils.toHex(key.data)
 
-proc `$`*(key: EdPublicKey): string = toHex(key.data)
+proc `$`*(key: EdPublicKey): string =
   ## Return string representation of ED25519 `private key`.
+  ncrutils.toHex(key.data)
 
-proc `$`*(sig: EdSignature): string = toHex(sig.data)
+proc `$`*(sig: EdSignature): string =
   ## Return string representation of ED25519 `signature`.
+  ncrutils.toHex(sig.data)
 
-proc init*(key: var EdPrivateKey, data: openarray[byte]): bool =
+proc init*(key: var EdPrivateKey, data: openArray[byte]): bool =
   ## Initialize ED25519 `private key` ``key`` from raw binary
   ## representation ``data``.
   ##
@@ -1754,7 +1763,7 @@ proc init*(key: var EdPrivateKey, data: openarray[byte]): bool =
     copyMem(addr key.data[0], unsafeAddr data[0], length)
     result = true
 
-proc init*(key: var EdPublicKey, data: openarray[byte]): bool =
+proc init*(key: var EdPublicKey, data: openArray[byte]): bool =
   ## Initialize ED25519 `public key` ``key`` from raw binary
   ## representation ``data``.
   ##
@@ -1764,7 +1773,7 @@ proc init*(key: var EdPublicKey, data: openarray[byte]): bool =
     copyMem(addr key.data[0], unsafeAddr data[0], length)
     result = true
 
-proc init*(sig: var EdSignature, data: openarray[byte]): bool =
+proc init*(sig: var EdSignature, data: openArray[byte]): bool =
   ## Initialize ED25519 `signature` ``sig`` from raw binary
   ## representation ``data``.
   ##
@@ -1779,32 +1788,24 @@ proc init*(key: var EdPrivateKey, data: string): bool =
   ## representation ``data``.
   ##
   ## Procedure returns ``true`` on success.
-  try:
-    init(key, fromHex(data))
-  except ValueError:
-    false
+  init(key, ncrutils.fromHex(data))
 
 proc init*(key: var EdPublicKey, data: string): bool =
   ## Initialize ED25519 `public key` ``key`` from hexadecimal string
   ## representation ``data``.
   ##
   ## Procedure returns ``true`` on success.
-  try:
-    init(key, fromHex(data))
-  except ValueError:
-    false
+  init(key, ncrutils.fromHex(data))
 
 proc init*(sig: var EdSignature, data: string): bool =
   ## Initialize ED25519 `signature` ``sig`` from hexadecimal string
   ## representation ``data``.
   ##
   ## Procedure returns ``true`` on success.
-  try:
-    init(sig, fromHex(data))
-  except ValueError:
-    false
+  init(sig, ncrutils.fromHex(data))
 
-proc init*(t: typedesc[EdPrivateKey], data: openarray[byte]): Result[EdPrivateKey, EdError] =
+proc init*(t: typedesc[EdPrivateKey],
+           data: openArray[byte]): Result[EdPrivateKey, EdError] =
   ## Initialize ED25519 `private key` from raw binary representation ``data``
   ## and return constructed object.
   var res: t
@@ -1813,7 +1814,8 @@ proc init*(t: typedesc[EdPrivateKey], data: openarray[byte]): Result[EdPrivateKe
   else:
     ok(res)
 
-proc init*(t: typedesc[EdPublicKey], data: openarray[byte]): Result[EdPublicKey, EdError] =
+proc init*(t: typedesc[EdPublicKey],
+           data: openArray[byte]): Result[EdPublicKey, EdError] =
   ## Initialize ED25519 `public key` from raw binary representation ``data``
   ## and return constructed object.
   var res: t
@@ -1822,7 +1824,8 @@ proc init*(t: typedesc[EdPublicKey], data: openarray[byte]): Result[EdPublicKey,
   else:
     ok(res)
 
-proc init*(t: typedesc[EdSignature], data: openarray[byte]): Result[EdSignature, EdError] =
+proc init*(t: typedesc[EdSignature],
+           data: openArray[byte]): Result[EdSignature, EdError] =
   ## Initialize ED25519 `signature` from raw binary representation ``data``
   ## and return constructed object.
   var res: t
@@ -1831,7 +1834,8 @@ proc init*(t: typedesc[EdSignature], data: openarray[byte]): Result[EdSignature,
   else:
     ok(res)
 
-proc init*(t: typedesc[EdPrivateKey], data: string): Result[EdPrivateKey, EdError] =
+proc init*(t: typedesc[EdPrivateKey],
+           data: string): Result[EdPrivateKey, EdError] =
   ## Initialize ED25519 `private key` from hexadecimal string representation
   ## ``data`` and return constructed object.
   var res: t
@@ -1840,7 +1844,8 @@ proc init*(t: typedesc[EdPrivateKey], data: string): Result[EdPrivateKey, EdErro
   else:
     ok(res)
 
-proc init*(t: typedesc[EdPublicKey], data: string): Result[EdPublicKey, EdError] =
+proc init*(t: typedesc[EdPublicKey],
+           data: string): Result[EdPublicKey, EdError] =
   ## Initialize ED25519 `public key` from hexadecimal string representation
   ## ``data`` and return constructed object.
   var res: t
@@ -1849,7 +1854,8 @@ proc init*(t: typedesc[EdPublicKey], data: string): Result[EdPublicKey, EdError]
   else:
     ok(res)
 
-proc init*(t: typedesc[EdSignature], data: string): Result[EdSignature, EdError] =
+proc init*(t: typedesc[EdSignature],
+           data: string): Result[EdSignature, EdError] =
   ## Initialize ED25519 `signature` from hexadecimal string representation
   ## ``data`` and return constructed object.
   var res: t
@@ -1876,7 +1882,7 @@ proc clear*(pair: var EdKeyPair) =
   burnMem(pair.pubkey.data)
 
 proc sign*[T: byte|char](key: EdPrivateKey,
-                         message: openarray[T]): EdSignature {.gcsafe, noinit.} =
+                         message: openArray[T]): EdSignature {.gcsafe, noinit.} =
   ## Create ED25519 signature of data ``message`` using private key ``key``.
   var ctx: sha512
   var r: GeP3
@@ -1909,7 +1915,7 @@ proc sign*[T: byte|char](key: EdPrivateKey,
   scMulAdd(result.data.toOpenArray(32, 63), hram.data.toOpenArray(0, 31),
            hash.data.toOpenArray(0, 31), nonce.data.toOpenArray(0, 31))
 
-proc verify*[T: byte|char](sig: EdSignature, message: openarray[T],
+proc verify*[T: byte|char](sig: EdSignature, message: openArray[T],
                            key: EdPublicKey): bool =
   ## Verify ED25519 signature ``sig`` using public key ``key`` and data
   ## ``message``.

libp2p/crypto/hkdf.nim

@@ -1,40 +1,36 @@
-## Nim-LibP2P
-## Copyright (c) 2020 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 # https://tools.ietf.org/html/rfc5869
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 import nimcrypto
-import bearssl
+import bearssl/[kdf, rand, hash]
 
-type
-  BearHKDFContext {.importc: "br_hkdf_context", header: "bearssl_kdf.h".} = object
-  HKDFResult*[len: static int] = array[len, byte]
+type HkdfResult*[len: static int] = array[len, byte]
 
-proc br_hkdf_init(ctx: ptr BearHKDFContext; hashClass: ptr HashClass; salt: pointer; len: csize_t) {.importc: "br_hkdf_init", header: "bearssl_kdf.h", raises: [].}
-proc br_hkdf_inject(ctx: ptr BearHKDFContext; ikm: pointer; len: csize_t) {.importc: "br_hkdf_inject", header: "bearssl_kdf.h", raises: [].}
-proc br_hkdf_flip(ctx: ptr BearHKDFContext) {.importc: "br_hkdf_flip", header: "bearssl_kdf.h", raises: [].}
-proc br_hkdf_produce(ctx: ptr BearHKDFContext; info: pointer; infoLen: csize_t; output: pointer; outputLen: csize_t) {.importc: "br_hkdf_produce", header: "bearssl_kdf.h", raises: [].}
-
-proc hkdf*[T: sha256; len: static int](_: type[T]; salt, ikm, info: openarray[byte]; outputs: var openarray[HKDFResult[len]]) =
+proc hkdf*[T: sha256; len: static int](_: type[T]; salt, ikm, info: openArray[byte]; outputs: var openArray[HkdfResult[len]]) =
   var
-    ctx: BearHKDFContext
-  br_hkdf_init(
-    addr ctx, addr sha256Vtable,
-    if salt.len > 0: unsafeaddr salt[0] else: nil, csize_t(salt.len))
-  br_hkdf_inject(
-    addr ctx, if ikm.len > 0: unsafeaddr ikm[0] else: nil, csize_t(ikm.len))
-  br_hkdf_flip(addr ctx)
+    ctx: HkdfContext
+  hkdfInit(
+    ctx, addr sha256Vtable,
+    if salt.len > 0: unsafeAddr salt[0] else: nil, csize_t(salt.len))
+  hkdfInject(
+    ctx, if ikm.len > 0: unsafeAddr ikm[0] else: nil, csize_t(ikm.len))
+  hkdfFlip(ctx)
   for i in 0..outputs.high:
-    br_hkdf_produce(
-      addr ctx,
-      if info.len > 0: unsafeaddr info[0]
+    discard hkdfProduce(
+      ctx,
+      if info.len > 0: unsafeAddr info[0]
       else: nil, csize_t(info.len),
       addr outputs[i][0], csize_t(outputs[i].len))

libp2p/crypto/minasn1.nim

@@ -1,19 +1,23 @@
-## Nim-Libp2p
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module implements minimal ASN.1 encoding/decoding primitives.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
-import stew/[endians2, results]
+import stew/[endians2, results, ctops]
 export results
-import nimcrypto/utils
+# We use `ncrutils` for constant-time hexadecimal encoding/decoding procedures.
+import nimcrypto/utils as ncrutils
 
 type
   Asn1Error* {.pure.} = enum
@@ -122,7 +126,7 @@ proc len*[T: Asn1Buffer|Asn1Composite](abc: T): int {.inline.} =
   len(abc.buffer) - abc.offset
 
 proc len*(field: Asn1Field): int {.inline.} =
-  result = field.length
+  field.length
 
 template getPtr*(field: untyped): pointer =
   cast[pointer](unsafeAddr field.buffer[field.offset])
@@ -153,68 +157,81 @@ proc code*(tag: Asn1Tag): byte {.inline.} =
   of Asn1Tag.Context:
     0xA0'u8
 
-proc asn1EncodeLength*(dest: var openarray[byte], length: int64): int =
+proc asn1EncodeLength*(dest: var openArray[byte], length: uint64): int =
   ## Encode ASN.1 DER length part of TLV triple and return number of bytes
   ## (octets) used.
   ##
   ## If length of ``dest`` is less then number of required bytes to encode
-  ## ``length`` value, then result of encoding will not be stored in ``dest``
+  ## ``length`` value, then result of encoding WILL NOT BE stored in ``dest``
   ## but number of bytes (octets) required will be returned.
-  if length < 0x80:
+  if length < 0x80'u64:
     if len(dest) >= 1:
-      dest[0] = cast[byte](length)
-      result = 1
+      dest[0] = byte(length and 0x7F'u64)
+    1
   else:
-    result = 0
+    var res = 1'u64
     var z = length
     while z != 0:
-      inc(result)
+      inc(res)
       z = z shr 8
-    if len(dest) >= result + 1:
-      dest[0] = cast[byte](0x80 + result)
+    if uint64(len(dest)) >= res:
+      dest[0] = byte((0x80'u64 + (res - 1'u64)) and 0xFF)
       var o = 1
-      for j in countdown(result - 1, 0):
-        dest[o] = cast[byte](length shr (j shl 3))
+      for j in countdown(res - 2, 0):
+        dest[o] = byte((length shr (j shl 3)) and 0xFF'u64)
         inc(o)
-    inc(result)
+    # Because our `length` argument is `uint64`, `res` could not be bigger
+    # then 9, so it is safe to convert it to `int`.
+    int(res)
 
-proc asn1EncodeInteger*(dest: var openarray[byte],
-                        value: openarray[byte]): int =
+proc asn1EncodeInteger*(dest: var openArray[byte],
+                        value: openArray[byte]): int =
   ## Encode big-endian binary representation of integer as ASN.1 DER `INTEGER`
   ## and return number of bytes (octets) used.
   ##
   ## If length of ``dest`` is less then number of required bytes to encode
-  ## ``value``, then result of encoding will not be stored in ``dest``
+  ## ``value``, then result of encoding WILL NOT BE stored in ``dest``
   ## but number of bytes (octets) required will be returned.
   var buffer: array[16, byte]
-  var o = 0
   var lenlen = 0
-  for i in 0..<len(value):
-    if value[o] != 0x00:
-      break
-    inc(o)
-  if len(value) > 0:
-    if o == len(value):
-      dec(o)
-    if value[o] >= 0x80'u8:
-      lenlen = asn1EncodeLength(buffer, len(value) - o + 1)
-      result = 1 + lenlen + 1 + (len(value) - o)
+
+  let offset =
+    block:
+      var o = 0
+      for i in 0 ..< len(value):
+        if value[o] != 0x00:
+          break
+        inc(o)
+      if o < len(value):
+        o
+      else:
+        o - 1
+
+  let destlen =
+    if len(value) > 0:
+      if value[offset] >= 0x80'u8:
+        lenlen = asn1EncodeLength(buffer, uint64(len(value) - offset + 1))
+        1 + lenlen + 1 + (len(value) - offset)
+      else:
+        lenlen = asn1EncodeLength(buffer, uint64(len(value) - offset))
+        1 + lenlen + (len(value) - offset)
     else:
-      lenlen = asn1EncodeLength(buffer, len(value) - o)
-      result = 1 + lenlen + (len(value) - o)
-  else:
-    result = 2
-  if len(dest) >= result:
-    var s = 1
+      2
+
+  if len(dest) >= destlen:
+    var shift = 1
     dest[0] = Asn1Tag.Integer.code()
     copyMem(addr dest[1], addr buffer[0], lenlen)
-    if value[o] >= 0x80'u8:
-      dest[1 + lenlen] = 0x00'u8
-      s = 2
-    if len(value) > 0:
-      copyMem(addr dest[s + lenlen], unsafeAddr value[o], len(value) - o)
+    # If ``destlen > 2`` it means that ``len(value) > 0`` too.
+    if destlen > 2:
+      if value[offset] >= 0x80'u8:
+        dest[1 + lenlen] = 0x00'u8
+        shift = 2
+      copyMem(addr dest[shift + lenlen], unsafeAddr value[offset],
+              len(value) - offset)
+  destlen
 
-proc asn1EncodeInteger*[T: SomeUnsignedInt](dest: var openarray[byte],
+proc asn1EncodeInteger*[T: SomeUnsignedInt](dest: var openArray[byte],
                                             value: T): int =
   ## Encode Nim's unsigned integer as ASN.1 DER `INTEGER` and return number of
   ## bytes (octets) used.
@@ -224,32 +241,34 @@ proc asn1EncodeInteger*[T: SomeUnsignedInt](dest: var openarray[byte],
   ## but number of bytes (octets) required will be returned.
   dest.asn1EncodeInteger(value.toBytesBE())
 
-proc asn1EncodeBoolean*(dest: var openarray[byte], value: bool): int =
+proc asn1EncodeBoolean*(dest: var openArray[byte], value: bool): int =
   ## Encode Nim's boolean as ASN.1 DER `BOOLEAN` and return number of bytes
   ## (octets) used.
   ##
   ## If length of ``dest`` is less then number of required bytes to encode
   ## ``value``, then result of encoding will not be stored in ``dest``
   ## but number of bytes (octets) required will be returned.
-  result = 3
-  if len(dest) >= result:
+  let res = 3
+  if len(dest) >= res:
     dest[0] = Asn1Tag.Boolean.code()
     dest[1] = 0x01'u8
     dest[2] = if value: 0xFF'u8 else: 0x00'u8
+  res
 
-proc asn1EncodeNull*(dest: var openarray[byte]): int =
+proc asn1EncodeNull*(dest: var openArray[byte]): int =
   ## Encode ASN.1 DER `NULL` and return number of bytes (octets) used.
   ##
   ## If length of ``dest`` is less then number of required bytes to encode
   ## ``value``, then result of encoding will not be stored in ``dest``
   ## but number of bytes (octets) required will be returned.
-  result = 2
-  if len(dest) >= result:
+  let res = 2
+  if len(dest) >= res:
     dest[0] = Asn1Tag.Null.code()
     dest[1] = 0x00'u8
+  res
 
-proc asn1EncodeOctetString*(dest: var openarray[byte],
-                          value: openarray[byte]): int =
+proc asn1EncodeOctetString*(dest: var openArray[byte],
+                            value: openArray[byte]): int =
   ## Encode array of bytes as ASN.1 DER `OCTET STRING` and return number of
   ## bytes (octets) used.
   ##
@@ -257,94 +276,101 @@ proc asn1EncodeOctetString*(dest: var openarray[byte],
   ## ``value``, then result of encoding will not be stored in ``dest``
   ## but number of bytes (octets) required will be returned.
   var buffer: array[16, byte]
-  var lenlen = asn1EncodeLength(buffer, len(value))
-  result = 1 + lenlen + len(value)
-  if len(dest) >= result:
+  let lenlen = asn1EncodeLength(buffer, uint64(len(value)))
+  let res = 1 + lenlen + len(value)
+  if len(dest) >= res:
     dest[0] = Asn1Tag.OctetString.code()
     copyMem(addr dest[1], addr buffer[0], lenlen)
     if len(value) > 0:
       copyMem(addr dest[1 + lenlen], unsafeAddr value[0], len(value))
+  res
 
-proc asn1EncodeBitString*(dest: var openarray[byte],
-                          value: openarray[byte], bits = 0): int =
+proc asn1EncodeBitString*(dest: var openArray[byte],
+                          value: openArray[byte], bits = 0): int =
   ## Encode array of bytes as ASN.1 DER `BIT STRING` and return number of bytes
   ## (octets) used.
   ##
-  ## ``bits`` number of used bits in ``value``. If ``bits == 0``, all the bits
-  ## from ``value`` are used, if ``bits != 0`` only number of ``bits`` will be
-  ## used.
+  ## ``bits`` number of unused bits in ``value``. If ``bits == 0``, all the bits
+  ## from ``value`` will be used.
   ##
   ## If length of ``dest`` is less then number of required bytes to encode
   ## ``value``, then result of encoding will not be stored in ``dest``
   ## but number of bytes (octets) required will be returned.
   var buffer: array[16, byte]
-  var lenlen = asn1EncodeLength(buffer, len(value) + 1)
-  var lbits = 0
-  if bits != 0:
-    lbits = len(value) shl 3 - bits
-  result = 1 + lenlen + 1 + len(value)
-  if len(dest) >= result:
+  let bitlen =
+    if bits != 0:
+      (len(value) shl 3) - bits
+    else:
+      (len(value) shl 3)
+
+  # Number of bytes used
+  let bytelen = (bitlen + 7) shr 3
+  # Number of unused bits
+  let unused = (8 - (bitlen and 7)) and 7
+  let mask = not((1'u8 shl unused) - 1'u8)
+  var lenlen = asn1EncodeLength(buffer, uint64(bytelen + 1))
+  let res = 1 + lenlen + 1 + len(value)
+  if len(dest) >= res:
     dest[0] = Asn1Tag.BitString.code()
     copyMem(addr dest[1], addr buffer[0], lenlen)
-    dest[1 + lenlen] = cast[byte](lbits)
-    if len(value) > 0:
-      copyMem(addr dest[2 + lenlen], unsafeAddr value[0], len(value))
+    dest[1 + lenlen] = byte(unused)
+    if bytelen > 0:
+      let lastbyte = value[bytelen - 1]
+      copyMem(addr dest[2 + lenlen], unsafeAddr value[0], bytelen)
+      # Set unused bits to zero
+      dest[2 + lenlen + bytelen - 1] = lastbyte and mask
+  res
 
-proc asn1EncodeTag[T: SomeUnsignedInt](dest: var openarray[byte],
+proc asn1EncodeTag[T: SomeUnsignedInt](dest: var openArray[byte],
                                        value: T): int =
   var v = value
   if value <= cast[T](0x7F):
     if len(dest) >= 1:
       dest[0] = cast[byte](value)
-    result = 1
+    1
   else:
     var s = 0
+    var res = 0
     while v != 0:
       v = v shr 7
       s += 7
-      inc(result)
-    if len(dest) >= result:
+      inc(res)
+    if len(dest) >= res:
       var k = 0
       while s != 0:
         s -= 7
         dest[k] = cast[byte](((value shr s) and cast[T](0x7F)) or cast[T](0x80))
         inc(k)
       dest[k - 1] = dest[k - 1] and 0x7F'u8
+    res
 
-proc asn1EncodeOid*(dest: var openarray[byte], value: openarray[int]): int =
+proc asn1EncodeOid*(dest: var openArray[byte], value: openArray[int]): int =
   ## Encode array of integers ``value`` as ASN.1 DER `OBJECT IDENTIFIER` and
   ## return number of bytes (octets) used.
   ##
-  ## OBJECT IDENTIFIER requirements for ``value`` elements:
-  ##   * len(value) >= 2
-  ##   * value[0] >= 1 and value[0] < 2
-  ##   * value[1] >= 1 and value[1] < 39
-  ##
   ## If length of ``dest`` is less then number of required bytes to encode
   ## ``value``, then result of encoding will not be stored in ``dest``
   ## but number of bytes (octets) required will be returned.
   var buffer: array[16, byte]
-  result = 1
-  doAssert(len(value) >= 2)
-  doAssert(value[0] >= 1 and value[0] < 2)
-  doAssert(value[1] >= 1 and value[1] <= 39)
+  var res = 1
   var oidlen = 1
   for i in 2..<len(value):
     oidlen += asn1EncodeTag(buffer, cast[uint64](value[i]))
-  result += asn1EncodeLength(buffer, oidlen)
-  result += oidlen
-  if len(dest) >= result:
+  res += asn1EncodeLength(buffer, uint64(oidlen))
+  res += oidlen
+  if len(dest) >= res:
     let last = dest.high
     var offset = 1
     dest[0] = Asn1Tag.Oid.code()
-    offset += asn1EncodeLength(dest.toOpenArray(offset, last), oidlen)
+    offset += asn1EncodeLength(dest.toOpenArray(offset, last), uint64(oidlen))
     dest[offset] = cast[byte](value[0] * 40 + value[1])
     offset += 1
     for i in 2..<len(value):
       offset += asn1EncodeTag(dest.toOpenArray(offset, last),
                               cast[uint64](value[i]))
+  res
 
-proc asn1EncodeOid*(dest: var openarray[byte], value: openarray[byte]): int =
+proc asn1EncodeOid*(dest: var openArray[byte], value: openArray[byte]): int =
   ## Encode array of bytes ``value`` as ASN.1 DER `OBJECT IDENTIFIER` and return
   ## number of bytes (octets) used.
   ##
@@ -355,15 +381,16 @@ proc asn1EncodeOid*(dest: var openarray[byte], value: openarray[byte]): int =
   ## ``value``, then result of encoding will not be stored in ``dest``
   ## but number of bytes (octets) required will be returned.
   var buffer: array[16, byte]
-  var lenlen = asn1EncodeLength(buffer, len(value))
-  result = 1 + lenlen + len(value)
-  if len(dest) >= result:
+  let lenlen = asn1EncodeLength(buffer, uint64(len(value)))
+  let res = 1 + lenlen + len(value)
+  if len(dest) >= res:
     dest[0] = Asn1Tag.Oid.code()
     copyMem(addr dest[1], addr buffer[0], lenlen)
     copyMem(addr dest[1 + lenlen], unsafeAddr value[0], len(value))
+  res
 
-proc asn1EncodeSequence*(dest: var openarray[byte],
-                         value: openarray[byte]): int =
+proc asn1EncodeSequence*(dest: var openArray[byte],
+                         value: openArray[byte]): int =
   ## Encode ``value`` as ASN.1 DER `SEQUENCE` and return number of bytes
   ## (octets) used.
   ##
@@ -371,14 +398,15 @@ proc asn1EncodeSequence*(dest: var openarray[byte],
   ## ``value``, then result of encoding will not be stored in ``dest``
   ## but number of bytes (octets) required will be returned.
   var buffer: array[16, byte]
-  var lenlen = asn1EncodeLength(buffer, len(value))
-  result = 1 + lenlen + len(value)
-  if len(dest) >= result:
+  let lenlen = asn1EncodeLength(buffer, uint64(len(value)))
+  let res = 1 + lenlen + len(value)
+  if len(dest) >= res:
     dest[0] = Asn1Tag.Sequence.code()
     copyMem(addr dest[1], addr buffer[0], lenlen)
     copyMem(addr dest[1 + lenlen], unsafeAddr value[0], len(value))
+  res
 
-proc asn1EncodeComposite*(dest: var openarray[byte],
+proc asn1EncodeComposite*(dest: var openArray[byte],
                           value: Asn1Composite): int =
   ## Encode composite value and return number of bytes (octets) used.
   ##
@@ -386,29 +414,34 @@ proc asn1EncodeComposite*(dest: var openarray[byte],
   ## ``value``, then result of encoding will not be stored in ``dest``
   ## but number of bytes (octets) required will be returned.
   var buffer: array[16, byte]
-  var lenlen = asn1EncodeLength(buffer, len(value.buffer))
-  result = 1 + lenlen + len(value.buffer)
-  if len(dest) >= result:
+  let lenlen = asn1EncodeLength(buffer, uint64(len(value.buffer)))
+  let res = 1 + lenlen + len(value.buffer)
+  if len(dest) >= res:
     dest[0] = value.tag.code()
     copyMem(addr dest[1], addr buffer[0], lenlen)
     copyMem(addr dest[1 + lenlen], unsafeAddr value.buffer[0],
             len(value.buffer))
+  res
 
-proc asn1EncodeContextTag*(dest: var openarray[byte], value: openarray[byte],
+proc asn1EncodeContextTag*(dest: var openArray[byte], value: openArray[byte],
                            tag: int): int =
   ## Encode ASN.1 DER `CONTEXT SPECIFIC TAG` ``tag`` for value ``value`` and
   ## return number of bytes (octets) used.
   ##
+  ## Note: Only values in [0, 15] range can be used as context tag ``tag``
+  ## values.
+  ##
   ## If length of ``dest`` is less then number of required bytes to encode
   ## ``value``, then result of encoding will not be stored in ``dest``
   ## but number of bytes (octets) required will be returned.
   var buffer: array[16, byte]
-  var lenlen = asn1EncodeLength(buffer, len(value))
-  result = 1 + lenlen + len(value)
-  if len(dest) >= result:
-    dest[0] = 0xA0'u8 or (cast[byte](tag) and 0x0F)
+  let lenlen = asn1EncodeLength(buffer, uint64(len(value)))
+  let res = 1 + lenlen + len(value)
+  if len(dest) >= res:
+    dest[0] = 0xA0'u8 or (byte(tag and 0xFF) and 0x0F'u8)
     copyMem(addr dest[1], addr buffer[0], lenlen)
     copyMem(addr dest[1 + lenlen], unsafeAddr value[0], len(value))
+  res
 
 proc getLength(ab: var Asn1Buffer): Asn1Result[uint64] =
   ## Decode length part of ASN.1 TLV triplet.
@@ -457,197 +490,300 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] =
     field: Asn1Field
     tag, ttag, offset: int
     length, tlength: uint64
-    klass: Asn1Class
+    aclass: Asn1Class
     inclass: bool
 
   inclass = false
   while true:
     offset = ab.offset
-    klass = ? ab.getTag(tag)
+    aclass = ? ab.getTag(tag)
 
-    if klass == Asn1Class.ContextSpecific:
+    case aclass
+    of Asn1Class.ContextSpecific:
       if inclass:
         return err(Asn1Error.Incorrect)
-
-      inclass = true
-      ttag = tag
-      tlength = ? ab.getLength()
-
-    elif klass == Asn1Class.Universal:
+      else:
+        inclass = true
+        ttag = tag
+        tlength = ? ab.getLength()
+    of Asn1Class.Universal:
       length = ? ab.getLength()
 
       if inclass:
         if length >= tlength:
           return err(Asn1Error.Incorrect)
 
-      if cast[byte](tag) == Asn1Tag.Boolean.code():
+      case byte(tag)
+      of Asn1Tag.Boolean.code():
         # BOOLEAN
         if length != 1:
           return err(Asn1Error.Incorrect)
-        if not ab.isEnough(cast[int](length)):
+
+        if not ab.isEnough(int(length)):
           return err(Asn1Error.Incomplete)
+
         let b = ab.buffer[ab.offset]
         if b != 0xFF'u8 and b != 0x00'u8:
            return err(Asn1Error.Incorrect)
 
-        field = Asn1Field(kind: Asn1Tag.Boolean, klass: klass,
-                          index: ttag, offset: cast[int](ab.offset),
+        field = Asn1Field(kind: Asn1Tag.Boolean, klass: aclass,
+                          index: ttag, offset: int(ab.offset),
                           length: 1)
         shallowCopy(field.buffer, ab.buffer)
         field.vbool = (b == 0xFF'u8)
         ab.offset += 1
         return ok(field)
-      elif cast[byte](tag) == Asn1Tag.Integer.code():
+
+      of Asn1Tag.Integer.code():
         # INTEGER
-        if not ab.isEnough(cast[int](length)):
-          return err(Asn1Error.Incomplete)
-        if ab.buffer[ab.offset] == 0x00'u8:
-          length -= 1
-          ab.offset += 1
-        field = Asn1Field(kind: Asn1Tag.Integer, klass: klass,
-                          index: ttag, offset: cast[int](ab.offset),
-                          length: cast[int](length))
-        shallowCopy(field.buffer, ab.buffer)
-        if length <= 8:
-          for i in 0..<int(length):
-            field.vint = (field.vint shl 8) or
-                         cast[uint64](ab.buffer[ab.offset + i])
-        ab.offset += cast[int](length)
-        return ok(field)
-      elif cast[byte](tag) == Asn1Tag.BitString.code():
+        if length == 0:
+          return err(Asn1Error.Incorrect)
+
+        if not ab.isEnough(int(length)):
+         return err(Asn1Error.Incomplete)
+
+        # Count number of leading zeroes
+        var zc = 0
+        while (zc < int(length)) and (ab.buffer[ab.offset + zc] == 0x00'u8):
+          inc(zc)
+
+        if zc > 1:
+          return err(Asn1Error.Incorrect)
+
+        if zc == 0:
+          # Negative or Positive integer
+          field = Asn1Field(kind: Asn1Tag.Integer, klass: aclass,
+                            index: ttag, offset: int(ab.offset),
+                            length: int(length))
+          shallowCopy(field.buffer, ab.buffer)
+          if (ab.buffer[ab.offset] and 0x80'u8) == 0x80'u8:
+            # Negative integer
+            if length <= 8:
+              # We need this transformation because our field.vint is uint64.
+              for i in 0 ..< 8:
+                if i < 8 - int(length):
+                  field.vint = (field.vint shl 8) or 0xFF'u64
+                else:
+                  let offset = ab.offset + i - (8 - int(length))
+                  field.vint = (field.vint shl 8) or uint64(ab.buffer[offset])
+          else:
+            # Positive integer
+            if length <= 8:
+              for i in 0 ..< int(length):
+                field.vint = (field.vint shl 8) or
+                              uint64(ab.buffer[ab.offset + i])
+          ab.offset += int(length)
+          return ok(field)
+        else:
+          if length == 1:
+            # Zero value integer
+            field = Asn1Field(kind: Asn1Tag.Integer, klass: aclass,
+                              index: ttag, offset: int(ab.offset),
+                              length: int(length), vint: 0'u64)
+            shallowCopy(field.buffer, ab.buffer)
+            ab.offset += int(length)
+            return ok(field)
+          else:
+            # Positive integer with leading zero
+            field = Asn1Field(kind: Asn1Tag.Integer, klass: aclass,
+                              index: ttag, offset: int(ab.offset) + 1,
+                              length: int(length) - 1)
+            shallowCopy(field.buffer, ab.buffer)
+            if length <= 9:
+              for i in 1 ..< int(length):
+                field.vint = (field.vint shl 8) or
+                              uint64(ab.buffer[ab.offset + i])
+            ab.offset += int(length)
+            return ok(field)
+
+      of Asn1Tag.BitString.code():
         # BIT STRING
-        if not ab.isEnough(cast[int](length)):
+        if length == 0:
+          # BIT STRING should include `unused` bits field, so length should be
+          # bigger then 1.
+          return err(Asn1Error.Incorrect)
+
+        elif length == 1:
+          if ab.buffer[ab.offset] != 0x00'u8:
+            return err(Asn1Error.Incorrect)
+          else:
+            # Zero-length BIT STRING.
+            field = Asn1Field(kind: Asn1Tag.BitString, klass: aclass,
+                              index: ttag, offset: int(ab.offset + 1),
+                              length: 0, ubits: 0)
+            shallowCopy(field.buffer, ab.buffer)
+            ab.offset += int(length)
+            return ok(field)
+
+        else:
+          if not ab.isEnough(int(length)):
+            return err(Asn1Error.Incomplete)
+
+          let unused = ab.buffer[ab.offset]
+          if unused > 0x07'u8:
+            # Number of unused bits should not be bigger then `7`.
+            return err(Asn1Error.Incorrect)
+
+          let mask = (1'u8 shl int(unused)) - 1'u8
+          if (ab.buffer[ab.offset + int(length) - 1] and mask) != 0x00'u8:
+            ## All unused bits should be set to `0`.
+            return err(Asn1Error.Incorrect)
+
+          field = Asn1Field(kind: Asn1Tag.BitString, klass: aclass,
+                            index: ttag, offset: int(ab.offset + 1),
+                            length: int(length - 1), ubits: int(unused))
+          shallowCopy(field.buffer, ab.buffer)
+          ab.offset += int(length)
+          return ok(field)
+
+      of Asn1Tag.OctetString.code():
+        # OCTET STRING
+        if not ab.isEnough(int(length)):
           return err(Asn1Error.Incomplete)
-        field = Asn1Field(kind: Asn1Tag.BitString, klass: klass,
-                          index: ttag, offset: cast[int](ab.offset + 1),
-                          length: cast[int](length - 1))
+
+        field = Asn1Field(kind: Asn1Tag.OctetString, klass: aclass,
+                          index: ttag, offset: int(ab.offset),
+                          length: int(length))
         shallowCopy(field.buffer, ab.buffer)
-        field.ubits = cast[int](((length - 1) shl 3) - ab.buffer[ab.offset])
-        ab.offset += cast[int](length)
+        ab.offset += int(length)
         return ok(field)
-      elif cast[byte](tag) == Asn1Tag.OctetString.code():
-        # OCT STRING
-        if not ab.isEnough(cast[int](length)):
-          return err(Asn1Error.Incomplete)
-        field = Asn1Field(kind: Asn1Tag.OctetString, klass: klass,
-                          index: ttag, offset: cast[int](ab.offset),
-                          length: cast[int](length))
-        shallowCopy(field.buffer, ab.buffer)
-        ab.offset += cast[int](length)
-        return ok(field)
-      elif cast[byte](tag) == Asn1Tag.Null.code():
+
+      of Asn1Tag.Null.code():
         # NULL
         if length != 0:
           return err(Asn1Error.Incorrect)
-        field = Asn1Field(kind: Asn1Tag.Null, klass: klass,
-                          index: ttag, offset: cast[int](ab.offset),
-                          length: 0)
+
+        field = Asn1Field(kind: Asn1Tag.Null, klass: aclass, index: ttag,
+                          offset: int(ab.offset), length: 0)
         shallowCopy(field.buffer, ab.buffer)
-        ab.offset += cast[int](length)
+        ab.offset += int(length)
         return ok(field)
-      elif cast[byte](tag) == Asn1Tag.Oid.code():
+
+      of Asn1Tag.Oid.code():
         # OID
-        if not ab.isEnough(cast[int](length)):
+        if not ab.isEnough(int(length)):
           return err(Asn1Error.Incomplete)
-        field = Asn1Field(kind: Asn1Tag.Oid, klass: klass,
-                          index: ttag, offset: cast[int](ab.offset),
-                          length: cast[int](length))
+
+        field = Asn1Field(kind: Asn1Tag.Oid, klass: aclass,
+                          index: ttag, offset: int(ab.offset),
+                          length: int(length))
         shallowCopy(field.buffer, ab.buffer)
-        ab.offset += cast[int](length)
+        ab.offset += int(length)
         return ok(field)
-      elif cast[byte](tag) == Asn1Tag.Sequence.code():
+
+      of Asn1Tag.Sequence.code():
         # SEQUENCE
-        if not ab.isEnough(cast[int](length)):
+        if not ab.isEnough(int(length)):
           return err(Asn1Error.Incomplete)
-        field = Asn1Field(kind: Asn1Tag.Sequence, klass: klass,
-                          index: ttag, offset: cast[int](ab.offset),
-                          length: cast[int](length))
+
+        field = Asn1Field(kind: Asn1Tag.Sequence, klass: aclass,
+                          index: ttag, offset: int(ab.offset),
+                          length: int(length))
         shallowCopy(field.buffer, ab.buffer)
-        ab.offset += cast[int](length)
+        ab.offset += int(length)
         return ok(field)
+
       else:
         return err(Asn1Error.NoSupport)
+
       inclass = false
       ttag = 0
     else:
       return err(Asn1Error.NoSupport)
 
-proc getBuffer*(field: Asn1Field): Asn1Buffer =
+proc getBuffer*(field: Asn1Field): Asn1Buffer {.inline.} =
   ## Return ``field`` as Asn1Buffer to enter composite types.
-  shallowCopy(result.buffer, field.buffer)
-  result.offset = field.offset
-  result.length = field.length
+  Asn1Buffer(buffer: field.buffer, offset: field.offset, length: field.length)
 
-proc `==`*(field: Asn1Field, data: openarray[byte]): bool =
+proc `==`*(field: Asn1Field, data: openArray[byte]): bool =
   ## Compares field ``field`` data with ``data`` and returns ``true`` if both
   ## buffers are equal.
   let length = len(field.buffer)
-  if length > 0:
-    if field.length == len(data):
-      result = equalMem(unsafeAddr field.buffer[field.offset],
-                        unsafeAddr data[0], field.length)
+  if length == 0 and len(data) == 0:
+    true
+  else:
+    if length > 0:
+      if field.length == len(data):
+        CT.isEqual(
+          field.buffer.toOpenArray(field.offset,
+                                   field.offset + field.length - 1),
+          data.toOpenArray(0, field.length - 1))
+      else:
+        false
+    else:
+      false
 
-proc init*(t: typedesc[Asn1Buffer], data: openarray[byte]): Asn1Buffer =
+proc init*(t: typedesc[Asn1Buffer], data: openArray[byte]): Asn1Buffer =
   ## Initialize ``Asn1Buffer`` from array of bytes ``data``.
-  result.buffer = @data
+  Asn1Buffer(buffer: @data)
 
 proc init*(t: typedesc[Asn1Buffer], data: string): Asn1Buffer =
   ## Initialize ``Asn1Buffer`` from hexadecimal string ``data``.
-  result.buffer = fromHex(data)
+  Asn1Buffer(buffer: ncrutils.fromHex(data))
 
 proc init*(t: typedesc[Asn1Buffer]): Asn1Buffer =
   ## Initialize empty ``Asn1Buffer``.
-  result.buffer = newSeq[byte]()
+  Asn1Buffer(buffer: newSeq[byte]())
 
 proc init*(t: typedesc[Asn1Composite], tag: Asn1Tag): Asn1Composite =
   ## Initialize ``Asn1Composite`` with tag ``tag``.
-  result.tag = tag
-  result.buffer = newSeq[byte]()
+  Asn1Composite(tag: tag, buffer: newSeq[byte]())
 
 proc init*(t: typedesc[Asn1Composite], idx: int): Asn1Composite =
   ## Initialize ``Asn1Composite`` with tag context-specific id ``id``.
-  result.tag = Asn1Tag.Context
-  result.idx = idx
-  result.buffer = newSeq[byte]()
+  Asn1Composite(tag: Asn1Tag.Context, idx: idx, buffer: newSeq[byte]())
 
 proc `$`*(buffer: Asn1Buffer): string =
   ## Return string representation of ``buffer``.
-  result = toHex(buffer.toOpenArray())
+  ncrutils.toHex(buffer.toOpenArray())
 
 proc `$`*(field: Asn1Field): string =
   ## Return string representation of ``field``.
-  result = "["
-  result.add($field.kind)
-  result.add("]")
-  if field.kind == Asn1Tag.NoSupport:
-    result.add(" ")
-    result.add(toHex(field.toOpenArray()))
-  elif field.kind == Asn1Tag.Boolean:
-    result.add(" ")
-    result.add($field.vbool)
-  elif field.kind == Asn1Tag.Integer:
-    result.add(" ")
+  var res = "["
+  res.add($field.kind)
+  res.add("]")
+  case field.kind
+  of Asn1Tag.Boolean:
+    res.add(" ")
+    res.add($field.vbool)
+    res
+  of Asn1Tag.Integer:
+    res.add(" ")
     if field.length <= 8:
-      result.add($field.vint)
+      res.add($field.vint)
     else:
-      result.add(toHex(field.toOpenArray()))
-  elif field.kind == Asn1Tag.BitString:
-    result.add(" ")
-    result.add("(")
-    result.add($field.ubits)
-    result.add(" bits) ")
-    result.add(toHex(field.toOpenArray()))
-  elif field.kind == Asn1Tag.OctetString:
-    result.add(" ")
-    result.add(toHex(field.toOpenArray()))
-  elif field.kind == Asn1Tag.Null:
-    result.add(" NULL")
-  elif field.kind == Asn1Tag.Oid:
-    result.add(" ")
-    result.add(toHex(field.toOpenArray()))
-  elif field.kind == Asn1Tag.Sequence:
-    result.add(" ")
-    result.add(toHex(field.toOpenArray()))
+      res.add(ncrutils.toHex(field.toOpenArray()))
+    res
+  of Asn1Tag.BitString:
+    res.add(" ")
+    res.add("(")
+    res.add($field.ubits)
+    res.add(" bits) ")
+    res.add(ncrutils.toHex(field.toOpenArray()))
+    res
+  of Asn1Tag.OctetString:
+    res.add(" ")
+    res.add(ncrutils.toHex(field.toOpenArray()))
+    res
+  of Asn1Tag.Null:
+    res.add(" NULL")
+    res
+  of Asn1Tag.Oid:
+    res.add(" ")
+    res.add(ncrutils.toHex(field.toOpenArray()))
+    res
+  of Asn1Tag.Sequence:
+    res.add(" ")
+    res.add(ncrutils.toHex(field.toOpenArray()))
+    res
+  of Asn1Tag.Context:
+    res.add(" ")
+    res.add(ncrutils.toHex(field.toOpenArray()))
+    res
+  else:
+    res.add(" ")
+    res.add(ncrutils.toHex(field.toOpenArray()))
+    res
 
 proc write*[T: Asn1Buffer|Asn1Composite](abc: var T, tag: Asn1Tag) =
   ## Write empty value to buffer or composite with ``tag``.
@@ -655,7 +791,7 @@ proc write*[T: Asn1Buffer|Asn1Composite](abc: var T, tag: Asn1Tag) =
   ## This procedure must be used to write `NULL`, `0` or empty `BIT STRING`,
   ## `OCTET STRING` types.
   doAssert(tag in {Asn1Tag.Null, Asn1Tag.Integer, Asn1Tag.BitString,
-                 Asn1Tag.OctetString})
+                   Asn1Tag.OctetString})
   var length: int
   if tag == Asn1Tag.Null:
     length = asn1EncodeNull(abc.toOpenArray())
@@ -692,7 +828,7 @@ proc write*[T: Asn1Buffer|Asn1Composite](abc: var T, value: bool) =
   abc.offset += length
 
 proc write*[T: Asn1Buffer|Asn1Composite](abc: var T, tag: Asn1Tag,
-                                         value: openarray[byte], bits = 0) =
+                                         value: openArray[byte], bits = 0) =
   ## Write array ``value`` using ``tag``.
   ##
   ## This procedure is used to write ASN.1 `INTEGER`, `OCTET STRING`,

libp2p/crypto/rsa.nim

@@ -1,11 +1,11 @@
-## Nim-Libp2p
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module implements constant-time RSA PKCS#1.5 DSA.
 ##
@@ -13,59 +13,65 @@
 ## BearSSL library <https://bearssl.org/>
 ## Copyright(C) 2018 Thomas Pornin <pornin@bolet.org>.
 
-{.push raises: Defect.}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
-import nimcrypto/utils
-import bearssl
+import bearssl/[rsa, rand, hash]
 import minasn1
-export Asn1Error
-import stew/results
-export results
+import stew/[results, ctops]
+# We use `ncrutils` for constant-time hexadecimal encoding/decoding procedures.
+import nimcrypto/utils as ncrutils
+
+export Asn1Error, results
 
 const
-  DefaultPublicExponent* = 3'u32
+  DefaultPublicExponent* = 65537'u32
     ## Default value for RSA public exponent.
-  MinKeySize* = 512
+    ## https://golang.org/src/crypto/rsa/rsa.go#226
+  MinKeySize* = 2048
     ## Minimal allowed RSA key size in bits.
-  DefaultKeySize* = 2048
+    ## https://github.com/libp2p/go-libp2p-core/blob/master/crypto/rsa_common.go#L13
+  DefaultKeySize* = 3072
     ## Default RSA key size in bits.
 
   RsaOidSha1* = [
-    0x05'u8, 0x2B'u8, 0x0E'u8, 0x03'u8, 0x02'u8, 0x1A'u8
+    byte 0x05, 0x2B, 0x0E, 0x03, 0x02, 0x1A
   ]
     ## RSA PKCS#1.5 SHA-1 hash object identifier.
   RsaOidSha224* = [
-    0x09'u8, 0x60'u8, 0x86'u8, 0x48'u8, 0x01'u8, 0x65'u8, 0x03'u8, 0x04'u8,
-    0x02'u8, 0x04'u8
+    byte 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04,
+    0x02, 0x04
   ]
     ## RSA PKCS#1.5 SHA-224 hash object identifier.
   RsaOidSha256* = [
-    0x09'u8, 0x60'u8, 0x86'u8, 0x48'u8, 0x01'u8, 0x65'u8, 0x03'u8, 0x04'u8,
-    0x02'u8, 0x01'u8
+    byte 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04,
+    0x02, 0x01
   ]
     ## RSA PKCS#1.5 SHA-256 hash object identifier.
   RsaOidSha384* = [
-    0x09'u8, 0x60'u8, 0x86'u8, 0x48'u8, 0x01'u8, 0x65'u8, 0x03'u8, 0x04'u8,
-    0x02'u8, 0x02'u8
+    byte 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04,
+    0x02, 0x02
   ]
     ## RSA PKCS#1.5 SHA-384 hash object identifier.
   RsaOidSha512* = [
-    0x09'u8, 0x60'u8, 0x86'u8, 0x48'u8, 0x01'u8, 0x65'u8, 0x03'u8, 0x04'u8,
-    0x02'u8, 0x03'u8
+    byte 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04,
+    0x02, 0x03
   ]
     ## RSA PKCS#1.5 SHA-512 hash object identifier.
 
 type
   RsaPrivateKey* = ref object
     buffer*: seq[byte]
-    seck*: BrRsaPrivateKey
-    pubk*: BrRsaPublicKey
-    pexp*: ptr cuchar
-    pexplen*: int
+    seck*: rsa.RsaPrivateKey
+    pubk*: rsa.RsaPublicKey
+    pexp*: ptr byte
+    pexplen*: uint
 
   RsaPublicKey* = ref object
     buffer*: seq[byte]
-    key*: BrRsaPublicKey
+    key*: rsa.RsaPublicKey
 
   RsaKeyPair* = RsaPrivateKey
 
@@ -78,7 +84,8 @@ type
   RsaError* = enum
     RsaGenError,
     RsaKeyIncorrectError,
-    RsaSignatureError
+    RsaSignatureError,
+    RsaLowSecurityError
 
   RsaResult*[T] = Result[T, RsaError]
 
@@ -96,8 +103,8 @@ template getFinish(bs, os, ls: untyped): untyped =
   var eo = -1
   if p >= s:
     let so = cast[int](p - s)
-    if so + ls <= len(bs):
-      eo = so + ls - 1
+    if so + int(ls) <= len(bs):
+      eo = so + int(ls) - 1
   eo
 
 template getArray*(bs, os, ls: untyped): untyped =
@@ -106,31 +113,34 @@ template getArray*(bs, os, ls: untyped): untyped =
 template trimZeroes(b: seq[byte], pt, ptlen: untyped) =
   var length = ptlen
   for i in 0..<length:
-    if pt[] != cast[cuchar](0x00'u8):
+    if pt[] != byte(0x00):
       break
-    pt = cast[ptr cuchar](cast[uint](pt) + 1)
+    pt = cast[ptr byte](cast[uint](pt) + 1)
     ptlen -= 1
 
-proc random*[T: RsaKP](t: typedesc[T], rng: var BrHmacDrbgContext,
+proc random*[T: RsaKP](t: typedesc[T], rng: var HmacDrbgContext,
                        bits = DefaultKeySize,
                        pubexp = DefaultPublicExponent): RsaResult[T] =
   ## Generate new random RSA private key using BearSSL's HMAC-SHA256-DRBG
   ## algorithm.
   ##
   ## ``bits`` number of bits in RSA key, must be in
-  ## range [512, 4096] (default = 2048).
+  ## range [2048, 4096] (default = 3072).
   ##
   ## ``pubexp`` is RSA public exponent, which must be prime (default = 3).
+  if bits < MinKeySize:
+    return err(RsaLowSecurityError)
+
   let
     sko = 0
-    pko = brRsaPrivateKeyBufferSize(bits)
-    eko = pko + brRsaPublicKeyBufferSize(bits)
+    pko = rsaKbufPrivSize(bits)
+    eko = pko + rsaKbufPubSize(bits)
     length = eko + ((bits + 7) shr 3)
 
   let res = new T
   res.buffer = newSeq[byte](length)
 
-  var keygen = brRsaKeygenGetDefault()
+  var keygen = rsaKeygenGetDefault()
 
   if keygen(addr rng.vtable,
             addr res.seck, addr res.buffer[sko],
@@ -139,12 +149,12 @@ proc random*[T: RsaKP](t: typedesc[T], rng: var BrHmacDrbgContext,
     return err(RsaGenError)
 
   let
-    compute = brRsaComputePrivexpGetDefault()
+    compute = rsaComputePrivexpGetDefault()
     computed = compute(addr res.buffer[eko], addr res.seck, pubexp)
   if computed == 0:
     return err(RsaGenError)
 
-  res.pexp = cast[ptr cuchar](addr res.buffer[eko])
+  res.pexp = addr res.buffer[eko]
   res.pexplen = computed
 
   trimZeroes(res.buffer, res.seck.p, res.seck.plen)
@@ -163,12 +173,12 @@ proc copy*[T: RsaPKI](key: T): T =
   doAssert(not isNil(key))
   when T is RsaPrivateKey:
     if len(key.buffer) > 0:
-      let length = key.seck.plen + key.seck.qlen + key.seck.dplen +
-                   key.seck.dqlen + key.seck.iqlen + key.pubk.nlen +
-                   key.pubk.elen + key.pexplen
+      let length = key.seck.plen.uint + key.seck.qlen.uint + key.seck.dplen.uint +
+                   key.seck.dqlen.uint + key.seck.iqlen.uint + key.pubk.nlen.uint +
+                   key.pubk.elen.uint + key.pexplen.uint
       result = new RsaPrivateKey
       result.buffer = newSeq[byte](length)
-      let po = 0
+      let po: uint = 0
       let qo = po + key.seck.plen
       let dpo = qo + key.seck.qlen
       let dqo = dpo + key.seck.dplen
@@ -184,14 +194,14 @@ proc copy*[T: RsaPKI](key: T): T =
       copyMem(addr result.buffer[no], key.pubk.n, key.pubk.nlen)
       copyMem(addr result.buffer[eo], key.pubk.e, key.pubk.elen)
       copyMem(addr result.buffer[peo], key.pexp, key.pexplen)
-      result.seck.p = cast[ptr cuchar](addr result.buffer[po])
-      result.seck.q = cast[ptr cuchar](addr result.buffer[qo])
-      result.seck.dp = cast[ptr cuchar](addr result.buffer[dpo])
-      result.seck.dq = cast[ptr cuchar](addr result.buffer[dqo])
-      result.seck.iq = cast[ptr cuchar](addr result.buffer[iqo])
-      result.pubk.n = cast[ptr cuchar](addr result.buffer[no])
-      result.pubk.e = cast[ptr cuchar](addr result.buffer[eo])
-      result.pexp = cast[ptr cuchar](addr result.buffer[peo])
+      result.seck.p = addr result.buffer[po]
+      result.seck.q = addr result.buffer[qo]
+      result.seck.dp = addr result.buffer[dpo]
+      result.seck.dq = addr result.buffer[dqo]
+      result.seck.iq = addr result.buffer[iqo]
+      result.pubk.n = addr result.buffer[no]
+      result.pubk.e = addr result.buffer[eo]
+      result.pexp = addr result.buffer[peo]
       result.seck.plen = key.seck.plen
       result.seck.qlen = key.seck.qlen
       result.seck.dplen = key.seck.dplen
@@ -210,8 +220,8 @@ proc copy*[T: RsaPKI](key: T): T =
       let eo = no + key.key.nlen
       copyMem(addr result.buffer[no], key.key.n, key.key.nlen)
       copyMem(addr result.buffer[eo], key.key.e, key.key.elen)
-      result.key.n = cast[ptr cuchar](addr result.buffer[no])
-      result.key.e = cast[ptr cuchar](addr result.buffer[eo])
+      result.key.n = cast[ptr char](addr result.buffer[no])
+      result.key.e = cast[ptr char](addr result.buffer[eo])
       result.key.nlen = key.key.nlen
       result.key.elen = key.key.elen
   elif T is RsaSignature:
@@ -219,14 +229,14 @@ proc copy*[T: RsaPKI](key: T): T =
       result = new RsaSignature
       result.buffer = key.buffer
 
-proc getKey*(key: RsaPrivateKey): RsaPublicKey =
+proc getPublicKey*(key: RsaPrivateKey): RsaPublicKey =
   ## Get RSA public key from RSA private key.
   doAssert(not isNil(key))
   let length = key.pubk.nlen + key.pubk.elen
   result = new RsaPublicKey
   result.buffer = newSeq[byte](length)
-  result.key.n = cast[ptr cuchar](addr result.buffer[0])
-  result.key.e = cast[ptr cuchar](addr result.buffer[key.pubk.nlen])
+  result.key.n = addr result.buffer[0]
+  result.key.e = addr result.buffer[key.pubk.nlen]
   copyMem(addr result.buffer[0], cast[pointer](key.pubk.n), key.pubk.nlen)
   copyMem(addr result.buffer[key.pubk.nlen], cast[pointer](key.pubk.e),
           key.pubk.elen)
@@ -239,7 +249,7 @@ proc seckey*(pair: RsaKeyPair): RsaPrivateKey {.inline.} =
 
 proc pubkey*(pair: RsaKeyPair): RsaPublicKey {.inline.} =
   ## Get RSA public key from pair ``pair``.
-  result = cast[RsaPrivateKey](pair).getKey()
+  result = cast[RsaPrivateKey](pair).getPublicKey()
 
 proc clear*[T: RsaPKI|RsaKeyPair](pki: var T) =
   ## Wipe and clear EC private key, public key or scalar object.
@@ -273,7 +283,7 @@ proc clear*[T: RsaPKI|RsaKeyPair](pki: var T) =
     burnMem(pki.buffer)
     pki.buffer.setLen(0)
 
-proc toBytes*(key: RsaPrivateKey, data: var openarray[byte]): RsaResult[int] =
+proc toBytes*(key: RsaPrivateKey, data: var openArray[byte]): RsaResult[int] =
   ## Serialize RSA private key ``key`` to ASN.1 DER binary form and store it
   ## to ``data``.
   ##
@@ -310,7 +320,7 @@ proc toBytes*(key: RsaPrivateKey, data: var openarray[byte]): RsaResult[int] =
   else:
     err(RsaKeyIncorrectError)
 
-proc toBytes*(key: RsaPublicKey, data: var openarray[byte]): RsaResult[int] =
+proc toBytes*(key: RsaPublicKey, data: var openArray[byte]): RsaResult[int] =
   ## Serialize RSA public key ``key`` to ASN.1 DER binary form and store it
   ## to ``data``.
   ##
@@ -344,7 +354,7 @@ proc toBytes*(key: RsaPublicKey, data: var openarray[byte]): RsaResult[int] =
   else:
     err(RsaKeyIncorrectError)
 
-proc toBytes*(sig: RsaSignature, data: var openarray[byte]): RSaResult[int] =
+proc toBytes*(sig: RsaSignature, data: var openArray[byte]): RsaResult[int] =
   ## Serialize RSA signature ``sig`` to raw binary form and store it
   ## to ``data``.
   ##
@@ -396,7 +406,7 @@ proc getBytes*(sig: RsaSignature): RsaResult[seq[byte]] =
   else:
     err(RsaSignatureError)
 
-proc init*(key: var RsaPrivateKey, data: openarray[byte]): Result[void, Asn1Error] =
+proc init*(key: var RsaPrivateKey, data: openArray[byte]): Result[void, Asn1Error] =
   ## Initialize RSA private key ``key`` from ASN.1 DER binary representation
   ## ``data``.
   ##
@@ -466,28 +476,28 @@ proc init*(key: var RsaPrivateKey, data: openarray[byte]): Result[void, Asn1Erro
      len(rawdp) > 0 and len(rawdq) > 0 and len(rawiq) > 0:
     key = new RsaPrivateKey
     key.buffer = @data
-    key.pubk.n = cast[ptr cuchar](addr key.buffer[rawn.offset])
-    key.pubk.e = cast[ptr cuchar](addr key.buffer[rawpube.offset])
-    key.seck.p = cast[ptr cuchar](addr key.buffer[rawp.offset])
-    key.seck.q = cast[ptr cuchar](addr key.buffer[rawq.offset])
-    key.seck.dp = cast[ptr cuchar](addr key.buffer[rawdp.offset])
-    key.seck.dq = cast[ptr cuchar](addr key.buffer[rawdq.offset])
-    key.seck.iq = cast[ptr cuchar](addr key.buffer[rawiq.offset])
-    key.pexp = cast[ptr cuchar](addr key.buffer[rawprie.offset])
-    key.pubk.nlen = len(rawn)
-    key.pubk.elen = len(rawpube)
-    key.seck.plen = len(rawp)
-    key.seck.qlen = len(rawq)
-    key.seck.dplen = len(rawdp)
-    key.seck.dqlen = len(rawdq)
-    key.seck.iqlen = len(rawiq)
-    key.pexplen = len(rawprie)
+    key.pubk.n = addr key.buffer[rawn.offset]
+    key.pubk.e = addr key.buffer[rawpube.offset]
+    key.seck.p = addr key.buffer[rawp.offset]
+    key.seck.q = addr key.buffer[rawq.offset]
+    key.seck.dp = addr key.buffer[rawdp.offset]
+    key.seck.dq = addr key.buffer[rawdq.offset]
+    key.seck.iq = addr key.buffer[rawiq.offset]
+    key.pexp = addr key.buffer[rawprie.offset]
+    key.pubk.nlen = uint(len(rawn))
+    key.pubk.elen = uint(len(rawpube))
+    key.seck.plen = uint(len(rawp))
+    key.seck.qlen = uint(len(rawq))
+    key.seck.dplen = uint(len(rawdp))
+    key.seck.dqlen = uint(len(rawdq))
+    key.seck.iqlen = uint(len(rawiq))
+    key.pexplen = uint(len(rawprie))
     key.seck.nBitlen = cast[uint32](len(rawn) shl 3)
     ok()
   else:
     err(Asn1Error.Incorrect)
 
-proc init*(key: var RsaPublicKey, data: openarray[byte]): Result[void, Asn1Error] =
+proc init*(key: var RsaPublicKey, data: openArray[byte]): Result[void, Asn1Error] =
   ## Initialize RSA public key ``key`` from ASN.1 DER binary representation
   ## ``data``.
   ##
@@ -548,15 +558,15 @@ proc init*(key: var RsaPublicKey, data: openarray[byte]): Result[void, Asn1Error
   if len(rawn) >= (MinKeySize shr 3) and len(rawe) > 0:
     key = new RsaPublicKey
     key.buffer = @data
-    key.key.n = cast[ptr cuchar](addr key.buffer[rawn.offset])
-    key.key.e = cast[ptr cuchar](addr key.buffer[rawe.offset])
-    key.key.nlen = len(rawn)
-    key.key.elen = len(rawe)
+    key.key.n = addr key.buffer[rawn.offset]
+    key.key.e = addr key.buffer[rawe.offset]
+    key.key.nlen = uint(len(rawn))
+    key.key.elen = uint(len(rawe))
     ok()
   else:
     err(Asn1Error.Incorrect)
 
-proc init*(sig: var RsaSignature, data: openarray[byte]): Result[void, Asn1Error] =
+proc init*(sig: var RsaSignature, data: openArray[byte]): Result[void, Asn1Error] =
   ## Initialize RSA signature ``sig`` from ASN.1 DER binary representation
   ## ``data``.
   ##
@@ -568,14 +578,16 @@ proc init*(sig: var RsaSignature, data: openarray[byte]): Result[void, Asn1Error
   else:
     err(Asn1Error.Incorrect)
 
-proc init*[T: RsaPKI](sospk: var T, data: string): Result[void, Asn1Error] {.inline.} =
+proc init*[T: RsaPKI](sospk: var T,
+                      data: string): Result[void, Asn1Error] {.inline.} =
   ## Initialize EC `private key`, `public key` or `scalar` ``sospk`` from
   ## hexadecimal string representation ``data``.
   ##
   ## Procedure returns ``Result[void, Asn1Status]``.
-  sospk.init(fromHex(data))
+  sospk.init(ncrutils.fromHex(data))
 
-proc init*(t: typedesc[RsaPrivateKey], data: openarray[byte]): RsaResult[RsaPrivateKey] =
+proc init*(t: typedesc[RsaPrivateKey],
+           data: openArray[byte]): RsaResult[RsaPrivateKey] =
   ## Initialize RSA private key from ASN.1 DER binary representation ``data``
   ## and return constructed object.
   var res: RsaPrivateKey
@@ -584,7 +596,8 @@ proc init*(t: typedesc[RsaPrivateKey], data: openarray[byte]): RsaResult[RsaPriv
   else:
     ok(res)
 
-proc init*(t: typedesc[RsaPublicKey], data: openarray[byte]): RsaResult[RsaPublicKey] =
+proc init*(t: typedesc[RsaPublicKey],
+           data: openArray[byte]): RsaResult[RsaPublicKey] =
   ## Initialize RSA public key from ASN.1 DER binary representation ``data``
   ## and return constructed object.
   var res: RsaPublicKey
@@ -593,7 +606,8 @@ proc init*(t: typedesc[RsaPublicKey], data: openarray[byte]): RsaResult[RsaPubli
   else:
     ok(res)
 
-proc init*(t: typedesc[RsaSignature], data: openarray[byte]): RsaResult[RsaSignature] =
+proc init*(t: typedesc[RsaSignature],
+           data: openArray[byte]): RsaResult[RsaSignature] =
   ## Initialize RSA signature from raw binary representation ``data`` and
   ## return constructed object.
   var res: RsaSignature
@@ -605,7 +619,7 @@ proc init*(t: typedesc[RsaSignature], data: openarray[byte]): RsaResult[RsaSigna
 proc init*[T: RsaPKI](t: typedesc[T], data: string): T {.inline.} =
   ## Initialize RSA `private key`, `public key` or `signature` from hexadecimal
   ## string representation ``data`` and return constructed object.
-  result = t.init(fromHex(data))
+  result = t.init(ncrutils.fromHex(data))
 
 proc `$`*(key: RsaPrivateKey): string =
   ## Return string representation of RSA private key.
@@ -616,21 +630,24 @@ proc `$`*(key: RsaPrivateKey): string =
     result.add($key.seck.nBitlen)
     result.add(" bits)\n")
     result.add("p   = ")
-    result.add(toHex(getArray(key.buffer, key.seck.p, key.seck.plen)))
+    result.add(ncrutils.toHex(getArray(key.buffer, key.seck.p, key.seck.plen)))
     result.add("\nq   = ")
-    result.add(toHex(getArray(key.buffer, key.seck.q, key.seck.qlen)))
+    result.add(ncrutils.toHex(getArray(key.buffer, key.seck.q, key.seck.qlen)))
     result.add("\ndp  = ")
-    result.add(toHex(getArray(key.buffer, key.seck.dp, key.seck.dplen)))
+    result.add(ncrutils.toHex(getArray(key.buffer, key.seck.dp,
+                                       key.seck.dplen)))
     result.add("\ndq  = ")
-    result.add(toHex(getArray(key.buffer, key.seck.dq, key.seck.dqlen)))
+    result.add(ncrutils.toHex(getArray(key.buffer, key.seck.dq,
+                                       key.seck.dqlen)))
     result.add("\niq  = ")
-    result.add(toHex(getArray(key.buffer, key.seck.iq, key.seck.iqlen)))
+    result.add(ncrutils.toHex(getArray(key.buffer, key.seck.iq,
+                                       key.seck.iqlen)))
     result.add("\npre = ")
-    result.add(toHex(getArray(key.buffer, key.pexp, key.pexplen)))
+    result.add(ncrutils.toHex(getArray(key.buffer, key.pexp, key.pexplen)))
     result.add("\nm   = ")
-    result.add(toHex(getArray(key.buffer, key.pubk.n, key.pubk.nlen)))
+    result.add(ncrutils.toHex(getArray(key.buffer, key.pubk.n, key.pubk.nlen)))
     result.add("\npue = ")
-    result.add(toHex(getArray(key.buffer, key.pubk.e, key.pubk.elen)))
+    result.add(ncrutils.toHex(getArray(key.buffer, key.pubk.e, key.pubk.elen)))
     result.add("\n")
 
 proc `$`*(key: RsaPublicKey): string =
@@ -642,9 +659,9 @@ proc `$`*(key: RsaPublicKey): string =
     result = "RSA key ("
     result.add($nbitlen)
     result.add(" bits)\nn = ")
-    result.add(toHex(getArray(key.buffer, key.key.n, key.key.nlen)))
+    result.add(ncrutils.toHex(getArray(key.buffer, key.key.n, key.key.nlen)))
     result.add("\ne = ")
-    result.add(toHex(getArray(key.buffer, key.key.e, key.key.elen)))
+    result.add(ncrutils.toHex(getArray(key.buffer, key.key.e, key.key.elen)))
     result.add("\n")
 
 proc `$`*(sig: RsaSignature): string =
@@ -653,114 +670,111 @@ proc `$`*(sig: RsaSignature): string =
     result = "Empty or uninitialized RSA signature"
   else:
     result = "RSA signature ("
-    result.add(toHex(sig.buffer))
+    result.add(ncrutils.toHex(sig.buffer))
     result.add(")")
 
-proc cmp(a: openarray[byte], b: openarray[byte]): bool =
-  let alen = len(a)
-  let blen = len(b)
-  if alen == blen:
-    if alen == 0:
-      true
-    else:
-      var n = alen
-      var res = 0
-      while n > 0:
-        dec(n)
-        res = res or int(a[n] xor b[n])
-      (res == 0)
-  else:
-    false
-
 proc `==`*(a, b: RsaPrivateKey): bool =
   ## Compare two RSA private keys for equality.
   ##
   ## Result is true if ``a`` and ``b`` are both ``nil`` or ``a`` and ``b`` are
   ## equal by value.
   if isNil(a) and isNil(b):
-    result = true
+    true
   elif isNil(a) and (not isNil(b)):
-    result = false
+    false
   elif isNil(b) and (not isNil(a)):
-    result = false
+    false
   else:
     if a.seck.nBitlen == b.seck.nBitlen:
       if cast[int](a.seck.nBitlen) > 0:
-        let r1 = cmp(getArray(a.buffer, a.seck.p, a.seck.plen),
-                     getArray(b.buffer, b.seck.p, b.seck.plen))
-        let r2 = cmp(getArray(a.buffer, a.seck.q, a.seck.qlen),
-                     getArray(b.buffer, b.seck.q, b.seck.qlen))
-        let r3 = cmp(getArray(a.buffer, a.seck.dp, a.seck.dplen),
-                     getArray(b.buffer, b.seck.dp, b.seck.dplen))
-        let r4 = cmp(getArray(a.buffer, a.seck.dq, a.seck.dqlen),
-                     getArray(b.buffer, b.seck.dq, b.seck.dqlen))
-        let r5 = cmp(getArray(a.buffer, a.seck.iq, a.seck.iqlen),
-                     getArray(b.buffer, b.seck.iq, b.seck.iqlen))
-        let r6 = cmp(getArray(a.buffer, a.pexp, a.pexplen),
-                     getArray(b.buffer, b.pexp, b.pexplen))
-        let r7 = cmp(getArray(a.buffer, a.pubk.n, a.pubk.nlen),
-                     getArray(b.buffer, b.pubk.n, b.pubk.nlen))
-        let r8 = cmp(getArray(a.buffer, a.pubk.e, a.pubk.elen),
-                     getArray(b.buffer, b.pubk.e, b.pubk.elen))
-        result = r1 and r2 and r3 and r4 and r5 and r6 and r7 and r8
+        let r1 = CT.isEqual(getArray(a.buffer, a.seck.p, a.seck.plen),
+                            getArray(b.buffer, b.seck.p, b.seck.plen))
+        let r2 = CT.isEqual(getArray(a.buffer, a.seck.q, a.seck.qlen),
+                            getArray(b.buffer, b.seck.q, b.seck.qlen))
+        let r3 = CT.isEqual(getArray(a.buffer, a.seck.dp, a.seck.dplen),
+                            getArray(b.buffer, b.seck.dp, b.seck.dplen))
+        let r4 = CT.isEqual(getArray(a.buffer, a.seck.dq, a.seck.dqlen),
+                            getArray(b.buffer, b.seck.dq, b.seck.dqlen))
+        let r5 = CT.isEqual(getArray(a.buffer, a.seck.iq, a.seck.iqlen),
+                            getArray(b.buffer, b.seck.iq, b.seck.iqlen))
+        let r6 = CT.isEqual(getArray(a.buffer, a.pexp, a.pexplen),
+                            getArray(b.buffer, b.pexp, b.pexplen))
+        let r7 = CT.isEqual(getArray(a.buffer, a.pubk.n, a.pubk.nlen),
+                            getArray(b.buffer, b.pubk.n, b.pubk.nlen))
+        let r8 = CT.isEqual(getArray(a.buffer, a.pubk.e, a.pubk.elen),
+                            getArray(b.buffer, b.pubk.e, b.pubk.elen))
+        r1 and r2 and r3 and r4 and r5 and r6 and r7 and r8
       else:
-        result = true
+        true
+    else:
+      false
 
 proc `==`*(a, b: RsaSignature): bool =
   ## Compare two RSA signatures for equality.
   if isNil(a) and isNil(b):
-    result = true
+    true
   elif isNil(a) and (not isNil(b)):
-    result = false
+    false
   elif isNil(b) and (not isNil(a)):
-    result = false
+    false
   else:
-    result = (a.buffer == b.buffer)
+    # We need to cover all the cases because Signature initialization procedure
+    # do not perform any checks.
+    if len(a.buffer) == 0 and len(b.buffer) == 0:
+      true
+    elif len(a.buffer) == 0 and len(b.buffer) != 0:
+      false
+    elif len(b.buffer) == 0 and len(a.buffer) != 0:
+      false
+    elif len(a.buffer) != len(b.buffer):
+      false
+    else:
+      CT.isEqual(a.buffer, b.buffer)
 
 proc `==`*(a, b: RsaPublicKey): bool =
   ## Compare two RSA public keys for equality.
   if isNil(a) and isNil(b):
-    result = true
+    true
   elif isNil(a) and (not isNil(b)):
-    result = false
+    false
   elif isNil(b) and (not isNil(a)):
-    result = false
+    false
   else:
-    let r1 = cmp(getArray(a.buffer, a.key.n, a.key.nlen),
-                 getArray(b.buffer, b.key.n, b.key.nlen))
-    let r2 = cmp(getArray(a.buffer, a.key.e, a.key.elen),
-                 getArray(b.buffer, b.key.e, b.key.elen))
-    result = r1 and r2
+    let r1 = CT.isEqual(getArray(a.buffer, a.key.n, a.key.nlen),
+                        getArray(b.buffer, b.key.n, b.key.nlen))
+    let r2 = CT.isEqual(getArray(a.buffer, a.key.e, a.key.elen),
+                        getArray(b.buffer, b.key.e, b.key.elen))
+    (r1 and r2)
 
 proc sign*[T: byte|char](key: RsaPrivateKey,
-                         message: openarray[T]): RsaResult[RsaSignature] {.gcsafe.} =
+                         message: openArray[T]): RsaResult[RsaSignature] {.gcsafe.} =
   ## Get RSA PKCS1.5 signature of data ``message`` using SHA256 and private
   ## key ``key``.
   if isNil(key):
     return err(RsaKeyIncorrectError)
 
-  var hc: BrHashCompatContext
+  var hc: HashCompatContext
   var hash: array[32, byte]
-  let impl = BrRsaPkcs1SignGetDefault()
+  let impl = rsaPkcs1SignGetDefault()
   var res = new RsaSignature
   res.buffer = newSeq[byte]((key.seck.nBitlen + 7) shr 3)
   var kv = addr sha256Vtable
   kv.init(addr hc.vtable)
   if len(message) > 0:
-    kv.update(addr hc.vtable, unsafeAddr message[0], len(message))
+    kv.update(addr hc.vtable, unsafeAddr message[0], uint(len(message)))
   else:
     kv.update(addr hc.vtable, nil, 0)
-  kv.output(addr hc.vtable, addr hash[0])
+  kv.out(addr hc.vtable, addr hash[0])
   var oid = RsaOidSha256
-  let implRes = impl(cast[ptr cuchar](addr oid[0]),
-                 cast[ptr cuchar](addr hash[0]), len(hash),
-                 addr key.seck, cast[ptr cuchar](addr res.buffer[0]))
+  let implRes = impl(addr oid[0],
+                 addr hash[0], uint(len(hash)),
+                 addr key.seck, addr res.buffer[0])
   if implRes == 0:
     err(RsaSignatureError)
   else:
     ok(res)
 
-proc verify*[T: byte|char](sig: RsaSignature, message: openarray[T],
+proc verify*[T: byte|char](sig: RsaSignature, message: openArray[T],
                            pubkey: RsaPublicKey): bool {.inline.} =
   ## Verify RSA signature ``sig`` using public key ``pubkey`` and data
   ## ``message``.
@@ -769,20 +783,20 @@ proc verify*[T: byte|char](sig: RsaSignature, message: openarray[T],
   ## verification failed.
   doAssert((not isNil(sig)) and (not isNil(pubkey)))
   if len(sig.buffer) > 0:
-    var hc: BrHashCompatContext
+    var hc: HashCompatContext
     var hash: array[32, byte]
     var check: array[32, byte]
-    var impl = BrRsaPkcs1VrfyGetDefault()
+    var impl = rsaPkcs1VrfyGetDefault()
     var kv = addr sha256Vtable
     kv.init(addr hc.vtable)
     if len(message) > 0:
-      kv.update(addr hc.vtable, unsafeAddr message[0], len(message))
+      kv.update(addr hc.vtable, unsafeAddr message[0], uint(len(message)))
     else:
       kv.update(addr hc.vtable, nil, 0)
-    kv.output(addr hc.vtable, addr hash[0])
+    kv.out(addr hc.vtable, addr hash[0])
     var oid = RsaOidSha256
-    let res = impl(cast[ptr cuchar](addr sig.buffer[0]), len(sig.buffer),
-                   cast[ptr cuchar](addr oid[0]),
-                   len(check), addr pubkey.key, cast[ptr cuchar](addr check[0]))
+    let res = impl(addr sig.buffer[0], uint(len(sig.buffer)),
+                   addr oid[0],
+                   uint(len(check)), addr pubkey.key, addr check[0])
     if res == 1:
       result = equalMem(addr check[0], addr hash[0], len(hash))

libp2p/crypto/secp.nim

@@ -1,20 +1,24 @@
-## Nim-Libp2p
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
+import bearssl/rand
 import
-  secp256k1, bearssl,
+  secp256k1,
   stew/[byteutils, results],
   nimcrypto/[hash, sha2]
 
-export sha2, results
+export sha2, results, rand
 
 const
   SkRawPrivateKeySize* = 256 div 8
@@ -34,17 +38,18 @@ type
 template pubkey*(v: SkKeyPair): SkPublicKey = SkPublicKey(secp256k1.SkKeyPair(v).pubkey)
 template seckey*(v: SkKeyPair): SkPrivateKey = SkPrivateKey(secp256k1.SkKeyPair(v).seckey)
 
-proc random*(t: typedesc[SkPrivateKey], rng: var BrHmacDrbgContext): SkPrivateKey =
-  let rngPtr = unsafeAddr rng # doesn't escape
+proc random*(t: typedesc[SkPrivateKey], rng: var HmacDrbgContext): SkPrivateKey =
+  #TODO is there a better way?
+  var rngPtr = addr rng
   proc callRng(data: var openArray[byte]) =
-    brHmacDrbgGenerate(rngPtr[], data)
+    hmacDrbgGenerate(rngPtr[], data)
 
   SkPrivateKey(SkSecretKey.random(callRng))
 
-proc random*(t: typedesc[SkKeyPair], rng: var BrHmacDrbgContext): SkKeyPair =
-  let rngPtr = unsafeAddr rng # doesn't escape
+proc random*(t: typedesc[SkKeyPair], rng: var HmacDrbgContext): SkKeyPair =
+  let rngPtr = addr rng
   proc callRng(data: var openArray[byte]) =
-    brHmacDrbgGenerate(rngPtr[], data)
+    hmacDrbgGenerate(rngPtr[], data)
 
   SkKeyPair(secp256k1.SkKeyPair.random(callRng))
 
@@ -54,7 +59,7 @@ template seckey*(v: SkKeyPair): SkPrivateKey =
 template pubkey*(v: SkKeyPair): SkPublicKey =
   SkPublicKey(secp256k1.SkKeyPair(v).pubkey)
 
-proc init*(key: var SkPrivateKey, data: openarray[byte]): SkResult[void] =
+proc init*(key: var SkPrivateKey, data: openArray[byte]): SkResult[void] =
   ## Initialize Secp256k1 `private key` ``key`` from raw binary
   ## representation ``data``.
   key = SkPrivateKey(? secp256k1.SkSecretKey.fromRaw(data))
@@ -66,7 +71,7 @@ proc init*(key: var SkPrivateKey, data: string): SkResult[void] =
   key = SkPrivateKey(? secp256k1.SkSecretKey.fromHex(data))
   ok()
 
-proc init*(key: var SkPublicKey, data: openarray[byte]): SkResult[void] =
+proc init*(key: var SkPublicKey, data: openArray[byte]): SkResult[void] =
   ## Initialize Secp256k1 `public key` ``key`` from raw binary
   ## representation ``data``.
   key = SkPublicKey(? secp256k1.SkPublicKey.fromRaw(data))
@@ -78,7 +83,7 @@ proc init*(key: var SkPublicKey, data: string): SkResult[void] =
   key = SkPublicKey(? secp256k1.SkPublicKey.fromHex(data))
   ok()
 
-proc init*(sig: var SkSignature, data: openarray[byte]): SkResult[void] =
+proc init*(sig: var SkSignature, data: openArray[byte]): SkResult[void] =
   ## Initialize Secp256k1 `signature` ``sig`` from raw binary
   ## representation ``data``.
   sig = SkSignature(? secp256k1.SkSignature.fromDer(data))
@@ -95,7 +100,7 @@ proc init*(sig: var SkSignature, data: string): SkResult[void] =
     return err("secp: Hex to bytes failed")
   init(sig, buffer)
 
-proc init*(t: typedesc[SkPrivateKey], data: openarray[byte]): SkResult[SkPrivateKey] =
+proc init*(t: typedesc[SkPrivateKey], data: openArray[byte]): SkResult[SkPrivateKey] =
   ## Initialize Secp256k1 `private key` from raw binary
   ## representation ``data``.
   ##
@@ -109,7 +114,7 @@ proc init*(t: typedesc[SkPrivateKey], data: string): SkResult[SkPrivateKey] =
   ## Procedure returns `private key` on success.
   SkSecretKey.fromHex(data).mapConvert(SkPrivateKey)
 
-proc init*(t: typedesc[SkPublicKey], data: openarray[byte]): SkResult[SkPublicKey] =
+proc init*(t: typedesc[SkPublicKey], data: openArray[byte]): SkResult[SkPublicKey] =
   ## Initialize Secp256k1 `public key` from raw binary
   ## representation ``data``.
   ##
@@ -125,7 +130,7 @@ proc init*(t: typedesc[SkPublicKey], data: string): SkResult[SkPublicKey] =
   var key: SkPublicKey
   key.init(data) and ok(key)
 
-proc init*(t: typedesc[SkSignature], data: openarray[byte]): SkResult[SkSignature] =
+proc init*(t: typedesc[SkSignature], data: openArray[byte]): SkResult[SkSignature] =
   ## Initialize Secp256k1 `signature` from raw binary
   ## representation ``data``.
   ##
@@ -141,11 +146,11 @@ proc init*(t: typedesc[SkSignature], data: string): SkResult[SkSignature] =
   var sig: SkSignature
   sig.init(data) and ok(sig)
 
-proc getKey*(key: SkPrivateKey): SkPublicKey =
+proc getPublicKey*(key: SkPrivateKey): SkPublicKey =
   ## Calculate and return Secp256k1 `public key` from `private key` ``key``.
   SkPublicKey(SkSecretKey(key).toPublicKey())
 
-proc toBytes*(key: SkPrivateKey, data: var openarray[byte]): SkResult[int] =
+proc toBytes*(key: SkPrivateKey, data: var openArray[byte]): SkResult[int] =
   ## Serialize Secp256k1 `private key` ``key`` to raw binary form and store it
   ## to ``data``.
   ##
@@ -157,7 +162,7 @@ proc toBytes*(key: SkPrivateKey, data: var openarray[byte]): SkResult[int] =
   else:
     err("secp: Not enough bytes")
 
-proc toBytes*(key: SkPublicKey, data: var openarray[byte]): SkResult[int] =
+proc toBytes*(key: SkPublicKey, data: var openArray[byte]): SkResult[int] =
   ## Serialize Secp256k1 `public key` ``key`` to raw binary form and store it
   ## to ``data``.
   ##
@@ -169,7 +174,7 @@ proc toBytes*(key: SkPublicKey, data: var openarray[byte]): SkResult[int] =
   else:
     err("secp: Not enough bytes")
 
-proc toBytes*(sig: SkSignature, data: var openarray[byte]): int =
+proc toBytes*(sig: SkSignature, data: var openArray[byte]): int =
   ## Serialize Secp256k1 `signature` ``sig`` to raw binary form and store it
   ## to ``data``.
   ##
@@ -191,24 +196,28 @@ proc getBytes*(sig: SkSignature): seq[byte] {.inline.} =
   let length = toBytes(sig, result)
   result.setLen(length)
 
-proc sign*[T: byte|char](key: SkPrivateKey, msg: openarray[T]): SkSignature =
+proc sign*[T: byte|char](key: SkPrivateKey, msg: openArray[T]): SkSignature =
   ## Sign message `msg` using private key `key` and return signature object.
   let h = sha256.digest(msg)
   SkSignature(sign(SkSecretKey(key), SkMessage(h.data)))
 
-proc verify*[T: byte|char](sig: SkSignature, msg: openarray[T],
+proc verify*[T: byte|char](sig: SkSignature, msg: openArray[T],
                            key: SkPublicKey): bool =
   let h = sha256.digest(msg)
   verify(secp256k1.SkSignature(sig), SkMessage(h.data), secp256k1.SkPublicKey(key))
 
-func clear*(key: var SkPrivateKey) {.borrow.}
+func clear*(key: var SkPrivateKey) = clear(secp256k1.SkSecretKey(key))
 
-proc `$`*(key: SkPrivateKey): string {.borrow.}
-proc `$`*(key: SkPublicKey): string {.borrow.}
-proc `$`*(key: SkSignature): string {.borrow.}
-proc `$`*(key: SkKeyPair): string {.borrow.}
+func `$`*(key: SkPrivateKey): string = $secp256k1.SkSecretKey(key)
+func `$`*(key: SkPublicKey): string = $secp256k1.SkPublicKey(key)
+func `$`*(key: SkSignature): string = $secp256k1.SkSignature(key)
+func `$`*(key: SkKeyPair): string = $secp256k1.SkKeyPair(key)
 
-proc `==`*(a, b: SkPrivateKey): bool {.borrow.}
-proc `==`*(a, b: SkPublicKey): bool {.borrow.}
-proc `==`*(a, b: SkSignature): bool {.borrow.}
-proc `==`*(a, b: SkKeyPair): bool {.borrow.}
+func `==`*(a, b: SkPrivateKey): bool =
+  secp256k1.SkSecretKey(a) == secp256k1.SkSecretKey(b)
+func `==`*(a, b: SkPublicKey): bool =
+  secp256k1.SkPublicKey(a) == secp256k1.SkPublicKey(b)
+func `==`*(a, b: SkSignature): bool =
+  secp256k1.SkSignature(a) == secp256k1.SkSignature(b)
+func `==`*(a, b: SkKeyPair): bool =
+  secp256k1.SkKeyPair(a) == secp256k1.SkKeyPair(b)

libp2p/daemon/daemonapi.nim

@@ -1,20 +1,26 @@
-## Nim-LibP2P
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 ## This module implementes API for `go-libp2p-daemon`.
-import os, osproc, strutils, tables, strtabs
-import chronos
+import std/[os, osproc, strutils, tables, strtabs, sequtils]
+import pkg/[chronos, chronicles]
 import ../varint, ../multiaddress, ../multicodec, ../cid, ../peerid
-import ../wire, ../multihash, ../protobuf/minprotobuf
+import ../wire, ../multihash, ../protobuf/minprotobuf, ../errors
 import ../crypto/crypto
 
-export peerid, multiaddress, multicodec, multihash, cid, crypto, wire
+export
+  peerid, multiaddress, multicodec, multihash, cid, crypto, wire, errors
 
 when not defined(windows):
   import posix
@@ -32,7 +38,7 @@ type
     Critical, Error, Warning, Notice, Info, Debug, Trace
 
   RequestType* {.pure.} = enum
-    IDENTITY = 0,
+    IDENTIFY = 0,
     CONNECT = 1,
     STREAM_OPEN = 2,
     STREAM_HANDLER = 3,
@@ -104,12 +110,12 @@ type
     RelayActive,   ## Enables active mode for relay.
     RelayDiscovery,## Enables passive discovery for relay.
     RelayHop,      ## Enables hop for relay.
-    NoInlinePeerID,## Disable inlining of peer ID (not yet in #master).
+    NoInlinePeerId,## Disable inlining of peer ID (not yet in #master).
     NoProcessCtrl  ## Process was not spawned.
 
   P2PStream* = ref object
     flags*: set[P2PStreamFlags]
-    peer*: PeerID
+    peer*: PeerId
     raddress*: MultiAddress
     protocol*: string
     transp*: StreamTransport
@@ -130,7 +136,7 @@ type
     userData*: RootRef
 
   PeerInfo* = object
-    peer*: PeerID
+    peer*: PeerId
     addresses*: seq[MultiAddress]
 
   PubsubTicket* = ref object
@@ -139,7 +145,7 @@ type
     transp*: StreamTransport
 
   PubSubMessage* = object
-    peer*: PeerID
+    peer*: PeerId
     data*: seq[byte]
     seqno*: seq[byte]
     topics*: seq[string]
@@ -147,114 +153,117 @@ type
     key*: PublicKey
 
   P2PStreamCallback* = proc(api: DaemonAPI,
-                            stream: P2PStream): Future[void] {.gcsafe.}
+                            stream: P2PStream): Future[void] {.gcsafe, raises: [Defect, CatchableError].}
   P2PPubSubCallback* = proc(api: DaemonAPI,
                             ticket: PubsubTicket,
-                            message: PubSubMessage): Future[bool] {.gcsafe.}
+                            message: PubSubMessage): Future[bool] {.gcsafe, raises: [Defect, CatchableError].}
 
-  DaemonRemoteError* = object of CatchableError
-  DaemonLocalError* = object of CatchableError
+  DaemonError* = object of LPError
+  DaemonRemoteError* = object of DaemonError
+  DaemonLocalError* = object of DaemonError
 
 var daemonsCount {.threadvar.}: int
 
+chronicles.formatIt(PeerInfo): shortLog(it)
+
 proc requestIdentity(): ProtoBuffer =
   ## https://github.com/libp2p/go-libp2p-daemon/blob/master/conn.go
   ## Processing function `doIdentify(req *pb.Request)`.
   result = initProtoBuffer({WithVarintLength})
-  result.write(initProtoField(1, cast[uint](RequestType.IDENTITY)))
+  result.write(1, cast[uint](RequestType.IDENTIFY))
   result.finish()
 
-proc requestConnect(peerid: PeerID,
-                    addresses: openarray[MultiAddress],
+proc requestConnect(peerid: PeerId,
+                    addresses: openArray[MultiAddress],
                     timeout = 0): ProtoBuffer =
   ## https://github.com/libp2p/go-libp2p-daemon/blob/master/conn.go
   ## Processing function `doConnect(req *pb.Request)`.
   result = initProtoBuffer({WithVarintLength})
   var msg = initProtoBuffer()
-  msg.write(initProtoField(1, peerid))
+  msg.write(1, peerid)
   for item in addresses:
-    msg.write(initProtoField(2, item.data.buffer))
+    msg.write(2, item.data.buffer)
   if timeout > 0:
-    msg.write(initProtoField(3, hint64(timeout)))
-  result.write(initProtoField(1, cast[uint](RequestType.CONNECT)))
-  result.write(initProtoField(2, msg))
+    msg.write(3, hint64(timeout))
+  result.write(1, cast[uint](RequestType.CONNECT))
+  result.write(2, msg)
   result.finish()
 
-proc requestDisconnect(peerid: PeerID): ProtoBuffer =
+proc requestDisconnect(peerid: PeerId): ProtoBuffer =
   ## https://github.com/libp2p/go-libp2p-daemon/blob/master/conn.go
   ## Processing function `doDisconnect(req *pb.Request)`.
   result = initProtoBuffer({WithVarintLength})
   var msg = initProtoBuffer()
-  msg.write(initProtoField(1, peerid))
-  result.write(initProtoField(1, cast[uint](RequestType.DISCONNECT)))
-  result.write(initProtoField(7, msg))
+  msg.write(1, peerid)
+  result.write(1, cast[uint](RequestType.DISCONNECT))
+  result.write(7, msg)
   result.finish()
 
-proc requestStreamOpen(peerid: PeerID,
-                       protocols: openarray[string],
+proc requestStreamOpen(peerid: PeerId,
+                       protocols: openArray[string],
                        timeout = 0): ProtoBuffer =
   ## https://github.com/libp2p/go-libp2p-daemon/blob/master/conn.go
   ## Processing function `doStreamOpen(req *pb.Request)`.
   result = initProtoBuffer({WithVarintLength})
   var msg = initProtoBuffer()
-  msg.write(initProtoField(1, peerid))
+  msg.write(1, peerid)
   for item in protocols:
-    msg.write(initProtoField(2, item))
+    msg.write(2, item)
   if timeout > 0:
-    msg.write(initProtoField(3, hint64(timeout)))
-  result.write(initProtoField(1, cast[uint](RequestType.STREAM_OPEN)))
-  result.write(initProtoField(3, msg))
+    msg.write(3, hint64(timeout))
+  result.write(1, cast[uint](RequestType.STREAM_OPEN))
+  result.write(3, msg)
   result.finish()
 
 proc requestStreamHandler(address: MultiAddress,
-                          protocols: openarray[MultiProtocol]): ProtoBuffer =
+                          protocols: openArray[MultiProtocol]): ProtoBuffer =
   ## https://github.com/libp2p/go-libp2p-daemon/blob/master/conn.go
   ## Processing function `doStreamHandler(req *pb.Request)`.
   result = initProtoBuffer({WithVarintLength})
   var msg = initProtoBuffer()
-  msg.write(initProtoField(1, address.data.buffer))
+  msg.write(1, address.data.buffer)
   for item in protocols:
-    msg.write(initProtoField(2, item))
-  result.write(initProtoField(1, cast[uint](RequestType.STREAM_HANDLER)))
-  result.write(initProtoField(4, msg))
+    msg.write(2, item)
+  result.write(1, cast[uint](RequestType.STREAM_HANDLER))
+  result.write(4, msg)
   result.finish()
 
 proc requestListPeers(): ProtoBuffer =
   ## https://github.com/libp2p/go-libp2p-daemon/blob/master/conn.go
   ## Processing function `doListPeers(req *pb.Request)`
   result = initProtoBuffer({WithVarintLength})
-  result.write(initProtoField(1, cast[uint](RequestType.LIST_PEERS)))
+  result.write(1, cast[uint](RequestType.LIST_PEERS))
   result.finish()
 
-proc requestDHTFindPeer(peer: PeerID, timeout = 0): ProtoBuffer =
+proc requestDHTFindPeer(peer: PeerId, timeout = 0): ProtoBuffer =
   ## https://github.com/libp2p/go-libp2p-daemon/blob/master/dht.go
   ## Processing function `doDHTFindPeer(req *pb.DHTRequest)`.
   let msgid = cast[uint](DHTRequestType.FIND_PEER)
   result = initProtoBuffer({WithVarintLength})
   var msg = initProtoBuffer()
-  msg.write(initProtoField(1, msgid))
-  msg.write(initProtoField(2, peer))
+  msg.write(1, msgid)
+  msg.write(2, peer)
   if timeout > 0:
-    msg.write(initProtoField(7, hint64(timeout)))
+    msg.write(7, hint64(timeout))
   msg.finish()
-  result.write(initProtoField(1, cast[uint](RequestType.DHT)))
-  result.write(initProtoField(5, msg))
+  result.write(1, cast[uint](RequestType.DHT))
+  result.write(5, msg)
   result.finish()
 
-proc requestDHTFindPeersConnectedToPeer(peer: PeerID,
+proc requestDHTFindPeersConnectedToPeer(peer: PeerId,
                                         timeout = 0): ProtoBuffer =
   ## https://github.com/libp2p/go-libp2p-daemon/blob/master/dht.go
   ## Processing function `doDHTFindPeersConnectedToPeer(req *pb.DHTRequest)`.
   let msgid = cast[uint](DHTRequestType.FIND_PEERS_CONNECTED_TO_PEER)
   result = initProtoBuffer({WithVarintLength})
   var msg = initProtoBuffer()
-  msg.write(initProtoField(1, msgid))
-  msg.write(initProtoField(2, peer))
+  msg.write(1, msgid)
+  msg.write(2, peer)
   if timeout > 0:
-    msg.write(initProtoField(7, hint64(timeout)))
+    msg.write(7, hint64(timeout))
   msg.finish()
-  result.write(initProtoField(1, cast[uint](RequestType.DHT)))
-  result.write(initProtoField(5, msg))
+  result.write(1, cast[uint](RequestType.DHT))
+  result.write(5, msg)
   result.finish()
 
 proc requestDHTFindProviders(cid: Cid,
@@ -264,14 +273,14 @@ proc requestDHTFindProviders(cid: Cid,
   let msgid = cast[uint](DHTRequestType.FIND_PROVIDERS)
   result = initProtoBuffer({WithVarintLength})
   var msg = initProtoBuffer()
-  msg.write(initProtoField(1, msgid))
-  msg.write(initProtoField(3, cid.data.buffer))
-  msg.write(initProtoField(6, count))
+  msg.write(1, msgid)
+  msg.write(3, cid.data.buffer)
+  msg.write(6, count)
   if timeout > 0:
-    msg.write(initProtoField(7, hint64(timeout)))
+    msg.write(7, hint64(timeout))
   msg.finish()
-  result.write(initProtoField(1, cast[uint](RequestType.DHT)))
-  result.write(initProtoField(5, msg))
+  result.write(1, cast[uint](RequestType.DHT))
+  result.write(5, msg)
   result.finish()
 
 proc requestDHTGetClosestPeers(key: string, timeout = 0): ProtoBuffer =
@@ -280,28 +289,28 @@ proc requestDHTGetClosestPeers(key: string, timeout = 0): ProtoBuffer =
   let msgid = cast[uint](DHTRequestType.GET_CLOSEST_PEERS)
   result = initProtoBuffer({WithVarintLength})
   var msg = initProtoBuffer()
-  msg.write(initProtoField(1, msgid))
-  msg.write(initProtoField(4, key))
+  msg.write(1, msgid)
+  msg.write(4, key)
   if timeout > 0:
-    msg.write(initProtoField(7, hint64(timeout)))
+    msg.write(7, hint64(timeout))
   msg.finish()
-  result.write(initProtoField(1, cast[uint](RequestType.DHT)))
-  result.write(initProtoField(5, msg))
+  result.write(1, cast[uint](RequestType.DHT))
+  result.write(5, msg)
   result.finish()
 
-proc requestDHTGetPublicKey(peer: PeerID, timeout = 0): ProtoBuffer =
+proc requestDHTGetPublicKey(peer: PeerId, timeout = 0): ProtoBuffer =
   ## https://github.com/libp2p/go-libp2p-daemon/blob/master/dht.go
   ## Processing function `doDHTGetPublicKey(req *pb.DHTRequest)`.
   let msgid = cast[uint](DHTRequestType.GET_PUBLIC_KEY)
   result = initProtoBuffer({WithVarintLength})
   var msg = initProtoBuffer()
-  msg.write(initProtoField(1, msgid))
-  msg.write(initProtoField(2, peer))
+  msg.write(1, msgid)
+  msg.write(2, peer)
   if timeout > 0:
-    msg.write(initProtoField(7, hint64(timeout)))
+    msg.write(7, hint64(timeout))
   msg.finish()
-  result.write(initProtoField(1, cast[uint](RequestType.DHT)))
-  result.write(initProtoField(5, msg))
+  result.write(1, cast[uint](RequestType.DHT))
+  result.write(5, msg)
   result.finish()
 
 proc requestDHTGetValue(key: string, timeout = 0): ProtoBuffer =
@@ -310,13 +319,13 @@ proc requestDHTGetValue(key: string, timeout = 0): ProtoBuffer =
   let msgid = cast[uint](DHTRequestType.GET_VALUE)
   result = initProtoBuffer({WithVarintLength})
   var msg = initProtoBuffer()
-  msg.write(initProtoField(1, msgid))
-  msg.write(initProtoField(4, key))
+  msg.write(1, msgid)
+  msg.write(4, key)
   if timeout > 0:
-    msg.write(initProtoField(7, hint64(timeout)))
+    msg.write(7, hint64(timeout))
   msg.finish()
-  result.write(initProtoField(1, cast[uint](RequestType.DHT)))
-  result.write(initProtoField(5, msg))
+  result.write(1, cast[uint](RequestType.DHT))
+  result.write(5, msg)
   result.finish()
 
 proc requestDHTSearchValue(key: string, timeout = 0): ProtoBuffer =
@@ -325,30 +334,30 @@ proc requestDHTSearchValue(key: string, timeout = 0): ProtoBuffer =
   let msgid = cast[uint](DHTRequestType.SEARCH_VALUE)
   result = initProtoBuffer({WithVarintLength})
   var msg = initProtoBuffer()
-  msg.write(initProtoField(1, msgid))
-  msg.write(initProtoField(4, key))
+  msg.write(1, msgid)
+  msg.write(4, key)
   if timeout > 0:
-    msg.write(initProtoField(7, hint64(timeout)))
+    msg.write(7, hint64(timeout))
   msg.finish()
-  result.write(initProtoField(1, cast[uint](RequestType.DHT)))
-  result.write(initProtoField(5, msg))
+  result.write(1, cast[uint](RequestType.DHT))
+  result.write(5, msg)
   result.finish()
 
-proc requestDHTPutValue(key: string, value: openarray[byte],
+proc requestDHTPutValue(key: string, value: openArray[byte],
                         timeout = 0): ProtoBuffer =
   ## https://github.com/libp2p/go-libp2p-daemon/blob/master/dht.go
   ## Processing function `doDHTPutValue(req *pb.DHTRequest)`.
   let msgid = cast[uint](DHTRequestType.PUT_VALUE)
   result = initProtoBuffer({WithVarintLength})
   var msg = initProtoBuffer()
-  msg.write(initProtoField(1, msgid))
-  msg.write(initProtoField(4, key))
-  msg.write(initProtoField(5, value))
+  msg.write(1, msgid)
+  msg.write(4, key)
+  msg.write(5, value)
   if timeout > 0:
-    msg.write(initProtoField(7, hint64(timeout)))
+    msg.write(7, hint64(timeout))
   msg.finish()
-  result.write(initProtoField(1, cast[uint](RequestType.DHT)))
-  result.write(initProtoField(5, msg))
+  result.write(1, cast[uint](RequestType.DHT))
+  result.write(5, msg)
   result.finish()
 
 proc requestDHTProvide(cid: Cid, timeout = 0): ProtoBuffer =
@@ -357,40 +366,40 @@ proc requestDHTProvide(cid: Cid, timeout = 0): ProtoBuffer =
   let msgid = cast[uint](DHTRequestType.PROVIDE)
   result = initProtoBuffer({WithVarintLength})
   var msg = initProtoBuffer()
-  msg.write(initProtoField(1, msgid))
-  msg.write(initProtoField(3, cid.data.buffer))
+  msg.write(1, msgid)
+  msg.write(3, cid.data.buffer)
   if timeout > 0:
-    msg.write(initProtoField(7, hint64(timeout)))
+    msg.write(7, hint64(timeout))
   msg.finish()
-  result.write(initProtoField(1, cast[uint](RequestType.DHT)))
-  result.write(initProtoField(5, msg))
+  result.write(1, cast[uint](RequestType.DHT))
+  result.write(5, msg)
   result.finish()
 
-proc requestCMTagPeer(peer: PeerID, tag: string, weight: int): ProtoBuffer =
+proc requestCMTagPeer(peer: PeerId, tag: string, weight: int): ProtoBuffer =
   ## https://github.com/libp2p/go-libp2p-daemon/blob/master/connmgr.go#L18
   let msgid = cast[uint](ConnManagerRequestType.TAG_PEER)
   result = initProtoBuffer({WithVarintLength})
   var msg = initProtoBuffer()
-  msg.write(initProtoField(1, msgid))
-  msg.write(initProtoField(2, peer))
-  msg.write(initProtoField(3, tag))
-  msg.write(initProtoField(4, hint64(weight)))
+  msg.write(1, msgid)
+  msg.write(2, peer)
+  msg.write(3, tag)
+  msg.write(4, hint64(weight))
   msg.finish()
-  result.write(initProtoField(1, cast[uint](RequestType.CONNMANAGER)))
-  result.write(initProtoField(6, msg))
+  result.write(1, cast[uint](RequestType.CONNMANAGER))
+  result.write(6, msg)
   result.finish()
 
-proc requestCMUntagPeer(peer: PeerID, tag: string): ProtoBuffer =
+proc requestCMUntagPeer(peer: PeerId, tag: string): ProtoBuffer =
   ## https://github.com/libp2p/go-libp2p-daemon/blob/master/connmgr.go#L33
   let msgid = cast[uint](ConnManagerRequestType.UNTAG_PEER)
   result = initProtoBuffer({WithVarintLength})
   var msg = initProtoBuffer()
-  msg.write(initProtoField(1, msgid))
-  msg.write(initProtoField(2, peer))
-  msg.write(initProtoField(3, tag))
+  msg.write(1, msgid)
+  msg.write(2, peer)
+  msg.write(3, tag)
   msg.finish()
-  result.write(initProtoField(1, cast[uint](RequestType.CONNMANAGER)))
-  result.write(initProtoField(6, msg))
+  result.write(1, cast[uint](RequestType.CONNMANAGER))
+  result.write(6, msg)
   result.finish()
 
 proc requestCMTrim(): ProtoBuffer =
@@ -398,10 +407,10 @@ proc requestCMTrim(): ProtoBuffer =
   let msgid = cast[uint](ConnManagerRequestType.TRIM)
   result = initProtoBuffer({WithVarintLength})
   var msg = initProtoBuffer()
-  msg.write(initProtoField(1, msgid))
+  msg.write(1, msgid)
   msg.finish()
-  result.write(initProtoField(1, cast[uint](RequestType.CONNMANAGER)))
-  result.write(initProtoField(6, msg))
+  result.write(1, cast[uint](RequestType.CONNMANAGER))
+  result.write(6, msg)
   result.finish()
 
 proc requestPSGetTopics(): ProtoBuffer =
@@ -410,10 +419,10 @@ proc requestPSGetTopics(): ProtoBuffer =
   let msgid = cast[uint](PSRequestType.GET_TOPICS)
   result = initProtoBuffer({WithVarintLength})
   var msg = initProtoBuffer()
-  msg.write(initProtoField(1, msgid))
+  msg.write(1, msgid)
   msg.finish()
-  result.write(initProtoField(1, cast[uint](RequestType.PUBSUB)))
-  result.write(initProtoField(8, msg))
+  result.write(1, cast[uint](RequestType.PUBSUB))
+  result.write(8, msg)
   result.finish()
 
 proc requestPSListPeers(topic: string): ProtoBuffer =
@@ -422,25 +431,25 @@ proc requestPSListPeers(topic: string): ProtoBuffer =
   let msgid = cast[uint](PSRequestType.LIST_PEERS)
   result = initProtoBuffer({WithVarintLength})
   var msg = initProtoBuffer()
-  msg.write(initProtoField(1, msgid))
-  msg.write(initProtoField(2, topic))
+  msg.write(1, msgid)
+  msg.write(2, topic)
   msg.finish()
-  result.write(initProtoField(1, cast[uint](RequestType.PUBSUB)))
-  result.write(initProtoField(8, msg))
+  result.write(1, cast[uint](RequestType.PUBSUB))
+  result.write(8, msg)
   result.finish()
 
-proc requestPSPublish(topic: string, data: openarray[byte]): ProtoBuffer =
+proc requestPSPublish(topic: string, data: openArray[byte]): ProtoBuffer =
   ## https://github.com/libp2p/go-libp2p-daemon/blob/master/pubsub.go
   ## Processing function `doPubsubPublish(req *pb.PSRequest)`.
   let msgid = cast[uint](PSRequestType.PUBLISH)
   result = initProtoBuffer({WithVarintLength})
   var msg = initProtoBuffer()
-  msg.write(initProtoField(1, msgid))
-  msg.write(initProtoField(2, topic))
-  msg.write(initProtoField(3, data))
+  msg.write(1, msgid)
+  msg.write(2, topic)
+  msg.write(3, data)
   msg.finish()
-  result.write(initProtoField(1, cast[uint](RequestType.PUBSUB)))
-  result.write(initProtoField(8, msg))
+  result.write(1, cast[uint](RequestType.PUBSUB))
+  result.write(8, msg)
   result.finish()
 
 proc requestPSSubscribe(topic: string): ProtoBuffer =
@@ -449,25 +458,26 @@ proc requestPSSubscribe(topic: string): ProtoBuffer =
   let msgid = cast[uint](PSRequestType.SUBSCRIBE)
   result = initProtoBuffer({WithVarintLength})
   var msg = initProtoBuffer()
-  msg.write(initProtoField(1, msgid))
-  msg.write(initProtoField(2, topic))
+  msg.write(1, msgid)
+  msg.write(2, topic)
   msg.finish()
-  result.write(initProtoField(1, cast[uint](RequestType.PUBSUB)))
-  result.write(initProtoField(8, msg))
+  result.write(1, cast[uint](RequestType.PUBSUB))
+  result.write(8, msg)
   result.finish()
 
-proc checkResponse(pb: var ProtoBuffer): ResponseKind {.inline.} =
+proc checkResponse(pb: ProtoBuffer): ResponseKind {.inline.} =
   result = ResponseKind.Malformed
   var value: uint64
-  if getVarintValue(pb, 1, value) > 0:
+  if getRequiredField(pb, 1, value).isOk():
     if value == 0:
       result = ResponseKind.Success
     else:
       result = ResponseKind.Error
 
-proc getErrorMessage(pb: var ProtoBuffer): string {.inline.} =
-  if pb.enterSubmessage() == cast[int](ResponseType.ERROR):
-    if pb.getString(1, result) == -1:
+proc getErrorMessage(pb: ProtoBuffer): string {.inline, raises: [Defect, DaemonLocalError].} =
+  var error: seq[byte]
+  if pb.getRequiredField(ResponseType.ERROR.int, error).isOk():
+    if initProtoBuffer(error).getRequiredField(1, result).isErr():
       raise newException(DaemonLocalError, "Error message is missing!")
 
 proc recvMessage(conn: StreamTransport): Future[seq[byte]] {.async.} =
@@ -493,7 +503,8 @@ proc recvMessage(conn: StreamTransport): Future[seq[byte]] {.async.} =
 
   result = buffer
 
-proc newConnection*(api: DaemonAPI): Future[StreamTransport] =
+proc newConnection*(api: DaemonAPI): Future[StreamTransport]
+  {.raises: [Defect, LPError].} =
   result = connect(api.address)
 
 proc closeConnection*(api: DaemonAPI, transp: StreamTransport): Future[void] =
@@ -643,7 +654,6 @@ proc newDaemonApi*(flags: set[P2PDaemonFlags] = {},
   api.servers = newSeq[P2PServer]()
   api.pattern = patternForChild
   api.ucounter = 1
-  api.handlers = initTable[string, P2PStreamCallback]()
 
   if len(sockpath) == 0:
     api.flags.excl(NoProcessCtrl)
@@ -718,8 +728,8 @@ proc newDaemonApi*(flags: set[P2PDaemonFlags] = {},
     args.add("-relayDiscovery=true")
   if RelayHop in api.flags:
     args.add("-relayHop=true")
-  if NoInlinePeerID in api.flags:
-    args.add("-noInlinePeerID=true")
+  if NoInlinePeerId in api.flags:
+    args.add("-noInlinePeerId=true")
   if len(bootstrapNodes) > 0:
     args.add("-bootstrapPeers=" & bootstrapNodes.join(","))
   if len(id) != 0:
@@ -737,16 +747,24 @@ proc newDaemonApi*(flags: set[P2PDaemonFlags] = {},
       opt.add $address
     args.add(opt)
   args.add("-noise=true")
+  args.add("-quic=false")
   args.add("-listen=" & $api.address)
 
   # We are trying to get absolute daemon path.
   let cmd = findExe(daemon)
+  trace "p2pd cmd", cmd, args
   if len(cmd) == 0:
     raise newException(DaemonLocalError, "Could not find daemon executable!")
 
   # Starting daemon process
   # echo "Starting ", cmd, " ", args.join(" ")
-  api.process = startProcess(cmd, "", args, env, {poParentStreams})
+  api.process = 
+    try:
+      startProcess(cmd, "", args, env, {poParentStreams})
+    except CatchableError as exc:
+      raise exc
+    except Exception as exc:
+      raiseAssert exc.msg
   # Waiting until daemon will not be bound to control socket.
   while true:
     if not api.process.running():
@@ -787,7 +805,7 @@ proc close*(api: DaemonAPI) {.async.} =
       pending.add(server.server.join())
     await allFutures(pending)
     for server in api.servers:
-      let address = initTAddress(server.address)
+      let address = initTAddress(server.address).tryGet()
       discard tryRemoveFile($address)
     api.servers.setLen(0)
   # Closing daemon's process.
@@ -798,7 +816,7 @@ proc close*(api: DaemonAPI) {.async.} =
       api.process.terminate()
     discard api.process.waitForExit()
   # Attempt to delete unix socket endpoint.
-  let address = initTAddress(api.address)
+  let address = initTAddress(api.address).tryGet()
   if address.family == AddressFamily.Unix:
     discard tryRemoveFile($address)
 
@@ -822,17 +840,14 @@ proc transactMessage(transp: StreamTransport,
     raise newException(DaemonLocalError, "Incorrect or empty message received!")
   result = initProtoBuffer(message)
 
-proc getPeerInfo(pb: var ProtoBuffer): PeerInfo =
+proc getPeerInfo(pb: ProtoBuffer): PeerInfo
+  {.raises: [Defect, DaemonLocalError].} =
   ## Get PeerInfo object from ``pb``.
   result.addresses = newSeq[MultiAddress]()
-  if pb.getValue(1, result.peer) == -1:
-    raise newException(DaemonLocalError, "Missing required field `peer`!")
-  var address = newSeq[byte]()
-  while pb.getBytes(2, address) != -1:
-    if len(address) != 0:
-      var copyaddr = address
-      result.addresses.add(MultiAddress.init(copyaddr).tryGet())
-      address.setLen(0)
+  if pb.getRequiredField(1, result.peer).isErr():
+    raise newException(DaemonLocalError, "Incorrect or empty message received!")
+
+  discard pb.getRepeatedField(2, result.addresses)
 
 proc identity*(api: DaemonAPI): Future[PeerInfo] {.async.} =
   ## Get Node identity information
@@ -840,13 +855,14 @@ proc identity*(api: DaemonAPI): Future[PeerInfo] {.async.} =
   try:
     var pb = await transactMessage(transp, requestIdentity())
     pb.withMessage() do:
-      let res = pb.enterSubmessage()
-      if res == cast[int](ResponseType.IDENTITY):
-        result = pb.getPeerInfo()
+      var res: seq[byte]
+      if pb.getRequiredField(ResponseType.IDENTITY.int, res).isOk():
+        var resPb = initProtoBuffer(res)
+        result = getPeerInfo(resPb)
   finally:
     await api.closeConnection(transp)
 
-proc connect*(api: DaemonAPI, peer: PeerID,
+proc connect*(api: DaemonAPI, peer: PeerId,
               addresses: seq[MultiAddress],
               timeout = 0) {.async.} =
   ## Connect to remote peer with id ``peer`` and addresses ``addresses``.
@@ -859,7 +875,7 @@ proc connect*(api: DaemonAPI, peer: PeerID,
   except:
     await api.closeConnection(transp)
 
-proc disconnect*(api: DaemonAPI, peer: PeerID) {.async.} =
+proc disconnect*(api: DaemonAPI, peer: PeerId) {.async.} =
   ## Disconnect from remote peer with id ``peer``.
   var transp = await api.newConnection()
   try:
@@ -869,7 +885,7 @@ proc disconnect*(api: DaemonAPI, peer: PeerID) {.async.} =
   finally:
     await api.closeConnection(transp)
 
-proc openStream*(api: DaemonAPI, peer: PeerID,
+proc openStream*(api: DaemonAPI, peer: PeerId,
                  protocols: seq[string],
                  timeout = 0): Future[P2PStream] {.async.} =
   ## Open new stream to peer ``peer`` using one of the protocols in
@@ -880,22 +896,20 @@ proc openStream*(api: DaemonAPI, peer: PeerID,
     var pb = await transp.transactMessage(requestStreamOpen(peer, protocols,
                                                             timeout))
     pb.withMessage() do:
-      var res = pb.enterSubmessage()
-      if res == cast[int](ResponseType.STREAMINFO):
+      var res: seq[byte]
+      if pb.getRequiredField(ResponseType.STREAMINFO.int, res).isOk():
+        let resPb = initProtoBuffer(res)
         # stream.peer = newSeq[byte]()
         var raddress = newSeq[byte]()
         stream.protocol = ""
-        if pb.getValue(1, stream.peer) == -1:
-          raise newException(DaemonLocalError, "Missing `peer` field!")
-        if pb.getLengthValue(2, raddress) == -1:
-          raise newException(DaemonLocalError, "Missing `address` field!")
+        resPb.getRequiredField(1, stream.peer).tryGet()
+        resPb.getRequiredField(2, raddress).tryGet()
         stream.raddress = MultiAddress.init(raddress).tryGet()
-        if pb.getLengthValue(3, stream.protocol) == -1:
-          raise newException(DaemonLocalError, "Missing `proto` field!")
+        resPb.getRequiredField(3, stream.protocol).tryGet()
         stream.flags.incl(Outbound)
         stream.transp = transp
         result = stream
-  except Exception as exc:
+  except CatchableError as exc:
     await api.closeConnection(transp)
     raise exc
 
@@ -906,22 +920,19 @@ proc streamHandler(server: StreamServer, transp: StreamTransport) {.async.} =
   var stream = new P2PStream
   var raddress = newSeq[byte]()
   stream.protocol = ""
-  if pb.getValue(1, stream.peer) == -1:
-    raise newException(DaemonLocalError, "Missing `peer` field!")
-  if pb.getLengthValue(2, raddress) == -1:
-    raise newException(DaemonLocalError, "Missing `address` field!")
+  pb.getRequiredField(1, stream.peer).tryGet()
+  pb.getRequiredField(2, raddress).tryGet()
   stream.raddress = MultiAddress.init(raddress).tryGet()
-  if pb.getLengthValue(3, stream.protocol) == -1:
-    raise newException(DaemonLocalError, "Missing `proto` field!")
+  pb.getRequiredField(3, stream.protocol).tryGet()
   stream.flags.incl(Inbound)
   stream.transp = transp
   if len(stream.protocol) > 0:
     var handler = api.handlers.getOrDefault(stream.protocol)
     if not isNil(handler):
-      asyncCheck handler(api, stream)
+      asyncSpawn handler(api, stream)
 
 proc addHandler*(api: DaemonAPI, protocols: seq[string],
-                 handler: P2PStreamCallback) {.async.} =
+                 handler: P2PStreamCallback) {.async, raises: [Defect, LPError].} =
   ## Add stream handler ``handler`` for set of protocols ``protocols``.
   var transp = await api.newConnection()
   let maddress = await getSocket(api.pattern, addr api.ucounter)
@@ -934,7 +945,7 @@ proc addHandler*(api: DaemonAPI, protocols: seq[string],
                                                                protocols))
     pb.withMessage() do:
       api.servers.add(P2PServer(server: server, address: maddress))
-  except Exception as exc:
+  except CatchableError as exc:
     for item in protocols:
       api.handlers.del(item)
     server.stop()
@@ -951,18 +962,15 @@ proc listPeers*(api: DaemonAPI): Future[seq[PeerInfo]] {.async.} =
     var pb = await transp.transactMessage(requestListPeers())
     pb.withMessage() do:
       result = newSeq[PeerInfo]()
-      var res = pb.enterSubmessage()
-      while res != 0:
-        if res == cast[int](ResponseType.PEERINFO):
-          var peer = pb.getPeerInfo()
+      var ress: seq[seq[byte]]
+      if pb.getRequiredRepeatedField(ResponseType.PEERINFO.int, ress).isOk():
+        for p in ress:
+          let peer = initProtoBuffer(p).getPeerInfo()
           result.add(peer)
-        else:
-          pb.skipSubmessage()
-        res = pb.enterSubmessage()
   finally:
     await api.closeConnection(transp)
 
-proc cmTagPeer*(api: DaemonAPI, peer: PeerID, tag: string,
+proc cmTagPeer*(api: DaemonAPI, peer: PeerId, tag: string,
               weight: int) {.async.} =
   ## Tag peer with id ``peer`` using ``tag`` and ``weight``.
   var transp = await api.newConnection()
@@ -973,7 +981,7 @@ proc cmTagPeer*(api: DaemonAPI, peer: PeerID, tag: string,
   finally:
     await api.closeConnection(transp)
 
-proc cmUntagPeer*(api: DaemonAPI, peer: PeerID, tag: string) {.async.} =
+proc cmUntagPeer*(api: DaemonAPI, peer: PeerId, tag: string) {.async.} =
   ## Remove tag ``tag`` from peer with id ``peer``.
   var transp = await api.newConnection()
   try:
@@ -993,44 +1001,61 @@ proc cmTrimPeers*(api: DaemonAPI) {.async.} =
   finally:
     await api.closeConnection(transp)
 
-proc dhtGetSinglePeerInfo(pb: var ProtoBuffer): PeerInfo =
-  if pb.enterSubmessage() == 2:
-    result = pb.getPeerInfo()
+proc dhtGetSinglePeerInfo(pb: ProtoBuffer): PeerInfo
+  {.raises: [Defect, DaemonLocalError].} =
+  var res: seq[byte]
+  if pb.getRequiredField(2, res).isOk():
+    result = initProtoBuffer(res).getPeerInfo()
   else:
     raise newException(DaemonLocalError, "Missing required field `peer`!")
 
-proc dhtGetSingleValue(pb: var ProtoBuffer): seq[byte] =
+proc dhtGetSingleValue(pb: ProtoBuffer): seq[byte]
+  {.raises: [Defect, DaemonLocalError].} =
   result = newSeq[byte]()
-  if pb.getLengthValue(3, result) == -1:
+  if pb.getRequiredField(3, result).isErr():
     raise newException(DaemonLocalError, "Missing field `value`!")
 
-proc dhtGetSinglePublicKey(pb: var ProtoBuffer): PublicKey =
-  if pb.getValue(3, result) == -1:
+proc dhtGetSinglePublicKey(pb: ProtoBuffer): PublicKey
+  {.raises: [Defect, DaemonLocalError].} =
+  if pb.getRequiredField(3, result).isErr():
     raise newException(DaemonLocalError, "Missing field `value`!")
 
-proc dhtGetSinglePeerID(pb: var ProtoBuffer): PeerID =
-  if pb.getValue(3, result) == -1:
+proc dhtGetSinglePeerId(pb: ProtoBuffer): PeerId
+  {.raises: [Defect, DaemonLocalError].} =
+  if pb.getRequiredField(3, result).isErr():
     raise newException(DaemonLocalError, "Missing field `value`!")
 
-proc enterDhtMessage(pb: var ProtoBuffer, rt: DHTResponseType) {.inline.} =
-  var dtype: uint
-  var res = pb.enterSubmessage()
-  if res == cast[int](ResponseType.DHT):
-    if pb.getVarintValue(1, dtype) == 0:
+proc enterDhtMessage(pb: ProtoBuffer, rt: DHTResponseType): ProtoBuffer
+  {.inline, raises: [Defect, DaemonLocalError].} =
+  var dhtResponse: seq[byte]
+  if pb.getRequiredField(ResponseType.DHT.int, dhtResponse).isOk():
+    var pbDhtResponse = initProtoBuffer(dhtResponse)
+    var dtype: uint
+    if pbDhtResponse.getRequiredField(1, dtype).isErr():
       raise newException(DaemonLocalError, "Missing required DHT field `type`!")
     if dtype != cast[uint](rt):
       raise newException(DaemonLocalError, "Wrong DHT answer type! ")
+
+    var value: seq[byte]
+    if pbDhtResponse.getRequiredField(3, value).isErr():
+      raise newException(DaemonLocalError, "Missing required DHT field `value`!")
+    
+    return initProtoBuffer(value)
   else:
     raise newException(DaemonLocalError, "Wrong message type!")
 
-proc enterPsMessage(pb: var ProtoBuffer) {.inline.} =
-  var res = pb.enterSubmessage()
-  if res != cast[int](ResponseType.PUBSUB):
+proc enterPsMessage(pb: ProtoBuffer): ProtoBuffer
+  {.inline, raises: [Defect, DaemonLocalError].} =
+  var res: seq[byte]
+  if pb.getRequiredField(ResponseType.PUBSUB.int, res).isErr():
     raise newException(DaemonLocalError, "Wrong message type!")
 
-proc getDhtMessageType(pb: var ProtoBuffer): DHTResponseType {.inline.} =
+  initProtoBuffer(res)
+
+proc getDhtMessageType(pb: ProtoBuffer): DHTResponseType
+  {.inline, raises: [Defect, DaemonLocalError].} =
   var dtype: uint
-  if pb.getVarintValue(1, dtype) == 0:
+  if pb.getRequiredField(1, dtype).isErr():
     raise newException(DaemonLocalError, "Missing required DHT field `type`!")
   if dtype == cast[uint](DHTResponseType.VALUE):
     result = DHTResponseType.VALUE
@@ -1039,7 +1064,7 @@ proc getDhtMessageType(pb: var ProtoBuffer): DHTResponseType {.inline.} =
   else:
     raise newException(DaemonLocalError, "Wrong DHT answer type!")
 
-proc dhtFindPeer*(api: DaemonAPI, peer: PeerID,
+proc dhtFindPeer*(api: DaemonAPI, peer: PeerId,
                   timeout = 0): Future[PeerInfo] {.async.} =
   ## Find peer with id ``peer`` and return peer information ``PeerInfo``.
   ##
@@ -1049,12 +1074,11 @@ proc dhtFindPeer*(api: DaemonAPI, peer: PeerID,
   try:
     var pb = await transp.transactMessage(requestDHTFindPeer(peer, timeout))
     withMessage(pb) do:
-      pb.enterDhtMessage(DHTResponseType.VALUE)
-      result = pb.dhtGetSinglePeerInfo()
+      result = pb.enterDhtMessage(DHTResponseType.VALUE).dhtGetSinglePeerInfo()
   finally:
     await api.closeConnection(transp)
 
-proc dhtGetPublicKey*(api: DaemonAPI, peer: PeerID,
+proc dhtGetPublicKey*(api: DaemonAPI, peer: PeerId,
                       timeout = 0): Future[PublicKey] {.async.} =
   ## Get peer's public key from peer with id ``peer``.
   ##
@@ -1064,8 +1088,7 @@ proc dhtGetPublicKey*(api: DaemonAPI, peer: PeerID,
   try:
     var pb = await transp.transactMessage(requestDHTGetPublicKey(peer, timeout))
     withMessage(pb) do:
-      pb.enterDhtMessage(DHTResponseType.VALUE)
-      result = pb.dhtGetSinglePublicKey()
+      result = pb.enterDhtMessage(DHTResponseType.VALUE).dhtGetSinglePublicKey()
   finally:
     await api.closeConnection(transp)
 
@@ -1079,8 +1102,7 @@ proc dhtGetValue*(api: DaemonAPI, key: string,
   try:
     var pb = await transp.transactMessage(requestDHTGetValue(key, timeout))
     withMessage(pb) do:
-      pb.enterDhtMessage(DHTResponseType.VALUE)
-      result = pb.dhtGetSingleValue()
+      result = pb.enterDhtMessage(DHTResponseType.VALUE).dhtGetSingleValue()
   finally:
     await api.closeConnection(transp)
 
@@ -1112,7 +1134,7 @@ proc dhtProvide*(api: DaemonAPI, cid: Cid, timeout = 0) {.async.} =
   finally:
     await api.closeConnection(transp)
 
-proc dhtFindPeersConnectedToPeer*(api: DaemonAPI, peer: PeerID,
+proc dhtFindPeersConnectedToPeer*(api: DaemonAPI, peer: PeerId,
                                  timeout = 0): Future[seq[PeerInfo]] {.async.} =
   ## Find peers which are connected to peer with id ``peer``.
   ##
@@ -1124,7 +1146,7 @@ proc dhtFindPeersConnectedToPeer*(api: DaemonAPI, peer: PeerID,
     let spb = requestDHTFindPeersConnectedToPeer(peer, timeout)
     var pb = await transp.transactMessage(spb)
     withMessage(pb) do:
-      pb.enterDhtMessage(DHTResponseType.BEGIN)
+      discard pb.enterDhtMessage(DHTResponseType.BEGIN)
       while true:
         var message = await transp.recvMessage()
         if len(message) == 0:
@@ -1138,18 +1160,18 @@ proc dhtFindPeersConnectedToPeer*(api: DaemonAPI, peer: PeerID,
     await api.closeConnection(transp)
 
 proc dhtGetClosestPeers*(api: DaemonAPI, key: string,
-                         timeout = 0): Future[seq[PeerID]] {.async.} =
+                         timeout = 0): Future[seq[PeerId]] {.async.} =
   ## Get closest peers for ``key``.
   ##
   ## You can specify timeout for DHT request with ``timeout`` value. ``0`` value
   ## means no timeout.
   var transp = await api.newConnection()
-  var list = newSeq[PeerID]()
+  var list = newSeq[PeerId]()
   try:
     let spb = requestDHTGetClosestPeers(key, timeout)
     var pb = await transp.transactMessage(spb)
     withMessage(pb) do:
-      pb.enterDhtMessage(DHTResponseType.BEGIN)
+      discard pb.enterDhtMessage(DHTResponseType.BEGIN)
       while true:
         var message = await transp.recvMessage()
         if len(message) == 0:
@@ -1157,7 +1179,7 @@ proc dhtGetClosestPeers*(api: DaemonAPI, key: string,
         var cpb = initProtoBuffer(message)
         if cpb.getDhtMessageType() == DHTResponseType.END:
           break
-        list.add(cpb.dhtGetSinglePeerID())
+        list.add(cpb.dhtGetSinglePeerId())
       result = list
   finally:
     await api.closeConnection(transp)
@@ -1174,7 +1196,7 @@ proc dhtFindProviders*(api: DaemonAPI, cid: Cid, count: uint32,
     let spb = requestDHTFindProviders(cid, count, timeout)
     var pb = await transp.transactMessage(spb)
     withMessage(pb) do:
-      pb.enterDhtMessage(DHTResponseType.BEGIN)
+      discard pb.enterDhtMessage(DHTResponseType.BEGIN)
       while true:
         var message = await transp.recvMessage()
         if len(message) == 0:
@@ -1198,7 +1220,7 @@ proc dhtSearchValue*(api: DaemonAPI, key: string,
   try:
     var pb = await transp.transactMessage(requestDHTSearchValue(key, timeout))
     withMessage(pb) do:
-      pb.enterDhtMessage(DHTResponseType.BEGIN)
+      discard pb.enterDhtMessage(DHTResponseType.BEGIN)
       while true:
         var message = await transp.recvMessage()
         if len(message) == 0:
@@ -1217,30 +1239,26 @@ proc pubsubGetTopics*(api: DaemonAPI): Future[seq[string]] {.async.} =
   try:
     var pb = await transp.transactMessage(requestPSGetTopics())
     withMessage(pb) do:
-      pb.enterPsMessage()
+      let innerPb = pb.enterPsMessage()
       var topics = newSeq[string]()
-      var topic = ""
-      while pb.getString(1, topic) != -1:
-        topics.add(topic)
-        topic.setLen(0)
+      discard innerPb.getRepeatedField(1, topics)
       result = topics
   finally:
     await api.closeConnection(transp)
 
 proc pubsubListPeers*(api: DaemonAPI,
-                      topic: string): Future[seq[PeerID]] {.async.} =
+                      topic: string): Future[seq[PeerId]] {.async.} =
   ## Get list of peers we are connected to and which also subscribed to topic
   ## ``topic``.
   var transp = await api.newConnection()
   try:
     var pb = await transp.transactMessage(requestPSListPeers(topic))
     withMessage(pb) do:
-      var peer: PeerID
-      pb.enterPsMessage()
-      var peers = newSeq[PeerID]()
-      while pb.getValue(2, peer) != -1:
-        peers.add(peer)
-      result = peers
+      var peer: PeerId
+      let innerPb = pb.enterPsMessage()
+      var peers = newSeq[seq[byte]]()
+      discard innerPb.getRepeatedField(2, peers)
+      result = peers.mapIt(PeerId.init(it).get())
   finally:
     await api.closeConnection(transp)
 
@@ -1255,24 +1273,15 @@ proc pubsubPublish*(api: DaemonAPI, topic: string,
   finally:
     await api.closeConnection(transp)
 
-proc getPubsubMessage*(pb: var ProtoBuffer): PubSubMessage =
+proc getPubsubMessage*(pb: ProtoBuffer): PubSubMessage =
   result.data = newSeq[byte]()
   result.seqno = newSeq[byte]()
-  discard pb.getValue(1, result.peer)
-  discard pb.getBytes(2, result.data)
-  discard pb.getBytes(3, result.seqno)
-  var item = newSeq[byte]()
-  while true:
-    if pb.getBytes(4, item) == -1:
-      break
-    var copyitem = item
-    var stritem = cast[string](copyitem)
-    if len(result.topics) == 0:
-      result.topics = newSeq[string]()
-    result.topics.add(stritem)
-    item.setLen(0)
-  discard pb.getValue(5, result.signature)
-  discard pb.getValue(6, result.key)
+  discard pb.getField(1, result.peer)
+  discard pb.getField(2, result.data)
+  discard pb.getField(3, result.seqno)
+  discard pb.getRepeatedField(4, result.topics)
+  discard pb.getField(5, result.signature)
+  discard pb.getField(6, result.key)
 
 proc pubsubLoop(api: DaemonAPI, ticket: PubsubTicket) {.async.} =
   while true:
@@ -1299,17 +1308,17 @@ proc pubsubSubscribe*(api: DaemonAPI, topic: string,
       ticket.topic = topic
       ticket.handler = handler
       ticket.transp = transp
-      asyncCheck pubsubLoop(api, ticket)
+      asyncSpawn pubsubLoop(api, ticket)
       result = ticket
-  except Exception as exc:
+  except CatchableError as exc:
     await api.closeConnection(transp)
     raise exc
 
-proc `$`*(pinfo: PeerInfo): string =
+proc shortLog*(pinfo: PeerInfo): string =
   ## Get string representation of ``PeerInfo`` object.
   result = newStringOfCap(128)
-  result.add("{PeerID: '")
-  result.add($pinfo.peer.pretty())
+  result.add("{PeerId: '")
+  result.add($pinfo.peer.shortLog())
   result.add("' Addresses: [")
   let length = len(pinfo.addresses)
   for i in 0..<length:

libp2p/daemon/transpool.nim

@@ -1,11 +1,16 @@
-## Nim-Libp2p
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 ## This module implements Pool of StreamTransport.
 import chronos

libp2p/debugutils.nim

@@ -0,0 +1,279 @@
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+## To enable dump of all incoming and outgoing unencrypted messages you need
+## to compile project with ``-d:libp2p_dump`` compile-time option. When this
+## option enabled ``nim-libp2p`` will create dumps of unencrypted messages for
+## every peer libp2p communicates.
+##
+## Every file is created with name "<PeerId>.pbcap". One file represents
+## all the communication with peer which identified by ``PeerId``.
+##
+## File can have multiple protobuf encoded messages of this format:
+##
+## Protobuf's message fields:
+##   1. SeqID: optional uint64
+##   2. Timestamp: required uint64
+##   3. Type: optional uint64
+##   4. Direction: required uint64
+##   5. LocalAddress: optional bytes
+##   6. RemoteAddress: optional bytes
+##   7. Message: required bytes
+import os, options
+import nimcrypto/utils, stew/endians2
+import protobuf/minprotobuf, stream/connection, protocols/secure/secure,
+       multiaddress, peerid, varint, muxers/mplex/coder
+
+from times import getTime, toUnix, fromUnix, nanosecond, format, Time,
+                  NanosecondRange, initTime
+from strutils import toHex, repeat
+export peerid, options, multiaddress
+
+type
+  FlowDirection* = enum
+    Outgoing, Incoming
+
+  ProtoMessage* = object
+    timestamp*: uint64
+    direction*: FlowDirection
+    message*: seq[byte]
+    seqID*: Option[uint64]
+    mtype*: Option[uint64]
+    local*: Option[MultiAddress]
+    remote*: Option[MultiAddress]
+
+const
+  libp2p_dump_dir* {.strdefine.} = "nim-libp2p"
+    ## default directory where all the dumps will be stored, if the path
+    ## relative it will be created in home directory. You can overload this path
+    ## using ``-d:libp2p_dump_dir=<otherpath>``.
+
+proc getTimestamp(): uint64 =
+  ## This procedure is present because `stdlib.times` missing it.
+  let time = getTime()
+  uint64(time.toUnix() * 1_000_000_000 + time.nanosecond())
+
+proc getTimedate(value: uint64): string =
+  ## This procedure is workaround for `stdlib.times` to just convert
+  ## nanoseconds' timestamp to DateTime object.
+  let time = initTime(int64(value div 1_000_000_000), value mod 1_000_000_000)
+  time.format("yyyy-MM-dd HH:mm:ss'.'fffzzz")
+
+proc dumpMessage*(conn: SecureConn, direction: FlowDirection,
+                  data: openArray[byte]) =
+  ## Store unencrypted message ``data`` to dump file, all the metadata will be
+  ## extracted from ``conn`` instance.
+  var pb = initProtoBuffer(options = {WithVarintLength})
+  pb.write(2, getTimestamp())
+  pb.write(4, uint64(direction))
+  pb.write(6, conn.observedAddr)
+  pb.write(7, data)
+  pb.finish()
+
+  let dirName =
+    if isAbsolute(libp2p_dump_dir):
+      libp2p_dump_dir
+    else:
+      getHomeDir() / libp2p_dump_dir
+
+  let fileName = $(conn.peerId) & ".pbcap"
+
+  # This is debugging procedure so it should not generate any exceptions,
+  # and we going to return at every possible OS error.
+  if not(dirExists(dirName)):
+    try:
+      createDir(dirName)
+    except CatchableError:
+      return
+
+  let pathName = dirName / fileName
+  var handle: File
+  try:
+    if open(handle, pathName, fmAppend):
+      discard writeBuffer(handle, addr pb.buffer[pb.offset], pb.getLen())
+  finally:
+    close(handle)
+
+proc decodeDumpMessage*(data: openArray[byte]): Option[ProtoMessage] =
+  ## Decode protobuf's message ProtoMessage from array of bytes ``data``.
+  var
+    pb = initProtoBuffer(data)
+    value: uint64
+    ma1, ma2: MultiAddress
+    pmsg: ProtoMessage
+
+  let res2 = pb.getField(2, pmsg.timestamp)
+  if res2.isErr() or not(res2.get()):
+    return none[ProtoMessage]()
+
+  let res4 = pb.getField(4, value)
+  if res4.isErr() or not(res4.get()):
+    return none[ProtoMessage]()
+
+  # `case` statement could not work here with an error "selector must be of an
+  # ordinal type, float or string"
+  pmsg.direction =
+    if value == uint64(Outgoing):
+      Outgoing
+    elif value == uint64(Incoming):
+      Incoming
+    else:
+      return none[ProtoMessage]()
+
+  let res7 = pb.getField(7, pmsg.message)
+  if res7.isErr() or not(res7.get()):
+    return none[ProtoMessage]()
+
+  value = 0'u64
+  let res1 = pb.getField(1, value)
+  if res1.isOk() and res1.get():
+    pmsg.seqID = some(value)
+  value = 0'u64
+  let res3 = pb.getField(3, value)
+  if res3.isOk() and res3.get():
+    pmsg.mtype = some(value)
+  let res5 = pb.getField(5, ma1)
+  if res5.isOk() and res5.get():
+    pmsg.local = some(ma1)
+  let res6 = pb.getField(6, ma2)
+  if res6.isOk() and res6.get():
+    pmsg.remote = some(ma2)
+
+  some(pmsg)
+
+iterator messages*(data: seq[byte]): Option[ProtoMessage] =
+  ## Iterate over sequence of bytes and decode all the ``ProtoMessage``
+  ## messages we found.
+  var value: uint64
+  var size: int
+  var offset = 0
+  while offset < len(data):
+    value = 0
+    size = 0
+    let res = PB.getUVarint(data.toOpenArray(offset, len(data) - 1),
+                            size, value)
+    if res.isOk():
+      if (value > 0'u64) and (value < uint64(len(data) - offset)):
+        offset += size
+        yield decodeDumpMessage(data.toOpenArray(offset,
+                                                 offset + int(value) - 1))
+        # value is previously checked to be less then len(data) which is `int`.
+        offset += int(value)
+      else:
+        break
+    else:
+      break
+
+proc dumpHex*(pbytes: openArray[byte], groupBy = 1, ascii = true): string =
+  ## Get hexadecimal dump of memory for array ``pbytes``.
+  var res = ""
+  var offset = 0
+  var ascii = ""
+
+  while offset < len(pbytes):
+    if (offset mod 16) == 0:
+      res = res & toHex(uint64(offset)) & ":  "
+
+    for k in 0 ..< groupBy:
+      let ch = pbytes[offset + k]
+      ascii.add(if ord(ch) > 31 and ord(ch) < 127: char(ch) else: '.')
+
+    let item =
+      case groupBy:
+      of 1:
+        toHex(pbytes[offset])
+      of 2:
+        toHex(uint16.fromBytes(pbytes.toOpenArray(offset, len(pbytes) - 1)))
+      of 4:
+        toHex(uint32.fromBytes(pbytes.toOpenArray(offset, len(pbytes) - 1)))
+      of 8:
+        toHex(uint64.fromBytes(pbytes.toOpenArray(offset, len(pbytes) - 1)))
+      else:
+        ""
+    res.add(item)
+    res.add(" ")
+    offset = offset + groupBy
+
+    if (offset mod 16) == 0:
+      res.add(" ")
+      res.add(ascii)
+      ascii.setLen(0)
+      res.add("\p")
+
+  if (offset mod 16) != 0:
+    let spacesCount = ((16 - (offset mod 16)) div groupBy) *
+                        (groupBy * 2 + 1) + 1
+    res = res & repeat(' ', spacesCount)
+    res = res & ascii
+
+  res.add("\p")
+  res
+
+proc mplexMessage*(data: seq[byte]): string =
+  var value = 0'u64
+  var size = 0
+  let res = PB.getUVarint(data, size, value)
+  if res.isOk():
+    if size < len(data) and data[size] == 0x00'u8:
+      let header = cast[MessageType](value and 0x07'u64)
+      let ident = (value shr 3)
+      "mplex: [" & $header & ", ident = " & $ident & "] "
+    else:
+      ""
+  else:
+    ""
+
+proc toString*(msg: ProtoMessage, dump = true): string =
+  ## Convert message ``msg`` to its string representation.
+  ## If ``dump`` is ``true`` (default) full hexadecimal dump with ascii will be
+  ## used, otherwise just a simple hexadecimal string will be printed.
+  var res = getTimedate(msg.timestamp)
+  let direction =
+    case msg.direction
+    of Incoming:
+      " << "
+    of Outgoing:
+      " >> "
+  let address =
+    block:
+      let local =
+        if msg.local.isSome():
+          "[" & $(msg.local.get()) & "]"
+        else:
+          "[LOCAL]"
+      let remote =
+        if msg.remote.isSome():
+          "[" & $(msg.remote.get()) & "]"
+        else:
+          "[REMOTE]"
+      local & direction & remote
+  let seqid =
+    if msg.seqID.isSome():
+      "seqID = " & $(msg.seqID.get()) & " "
+    else:
+      ""
+  let mtype =
+    if msg.mtype.isSome():
+      "type = " & $(msg.mtype.get()) & " "
+    else:
+      ""
+  res.add(" ")
+  res.add(address)
+  res.add(" ")
+  res.add(mtype)
+  res.add(seqid)
+  res.add(mplexMessage(msg.message))
+  res.add(" ")
+  res.add("\p")
+  if dump:
+    res.add(dumpHex(msg.message))
+  else:
+    res.add(utils.toHex(msg.message))
+    res.add("\p")
+  res

libp2p/dial.nim

@@ -0,0 +1,76 @@
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import chronos
+import stew/results
+import peerid,
+       stream/connection,
+       transports/transport
+
+export results
+
+type
+  Dial* = ref object of RootObj
+
+method connect*(
+  self: Dial,
+  peerId: PeerId,
+  addrs: seq[MultiAddress],
+  forceDial = false) {.async, base.} =
+  ## connect remote peer without negotiating
+  ## a protocol
+  ##
+
+  doAssert(false, "Not implemented!")
+
+method connect*(
+  self: Dial,
+  addrs: seq[MultiAddress]): Future[PeerId] {.async, base.} =
+  ## Connects to a peer and retrieve its PeerId
+
+  doAssert(false, "Not implemented!")
+
+method dial*(
+  self: Dial,
+  peerId: PeerId,
+  protos: seq[string],
+  ): Future[Connection] {.async, base.} =
+  ## create a protocol stream over an
+  ## existing connection
+  ##
+
+  doAssert(false, "Not implemented!")
+
+method dial*(
+  self: Dial,
+  peerId: PeerId,
+  addrs: seq[MultiAddress],
+  protos: seq[string],
+  forceDial = false): Future[Connection] {.async, base.} =
+  ## create a protocol stream and establish
+  ## a connection if one doesn't exist already
+  ##
+
+  doAssert(false, "Not implemented!")
+
+method addTransport*(
+  self: Dial,
+  transport: Transport) {.base.} =
+  doAssert(false, "Not implemented!")
+
+method tryDial*(
+  self: Dial,
+  peerId: PeerId,
+  addrs: seq[MultiAddress]): Future[Opt[MultiAddress]] {.async, base.} =
+  doAssert(false, "Not implemented!")

libp2p/dialer.nim

@@ -0,0 +1,332 @@
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+import std/[sugar, tables, sequtils]
+
+import stew/results
+import pkg/[chronos,
+            chronicles,
+            metrics]
+
+import dial,
+       peerid,
+       peerinfo,
+       multicodec,
+       multistream,
+       connmanager,
+       stream/connection,
+       transports/transport,
+       nameresolving/nameresolver,
+       upgrademngrs/upgrade,
+       errors
+
+export dial, errors, results
+
+logScope:
+  topics = "libp2p dialer"
+
+declareCounter(libp2p_total_dial_attempts, "total attempted dials")
+declareCounter(libp2p_successful_dials, "dialed successful peers")
+declareCounter(libp2p_failed_dials, "failed dials")
+declareCounter(libp2p_failed_upgrades_outgoing, "outgoing connections failed upgrades")
+
+type
+  DialFailedError* = object of LPError
+
+  Dialer* = ref object of Dial
+    localPeerId*: PeerId
+    ms: MultistreamSelect
+    connManager: ConnManager
+    dialLock: Table[PeerId, AsyncLock]
+    transports: seq[Transport]
+    nameResolver: NameResolver
+
+proc dialAndUpgrade(
+  self: Dialer,
+  peerId: Opt[PeerId],
+  hostname: string,
+  address: MultiAddress):
+  Future[Connection] {.async.} =
+
+  for transport in self.transports: # for each transport
+    if transport.handles(address):   # check if it can dial it
+      trace "Dialing address", address, peerId, hostname
+      let dialed =
+        try:
+          libp2p_total_dial_attempts.inc()
+          await transport.dial(hostname, address)
+        except CancelledError as exc:
+          debug "Dialing canceled", msg = exc.msg, peerId
+          raise exc
+        except CatchableError as exc:
+          debug "Dialing failed", msg = exc.msg, peerId
+          libp2p_failed_dials.inc()
+          return nil # Try the next address
+
+      # also keep track of the connection's bottom unsafe transport direction
+      # required by gossipsub scoring
+      dialed.transportDir = Direction.Out
+
+      libp2p_successful_dials.inc()
+
+      let conn =
+        try:
+          await transport.upgradeOutgoing(dialed, peerId)
+        except CatchableError as exc:
+          # If we failed to establish the connection through one transport,
+          # we won't succeeded through another - no use in trying again
+          await dialed.close()
+          debug "Upgrade failed", msg = exc.msg, peerId
+          if exc isnot CancelledError:
+            libp2p_failed_upgrades_outgoing.inc()
+
+          # Try other address
+          return nil
+
+      doAssert not isNil(conn), "connection died after upgradeOutgoing"
+      debug "Dial successful", conn, peerId = conn.peerId
+      return conn
+  return nil
+
+proc expandDnsAddr(
+  self: Dialer,
+  peerId: Opt[PeerId],
+  address: MultiAddress): Future[seq[(MultiAddress, Opt[PeerId])]] {.async.} =
+
+  if not DNSADDR.matchPartial(address): return @[(address, peerId)]
+  if isNil(self.nameResolver):
+    info "Can't resolve DNSADDR without NameResolver", ma=address
+    return @[]
+
+  let
+    toResolve =
+      if peerId.isSome:
+        address & MultiAddress.init(multiCodec("p2p"), peerId.tryGet()).tryGet()
+      else:
+        address
+    resolved = await self.nameResolver.resolveDnsAddr(toResolve)
+
+  for resolvedAddress in resolved:
+    let lastPart = resolvedAddress[^1].tryGet()
+    if lastPart.protoCode == Result[MultiCodec, string].ok(multiCodec("p2p")):
+      let
+        peerIdBytes = lastPart.protoArgument().tryGet()
+        addrPeerId = PeerId.init(peerIdBytes).tryGet()
+      result.add((resolvedAddress[0..^2].tryGet(), Opt.some(addrPeerId)))
+    else:
+      result.add((resolvedAddress, peerId))
+
+proc dialAndUpgrade(
+  self: Dialer,
+  peerId: Opt[PeerId],
+  addrs: seq[MultiAddress]):
+  Future[Connection] {.async.} =
+
+  debug "Dialing peer", peerId
+
+  for rawAddress in addrs:
+    # resolve potential dnsaddr
+    let addresses = await self.expandDnsAddr(peerId, rawAddress)
+
+    for (expandedAddress, addrPeerId) in addresses:
+      # DNS resolution
+      let
+        hostname = expandedAddress.getHostname()
+        resolvedAddresses =
+          if isNil(self.nameResolver): @[expandedAddress]
+          else: await self.nameResolver.resolveMAddress(expandedAddress)
+
+      for resolvedAddress in resolvedAddresses:
+        result = await self.dialAndUpgrade(addrPeerId, hostname, resolvedAddress)
+        if not isNil(result):
+          return result
+
+proc internalConnect(
+  self: Dialer,
+  peerId: Opt[PeerId],
+  addrs: seq[MultiAddress],
+  forceDial: bool):
+  Future[Connection] {.async.} =
+  if Opt.some(self.localPeerId) == peerId:
+    raise newException(CatchableError, "can't dial self!")
+
+  # Ensure there's only one in-flight attempt per peer
+  let lock = self.dialLock.mgetOrPut(peerId.get(default(PeerId)), newAsyncLock())
+  try:
+    await lock.acquire()
+
+    # Check if we have a connection already and try to reuse it
+    var conn =
+      if peerId.isSome: self.connManager.selectConn(peerId.get())
+      else: nil
+    if conn != nil:
+      if conn.atEof or conn.closed:
+        # This connection should already have been removed from the connection
+        # manager - it's essentially a bug that we end up here - we'll fail
+        # for now, hoping that this will clean themselves up later...
+        warn "dead connection in connection manager", conn
+        await conn.close()
+        raise newException(DialFailedError, "Zombie connection encountered")
+
+      trace "Reusing existing connection", conn, direction = $conn.dir
+      return conn
+
+    let slot = await self.connManager.getOutgoingSlot(forceDial)
+    conn =
+      try:
+        await self.dialAndUpgrade(peerId, addrs)
+      except CatchableError as exc:
+        slot.release()
+        raise exc
+    slot.trackConnection(conn)
+    if isNil(conn): # None of the addresses connected
+      raise newException(DialFailedError, "Unable to establish outgoing link")
+
+    # A disconnect could have happened right after
+    # we've added the connection so we check again
+    # to prevent races due to that.
+    if conn.closed() or conn.atEof():
+      # This can happen when the other ends drops us
+      # before we get a chance to return the connection
+      # back to the dialer.
+      trace "Connection dead on arrival", conn
+      raise newLPStreamClosedError()
+
+    return conn
+  finally:
+    if lock.locked():
+      lock.release()
+
+method connect*(
+  self: Dialer,
+  peerId: PeerId,
+  addrs: seq[MultiAddress],
+  forceDial = false) {.async.} =
+  ## connect remote peer without negotiating
+  ## a protocol
+  ##
+
+  if self.connManager.connCount(peerId) > 0:
+    return
+
+  discard await self.internalConnect(Opt.some(peerId), addrs, forceDial)
+
+method connect*(
+  self: Dialer,
+  addrs: seq[MultiAddress],
+  ): Future[PeerId] {.async.} =
+  ## Connects to a peer and retrieve its PeerId
+
+  return (await self.internalConnect(Opt.none(PeerId), addrs, false)).peerId
+
+proc negotiateStream(
+  self: Dialer,
+  conn: Connection,
+  protos: seq[string]): Future[Connection] {.async.} =
+  trace "Negotiating stream", conn, protos
+  let selected = await self.ms.select(conn, protos)
+  if not protos.contains(selected):
+    await conn.closeWithEOF()
+    raise newException(DialFailedError, "Unable to select sub-protocol " & $protos)
+
+  return conn
+
+method tryDial*(
+  self: Dialer,
+  peerId: PeerId,
+  addrs: seq[MultiAddress]): Future[Opt[MultiAddress]] {.async.} =
+  ## Create a protocol stream in order to check
+  ## if a connection is possible.
+  ## Doesn't use the Connection Manager to save it.
+  ##
+
+  trace "Check if it can dial", peerId, addrs
+  try:
+    let conn = await self.dialAndUpgrade(Opt.some(peerId), addrs)
+    if conn.isNil():
+      raise newException(DialFailedError, "No valid multiaddress")
+    await conn.close()
+    return conn.observedAddr
+  except CancelledError as exc:
+    raise exc
+  except CatchableError as exc:
+    raise newException(DialFailedError, exc.msg)
+
+method dial*(
+  self: Dialer,
+  peerId: PeerId,
+  protos: seq[string]): Future[Connection] {.async.} =
+  ## create a protocol stream over an
+  ## existing connection
+  ##
+
+  trace "Dialing (existing)", peerId, protos
+  let stream = await self.connManager.getStream(peerId)
+  if stream.isNil:
+    raise newException(DialFailedError, "Couldn't get muxed stream")
+
+  return await self.negotiateStream(stream, protos)
+
+method dial*(
+  self: Dialer,
+  peerId: PeerId,
+  addrs: seq[MultiAddress],
+  protos: seq[string],
+  forceDial = false): Future[Connection] {.async.} =
+  ## create a protocol stream and establish
+  ## a connection if one doesn't exist already
+  ##
+
+  var
+    conn: Connection
+    stream: Connection
+
+  proc cleanup() {.async.} =
+    if not(isNil(stream)):
+      await stream.closeWithEOF()
+
+    if not(isNil(conn)):
+      await conn.close()
+
+  try:
+    trace "Dialing (new)", peerId, protos
+    conn = await self.internalConnect(Opt.some(peerId), addrs, forceDial)
+    trace "Opening stream", conn
+    stream = await self.connManager.getStream(conn)
+
+    if isNil(stream):
+      raise newException(DialFailedError,
+        "Couldn't get muxed stream")
+
+    return await self.negotiateStream(stream, protos)
+  except CancelledError as exc:
+    trace "Dial canceled", conn
+    await cleanup()
+    raise exc
+  except CatchableError as exc:
+    debug "Error dialing", conn, msg = exc.msg
+    await cleanup()
+    raise exc
+
+method addTransport*(self: Dialer, t: Transport) =
+  self.transports &= t
+
+proc new*(
+  T: type Dialer,
+  localPeerId: PeerId,
+  connManager: ConnManager,
+  transports: seq[Transport],
+  ms: MultistreamSelect,
+  nameResolver: NameResolver = nil): Dialer =
+
+  T(localPeerId: localPeerId,
+    connManager: connManager,
+    transports: transports,
+    ms: ms,
+    nameResolver: nameResolver)

libp2p/discovery/discoverymngr.nim

@@ -0,0 +1,163 @@
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import std/sequtils
+import chronos, chronicles, stew/results
+import ../errors
+
+type
+  BaseAttr = ref object of RootObj
+    comparator: proc(f, c: BaseAttr): bool {.gcsafe, raises: [Defect].}
+
+  Attribute[T] = ref object of BaseAttr
+    value: T
+
+  PeerAttributes* = object
+    attributes: seq[BaseAttr]
+
+  DiscoveryService* = distinct string
+
+proc `==`*(a, b: DiscoveryService): bool {.borrow.}
+
+proc ofType*[T](f: BaseAttr, _: type[T]): bool =
+  return f of Attribute[T]
+
+proc to*[T](f: BaseAttr, _: type[T]): T =
+  Attribute[T](f).value
+
+proc add*[T](pa: var PeerAttributes,
+             value: T) =
+  pa.attributes.add(Attribute[T](
+      value: value,
+      comparator: proc(f: BaseAttr, c: BaseAttr): bool =
+        f.ofType(T) and c.ofType(T) and f.to(T) == c.to(T)
+    )
+  )
+
+iterator items*(pa: PeerAttributes): BaseAttr =
+  for f in pa.attributes:
+    yield f
+
+proc getAll*[T](pa: PeerAttributes, t: typedesc[T]): seq[T] =
+  for f in pa.attributes:
+    if f.ofType(T):
+      result.add(f.to(T))
+
+proc `{}`*[T](pa: PeerAttributes, t: typedesc[T]): Opt[T] =
+  for f in pa.attributes:
+    if f.ofType(T):
+      return Opt.some(f.to(T))
+  Opt.none(T)
+
+proc `[]`*[T](pa: PeerAttributes, t: typedesc[T]): T {.raises: [Defect, KeyError].} =
+  pa{T}.valueOr: raise newException(KeyError, "Attritute not found")
+
+proc match*(pa, candidate: PeerAttributes): bool =
+  for f in pa.attributes:
+    block oneAttribute:
+      for field in candidate.attributes:
+        if field.comparator(field, f):
+          break oneAttribute
+      return false
+  return true
+
+type
+  PeerFoundCallback* = proc(pa: PeerAttributes) {.raises: [Defect], gcsafe.}
+
+  DiscoveryInterface* = ref object of RootObj
+    onPeerFound*: PeerFoundCallback
+    toAdvertise*: PeerAttributes
+    advertisementUpdated*: AsyncEvent
+    advertiseLoop*: Future[void]
+
+method request*(self: DiscoveryInterface, pa: PeerAttributes) {.async, base.} =
+  doAssert(false, "Not implemented!")
+
+method advertise*(self: DiscoveryInterface) {.async, base.} =
+  doAssert(false, "Not implemented!")
+
+type
+  DiscoveryError* = object of LPError
+
+  DiscoveryQuery* = ref object
+    attr: PeerAttributes
+    peers: AsyncQueue[PeerAttributes]
+    futs: seq[Future[void]]
+
+  DiscoveryManager* = ref object
+    interfaces: seq[DiscoveryInterface]
+    queries: seq[DiscoveryQuery]
+
+proc add*(dm: DiscoveryManager, di: DiscoveryInterface) =
+  dm.interfaces &= di
+
+  di.onPeerFound = proc (pa: PeerAttributes) =
+    for query in dm.queries:
+      if query.attr.match(pa):
+        try:
+          query.peers.putNoWait(pa)
+        except AsyncQueueFullError as exc:
+          debug "Cannot push discovered peer to queue"
+
+proc request*(dm: DiscoveryManager, pa: PeerAttributes): DiscoveryQuery =
+  var query = DiscoveryQuery(attr: pa, peers: newAsyncQueue[PeerAttributes]())
+  for i in dm.interfaces:
+    query.futs.add(i.request(pa))
+  dm.queries.add(query)
+  dm.queries.keepItIf(it.futs.anyIt(not it.finished()))
+  return query
+
+proc request*[T](dm: DiscoveryManager, value: T): DiscoveryQuery =
+  var pa: PeerAttributes
+  pa.add(value)
+  return dm.request(pa)
+
+proc advertise*(dm: DiscoveryManager, pa: PeerAttributes) =
+  for i in dm.interfaces:
+    i.toAdvertise = pa
+    if i.advertiseLoop.isNil:
+      i.advertisementUpdated = newAsyncEvent()
+      i.advertiseLoop = i.advertise()
+    else:
+      i.advertisementUpdated.fire()
+
+proc advertise*[T](dm: DiscoveryManager, value: T) =
+  var pa: PeerAttributes
+  pa.add(value)
+  dm.advertise(pa)
+
+proc stop*(query: DiscoveryQuery) =
+  for r in query.futs:
+    if not r.finished(): r.cancel()
+
+proc stop*(dm: DiscoveryManager) =
+  for q in dm.queries:
+    q.stop()
+  for i in dm.interfaces:
+    if isNil(i.advertiseLoop): continue
+    i.advertiseLoop.cancel()
+
+proc getPeer*(query: DiscoveryQuery): Future[PeerAttributes] {.async.} =
+  let getter = query.peers.popFirst()
+
+  try:
+    await getter or allFinished(query.futs)
+  except CancelledError as exc:
+    getter.cancel()
+    raise exc
+
+  if not finished(getter):
+    # discovery loops only finish when they don't handle the query
+    raise newException(DiscoveryError, "Unable to find any peer matching this request")
+  return await getter

libp2p/discovery/rendezvousinterface.nim

@@ -0,0 +1,77 @@
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import sequtils
+import chronos
+import ./discoverymngr,
+       ../protocols/rendezvous,
+       ../peerid
+
+type
+  RendezVousInterface* = ref object of DiscoveryInterface
+    rdv*: RendezVous
+    timeToRequest: Duration
+    timeToAdvertise: Duration
+
+  RdvNamespace* = distinct string
+
+proc `==`*(a, b: RdvNamespace): bool {.borrow.}
+
+method request*(self: RendezVousInterface, pa: PeerAttributes) {.async.} =
+  var namespace = ""
+  for attr in pa:
+    if attr.ofType(RdvNamespace):
+      namespace = string attr.to(RdvNamespace)
+    elif attr.ofType(DiscoveryService):
+      namespace = string attr.to(DiscoveryService)
+    elif attr.ofType(PeerId):
+      namespace = $attr.to(PeerId)
+    else:
+      # unhandled type
+      return
+  while true:
+    for pr in await self.rdv.request(namespace):
+      var peer: PeerAttributes
+      peer.add(pr.peerId)
+      for address in pr.addresses:
+        peer.add(address.address)
+
+      peer.add(DiscoveryService(namespace))
+      peer.add(RdvNamespace(namespace))
+      self.onPeerFound(peer)
+
+    await sleepAsync(self.timeToRequest)
+
+method advertise*(self: RendezVousInterface) {.async.} =
+  while true:
+    var toAdvertise: seq[string]
+    for attr in self.toAdvertise:
+      if attr.ofType(RdvNamespace):
+        toAdvertise.add string attr.to(RdvNamespace)
+      elif attr.ofType(DiscoveryService):
+        toAdvertise.add string attr.to(DiscoveryService)
+      elif attr.ofType(PeerId):
+        toAdvertise.add $attr.to(PeerId)
+
+    self.advertisementUpdated.clear()
+    for toAdv in toAdvertise:
+      await self.rdv.advertise(toAdv, self.timeToAdvertise)
+
+    await sleepAsync(self.timeToAdvertise) or self.advertisementUpdated.wait()
+
+proc new*(T: typedesc[RendezVousInterface],
+          rdv: RendezVous,
+          ttr: Duration = 1.minutes,
+          tta: Duration = MinimumDuration): RendezVousInterface =
+  T(rdv: rdv, timeToRequest: ttr, timeToAdvertise: tta)

libp2p/errors.nim

@@ -5,11 +5,20 @@ import chronos
 import chronicles
 import macros
 
-# could not figure how to make it with a simple template
-# sadly nim needs more love for hygenic templates
+type
+  # Base exception type for libp2p
+  LPError* = object of CatchableError
+
+func toException*(e: cstring): ref LPError =
+  (ref LPError)(msg: $e)
+
+func toException*(e: string): ref LPError =
+  (ref LPError)(msg: e)
+
+# TODO: could not figure how to make it with a simple template
+# sadly nim needs more love for hygienic templates
 # so here goes the macro, its based on the proc/template version
 # and uses quote do so it's quite readable
-
 macro checkFutures*[T](futs: seq[Future[T]], exclude: untyped = []): untyped =
   let nexclude = exclude.len
   case nexclude
@@ -19,8 +28,8 @@ macro checkFutures*[T](futs: seq[Future[T]], exclude: untyped = []): untyped =
         if res.failed:
           let exc = res.readError()
           # We still don't abort but warn
-          warn "A future has failed, enable trace logging for details", error=exc.name
-          trace "Exception message", msg=exc.msg
+          debug "A future has failed, enable trace logging for details", error = exc.name
+          trace "Exception message", msg= exc.msg, stack = getStackTrace()
   else:
     quote do:
       for res in `futs`:
@@ -32,7 +41,7 @@ macro checkFutures*[T](futs: seq[Future[T]], exclude: untyped = []): untyped =
                 trace "A future has failed", error=exc.name, msg=exc.msg
                 break check
             # We still don't abort but warn
-            warn "A future has failed, enable trace logging for details", error=exc.name
+            debug "A future has failed, enable trace logging for details", error=exc.name
             trace "Exception details", msg=exc.msg
 
 proc allFuturesThrowing*[T](args: varargs[Future[T]]): Future[void] =
@@ -40,7 +49,7 @@ proc allFuturesThrowing*[T](args: varargs[Future[T]]): Future[void] =
   for fut in args:
     futs &= fut
   proc call() {.async.} =
-    var first: ref Exception = nil
+    var first: ref CatchableError = nil
     futs = await allFinished(futs)
     for fut in futs:
       if fut.failed:
@@ -48,6 +57,8 @@ proc allFuturesThrowing*[T](args: varargs[Future[T]]): Future[void] =
         if err of Defect:
           raise err
         else:
+          if err of CancelledError:
+            raise err
           if isNil(first):
             first = err
     if not isNil(first):
@@ -61,5 +72,5 @@ template tryAndWarn*(message: static[string]; body: untyped): untyped =
   except CancelledError as exc:
     raise exc
   except CatchableError as exc:
-    warn "An exception has ocurred, enable trace logging for details", name = exc.name, msg = message
+    debug "An exception has ocurred, enable trace logging for details", name = exc.name, msg = message
     trace "Exception details", exc = exc.msg

libp2p/multiaddress.nim

@@ -1,23 +1,27 @@
-## Nim-Libp2p
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module implements MultiAddress.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+{.push public.}
 
-import nativesockets
-import tables, strutils, stew/shims/net
-import chronos
+import pkg/chronos
+import std/[nativesockets, hashes]
+import tables, strutils, sets, stew/shims/net
 import multicodec, multihash, multibase, transcoder, vbuffer, peerid,
-       protobuf/minprotobuf
+       protobuf/minprotobuf, errors, utility
 import stew/[base58, base32, endians2, results]
-export results, minprotobuf, vbuffer
+export results, minprotobuf, vbuffer, utility
 
 type
   MAKind* = enum
@@ -46,6 +50,9 @@ type
 
   MaResult*[T] = Result[T, string]
 
+  MaError* = object of LPError
+  MaInvalidAddress* = object of MaError
+
   IpTransportProtocol* = enum
     tcpProtocol
     udpProtocol
@@ -56,6 +63,12 @@ const
   IPPROTO_TCP = Protocol.IPPROTO_TCP
   IPPROTO_UDP = Protocol.IPPROTO_UDP
 
+proc hash*(a: MultiAddress): Hash =
+  var h: Hash = 0
+  h = h !& hash(a.data.buffer)
+  h = h !& hash(a.data.offset)
+  !$h
+
 proc ip4StB(s: string, vb: var VBuffer): bool =
   ## IPv4 stringToBuffer() implementation.
   try:
@@ -209,6 +222,40 @@ proc onionVB(vb: var VBuffer): bool =
   if vb.readArray(buf) == 12:
     result = true
 
+proc onion3StB(s: string, vb: var VBuffer): bool =
+  try:
+    var parts = s.split(':')
+    if len(parts) != 2:
+      return false
+    if len(parts[0]) != 56:
+      return false
+    var address = Base32Lower.decode(parts[0].toLowerAscii())
+    var nport = parseInt(parts[1])
+    if (nport > 0 and nport < 65536) and len(address) == 35:
+      address.setLen(37)
+      address[35] = cast[byte]((nport shr 8) and 0xFF)
+      address[36] = cast[byte](nport and 0xFF)
+      vb.writeArray(address)
+      result = true
+  except:
+    discard
+
+proc onion3BtS(vb: var VBuffer, s: var string): bool =
+  ## ONION address bufferToString() implementation.
+  var buf: array[37, byte]
+  if vb.readArray(buf) == 37:
+    var nport = (cast[uint16](buf[35]) shl 8) or cast[uint16](buf[36])
+    s = Base32Lower.encode(buf.toOpenArray(0, 34))
+    s.add(":")
+    s.add($nport)
+    result = true
+
+proc onion3VB(vb: var VBuffer): bool =
+  ## ONION address validateBuffer() implementation.
+  var buf: array[37, byte]
+  if vb.readArray(buf) == 37:
+    result = true
+
 proc unixStB(s: string, vb: var VBuffer): bool =
   ## Unix socket name stringToBuffer() implementation.
   if len(s) > 0:
@@ -297,6 +344,11 @@ const
     bufferToString: onionBtS,
     validateBuffer: onionVB
   )
+  TranscoderOnion3* = Transcoder(
+    stringToBuffer: onion3StB,
+    bufferToString: onion3BtS,
+    validateBuffer: onion3VB
+  )
   TranscoderDNS* = Transcoder(
     stringToBuffer: dnsStB,
     bufferToString: dnsBtS,
@@ -350,9 +402,16 @@ const
       mcodec: multiCodec("onion"), kind: Fixed, size: 10,
       coder: TranscoderOnion
     ),
+    MAProtocol(
+      mcodec: multiCodec("onion3"), kind: Fixed, size: 37,
+      coder: TranscoderOnion3
+    ),
     MAProtocol(
       mcodec: multiCodec("ws"), kind: Marker, size: 0
     ),
+    MAProtocol(
+      mcodec: multiCodec("wss"), kind: Marker, size: 0
+    ),
     MAProtocol(
       mcodec: multiCodec("ipfs"), kind: Length, size: 0,
       coder: TranscoderP2P
@@ -365,6 +424,10 @@ const
       mcodec: multiCodec("unix"), kind: Path, size: 0,
       coder: TranscoderUnix
     ),
+    MAProtocol(
+      mcodec: multiCodec("dns"), kind: Length, size: 0,
+      coder: TranscoderDNS
+    ),
     MAProtocol(
       mcodec: multiCodec("dns4"), kind: Length, size: 0,
       coder: TranscoderDNS
@@ -391,23 +454,30 @@ const
     )
   ]
 
+  DNSANY* = mapEq("dns")
   DNS4* = mapEq("dns4")
   DNS6* = mapEq("dns6")
+  DNSADDR* = mapEq("dnsaddr")
   IP4* = mapEq("ip4")
   IP6* = mapEq("ip6")
-  DNS* = mapOr(mapEq("dnsaddr"), DNS4, DNS6)
+  DNS* = mapOr(DNSANY, DNS4, DNS6, DNSADDR)
   IP* = mapOr(IP4, IP6)
   TCP* = mapOr(mapAnd(DNS, mapEq("tcp")), mapAnd(IP, mapEq("tcp")))
   UDP* = mapOr(mapAnd(DNS, mapEq("udp")), mapAnd(IP, mapEq("udp")))
   UTP* = mapAnd(UDP, mapEq("utp"))
   QUIC* = mapAnd(UDP, mapEq("quic"))
   UNIX* = mapEq("unix")
+  WS* = mapAnd(TCP, mapEq("ws"))
+  WSS* = mapAnd(TCP, mapEq("wss"))
+  WebSockets* = mapOr(WS, WSS)
 
   Unreliable* = mapOr(UDP)
 
-  Reliable* = mapOr(TCP, UTP, QUIC)
+  Reliable* = mapOr(TCP, UTP, QUIC, WebSockets)
 
-  IPFS* = mapAnd(Reliable, mapEq("p2p"))
+  P2PPattern* = mapEq("p2p")
+
+  IPFS* = mapAnd(Reliable, P2PPattern)
 
   HTTP* = mapOr(
     mapAnd(TCP, mapEq("http")),
@@ -426,9 +496,10 @@ const
     mapAnd(HTTPS, mapEq("p2p-webrtc-direct"))
   )
 
+  CircuitRelay* = mapEq("p2p-circuit")
+
 proc initMultiAddressCodeTable(): Table[MultiCodec,
                                         MAProtocol] {.compileTime.} =
-  result = initTable[MultiCodec, MAProtocol]()
   for item in ProtocolsList:
     result[item.mcodec] = item
 
@@ -448,7 +519,6 @@ proc trimRight(s: string, ch: char): string =
 proc shcopy*(m1: var MultiAddress, m2: MultiAddress) =
   shallowCopy(m1.data.buffer, m2.data.buffer)
   m1.data.offset = m2.data.offset
-  m1.data.length = m2.data.length
 
 proc protoCode*(ma: MultiAddress): MaResult[MultiCodec] =
   ## Returns MultiAddress ``ma`` protocol code.
@@ -479,7 +549,7 @@ proc protoName*(ma: MultiAddress): MaResult[string] =
       ok($(proto.mcodec))
 
 proc protoArgument*(ma: MultiAddress,
-                    value: var openarray[byte]): MaResult[int] =
+                    value: var openArray[byte]): MaResult[int] =
   ## Returns MultiAddress ``ma`` protocol argument value.
   ##
   ## If current MultiAddress do not have argument value, then result will be
@@ -503,7 +573,7 @@ proc protoArgument*(ma: MultiAddress,
           err("multiaddress: Decoding protocol error")
         else:
           ok(res)
-      elif proto.kind in {Length, Path}:
+      elif proto.kind in {MAKind.Length, Path}:
         if vb.data.readSeq(buffer) == -1:
           err("multiaddress: Decoding protocol error")
         else:
@@ -524,6 +594,13 @@ proc protoAddress*(ma: MultiAddress): MaResult[seq[byte]] =
   buffer.setLen(res)
   ok(buffer)
 
+proc protoArgument*(ma: MultiAddress): MaResult[seq[byte]] =
+  ## Returns MultiAddress ``ma`` protocol address binary blob.
+  ##
+  ## If current MultiAddress do not have argument value, then result array will
+  ## be empty.
+  ma.protoAddress()
+
 proc getPart(ma: MultiAddress, index: int): MaResult[MultiAddress] =
   var header: uint64
   var data = newSeq[byte]()
@@ -531,6 +608,9 @@ proc getPart(ma: MultiAddress, index: int): MaResult[MultiAddress] =
   var vb = ma
   var res: MultiAddress
   res.data = initVBuffer()
+
+  if index < 0: return err("multiaddress: negative index gived to getPart")
+
   while offset <= index:
     if vb.data.readVarint(header) == -1:
       return err("multiaddress: Malformed binary address!")
@@ -548,7 +628,7 @@ proc getPart(ma: MultiAddress, index: int): MaResult[MultiAddress] =
         res.data.writeVarint(header)
         res.data.writeArray(data)
         res.data.finish()
-    elif proto.kind in {Length, Path}:
+    elif proto.kind in {MAKind.Length, Path}:
       if vb.data.readSeq(data) == -1:
         return err("multiaddress: Decoding protocol error")
 
@@ -563,9 +643,31 @@ proc getPart(ma: MultiAddress, index: int): MaResult[MultiAddress] =
     inc(offset)
   ok(res)
 
-proc `[]`*(ma: MultiAddress, i: int): MaResult[MultiAddress] {.inline.} =
+proc getParts[U, V](ma: MultiAddress, slice: HSlice[U, V]): MaResult[MultiAddress] =
+  when slice.a is BackwardsIndex or slice.b is BackwardsIndex:
+    let maLength = ? len(ma)
+  template normalizeIndex(index): int =
+    when index is BackwardsIndex: maLength - int(index)
+    else: int(index)
+  let
+    indexStart = normalizeIndex(slice.a)
+    indexEnd = normalizeIndex(slice.b)
+  var res: MultiAddress
+  for i in indexStart..indexEnd:
+    ? res.append(? ma[i])
+  ok(res)
+
+proc `[]`*(ma: MultiAddress, i: int | BackwardsIndex): MaResult[MultiAddress] {.inline.} =
   ## Returns part with index ``i`` of MultiAddress ``ma``.
-  ma.getPart(i)
+  when i is BackwardsIndex:
+    let maLength = ? len(ma)
+    ma.getPart(maLength - int(i))
+  else:
+    ma.getPart(i)
+
+proc `[]`*(ma: MultiAddress, slice: HSlice): MaResult[MultiAddress] {.inline.} =
+  ## Returns parts with slice ``slice`` of MultiAddress ``ma``.
+  ma.getParts(slice)
 
 iterator items*(ma: MultiAddress): MaResult[MultiAddress] =
   ## Iterates over all addresses inside of MultiAddress ``ma``.
@@ -592,7 +694,7 @@ iterator items*(ma: MultiAddress): MaResult[MultiAddress] =
 
       res.data.writeVarint(header)
       res.data.writeArray(data)
-    elif proto.kind in {Length, Path}:
+    elif proto.kind in {MAKind.Length, Path}:
       if vb.data.readSeq(data) == -1:
         yield err(MaResult[MultiAddress], "Decoding protocol error")
 
@@ -603,6 +705,13 @@ iterator items*(ma: MultiAddress): MaResult[MultiAddress] =
     res.data.finish()
     yield ok(MaResult[MultiAddress], res)
 
+proc len*(ma: MultiAddress): MaResult[int] =
+  var counter: int
+  for part in ma:
+    if part.isErr: return err(part.error)
+    counter.inc()
+  ok(counter)
+
 proc contains*(ma: MultiAddress, codec: MultiCodec): MaResult[bool] {.inline.} =
   ## Returns ``true``, if address with MultiCodec ``codec`` present in
   ## MultiAddress ``ma``.
@@ -704,7 +813,7 @@ proc validate*(ma: MultiAddress): bool =
 
 proc init*(
     mtype: typedesc[MultiAddress], protocol: MultiCodec,
-    value: openarray[byte] = []): MaResult[MultiAddress] =
+    value: openArray[byte] = []): MaResult[MultiAddress] =
   ## Initialize MultiAddress object from protocol id ``protocol`` and array
   ## of bytes ``value``.
   let proto = CodeAddresses.getOrDefault(protocol)
@@ -735,7 +844,7 @@ proc init*(
       raiseAssert "None checked above"
 
 proc init*(mtype: typedesc[MultiAddress], protocol: MultiCodec,
-           value: PeerID): MaResult[MultiAddress] {.inline.} =
+           value: PeerId): MaResult[MultiAddress] {.inline.} =
   ## Initialize MultiAddress object from protocol id ``protocol`` and peer id
   ## ``value``.
   init(mtype, protocol, value.data)
@@ -813,7 +922,7 @@ proc init*(mtype: typedesc[MultiAddress],
     ok(res)
 
 proc init*(mtype: typedesc[MultiAddress],
-           data: openarray[byte]): MaResult[MultiAddress] =
+           data: openArray[byte]): MaResult[MultiAddress] =
   ## Initialize MultiAddress with array of bytes ``data``.
   if len(data) == 0:
     err("multiaddress: Address could not be empty!")
@@ -831,31 +940,27 @@ proc init*(mtype: typedesc[MultiAddress]): MultiAddress =
   ## Initialize empty MultiAddress.
   result.data = initVBuffer()
 
-proc init*(mtype: typedesc[MultiAddress],
-           address: ValidIpAddress,
-           protocol: IpTransportProtocol,
-           port: Port): MultiAddress =
+proc init*(mtype: typedesc[MultiAddress], address: ValidIpAddress,
+           protocol: IpTransportProtocol, port: Port): MultiAddress =
+  var res: MultiAddress
+  res.data = initVBuffer()
   let
-    familyProto = case address.family
+    networkProto = case address.family
       of IpAddressFamily.IPv4: getProtocol("ip4")
       of IpAddressFamily.IPv6: getProtocol("ip6")
 
-    protoProto = case protocol
+    transportProto = case protocol
       of tcpProtocol: getProtocol("tcp")
       of udpProtocol: getProtocol("udp")
 
-  var data = initVBuffer()
-  data.write(familyProto.mcodec)
-  var written = familyProto.coder.stringToBuffer($address, data)
-  doAssert written,
-           "Merely writing a string to a buffer should always be possible"
-  data.write(protoProto.mcodec)
-  written = protoProto.coder.stringToBuffer($port, data)
-  doAssert written,
-           "Merely writing a string to a buffer should always be possible"
-  data.finish()
-
-  MultiAddress(data: data)
+  res.data.write(networkProto.mcodec)
+  case address.family
+    of IpAddressFamily.IPv4: res.data.writeArray(address.address_v4)
+    of IpAddressFamily.IPv6: res.data.writeArray(address.address_v6)
+  res.data.write(transportProto.mcodec)
+  res.data.writeArray(toBytesBE(uint16(port)))
+  res.data.finish()
+  res
 
 proc init*(mtype: typedesc[MultiAddress], address: TransportAddress,
            protocol = IPPROTO_TCP): MaResult[MultiAddress] =
@@ -866,7 +971,9 @@ proc init*(mtype: typedesc[MultiAddress], address: TransportAddress,
   let protoProto = case protocol
                    of IPPROTO_TCP: getProtocol("tcp")
                    of IPPROTO_UDP: getProtocol("udp")
-                   else: return err("multiaddress: protocol should be either TCP or UDP")
+                   else: default(MAProtocol)
+  if protoProto.size == 0:
+    return err("multiaddress: protocol should be either TCP or UDP")
   if address.family == AddressFamily.IPv4:
     res.data.write(getProtocol("ip4").mcodec)
     res.data.writeArray(address.address_v4)
@@ -904,77 +1011,41 @@ proc append*(m1: var MultiAddress, m2: MultiAddress): MaResult[void] =
     ok()
 
 proc `&`*(m1, m2: MultiAddress): MultiAddress {.
-     raises: [Defect, ResultError[string]].} =
+     raises: [Defect, LPError].} =
   ## Concatenates two addresses ``m1`` and ``m2``, and returns result.
   ##
   ## This procedure performs validation of concatenated result and can raise
   ## exception on error.
+  ##
+
   concat(m1, m2).tryGet()
 
 proc `&=`*(m1: var MultiAddress, m2: MultiAddress) {.
-     raises: [Defect, ResultError[string]].} =
+     raises: [Defect, LPError].} =
   ## Concatenates two addresses ``m1`` and ``m2``.
   ##
   ## This procedure performs validation of concatenated result and can raise
   ## exception on error.
   ##
+
   m1.append(m2).tryGet()
 
-proc isWire*(ma: MultiAddress): bool =
-  ## Returns ``true`` if MultiAddress ``ma`` is one of:
-  ## - {IP4}/{TCP, UDP}
-  ## - {IP6}/{TCP, UDP}
-  ## - {UNIX}/{PATH}
-  var
-    state = 0
-  try:
-    for rpart in ma.items():
-      if rpart.isErr():
-        return false
-      let part = rpart.get()
-
-      if state == 0:
-        let rcode = part.protoCode()
-        if rcode.isErr():
-          return false
-        let code = rcode.get()
-
-        if code == multiCodec("ip4") or code == multiCodec("ip6"):
-          inc(state)
-          continue
-        elif code == multiCodec("unix"):
-          result = true
-          break
-        else:
-          result = false
-          break
-      elif state == 1:
-        let rcode = part.protoCode()
-        if rcode.isErr():
-          return false
-        let code = rcode.get()
-
-        if code == multiCodec("tcp") or code == multiCodec("udp"):
-          inc(state)
-          result = true
-        else:
-          result = false
-          break
-      else:
-        result = false
-        break
-  except:
-    result = false
+proc `==`*(m1: var MultiAddress, m2: MultiAddress): bool =
+  ## Check of two MultiAddress are equal
+  m1.data == m2.data
 
 proc matchPart(pat: MaPattern, protos: seq[MultiCodec]): MaPatResult =
   var empty: seq[MultiCodec]
   var pcs = protos
   if pat.operator == Or:
+    result = MaPatResult(flag: false, rem: empty)
     for a in pat.args:
       let res = a.matchPart(pcs)
       if res.flag:
-        return MaPatResult(flag: true, rem: res.rem)
-    result = MaPatResult(flag: false, rem: empty)
+        #Greedy Or
+        if result.flag == false or
+             result.rem.len > res.rem.len:
+          result = res
   elif pat.operator == And:
     if len(pcs) < len(pat.args):
       return MaPatResult(flag: false, rem: empty)
@@ -1024,32 +1095,35 @@ proc `$`*(pat: MaPattern): string =
 proc write*(pb: var ProtoBuffer, field: int, value: MultiAddress) {.inline.} =
   write(pb, field, value.data.buffer)
 
-proc getField*(pb: var ProtoBuffer, field: int,
-               value: var MultiAddress): bool {.inline.} =
+proc getField*(pb: ProtoBuffer, field: int,
+               value: var MultiAddress): ProtoResult[bool] {.
+     inline.} =
   var buffer: seq[byte]
-  if not(getField(pb, field, buffer)):
-    return false
-  if len(buffer) == 0:
-    return false
-  let ma = MultiAddress.init(buffer)
-  if ma.isOk():
-    value = ma.get()
-    true
+  let res = ? pb.getField(field, buffer)
+  if not(res):
+    ok(false)
   else:
-    false
+    let ma = MultiAddress.init(buffer)
+    if ma.isOk():
+      value = ma.get()
+      ok(true)
+    else:
+      err(ProtoError.IncorrectBlob)
 
-proc getRepeatedField*(pb: var ProtoBuffer, field: int,
-                       value: var seq[MultiAddress]): bool {.inline.} =
+proc getRepeatedField*(pb: ProtoBuffer, field: int,
+                       value: var seq[MultiAddress]): ProtoResult[bool] {.
+     inline.} =
   var items: seq[seq[byte]]
   value.setLen(0)
-  if not(getRepeatedField(pb, field, items)):
-    return false
-  if len(items) == 0:
-    return true
-  for item in items:
-    let ma = MultiAddress.init(item)
-    if ma.isOk():
-      value.add(ma.get())
-    else:
-      value.setLen(0)
-      return false
+  let res = ? pb.getRepeatedField(field, items)
+  if not(res):
+    ok(false)
+  else:
+    for item in items:
+      let ma = MultiAddress.init(item)
+      if ma.isOk():
+        value.add(ma.get())
+      else:
+        value.setLen(0)
+        return err(ProtoError.IncorrectBlob)
+    ok(true)

libp2p/multibase.nim

@@ -1,11 +1,11 @@
-## Nim-Libp2p
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module implements MultiBase.
 ##
@@ -13,13 +13,16 @@
 ## 1. base32z
 ##
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 import tables
 import stew/[base32, base58, base64, results]
 
 type
-  MultibaseStatus* {.pure.} = enum
+  MultiBaseStatus* {.pure.} = enum
     Error, Success, Overrun, Incorrect, BadCodec, NotSupported
 
   MultiBase* = object
@@ -29,169 +32,169 @@ type
   MBCodec = object
     code: char
     name: string
-    encr: proc(inbytes: openarray[byte],
-               outbytes: var openarray[char],
-               outlen: var int): MultibaseStatus {.nimcall, gcsafe, noSideEffect, raises: [Defect].}
-    decr: proc(inbytes: openarray[char],
-               outbytes: var openarray[byte],
-               outlen: var int): MultibaseStatus {.nimcall, gcsafe, noSideEffect, raises: [Defect].}
+    encr: proc(inbytes: openArray[byte],
+               outbytes: var openArray[char],
+               outlen: var int): MultiBaseStatus {.nimcall, gcsafe, noSideEffect, raises: [Defect].}
+    decr: proc(inbytes: openArray[char],
+               outbytes: var openArray[byte],
+               outlen: var int): MultiBaseStatus {.nimcall, gcsafe, noSideEffect, raises: [Defect].}
     encl: MBCodeSize
     decl: MBCodeSize
 
-proc idd(inbytes: openarray[char], outbytes: var openarray[byte],
-         outlen: var int): MultibaseStatus =
+proc idd(inbytes: openArray[char], outbytes: var openArray[byte],
+         outlen: var int): MultiBaseStatus =
   let length = len(inbytes)
   if length > len(outbytes):
     outlen = length
-    result = MultibaseStatus.Overrun
+    result = MultiBaseStatus.Overrun
   else:
     copyMem(addr outbytes[0], unsafeAddr inbytes[0], length)
     outlen = length
-    result = MultibaseStatus.Success
+    result = MultiBaseStatus.Success
 
-proc ide(inbytes: openarray[byte],
-         outbytes: var openarray[char],
-         outlen: var int): MultibaseStatus =
+proc ide(inbytes: openArray[byte],
+         outbytes: var openArray[char],
+         outlen: var int): MultiBaseStatus =
   let length = len(inbytes)
   if length > len(outbytes):
     outlen = length
-    result = MultibaseStatus.Overrun
+    result = MultiBaseStatus.Overrun
   else:
     copyMem(addr outbytes[0], unsafeAddr inbytes[0], length)
     outlen = length
-    result = MultibaseStatus.Success
+    result = MultiBaseStatus.Success
 
 proc idel(length: int): int = length
 proc iddl(length: int): int = length
 
-proc b16d(inbytes: openarray[char],
-          outbytes: var openarray[byte],
-          outlen: var int): MultibaseStatus =
+proc b16d(inbytes: openArray[char],
+          outbytes: var openArray[byte],
+          outlen: var int): MultiBaseStatus =
   discard
 
-proc b16e(inbytes: openarray[byte],
-          outbytes: var openarray[char],
-          outlen: var int): MultibaseStatus =
+proc b16e(inbytes: openArray[byte],
+          outbytes: var openArray[char],
+          outlen: var int): MultiBaseStatus =
   discard
 
-proc b16ud(inbytes: openarray[char],
-           outbytes: var openarray[byte],
-           outlen: var int): MultibaseStatus =
+proc b16ud(inbytes: openArray[char],
+           outbytes: var openArray[byte],
+           outlen: var int): MultiBaseStatus =
   discard
 
-proc b16ue(inbytes: openarray[byte],
-           outbytes: var openarray[char],
-           outlen: var int): MultibaseStatus =
+proc b16ue(inbytes: openArray[byte],
+           outbytes: var openArray[char],
+           outlen: var int): MultiBaseStatus =
   discard
 
 proc b16el(length: int): int = length shl 1
 proc b16dl(length: int): int = (length + 1) div 2
 
-proc b32ce(r: Base32Status): MultibaseStatus {.inline.} =
-  result = MultibaseStatus.Error
+proc b32ce(r: Base32Status): MultiBaseStatus {.inline.} =
+  result = MultiBaseStatus.Error
   if r == Base32Status.Incorrect:
-    result = MultibaseStatus.Incorrect
+    result = MultiBaseStatus.Incorrect
   elif r == Base32Status.Overrun:
-    result = MultibaseStatus.Overrun
+    result = MultiBaseStatus.Overrun
   elif r == Base32Status.Success:
-    result = MultibaseStatus.Success
+    result = MultiBaseStatus.Success
 
-proc b58ce(r: Base58Status): MultibaseStatus {.inline.} =
-  result = MultibaseStatus.Error
+proc b58ce(r: Base58Status): MultiBaseStatus {.inline.} =
+  result = MultiBaseStatus.Error
   if r == Base58Status.Incorrect:
-    result = MultibaseStatus.Incorrect
+    result = MultiBaseStatus.Incorrect
   elif r == Base58Status.Overrun:
-    result = MultibaseStatus.Overrun
+    result = MultiBaseStatus.Overrun
   elif r == Base58Status.Success:
-    result = MultibaseStatus.Success
+    result = MultiBaseStatus.Success
 
-proc b64ce(r: Base64Status): MultibaseStatus {.inline.} =
+proc b64ce(r: Base64Status): MultiBaseStatus {.inline.} =
   result = MultiBaseStatus.Error
   if r == Base64Status.Incorrect:
-    result = MultibaseStatus.Incorrect
+    result = MultiBaseStatus.Incorrect
   elif r == Base64Status.Overrun:
     result = MultiBaseStatus.Overrun
   elif r == Base64Status.Success:
-    result = MultibaseStatus.Success
+    result = MultiBaseStatus.Success
 
-proc b32hd(inbytes: openarray[char],
-           outbytes: var openarray[byte],
-           outlen: var int): MultibaseStatus =
+proc b32hd(inbytes: openArray[char],
+           outbytes: var openArray[byte],
+           outlen: var int): MultiBaseStatus =
   result = b32ce(HexBase32Lower.decode(inbytes, outbytes, outlen))
 
-proc b32he(inbytes: openarray[byte],
-           outbytes: var openarray[char],
-           outlen: var int): MultibaseStatus =
+proc b32he(inbytes: openArray[byte],
+           outbytes: var openArray[char],
+           outlen: var int): MultiBaseStatus =
   result = b32ce(HexBase32Lower.encode(inbytes, outbytes, outlen))
 
-proc b32hud(inbytes: openarray[char],
-            outbytes: var openarray[byte],
-            outlen: var int): MultibaseStatus =
+proc b32hud(inbytes: openArray[char],
+            outbytes: var openArray[byte],
+            outlen: var int): MultiBaseStatus =
   result = b32ce(HexBase32Upper.decode(inbytes, outbytes, outlen))
 
-proc b32hue(inbytes: openarray[byte],
-            outbytes: var openarray[char],
-            outlen: var int): MultibaseStatus =
+proc b32hue(inbytes: openArray[byte],
+            outbytes: var openArray[char],
+            outlen: var int): MultiBaseStatus =
   result = b32ce(HexBase32Upper.encode(inbytes, outbytes, outlen))
 
-proc b32hpd(inbytes: openarray[char],
-            outbytes: var openarray[byte],
-            outlen: var int): MultibaseStatus =
+proc b32hpd(inbytes: openArray[char],
+            outbytes: var openArray[byte],
+            outlen: var int): MultiBaseStatus =
   result = b32ce(HexBase32LowerPad.decode(inbytes, outbytes, outlen))
 
-proc b32hpe(inbytes: openarray[byte],
-            outbytes: var openarray[char],
-            outlen: var int): MultibaseStatus =
+proc b32hpe(inbytes: openArray[byte],
+            outbytes: var openArray[char],
+            outlen: var int): MultiBaseStatus =
   result = b32ce(HexBase32LowerPad.encode(inbytes, outbytes, outlen))
 
-proc b32hpud(inbytes: openarray[char],
-             outbytes: var openarray[byte],
-             outlen: var int): MultibaseStatus =
+proc b32hpud(inbytes: openArray[char],
+             outbytes: var openArray[byte],
+             outlen: var int): MultiBaseStatus =
   result = b32ce(HexBase32UpperPad.decode(inbytes, outbytes, outlen))
 
-proc b32hpue(inbytes: openarray[byte],
-             outbytes: var openarray[char],
-             outlen: var int): MultibaseStatus =
+proc b32hpue(inbytes: openArray[byte],
+             outbytes: var openArray[char],
+             outlen: var int): MultiBaseStatus =
   result = b32ce(HexBase32UpperPad.encode(inbytes, outbytes, outlen))
 
-proc b32d(inbytes: openarray[char],
-          outbytes: var openarray[byte],
-          outlen: var int): MultibaseStatus =
+proc b32d(inbytes: openArray[char],
+          outbytes: var openArray[byte],
+          outlen: var int): MultiBaseStatus =
   result = b32ce(Base32Lower.decode(inbytes, outbytes, outlen))
 
-proc b32e(inbytes: openarray[byte],
-          outbytes: var openarray[char],
-          outlen: var int): MultibaseStatus =
+proc b32e(inbytes: openArray[byte],
+          outbytes: var openArray[char],
+          outlen: var int): MultiBaseStatus =
   result = b32ce(Base32Lower.encode(inbytes, outbytes, outlen))
 
-proc b32ud(inbytes: openarray[char],
-           outbytes: var openarray[byte],
-           outlen: var int): MultibaseStatus =
+proc b32ud(inbytes: openArray[char],
+           outbytes: var openArray[byte],
+           outlen: var int): MultiBaseStatus =
   result = b32ce(Base32Upper.decode(inbytes, outbytes, outlen))
 
-proc b32ue(inbytes: openarray[byte],
-           outbytes: var openarray[char],
-           outlen: var int): MultibaseStatus =
+proc b32ue(inbytes: openArray[byte],
+           outbytes: var openArray[char],
+           outlen: var int): MultiBaseStatus =
   result = b32ce(Base32Upper.encode(inbytes, outbytes, outlen))
 
-proc b32pd(inbytes: openarray[char],
-           outbytes: var openarray[byte],
-           outlen: var int): MultibaseStatus =
+proc b32pd(inbytes: openArray[char],
+           outbytes: var openArray[byte],
+           outlen: var int): MultiBaseStatus =
   result = b32ce(Base32LowerPad.decode(inbytes, outbytes, outlen))
 
-proc b32pe(inbytes: openarray[byte],
-           outbytes: var openarray[char],
-           outlen: var int): MultibaseStatus =
+proc b32pe(inbytes: openArray[byte],
+           outbytes: var openArray[char],
+           outlen: var int): MultiBaseStatus =
   result = b32ce(Base32LowerPad.encode(inbytes, outbytes, outlen))
 
-proc b32pud(inbytes: openarray[char],
-            outbytes: var openarray[byte],
-            outlen: var int): MultibaseStatus =
+proc b32pud(inbytes: openArray[char],
+            outbytes: var openArray[byte],
+            outlen: var int): MultiBaseStatus =
   result = b32ce(Base32UpperPad.decode(inbytes, outbytes, outlen))
 
-proc b32pue(inbytes: openarray[byte],
-            outbytes: var openarray[char],
-            outlen: var int): MultibaseStatus =
+proc b32pue(inbytes: openArray[byte],
+            outbytes: var openArray[char],
+            outlen: var int): MultiBaseStatus =
   result = b32ce(Base32UpperPad.encode(inbytes, outbytes, outlen))
 
 proc b32el(length: int): int = Base32Lower.encodedLength(length)
@@ -199,24 +202,24 @@ proc b32dl(length: int): int = Base32Lower.decodedLength(length)
 proc b32pel(length: int): int = Base32LowerPad.encodedLength(length)
 proc b32pdl(length: int): int = Base32LowerPad.decodedLength(length)
 
-proc b58fd(inbytes: openarray[char],
-           outbytes: var openarray[byte],
-           outlen: var int): MultibaseStatus =
+proc b58fd(inbytes: openArray[char],
+           outbytes: var openArray[byte],
+           outlen: var int): MultiBaseStatus =
   result = b58ce(FLCBase58.decode(inbytes, outbytes, outlen))
 
-proc b58fe(inbytes: openarray[byte],
-           outbytes: var openarray[char],
-           outlen: var int): MultibaseStatus =
+proc b58fe(inbytes: openArray[byte],
+           outbytes: var openArray[char],
+           outlen: var int): MultiBaseStatus =
   result = b58ce(FLCBase58.encode(inbytes, outbytes, outlen))
 
-proc b58bd(inbytes: openarray[char],
-           outbytes: var openarray[byte],
-           outlen: var int): MultibaseStatus =
+proc b58bd(inbytes: openArray[char],
+           outbytes: var openArray[byte],
+           outlen: var int): MultiBaseStatus =
   result = b58ce(BTCBase58.decode(inbytes, outbytes, outlen))
 
-proc b58be(inbytes: openarray[byte],
-           outbytes: var openarray[char],
-           outlen: var int): MultibaseStatus =
+proc b58be(inbytes: openArray[byte],
+           outbytes: var openArray[char],
+           outlen: var int): MultiBaseStatus =
   result = b58ce(BTCBase58.encode(inbytes, outbytes, outlen))
 
 proc b58el(length: int): int = Base58.encodedLength(length)
@@ -227,48 +230,48 @@ proc b64dl(length: int): int = Base64.decodedLength(length)
 proc b64pel(length: int): int = Base64Pad.encodedLength(length)
 proc b64pdl(length: int): int = Base64Pad.decodedLength(length)
 
-proc b64e(inbytes: openarray[byte],
-          outbytes: var openarray[char],
-          outlen: var int): MultibaseStatus =
+proc b64e(inbytes: openArray[byte],
+          outbytes: var openArray[char],
+          outlen: var int): MultiBaseStatus =
   result = b64ce(Base64.encode(inbytes, outbytes, outlen))
 
-proc b64d(inbytes: openarray[char],
-          outbytes: var openarray[byte],
-          outlen: var int): MultibaseStatus =
+proc b64d(inbytes: openArray[char],
+          outbytes: var openArray[byte],
+          outlen: var int): MultiBaseStatus =
   result = b64ce(Base64.decode(inbytes, outbytes, outlen))
 
-proc b64pe(inbytes: openarray[byte],
-           outbytes: var openarray[char],
-           outlen: var int): MultibaseStatus =
+proc b64pe(inbytes: openArray[byte],
+           outbytes: var openArray[char],
+           outlen: var int): MultiBaseStatus =
   result = b64ce(Base64Pad.encode(inbytes, outbytes, outlen))
 
-proc b64pd(inbytes: openarray[char],
-           outbytes: var openarray[byte],
-           outlen: var int): MultibaseStatus =
+proc b64pd(inbytes: openArray[char],
+           outbytes: var openArray[byte],
+           outlen: var int): MultiBaseStatus =
   result = b64ce(Base64Pad.decode(inbytes, outbytes, outlen))
 
-proc b64ue(inbytes: openarray[byte],
-           outbytes: var openarray[char],
-           outlen: var int): MultibaseStatus =
+proc b64ue(inbytes: openArray[byte],
+           outbytes: var openArray[char],
+           outlen: var int): MultiBaseStatus =
   result = b64ce(Base64Url.encode(inbytes, outbytes, outlen))
 
-proc b64ud(inbytes: openarray[char],
-           outbytes: var openarray[byte],
-           outlen: var int): MultibaseStatus =
+proc b64ud(inbytes: openArray[char],
+           outbytes: var openArray[byte],
+           outlen: var int): MultiBaseStatus =
   result = b64ce(Base64Url.decode(inbytes, outbytes, outlen))
 
-proc b64upe(inbytes: openarray[byte],
-          outbytes: var openarray[char],
-          outlen: var int): MultibaseStatus =
+proc b64upe(inbytes: openArray[byte],
+          outbytes: var openArray[char],
+          outlen: var int): MultiBaseStatus =
   result = b64ce(Base64UrlPad.encode(inbytes, outbytes, outlen))
 
-proc b64upd(inbytes: openarray[char],
-            outbytes: var openarray[byte],
-            outlen: var int): MultibaseStatus =
+proc b64upd(inbytes: openArray[char],
+            outbytes: var openArray[byte],
+            outlen: var int): MultiBaseStatus =
   result = b64ce(Base64UrlPad.decode(inbytes, outbytes, outlen))
 
 const
-  MultibaseCodecs = [
+  MultiBaseCodecs = [
     MBCodec(name: "identity", code: chr(0x00),
       decr: idd, encr: ide, decl: iddl, encl: idel
     ),
@@ -328,18 +331,16 @@ const
   ]
 
 proc initMultiBaseCodeTable(): Table[char, MBCodec] {.compileTime.} =
-  result = initTable[char, MBCodec]()
-  for item in MultibaseCodecs:
+  for item in MultiBaseCodecs:
     result[item.code] = item
 
 proc initMultiBaseNameTable(): Table[string, MBCodec] {.compileTime.} =
-  result = initTable[string, MBCodec]()
-  for item in MultibaseCodecs:
+  for item in MultiBaseCodecs:
     result[item.name] = item
 
 const
-  CodeMultibases = initMultiBaseCodeTable()
-  NameMultibases = initMultiBaseNameTable()
+  CodeMultiBases = initMultiBaseCodeTable()
+  NameMultiBases = initMultiBaseNameTable()
 
 proc encodedLength*(mbtype: typedesc[MultiBase], encoding: string,
                     length: int): int =
@@ -348,7 +349,7 @@ proc encodedLength*(mbtype: typedesc[MultiBase], encoding: string,
   ##
   ## Procedure returns ``-1`` if ``encoding`` scheme is not supported or
   ## not present.
-  let mb = NameMultibases.getOrDefault(encoding)
+  let mb = NameMultiBases.getOrDefault(encoding)
   if len(mb.name) == 0 or isNil(mb.encl):
     result = -1
   else:
@@ -361,7 +362,7 @@ proc decodedLength*(mbtype: typedesc[MultiBase], encoding: char,
                     length: int): int =
   ## Return estimated size of buffer to store MultiBase decoded value with
   ## encoding character ``encoding`` of length ``length``.
-  let mb = CodeMultibases.getOrDefault(encoding)
+  let mb = CodeMultiBases.getOrDefault(encoding)
   if len(mb.name) == 0 or isNil(mb.decl) or length == 0:
     result = -1
   else:
@@ -371,8 +372,8 @@ proc decodedLength*(mbtype: typedesc[MultiBase], encoding: char,
       result = mb.decl(length - 1)
 
 proc encode*(mbtype: typedesc[MultiBase], encoding: string,
-             inbytes: openarray[byte], outbytes: var openarray[char],
-             outlen: var int): MultibaseStatus =
+             inbytes: openArray[byte], outbytes: var openArray[char],
+             outlen: var int): MultiBaseStatus =
   ## Encode array ``inbytes`` using MultiBase encoding scheme ``encoding`` and
   ## store encoded value to ``outbytes``.
   ##
@@ -388,11 +389,11 @@ proc encode*(mbtype: typedesc[MultiBase], encoding: string,
   ##
   ## On successfull encoding ``MultiBaseStatus.Success`` will be returned and
   ## ``outlen`` will be set to number of encoded octets (bytes).
-  let mb = NameMultibases.getOrDefault(encoding)
+  let mb = NameMultiBases.getOrDefault(encoding)
   if len(mb.name) == 0:
-    return MultibaseStatus.BadCodec
+    return MultiBaseStatus.BadCodec
   if isNil(mb.encr) or isNil(mb.encl):
-    return MultibaseStatus.NotSupported
+    return MultiBaseStatus.NotSupported
   if len(outbytes) > 1:
     result = mb.encr(inbytes, outbytes.toOpenArray(1, outbytes.high),
                      outlen)
@@ -410,8 +411,8 @@ proc encode*(mbtype: typedesc[MultiBase], encoding: string,
       result = MultiBaseStatus.Overrun
       outlen = mb.encl(len(inbytes)) + 1
 
-proc decode*(mbtype: typedesc[MultiBase], inbytes: openarray[char],
-             outbytes: var openarray[byte], outlen: var int): MultibaseStatus =
+proc decode*(mbtype: typedesc[MultiBase], inbytes: openArray[char],
+             outbytes: var openArray[byte], outlen: var int): MultiBaseStatus =
   ## Decode array ``inbytes`` using MultiBase encoding and store decoded value
   ## to ``outbytes``.
   ##
@@ -428,24 +429,24 @@ proc decode*(mbtype: typedesc[MultiBase], inbytes: openarray[char],
   ## ``outlen`` will be set to number of encoded octets (bytes).
   let length = len(inbytes)
   if length == 0:
-    return MultibaseStatus.Incorrect
-  let mb = CodeMultibases.getOrDefault(inbytes[0])
+    return MultiBaseStatus.Incorrect
+  let mb = CodeMultiBases.getOrDefault(inbytes[0])
   if len(mb.name) == 0:
-    return MultibaseStatus.BadCodec
+    return MultiBaseStatus.BadCodec
   if isNil(mb.decr) or isNil(mb.decl):
-    return MultibaseStatus.NotSupported
+    return MultiBaseStatus.NotSupported
   if length == 1:
     outlen = 0
-    result = MultibaseStatus.Success
+    result = MultiBaseStatus.Success
   else:
     result = mb.decr(inbytes.toOpenArray(1, length - 1), outbytes, outlen)
 
 proc encode*(mbtype: typedesc[MultiBase], encoding: string,
-             inbytes: openarray[byte]): Result[string, string] =
+             inbytes: openArray[byte]): Result[string, string] =
   ## Encode array ``inbytes`` using MultiBase encoding scheme ``encoding`` and
   ## return encoded string.
   let length = len(inbytes)
-  let mb = NameMultibases.getOrDefault(encoding)
+  let mb = NameMultiBases.getOrDefault(encoding)
   if len(mb.name) == 0:
     return err("multibase: Encoding scheme is incorrect!")
   if isNil(mb.encr) or isNil(mb.encl):
@@ -464,13 +465,13 @@ proc encode*(mbtype: typedesc[MultiBase], encoding: string,
     buffer[0] = mb.code
   ok(buffer)
 
-proc decode*(mbtype: typedesc[MultiBase], inbytes: openarray[char]): Result[seq[byte], string] =
+proc decode*(mbtype: typedesc[MultiBase], inbytes: openArray[char]): Result[seq[byte], string] =
   ## Decode MultiBase encoded array ``inbytes`` and return decoded sequence of
   ## bytes.
   let length = len(inbytes)
   if length == 0:
     return err("multibase: Could not decode zero-length string")
-  let mb = CodeMultibases.getOrDefault(inbytes[0])
+  let mb = CodeMultiBases.getOrDefault(inbytes[0])
   if len(mb.name) == 0:
     return err("multibase: MultiBase scheme is incorrect!")
   if isNil(mb.decr) or isNil(mb.decl):

libp2p/multicodec.nim

@@ -1,15 +1,18 @@
-## Nim-Libp2p
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not BE copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not BE copied, modified, or distributed except according to
+# those terms.
 
 ## This module implements MultiCodec.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 import tables, hashes
 import varint, vbuffer
@@ -200,7 +203,10 @@ const MultiCodecList = [
   ("p2p-webrtc-star", 0x0113), # not in multicodec list
   ("p2p-webrtc-direct", 0x0114), # not in multicodec list
   ("onion", 0x01BC),
+  ("onion3", 0x01BD),
   ("p2p-circuit", 0x0122),
+  ("libp2p-peer-record", 0x0301),
+  ("dns", 0x35),
   ("dns4", 0x36),
   ("dns6", 0x37),
   ("dnsaddr", 0x38),
@@ -242,12 +248,10 @@ const
   InvalidMultiCodec* = MultiCodec(-1)
 
 proc initMultiCodecNameTable(): Table[string, int] {.compileTime.} =
-  result = initTable[string, int]()
   for item in MultiCodecList:
     result[item[0]] = item[1]
 
 proc initMultiCodecCodeTable(): Table[int, string] {.compileTime.} =
-  result = initTable[int, string]()
   for item in MultiCodecList:
     result[item[1]] = item[0]
 
@@ -271,7 +275,7 @@ proc `$`*(mc: MultiCodec): string =
   ## Returns string representation of MultiCodec ``mc``.
   let name = CodeCodecs.getOrDefault(int(mc), "")
   doAssert(name != "")
-  name  
+  name
 
 proc `==`*(mc: MultiCodec, name: string): bool {.inline.} =
   ## Compares MultiCodec ``mc`` with string ``name``.

libp2p/multihash.nim

@@ -1,11 +1,11 @@
-## Nim-Libp2p
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module implements MultiHash.
 ## Supported hashes are:
@@ -21,7 +21,10 @@
 ## 1. SKEIN
 ## 2. MURMUR
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 import tables
 import nimcrypto/[sha, sha2, keccak, blake2, hash, utils]
@@ -41,8 +44,8 @@ const
   ErrParseError = "Parse error fromHex"
 
 type
-  MHashCoderProc* = proc(data: openarray[byte],
-                         output: var openarray[byte]) {.nimcall, gcsafe, noSideEffect, raises: [Defect].}
+  MHashCoderProc* = proc(data: openArray[byte],
+                         output: var openArray[byte]) {.nimcall, gcsafe, noSideEffect, raises: [Defect].}
   MHash* = object
     mcodec*: MultiCodec
     size*: int
@@ -56,20 +59,20 @@ type
 
   MhResult*[T] = Result[T, cstring]
 
-proc identhash(data: openarray[byte], output: var openarray[byte]) =
+proc identhash(data: openArray[byte], output: var openArray[byte]) =
   if len(output) > 0:
     var length = if len(data) > len(output): len(output)
                  else: len(data)
     copyMem(addr output[0], unsafeAddr data[0], length)
 
-proc sha1hash(data: openarray[byte], output: var openarray[byte]) =
+proc sha1hash(data: openArray[byte], output: var openArray[byte]) =
   if len(output) > 0:
     var digest = sha1.digest(data)
     var length = if sha1.sizeDigest > len(output): len(output)
                  else: sha1.sizeDigest
     copyMem(addr output[0], addr digest.data[0], length)
 
-proc dblsha2_256hash(data: openarray[byte], output: var openarray[byte]) =
+proc dblsha2_256hash(data: openArray[byte], output: var openArray[byte]) =
   if len(output) > 0:
     var digest1 = sha256.digest(data)
     var digest2 = sha256.digest(digest1.data)
@@ -77,91 +80,91 @@ proc dblsha2_256hash(data: openarray[byte], output: var openarray[byte]) =
                  else: sha256.sizeDigest
     copyMem(addr output[0], addr digest2.data[0], length)
 
-proc blake2Bhash(data: openarray[byte], output: var openarray[byte]) =
+proc blake2Bhash(data: openArray[byte], output: var openArray[byte]) =
   if len(output) > 0:
     var digest = blake2_512.digest(data)
     var length = if blake2_512.sizeDigest > len(output): len(output)
                  else: blake2_512.sizeDigest
     copyMem(addr output[0], addr digest.data[0], length)
 
-proc blake2Shash(data: openarray[byte], output: var openarray[byte]) =
+proc blake2Shash(data: openArray[byte], output: var openArray[byte]) =
   if len(output) > 0:
     var digest = blake2_256.digest(data)
     var length = if blake2_256.sizeDigest > len(output): len(output)
                  else: blake2_256.sizeDigest
     copyMem(addr output[0], addr digest.data[0], length)
 
-proc sha2_256hash(data: openarray[byte], output: var openarray[byte]) =
+proc sha2_256hash(data: openArray[byte], output: var openArray[byte]) =
   if len(output) > 0:
     var digest = sha256.digest(data)
     var length = if sha256.sizeDigest > len(output): len(output)
                  else: sha256.sizeDigest
     copyMem(addr output[0], addr digest.data[0], length)
 
-proc sha2_512hash(data: openarray[byte], output: var openarray[byte]) =
+proc sha2_512hash(data: openArray[byte], output: var openArray[byte]) =
   if len(output) > 0:
     var digest = sha512.digest(data)
     var length = if sha512.sizeDigest > len(output): len(output)
                  else: sha512.sizeDigest
     copyMem(addr output[0], addr digest.data[0], length)
 
-proc sha3_224hash(data: openarray[byte], output: var openarray[byte]) =
+proc sha3_224hash(data: openArray[byte], output: var openArray[byte]) =
   if len(output) > 0:
     var digest = sha3_224.digest(data)
     var length = if sha3_224.sizeDigest > len(output): len(output)
                  else: sha3_224.sizeDigest
     copyMem(addr output[0], addr digest.data[0], length)
 
-proc sha3_256hash(data: openarray[byte], output: var openarray[byte]) =
+proc sha3_256hash(data: openArray[byte], output: var openArray[byte]) =
   if len(output) > 0:
     var digest = sha3_256.digest(data)
     var length = if sha3_256.sizeDigest > len(output): len(output)
                  else: sha3_256.sizeDigest
     copyMem(addr output[0], addr digest.data[0], length)
 
-proc sha3_384hash(data: openarray[byte], output: var openarray[byte]) =
+proc sha3_384hash(data: openArray[byte], output: var openArray[byte]) =
   if len(output) > 0:
     var digest = sha3_384.digest(data)
     var length = if sha3_384.sizeDigest > len(output): len(output)
                  else: sha3_384.sizeDigest
     copyMem(addr output[0], addr digest.data[0], length)
 
-proc sha3_512hash(data: openarray[byte], output: var openarray[byte]) =
+proc sha3_512hash(data: openArray[byte], output: var openArray[byte]) =
   if len(output) > 0:
     var digest = sha3_512.digest(data)
     var length = if sha3_512.sizeDigest > len(output): len(output)
                  else: sha3_512.sizeDigest
     copyMem(addr output[0], addr digest.data[0], length)
 
-proc keccak_224hash(data: openarray[byte], output: var openarray[byte]) =
+proc keccak_224hash(data: openArray[byte], output: var openArray[byte]) =
   if len(output) > 0:
     var digest = keccak224.digest(data)
     var length = if keccak224.sizeDigest > len(output): len(output)
                  else: keccak224.sizeDigest
     copyMem(addr output[0], addr digest.data[0], length)
 
-proc keccak_256hash(data: openarray[byte], output: var openarray[byte]) =
+proc keccak_256hash(data: openArray[byte], output: var openArray[byte]) =
   if len(output) > 0:
     var digest = keccak256.digest(data)
     var length = if keccak256.sizeDigest > len(output): len(output)
                  else: keccak256.sizeDigest
     copyMem(addr output[0], addr digest.data[0], length)
 
-proc keccak_384hash(data: openarray[byte], output: var openarray[byte]) =
+proc keccak_384hash(data: openArray[byte], output: var openArray[byte]) =
   if len(output) > 0:
     var digest = keccak384.digest(data)
     var length = if keccak384.sizeDigest > len(output): len(output)
                  else: keccak384.sizeDigest
     copyMem(addr output[0], addr digest.data[0], length)
 
-proc keccak_512hash(data: openarray[byte], output: var openarray[byte]) =
+proc keccak_512hash(data: openArray[byte], output: var openArray[byte]) =
   if len(output) > 0:
     var digest = keccak512.digest(data)
     var length = if keccak512.sizeDigest > len(output): len(output)
                  else: keccak512.sizeDigest
     copyMem(addr output[0], addr digest.data[0], length)
 
-proc shake_128hash(data: openarray[byte], output: var openarray[byte]) =
+proc shake_128hash(data: openArray[byte], output: var openArray[byte]) =
   var sctx: shake128
   if len(output) > 0:
     sctx.init()
@@ -170,7 +173,7 @@ proc shake_128hash(data: openarray[byte], output: var openarray[byte]) =
     discard sctx.output(addr output[0], uint(len(output)))
     sctx.clear()
 
-proc shake_256hash(data: openarray[byte], output: var openarray[byte]) =
+proc shake_256hash(data: openArray[byte], output: var openArray[byte]) =
   var sctx: shake256
   if len(output) > 0:
     sctx.init()
@@ -208,16 +211,16 @@ const
     ),
     MHash(mcodec: multiCodec("shake-128"), size: 32, coder: shake_128hash),
     MHash(mcodec: multiCodec("shake-256"), size: 64, coder: shake_256hash),
-    MHash(mcodec: multiCodec("keccak-224"), size: keccak_224.sizeDigest,
+    MHash(mcodec: multiCodec("keccak-224"), size: keccak224.sizeDigest,
           coder: keccak_224hash
     ),
-    MHash(mcodec: multiCodec("keccak-256"), size: keccak_256.sizeDigest,
+    MHash(mcodec: multiCodec("keccak-256"), size: keccak256.sizeDigest,
           coder: keccak_256hash
     ),
-    MHash(mcodec: multiCodec("keccak-384"), size: keccak_384.sizeDigest,
+    MHash(mcodec: multiCodec("keccak-384"), size: keccak384.sizeDigest,
           coder: keccak_384hash
     ),
-    MHash(mcodec: multiCodec("keccak-512"), size: keccak_512.sizeDigest,
+    MHash(mcodec: multiCodec("keccak-512"), size: keccak512.sizeDigest,
           coder: keccak_512hash
     ),
     MHash(mcodec: multiCodec("blake2b-8"), size: 1, coder: blake2Bhash),
@@ -319,14 +322,13 @@ const
   ]
 
 proc initMultiHashCodeTable(): Table[MultiCodec, MHash] {.compileTime.} =
-  result = initTable[MultiCodec, MHash]()
   for item in HashesList:
     result[item.mcodec] = item
 
 const
   CodeHashes = initMultiHashCodeTable()
 
-proc digestImplWithHash(hash: MHash, data: openarray[byte]): MultiHash =
+proc digestImplWithHash(hash: MHash, data: openArray[byte]): MultiHash =
   var buffer: array[MaxHashSize, byte]
   result.data = initVBuffer()
   result.mcodec = hash.mcodec
@@ -344,7 +346,7 @@ proc digestImplWithHash(hash: MHash, data: openarray[byte]): MultiHash =
     result.size = hash.size
   result.data.finish()
 
-proc digestImplWithoutHash(hash: MHash, data: openarray[byte]): MultiHash =
+proc digestImplWithoutHash(hash: MHash, data: openArray[byte]): MultiHash =
   result.data = initVBuffer()
   result.mcodec = hash.mcodec
   result.size = len(data)
@@ -355,7 +357,7 @@ proc digestImplWithoutHash(hash: MHash, data: openarray[byte]): MultiHash =
   result.data.finish()
 
 proc digest*(mhtype: typedesc[MultiHash], hashname: string,
-             data: openarray[byte]): MhResult[MultiHash] {.inline.} =
+             data: openArray[byte]): MhResult[MultiHash] {.inline.} =
   ## Perform digest calculation using hash algorithm with name ``hashname`` on
   ## data array ``data``.
   let mc = MultiCodec.codec(hashname)
@@ -369,7 +371,7 @@ proc digest*(mhtype: typedesc[MultiHash], hashname: string,
       ok(digestImplWithHash(hash, data))
 
 proc digest*(mhtype: typedesc[MultiHash], hashcode: int,
-             data: openarray[byte]): MhResult[MultiHash] {.inline.} =
+             data: openArray[byte]): MhResult[MultiHash] {.inline.} =
   ## Perform digest calculation using hash algorithm with code ``hashcode`` on
   ## data array ``data``.
   let hash = CodeHashes.getOrDefault(hashcode)
@@ -407,7 +409,7 @@ proc init*[T](mhtype: typedesc[MultiHash], hashcode: MultiCodec,
     ok(digestImplWithoutHash(hash, mdigest.data))
 
 proc init*(mhtype: typedesc[MultiHash], hashname: string,
-           bdigest: openarray[byte]): MhResult[MultiHash] {.inline.} =
+           bdigest: openArray[byte]): MhResult[MultiHash] {.inline.} =
   ## Create MultiHash from array of bytes ``bdigest`` and hash algorithm code
   ## ``hashcode``.
   let mc = MultiCodec.codec(hashname)
@@ -423,7 +425,7 @@ proc init*(mhtype: typedesc[MultiHash], hashname: string,
       ok(digestImplWithoutHash(hash, bdigest))
 
 proc init*(mhtype: typedesc[MultiHash], hashcode: MultiCodec,
-           bdigest: openarray[byte]): MhResult[MultiHash] {.inline.} =
+           bdigest: openArray[byte]): MhResult[MultiHash] {.inline.} =
   ## Create MultiHash from array of bytes ``bdigest`` and hash algorithm code
   ## ``hashcode``.
   let hash = CodeHashes.getOrDefault(hashcode)
@@ -434,7 +436,7 @@ proc init*(mhtype: typedesc[MultiHash], hashcode: MultiCodec,
   else:
     ok(digestImplWithoutHash(hash, bdigest))
 
-proc decode*(mhtype: typedesc[MultiHash], data: openarray[byte],
+proc decode*(mhtype: typedesc[MultiHash], data: openArray[byte],
              mhash: var MultiHash): MhResult[int] =
   ## Decode MultiHash value from array of bytes ``data``.
   ##
@@ -479,7 +481,7 @@ proc decode*(mhtype: typedesc[MultiHash], data: openarray[byte],
                                                vb.offset + int(size) - 1))
   ok(vb.offset + int(size))
 
-proc validate*(mhtype: typedesc[MultiHash], data: openarray[byte]): bool =
+proc validate*(mhtype: typedesc[MultiHash], data: openArray[byte]): bool =
   ## Returns ``true`` if array of bytes ``data`` has correct MultiHash inside.
   var code, size: uint64
   var res: VarintResult[void]
@@ -510,7 +512,7 @@ proc validate*(mhtype: typedesc[MultiHash], data: openarray[byte]): bool =
   result = true
 
 proc init*(mhtype: typedesc[MultiHash],
-           data: openarray[byte]): MhResult[MultiHash] {.inline.} =
+           data: openArray[byte]): MhResult[MultiHash] {.inline.} =
   ## Create MultiHash from bytes array ``data``.
   var hash: MultiHash
   discard ? MultiHash.decode(data, hash)
@@ -531,7 +533,7 @@ proc init58*(mhtype: typedesc[MultiHash],
   if MultiHash.decode(Base58.decode(data), result) == -1:
     raise newException(MultihashError, "Incorrect MultiHash binary format")
 
-proc cmp(a: openarray[byte], b: openarray[byte]): bool {.inline.} =
+proc cmp(a: openArray[byte], b: openArray[byte]): bool {.inline.} =
   if len(a) != len(b):
     return false
   var n = len(a)

libp2p/multistream.nim

@@ -1,21 +1,24 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-import strutils
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import std/[strutils, sequtils]
 import chronos, chronicles, stew/byteutils
 import stream/connection,
-       vbuffer,
-       errors,
        protocols/protocol
 
 logScope:
-  topics = "multistream"
+  topics = "libp2p multistream"
 
 const
   MsgSize* = 64*1024
@@ -26,10 +29,12 @@ const
   Ls* = "\x03ls\n"
 
 type
-  Matcher* = proc (proto: string): bool {.gcsafe.}
+  Matcher* = proc (proto: string): bool {.gcsafe, raises: [Defect].}
+
+  MultiStreamError* = object of LPError
 
   HandlerHolder* = object
-    proto*: string
+    protos*: seq[string]
     protocol*: LPProtocol
     match*: Matcher
 
@@ -37,47 +42,68 @@ type
     handlers*: seq[HandlerHolder]
     codec*: string
 
-proc newMultistream*(): MultistreamSelect =
-  new result
-  result.codec = MSCodec
+proc new*(T: typedesc[MultistreamSelect]): T =
+  T(codec: MSCodec)
+
+template validateSuffix(str: string): untyped =
+    if str.endsWith("\n"):
+      str.removeSuffix("\n")
+    else:
+      raise newException(MultiStreamError, "MultistreamSelect failed, malformed message")
 
 proc select*(m: MultistreamSelect,
              conn: Connection,
              proto: seq[string]):
              Future[string] {.async.} =
-  trace "initiating handshake", codec = m.codec
+  trace "initiating handshake", conn, codec = m.codec
   ## select a remote protocol
   await conn.write(m.codec) # write handshake
   if proto.len() > 0:
-    trace "selecting proto", proto = proto[0]
+    trace "selecting proto", conn, proto = proto[0]
     await conn.writeLp((proto[0] & "\n")) # select proto
 
-  var s = string.fromBytes((await conn.readLp(1024))) # read ms header
-  s.removeSuffix("\n")
-  if s != Codec:
-    notice "handshake failed", codec = s.toHex()
-    return ""
+  var s = string.fromBytes((await conn.readLp(MsgSize))) # read ms header
+  validateSuffix(s)
 
+  if s != Codec:
+    notice "handshake failed", conn, codec = s
+    raise newException(MultiStreamError, "MultistreamSelect handshake failed")
+  else:
+    trace "multistream handshake success", conn
+
+  echo " 0 ", proto
   if proto.len() == 0: # no protocols, must be a handshake call
     return Codec
   else:
-    s = string.fromBytes(await conn.readLp(1024)) # read the first proto
-    trace "reading first requested proto"
-    s.removeSuffix("\n")
+    echo " 1"
+    s = string.fromBytes(await conn.readLp(MsgSize)) # read the first proto
+    echo " 2"
+    validateSuffix(s)
+    echo " 3"
+    trace "reading first requested proto", conn
     if s == proto[0]:
-      trace "successfully selected ", proto = proto[0]
+      echo "  4.1"
+      trace "successfully selected ", conn, proto = proto[0]
+      conn.protocol = proto[0]
+      echo "  4.2 ", proto[0]
       return proto[0]
     elif proto.len > 1:
+      echo "  5.1"
       # Try to negotiate alternatives
       let protos = proto[1..<proto.len()]
-      trace "selecting one of several protos", protos = protos
+      trace "selecting one of several protos", conn, protos = protos
+      echo "  5.2"
       for p in protos:
-        trace "selecting proto", proto = p
+        echo "   5.2.1"
+        trace "selecting proto", conn, proto = p
         await conn.writeLp((p & "\n")) # select proto
-        s = string.fromBytes(await conn.readLp(1024)) # read the first proto
-        s.removeSuffix("\n")
+        echo "   5.2.2"
+        s = string.fromBytes(await conn.readLp(MsgSize)) # read the first proto
+        validateSuffix(s)
+        echo "   5.2.3"
         if s == p:
-          trace "selected protocol", protocol = s
+          trace "selected protocol", conn, protocol = s
+          conn.protocol = s
           return s
       return ""
     else:
@@ -87,6 +113,7 @@ proc select*(m: MultistreamSelect,
 proc select*(m: MultistreamSelect,
              conn: Connection,
              proto: string): Future[bool] {.async.} =
+  echo proto
   if proto.len > 0:
     return (await m.select(conn, @[proto])) == proto
   else:
@@ -104,83 +131,102 @@ proc list*(m: MultistreamSelect,
   await conn.write(Ls) # send ls
 
   var list = newSeq[string]()
-  let ms = string.fromBytes(await conn.readLp(1024))
+  let ms = string.fromBytes(await conn.readLp(MsgSize))
   for s in ms.split("\n"):
     if s.len() > 0:
       list.add(s)
 
   result = list
 
-proc handle*(m: MultistreamSelect, conn: Connection) {.async, gcsafe.} =
-  trace "handle: starting multistream handling"
+proc handle*(m: MultistreamSelect, conn: Connection, active: bool = false) {.async, gcsafe.} =
+  trace "Starting multistream handler", conn, handshaked = active
+  var handshaked = active
   try:
-    while not conn.closed:
-      var ms = string.fromBytes(await conn.readLp(1024))
-      ms.removeSuffix("\n")
+    while not conn.atEof:
+      var ms = string.fromBytes(await conn.readLp(MsgSize))
+      validateSuffix(ms)
 
-      trace "handle: got request for ", ms
+      if not handshaked and ms != Codec:
+        notice "expected handshake message", conn, instead=ms
+        raise newException(CatchableError,
+                           "MultistreamSelect handling failed, invalid first message")
+
+      trace "handle: got request", conn, ms
       if ms.len() <= 0:
-        trace "handle: invalid proto"
+        trace "handle: invalid proto", conn
         await conn.write(Na)
 
       if m.handlers.len() == 0:
-        trace "handle: sending `na` for protocol ", protocol = ms
+        trace "handle: sending `na` for protocol", conn, protocol = ms
         await conn.write(Na)
         continue
 
       case ms:
-        of "ls":
-          trace "handle: listing protos"
-          var protos = ""
-          for h in m.handlers:
-            protos &= (h.proto & "\n")
-          await conn.writeLp(protos)
-        of Codec:
+      of "ls":
+        trace "handle: listing protos", conn
+        var protos = ""
+        for h in m.handlers:
+          for proto in h.protos:
+            protos &= (proto & "\n")
+        await conn.writeLp(protos)
+      of Codec:
+        if not handshaked:
           await conn.write(m.codec)
+          handshaked = true
         else:
-          for h in m.handlers:
-            if (not isNil(h.match) and h.match(ms)) or ms == h.proto:
-              trace "found handler for", protocol = ms
-              await conn.writeLp((h.proto & "\n"))
-              await h.protocol.handler(conn, ms)
-              return
-          debug "no handlers for ", protocol = ms
+          trace "handle: sending `na` for duplicate handshake while handshaked",
+            conn
           await conn.write(Na)
+      else:
+        for h in m.handlers:
+          if (not isNil(h.match) and h.match(ms)) or h.protos.contains(ms):
+            trace "found handler", conn, protocol = ms
+            await conn.writeLp(ms & "\n")
+            conn.protocol = ms
+            await h.protocol.handler(conn, ms)
+            return
+        debug "no handlers", conn, protocol = ms
+        await conn.write(Na)
   except CancelledError as exc:
-    await conn.close()
     raise exc
   except CatchableError as exc:
-    trace "exception in multistream", exc = exc.msg
-    await conn.close()
+    trace "Exception in multistream", conn, msg = exc.msg
   finally:
-    trace "leaving multistream loop"
+    await conn.close()
 
-proc addHandler*[T: LPProtocol](m: MultistreamSelect,
-                                codec: string,
-                                protocol: T,
-                                matcher: Matcher = nil) =
-  ## register a protocol
-  # TODO: This is a bug in chronicles,
-  # it break if I uncomment this line.
-  # Which is almost the same as the
-  # one on the next override of addHandler
-  #
-  # trace "registering protocol", codec = codec
-  m.handlers.add(HandlerHolder(proto: codec,
+  trace "Stopped multistream handler", conn
+
+proc addHandler*(m: MultistreamSelect,
+                 codecs: seq[string],
+                 protocol: LPProtocol,
+                 matcher: Matcher = nil) =
+  trace "registering protocols", protos = codecs
+  m.handlers.add(HandlerHolder(protos: codecs,
                                protocol: protocol,
                                match: matcher))
 
-proc addHandler*[T: LPProtoHandler](m: MultistreamSelect,
-                                    codec: string,
-                                    handler: T,
-                                    matcher: Matcher = nil) =
-  ## helper to allow registering pure handlers
+proc addHandler*(m: MultistreamSelect,
+                 codec: string,
+                 protocol: LPProtocol,
+                 matcher: Matcher = nil) =
+  addHandler(m, @[codec], protocol, matcher)
 
-  trace "registering proto handler", codec = codec
+proc addHandler*(m: MultistreamSelect,
+                 codec: string,
+                 handler: LPProtoHandler,
+                 matcher: Matcher = nil) =
+  ## helper to allow registering pure handlers
+  trace "registering proto handler", proto = codec
   let protocol = new LPProtocol
   protocol.codec = codec
   protocol.handler = handler
 
-  m.handlers.add(HandlerHolder(proto: codec,
+  m.handlers.add(HandlerHolder(protos: @[codec],
                                protocol: protocol,
                                match: matcher))
+
+proc start*(m: MultistreamSelect) {.async.} =
+  await allFutures(m.handlers.mapIt(it.protocol.start()))
+
+proc stop*(m: MultistreamSelect) {.async.} =
+  await allFutures(m.handlers.mapIt(it.protocol.stop()))

libp2p/muxers/mplex/coder.nim

@@ -1,79 +1,94 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-import chronos
-import nimcrypto/utils, chronicles, stew/byteutils
-import types,
-       ../../stream/connection,
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import pkg/[chronos, nimcrypto/utils, chronicles, stew/byteutils]
+import ../../stream/connection,
        ../../utility,
        ../../varint,
-       ../../vbuffer
+       ../../vbuffer,
+       ../muxer
 
 logScope:
-  topics = "mplexcoder"
+  topics = "libp2p mplexcoder"
 
 type
+  MessageType* {.pure.} = enum
+    New,
+    MsgIn,
+    MsgOut,
+    CloseIn,
+    CloseOut,
+    ResetIn,
+    ResetOut
+
   Msg* = tuple
     id: uint64
     msgType: MessageType
     data: seq[byte]
 
-  InvalidMplexMsgType = object of CatchableError
+  InvalidMplexMsgType* = object of MuxerError
+
+# https://github.com/libp2p/specs/tree/master/mplex#writing-to-a-stream
+const MaxMsgSize* = 1 shl 20 # 1mb
 
 proc newInvalidMplexMsgType*(): ref InvalidMplexMsgType =
   newException(InvalidMplexMsgType, "invalid message type")
 
 proc readMsg*(conn: Connection): Future[Msg] {.async, gcsafe.} =
   let header = await conn.readVarint()
-  trace "read header varint", varint = header
+  trace "read header varint", varint = header, conn
 
   let data = await conn.readLp(MaxMsgSize)
-  trace "read data", dataLen = data.len, data = shortLog(data)
+  trace "read data", dataLen = data.len, data = shortLog(data), conn
 
   let msgType = header and 0x7
   if msgType.int > ord(MessageType.ResetOut):
     raise newInvalidMplexMsgType()
 
-  result = (header shr 3, MessageType(msgType), data)
+  return (header shr 3, MessageType(msgType), data)
 
 proc writeMsg*(conn: Connection,
                id: uint64,
                msgType: MessageType,
-               data: seq[byte] = @[]) {.async, gcsafe.} =
-  trace "sending data over mplex", id,
-                                   msgType,
-                                   data = data.len
+               data: seq[byte] = @[]): Future[void] =
   var
     left = data.len
     offset = 0
+    buf = initVBuffer()
+
+  # Split message into length-prefixed chunks
   while left > 0 or data.len == 0:
     let
       chunkSize = if left > MaxMsgSize: MaxMsgSize - 64 else: left
-      chunk = if chunkSize > 0 : data[offset..(offset + chunkSize - 1)] else: data
-    ## write length prefixed
-    var buf = initVBuffer()
+
     buf.writePBVarint(id shl 3 or ord(msgType).uint64)
-    buf.writePBVarint(chunkSize.uint64) # size should be always sent
-    buf.finish()
+    buf.writeSeq(data.toOpenArray(offset, offset + chunkSize - 1))
     left = left - chunkSize
     offset = offset + chunkSize
-    await conn.write(buf.buffer & chunk)
 
     if data.len == 0:
-      return
+      break
+
+  trace "writing mplex message",
+    conn, id, msgType, data = data.len, encoded = buf.buffer.len
+
+  # Write all chunks in a single write to avoid async races where a close
+  # message gets written before some of the chunks
+  conn.write(buf.buffer)
 
 proc writeMsg*(conn: Connection,
                id: uint64,
                msgType: MessageType,
-               data: string) {.async, gcsafe.} =
-  # TODO: changing this to
-  #`await conn.writeMsg(id, msgType, data.toBytes())`
-  # causes all sorts of race conditions and hangs.
-  # DON'T DO IT!
-  result = conn.writeMsg(id, msgType, data.toBytes())
+               data: string): Future[void] =
+  conn.writeMsg(id, msgType, data.toBytes())

libp2p/muxers/mplex/lpchannel.nim

@@ -1,27 +1,37 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-import oids, deques
-import chronos, chronicles, metrics
-import types,
-       coder,
-       nimcrypto/utils,
-       ../../stream/connection,
-       ../../stream/bufferstream,
-       ../../utility,
-       ../../errors,
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import std/[oids, strformat]
+import pkg/[chronos, chronicles, metrics, nimcrypto/utils]
+import ./coder,
+       ../muxer,
+       ../../stream/[bufferstream, connection, streamseq],
        ../../peerinfo
 
 export connection
 
 logScope:
-  topics = "mplexchannel"
+  topics = "libp2p mplexchannel"
+
+when defined(libp2p_mplex_metrics):
+  declareHistogram libp2p_mplex_qlen, "message queue length",
+    buckets = [0.0, 1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0]
+  declareCounter libp2p_mplex_qlenclose, "closed because of max queuelen"
+  declareHistogram libp2p_mplex_qtime, "message queuing time"
+
+when defined(libp2p_network_protocols_metrics):
+  declareCounter libp2p_protocols_bytes, "total sent or received bytes", ["protocol", "direction"]
 
 ## Channel half-closed states
 ##
@@ -30,19 +40,15 @@ logScope:
 ## | Read     | Yes (until EOF)   | No
 ## | Write    | No                | Yes
 ##
+## Channels are considered fully closed when both outgoing and incoming
+## directions are closed and when the reader of the channel has read the
+## EOF marker
 
-# TODO: this is one place where we need to use
-# a proper state machine, but I've opted out of
-# it for now for two reasons:
-#
-# 1) we don't have that many states to manage
-# 2) I'm not sure if adding the state machine
-# would have simplified or complicated the code
-#
-# But now that this is in place, we should perhaps
-# reconsider reworking it again, this time with a
-# more formal approach.
-#
+const
+  MaxWrites = 1024 ##\
+    ## Maximum number of in-flight writes - after this, we disconnect the peer
+
+  LPChannelTrackerName* = "LPChannel"
 
 type
   LPChannel* = ref object of BufferStream
@@ -50,218 +56,249 @@ type
     name*: string                 # name of the channel (for debugging)
     conn*: Connection             # wrapped connection used to for writing
     initiator*: bool              # initiated remotely or locally flag
-    isLazy*: bool                 # is channel lazy
-    isOpen*: bool                 # has channel been opened (only used with isLazy)
+    isOpen*: bool                 # has channel been opened
     closedLocal*: bool            # has channel been closed locally
+    remoteReset*: bool            # has channel been remotely reset
+    localReset*: bool             # has channel been reset locally
     msgCode*: MessageType         # cached in/out message code
     closeCode*: MessageType       # cached in/out close code
     resetCode*: MessageType       # cached in/out reset code
+    writes*: int                  # In-flight writes
 
-proc open*(s: LPChannel) {.async, gcsafe.}
-
-template withWriteLock(lock: AsyncLock, body: untyped): untyped =
+func shortLog*(s: LPChannel): auto =
   try:
-    await lock.acquire()
-    body
-  finally:
-    if not(isNil(lock)) and lock.locked:
-      lock.release()
+    if s.isNil: "LPChannel(nil)"
+    elif s.name != $s.oid and s.name.len > 0:
+      &"{shortLog(s.conn.peerId)}:{s.oid}:{s.name}"
+    else: &"{shortLog(s.conn.peerId)}:{s.oid}"
+  except ValueError as exc:
+    raise newException(Defect, exc.msg)
 
-template withEOFExceptions(body: untyped): untyped =
+chronicles.formatIt(LPChannel): shortLog(it)
+
+proc open*(s: LPChannel) {.async, gcsafe.} =
+  trace "Opening channel", s, conn = s.conn
+  if s.conn.isClosed:
+    return
   try:
-      body
-  except LPStreamEOFError as exc:
-    trace "muxed connection EOF", exc = exc.msg
-  except LPStreamClosedError as exc:
-    trace "muxed connection closed", exc = exc.msg
-  except LPStreamIncompleteError as exc:
-    trace "incomplete message", exc = exc.msg
+    await s.conn.writeMsg(s.id, MessageType.New, s.name)
+    s.isOpen = true
+  except CancelledError as exc:
+    raise exc
+  except CatchableError as exc:
+    await s.conn.close()
+    raise exc
 
-method reset*(s: LPChannel) {.base, async, gcsafe.}
+method closed*(s: LPChannel): bool =
+  s.closedLocal
+
+proc closeUnderlying(s: LPChannel): Future[void] {.async.} =
+  ## Channels may be closed for reading and writing in any order - we'll close
+  ## the underlying bufferstream when both directions are closed
+  if s.closedLocal and s.atEof():
+    await procCall BufferStream(s).close()
+
+proc reset*(s: LPChannel) {.async, gcsafe.} =
+  if s.isClosed:
+    trace "Already closed", s
+    return
+
+  s.isClosed = true
+  s.closedLocal = true
+  s.localReset = not s.remoteReset
+
+  trace "Resetting channel", s, len = s.len
+
+  if s.isOpen and not s.conn.isClosed:
+    # If the connection is still active, notify the other end
+    proc resetMessage() {.async.} =
+      try:
+        trace "sending reset message", s, conn = s.conn
+        await s.conn.writeMsg(s.id, s.resetCode) # write reset
+      except CatchableError as exc:
+        # No cancellations
+        await s.conn.close()
+        trace "Can't send reset message", s, conn = s.conn, msg = exc.msg
+
+    asyncSpawn resetMessage()
+
+  await s.closeImpl() # noraises, nocancels
+
+  trace "Channel reset", s
+
+method close*(s: LPChannel) {.async, gcsafe.} =
+  ## Close channel for writing - a message will be sent to the other peer
+  ## informing them that the channel is closed and that we're waiting for
+  ## their acknowledgement.
+  if s.closedLocal:
+    trace "Already closed", s
+    return
+  s.closedLocal = true
+
+  trace "Closing channel", s, conn = s.conn, len = s.len
+
+  if s.isOpen and not s.conn.isClosed:
+    try:
+      await s.conn.writeMsg(s.id, s.closeCode) # write close
+    except CancelledError as exc:
+      await s.conn.close()
+      raise exc
+    except CatchableError as exc:
+      # It's harmless that close message cannot be sent - the connection is
+      # likely down already
+      await s.conn.close()
+      trace "Cannot send close message", s, id = s.id, msg = exc.msg
+
+  await s.closeUnderlying() # maybe already eofed
+
+  trace "Closed channel", s, len = s.len
 
 method initStream*(s: LPChannel) =
   if s.objName.len == 0:
-    s.objName = "LPChannel"
+    s.objName = LPChannelTrackerName
+
+  s.timeoutHandler = proc(): Future[void] {.gcsafe.} =
+    trace "Idle timeout expired, resetting LPChannel", s
+    s.reset()
 
   procCall BufferStream(s).initStream()
 
-proc newChannel*(id: uint64,
-                 conn: Connection,
-                 initiator: bool,
-                 name: string = "",
-                 size: int = DefaultBufferSize,
-                 lazy: bool = false): LPChannel =
-  result = LPChannel(id: id,
-                     name: name,
-                     conn: conn,
-                     initiator: initiator,
-                     msgCode: if initiator: MessageType.MsgOut else: MessageType.MsgIn,
-                     closeCode: if initiator: MessageType.CloseOut else: MessageType.CloseIn,
-                     resetCode: if initiator: MessageType.ResetOut else: MessageType.ResetIn,
-                     isLazy: lazy)
+method readOnce*(s: LPChannel,
+                 pbytes: pointer,
+                 nbytes: int):
+                 Future[int] {.async.} =
+  ## Mplex relies on reading being done regularly from every channel, or all
+  ## channels are blocked - in particular, this means that reading from one
+  ## channel must not be done from within a callback / read handler of another
+  ## or the reads will lock each other.
+  if s.remoteReset:
+    raise newLPStreamResetError()
+  if s.localReset:
+    raise newLPStreamClosedError()
+  if s.atEof():
+    raise newLPStreamRemoteClosedError()
+  if s.conn.closed:
+    raise newLPStreamConnDownError()
+  try:
+    let bytes = await procCall BufferStream(s).readOnce(pbytes, nbytes)
+    when defined(libp2p_network_protocols_metrics):
+      if s.protocol.len > 0:
+        libp2p_protocols_bytes.inc(bytes.int64, labelValues=[s.protocol, "in"])
 
-  let chan = result
-  logScope:
-    id = chan.id
-    initiator = chan.initiator
-    name = chan.name
-    oid = $chan.oid
-    peer = $chan.conn.peerInfo
-    # stack = getStackTrace()
-
-  proc writeHandler(data: seq[byte]): Future[void] {.async, gcsafe.} =
-    try:
-      if chan.isLazy and not(chan.isOpen):
-        await chan.open()
-
-      # writes should happen in sequence
-      trace "sending data"
-
-      await conn.writeMsg(chan.id,
-                          chan.msgCode,
-                          data).wait(2.minutes) # write header
-    except CatchableError as exc:
-      trace "exception in lpchannel write handler", exc = exc.msg
-      await chan.reset()
-      raise exc
-
-  result.initBufferStream(writeHandler, size)
-  when chronicles.enabledLogLevel == LogLevel.TRACE:
-    result.name = if result.name.len > 0: result.name else: $result.oid
-
-  trace "created new lpchannel"
-
-proc closeMessage(s: LPChannel) {.async.} =
-  logScope:
-    id = s.id
-    initiator = s.initiator
-    name = s.name
-    oid = $s.oid
-    peer = $s.conn.peerInfo
-    # stack = getStackTrace()
-
-  ## send close message - this will not raise
-  ## on EOF or Closed
-  withWriteLock(s.writeLock):
-    trace "sending close message"
-
-    await s.conn.writeMsg(s.id, s.closeCode) # write close
-
-proc resetMessage(s: LPChannel) {.async.} =
-  logScope:
-    id = s.id
-    initiator = s.initiator
-    name = s.name
-    oid = $s.oid
-    peer = $s.conn.peerInfo
-    # stack = getStackTrace()
-
-  ## send reset message - this will not raise
-  withEOFExceptions:
-    withWriteLock(s.writeLock):
-      trace "sending reset message"
-
-      await s.conn.writeMsg(s.id, s.resetCode) # write reset
-
-proc open*(s: LPChannel) {.async, gcsafe.} =
-  logScope:
-    id = s.id
-    initiator = s.initiator
-    name = s.name
-    oid = $s.oid
-    peer = $s.conn.peerInfo
-    # stack = getStackTrace()
-
-  ## NOTE: Don't call withExcAndLock or withWriteLock,
-  ## because this already gets called from writeHandler
-  ## which is locked
-  await s.conn.writeMsg(s.id, MessageType.New, s.name)
-  trace "opened channel"
-  s.isOpen = true
-
-proc closeRemote*(s: LPChannel) {.async.} =
-  logScope:
-    id = s.id
-    initiator = s.initiator
-    name = s.name
-    oid = $s.oid
-    peer = $s.conn.peerInfo
-    # stack = getStackTrace()
-
-  trace "got EOF, closing channel"
-  await s.drainBuffer()
-
-  s.isEof = true # set EOF immediately to prevent further reads
-  await s.close() # close local end
-
-  # call to avoid leaks
-  await procCall BufferStream(s).close() # close parent bufferstream
-  trace "channel closed on EOF"
-
-method closed*(s: LPChannel): bool =
-  ## this emulates half-closed behavior
-  ## when closed locally writing is
-  ## disabled - see the table in the
-  ## header of the file
-  s.closedLocal
-
-method reset*(s: LPChannel) {.base, async, gcsafe.} =
-  logScope:
-    id = s.id
-    initiator = s.initiator
-    name = s.name
-    oid = $s.oid
-    peer = $s.conn.peerInfo
-    # stack = getStackTrace()
-
-  trace "resetting channel"
-
-  if s.closedLocal and s.isEof:
-    trace "channel already closed or reset"
-    return
-
-  # we asyncCheck here because the other end
-  # might be dead already - reset is always
-  # optimistic
-  asyncCheck s.resetMessage()
-
-  # drain the buffer before closing
-  await s.drainBuffer()
-  await procCall BufferStream(s).close()
-
-  s.isEof = true
-  s.closedLocal = true
-
-  trace "channel reset"
-
-method close*(s: LPChannel) {.async, gcsafe.} =
-  logScope:
-    id = s.id
-    initiator = s.initiator
-    name = s.name
-    oid = $s.oid
-    peer = $s.conn.peerInfo
-    # stack = getStackTrace()
+    trace "readOnce", s, bytes
+    if bytes == 0:
+      await s.closeUnderlying()
+    return bytes
+  except CatchableError as exc:
+    # readOnce in BufferStream generally raises on EOF or cancellation - for
+    # the former, resetting is harmless, for the latter it's necessary because
+    # data has been lost in s.readBuf and there's no way to gracefully recover /
+    # use the channel any more
+    await s.reset()
+    raise newLPStreamConnDownError(exc)
 
+proc prepareWrite(s: LPChannel, msg: seq[byte]): Future[void] {.async.} =
+  # prepareWrite is the slow path of writing a message - see conditions in
+  # write
+  if s.remoteReset:
+    raise newLPStreamResetError()
   if s.closedLocal:
-    trace "channel already closed"
+    raise newLPStreamClosedError()
+  if s.conn.closed:
+    raise newLPStreamConnDownError()
+
+  if msg.len == 0:
     return
 
-  trace "closing local lpchannel"
+  if s.writes >= MaxWrites:
+    debug "Closing connection, too many in-flight writes on channel",
+      s, conn = s.conn, writes = s.writes
+    when defined(libp2p_mplex_metrics):
+        libp2p_mplex_qlenclose.inc()
+    await s.reset()
+    await s.conn.close()
+    return
 
-  proc closeInternal() {.async.} =
-    try:
-      await s.closeMessage().wait(2.minutes)
-      if s.atEof: # already closed by remote close parent buffer immediately
-        await procCall BufferStream(s).close()
-    except CancelledError as exc:
-      await s.reset()
-      raise exc
-    except CatchableError as exc:
-      trace "exception closing channel"
-      await s.reset()
+  if not s.isOpen:
+    await s.open()
 
-    trace "lpchannel closed local"
+  await s.conn.writeMsg(s.id, s.msgCode, msg)
 
-  s.closedLocal = true
-  asyncCheck closeInternal()
+proc completeWrite(
+    s: LPChannel, fut: Future[void], msgLen: int): Future[void] {.async.} =
+  try:
+    s.writes += 1
+
+    when defined(libp2p_mplex_metrics):
+      libp2p_mplex_qlen.observe(s.writes.int64 - 1)
+      libp2p_mplex_qtime.time:
+        await fut
+    else:
+      await fut
+
+    when defined(libp2p_network_protocol_metrics):
+      if s.protocol.len > 0:
+        libp2p_protocols_bytes.inc(msgLen.int64, labelValues=[s.protocol, "out"])
+
+    s.activity = true
+  except CancelledError as exc:
+    # Chronos may still send the data
+    raise exc
+  except LPStreamClosedError as exc:
+    raise exc
+  except CatchableError as exc:
+    trace "exception in lpchannel write handler", s, msg = exc.msg
+    await s.reset()
+    await s.conn.close()
+    raise newLPStreamConnDownError(exc)
+  finally:
+    s.writes -= 1
+
+method write*(s: LPChannel, msg: seq[byte]): Future[void] =
+  ## Write to mplex channel - there may be up to MaxWrite concurrent writes
+  ## pending after which the peer is disconnected
+
+  let
+    closed = s.closedLocal or s.conn.closed
+
+  let fut =
+    if (not closed) and msg.len > 0 and s.writes < MaxWrites and s.isOpen:
+      # Fast path: Avoid a copy of msg being kept in the closure created by
+      # `{.async.}` as this drives up memory usage - the conditions are laid out
+      # in prepareWrite
+      s.conn.writeMsg(s.id, s.msgCode, msg)
+    else:
+      prepareWrite(s, msg)
+
+  s.completeWrite(fut, msg.len)
+
+method getWrapped*(s: LPChannel): Connection = s.conn
+
+proc init*(
+  L: type LPChannel,
+  id: uint64,
+  conn: Connection,
+  initiator: bool,
+  name: string = "",
+  timeout: Duration = DefaultChanTimeout): LPChannel =
+
+  let chann = L(
+    id: id,
+    name: name,
+    conn: conn,
+    initiator: initiator,
+    timeout: timeout,
+    isOpen: if initiator: false else: true,
+    msgCode: if initiator: MessageType.MsgOut else: MessageType.MsgIn,
+    closeCode: if initiator: MessageType.CloseOut else: MessageType.CloseIn,
+    resetCode: if initiator: MessageType.ResetOut else: MessageType.ResetIn,
+    dir: if initiator: Direction.Out else: Direction.In)
+
+  chann.initStream()
+
+  when chronicles.enabledLogLevel == LogLevel.TRACE:
+    chann.name = if chann.name.len > 0: chann.name else: $chann.oid
+
+  trace "Created new lpchannel", s = chann, id, initiator
+
+  return chann

libp2p/muxers/mplex/mplex.nim

@@ -1,11 +1,16 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 import tables, sequtils, oids
 import chronos, chronicles, stew/byteutils, metrics
@@ -13,219 +18,227 @@ import ../muxer,
        ../../stream/connection,
        ../../stream/bufferstream,
        ../../utility,
-       ../../errors,
        ../../peerinfo,
-       coder,
-       types,
-       lpchannel
+       ./coder,
+       ./lpchannel
+
+export muxer
 
 logScope:
-  topics = "mplex"
+  topics = "libp2p mplex"
 
-declareGauge(libp2p_mplex_channels, "mplex channels", labels = ["initiator", "peer"])
+const MplexCodec* = "/mplex/6.7.0"
+
+const
+  MaxChannelCount = 200
+
+when defined(libp2p_expensive_metrics):
+  declareGauge(libp2p_mplex_channels,
+    "mplex channels", labels = ["initiator", "peer"])
 
 type
-  Mplex* = ref object of Muxer
-    remote: Table[uint64, LPChannel]
-    local: Table[uint64, LPChannel]
-    currentId*: uint64
-    maxChannels*: uint64
-    isClosed: bool
-    when chronicles.enabledLogLevel == LogLevel.TRACE:
-      oid*: Oid
+  InvalidChannelIdError* = object of MuxerError
 
-proc getChannelList(m: Mplex, initiator: bool): var Table[uint64, LPChannel] =
-  if initiator:
-    trace "picking local channels", initiator = initiator, oid = $m.oid
-    result = m.local
-  else:
-    trace "picking remote channels", initiator = initiator, oid = $m.oid
-    result = m.remote
+  Mplex* = ref object of Muxer
+    channels: array[bool, Table[uint64, LPChannel]]
+    currentId: uint64
+    inChannTimeout: Duration
+    outChannTimeout: Duration
+    isClosed: bool
+    oid*: Oid
+    maxChannCount: int
+
+func shortLog*(m: Mplex): auto =
+  shortLog(m.connection)
+
+chronicles.formatIt(Mplex): shortLog(it)
+
+proc newTooManyChannels(): ref TooManyChannels =
+  newException(TooManyChannels, "max allowed channel count exceeded")
+
+proc newInvalidChannelIdError(): ref InvalidChannelIdError =
+  newException(InvalidChannelIdError, "max allowed channel count exceeded")
+
+proc cleanupChann(m: Mplex, chann: LPChannel) {.async, inline.} =
+  ## remove the local channel from the internal tables
+  ##
+  try:
+    await chann.join()
+    m.channels[chann.initiator].del(chann.id)
+    trace "cleaned up channel", m, chann
+
+    when defined(libp2p_expensive_metrics):
+      libp2p_mplex_channels.set(
+        m.channels[chann.initiator].len.int64,
+        labelValues = [$chann.initiator, $m.connection.peerId])
+  except CatchableError as exc:
+    warn "Error cleaning up mplex channel", m, chann, msg = exc.msg
 
 proc newStreamInternal*(m: Mplex,
                         initiator: bool = true,
                         chanId: uint64 = 0,
                         name: string = "",
-                        lazy: bool = false):
-                        Future[LPChannel] {.async, gcsafe.} =
+                        timeout: Duration): LPChannel
+                        {.gcsafe, raises: [Defect, InvalidChannelIdError].} =
   ## create new channel/stream
   ##
   let id = if initiator:
     m.currentId.inc(); m.currentId
     else: chanId
 
-  trace "creating new channel", channelId = id,
-                                initiator = initiator,
-                                name = name,
-                                oid = $m.oid
-  result = newChannel(id,
-                      m.connection,
-                      initiator,
-                      name,
-                      lazy = lazy)
+  if id in m.channels[initiator]:
+    raise newInvalidChannelIdError()
 
-  result.peerInfo = m.connection.peerInfo
+  result = LPChannel.init(
+    id,
+    m.connection,
+    initiator,
+    name,
+    timeout = timeout)
+
+  result.peerId = m.connection.peerId
   result.observedAddr = m.connection.observedAddr
+  result.transportDir = m.connection.transportDir
+  when defined(libp2p_agents_metrics):
+    result.shortAgent = m.connection.shortAgent
 
-  doAssert(id notin m.getChannelList(initiator),
-    "channel slot already taken!")
+  trace "Creating new channel", m, channel = result, id, initiator, name
 
-  m.getChannelList(initiator)[id] = result
-  libp2p_mplex_channels.set(
-    m.getChannelList(initiator).len.int64,
-    labelValues = [$initiator,
-                   $m.connection.peerInfo])
+  m.channels[initiator][id] = result
 
-proc cleanupChann(m: Mplex, chann: LPChannel) {.async, inline.} =
-  ## remove the local channel from the internal tables
-  ##
-  await chann.closeEvent.wait()
-  if not isNil(chann):
-    m.getChannelList(chann.initiator).del(chann.id)
-    trace "cleaned up channel", id = chann.id
+  # All the errors are handled inside `cleanupChann()` procedure.
+  asyncSpawn m.cleanupChann(result)
 
-  libp2p_mplex_channels.set(
-    m.getChannelList(chann.initiator).len.int64,
-    labelValues = [$chann.initiator,
-                    $m.connection.peerInfo])
+  when defined(libp2p_expensive_metrics):
+    libp2p_mplex_channels.set(
+      m.channels[initiator].len.int64,
+      labelValues = [$initiator, $m.connection.peerId])
 
 proc handleStream(m: Mplex, chann: LPChannel) {.async.} =
   ## call the muxer stream handler for this channel
   ##
   try:
     await m.streamHandler(chann)
-    trace "finished handling stream"
+    trace "finished handling stream", m, chann
     doAssert(chann.closed, "connection not closed by handler!")
-  except CancelledError as exc:
-    trace "cancling stream handler", exc = exc.msg
-    await chann.reset()
-    raise
   except CatchableError as exc:
-    trace "exception in stream handler", exc = exc.msg
+    trace "Exception in mplex stream handler", m, chann, msg = exc.msg
     await chann.reset()
-    await m.cleanupChann(chann)
 
 method handle*(m: Mplex) {.async, gcsafe.} =
-  trace "starting mplex main loop", oid = $m.oid
+  trace "Starting mplex handler", m
   try:
-    defer:
-      trace "stopping mplex main loop", oid = $m.oid
-      await m.close()
-
-    while not m.connection.closed:
-      trace "waiting for data", oid = $m.oid
-      let (id, msgType, data) = await m.connection.readMsg()
-      trace "read message from connection", id = id,
-                                            msgType = msgType,
-                                            data = data.shortLog,
-                                            oid = $m.oid
-
-      let initiator = bool(ord(msgType) and 1)
-      var channel: LPChannel
-      if MessageType(msgType) != MessageType.New:
-        let channels = m.getChannelList(initiator)
-        if id notin channels:
-
-          trace "Channel not found, skipping", id = id,
-                                                initiator = initiator,
-                                                msg = msgType,
-                                                oid = $m.oid
-          continue
-        channel = channels[id]
+    while not m.connection.atEof:
+      trace "waiting for data", m
+      let
+        (id, msgType, data) = await m.connection.readMsg()
+        initiator = bool(ord(msgType) and 1)
 
       logScope:
         id = id
         initiator = initiator
         msgType = msgType
         size = data.len
-        muxer_oid = $m.oid
+
+      trace "read message from connection", m, data = data.shortLog
+
+      var channel =
+        if MessageType(msgType) != MessageType.New:
+          let tmp = m.channels[initiator].getOrDefault(id, nil)
+          if tmp == nil:
+            trace "Channel not found, skipping", m
+            continue
+
+          tmp
+        else:
+          if m.channels[false].len > m.maxChannCount - 1:
+            warn "too many channels created by remote peer",
+                  allowedMax = MaxChannelCount, m
+            raise newTooManyChannels()
+
+          let name = string.fromBytes(data)
+          m.newStreamInternal(false, id, name, timeout = m.outChannTimeout)
+
+      trace "Processing channel message", m, channel, data = data.shortLog
 
       case msgType:
         of MessageType.New:
-          let name = string.fromBytes(data)
-          channel = await m.newStreamInternal(false, id, name)
-
-          trace "created channel", name = channel.name,
-                                    oid = $channel.oid
+          trace "created channel", m, channel
 
           if not isNil(m.streamHandler):
-            # launch handler task
-            asyncCheck m.handleStream(channel)
+            # Launch handler task
+            # All the errors are handled inside `handleStream()` procedure.
+            asyncSpawn m.handleStream(channel)
 
         of MessageType.MsgIn, MessageType.MsgOut:
-          logScope:
-            name = channel.name
-            oid = $channel.oid
-
-          trace "pushing data to channel"
-
           if data.len > MaxMsgSize:
+            warn "attempting to send a packet larger than allowed",
+                 allowed = MaxMsgSize, channel
             raise newLPStreamLimitError()
-          await channel.pushTo(data)
+
+          trace "pushing data to channel", m, channel, len = data.len
+          await channel.pushData(data)
+          trace "pushed data to channel", m, channel, len = data.len
 
         of MessageType.CloseIn, MessageType.CloseOut:
-          logScope:
-            name = channel.name
-            oid = $channel.oid
-
-          trace "closing channel"
-
-          await channel.closeRemote()
-          await m.cleanupChann(channel)
-
-          trace "deleted channel"
+          await channel.pushEof()
         of MessageType.ResetIn, MessageType.ResetOut:
-          logScope:
-            name = channel.name
-            oid = $channel.oid
-
-          trace "resetting channel"
-
+          channel.remoteReset = true
           await channel.reset()
-          await m.cleanupChann(channel)
-
-          trace "deleted channel"
-  except CancelledError as exc:
-    raise exc
+  except CancelledError:
+    debug "Unexpected cancellation in mplex handler", m
+  except LPStreamEOFError as exc:
+    trace "Stream EOF", m, msg = exc.msg
   except CatchableError as exc:
-    trace "Exception occurred", exception = exc.msg, oid = $m.oid
+    debug "Unexpected exception in mplex read loop", m, msg = exc.msg
+  finally:
+    await m.close()
+  trace "Stopped mplex handler", m
 
-proc newMplex*(conn: Connection,
-               maxChanns: uint = MaxChannels): Mplex =
-  new result
-  result.connection = conn
-  result.maxChannels = maxChanns
-  result.remote = initTable[uint64, LPChannel]()
-  result.local = initTable[uint64, LPChannel]()
-
-  when chronicles.enabledLogLevel == LogLevel.TRACE:
-    result.oid = genOid()
+proc new*(M: type Mplex,
+           conn: Connection,
+           inTimeout, outTimeout: Duration = DefaultChanTimeout,
+           maxChannCount: int = MaxChannelCount): Mplex =
+  M(connection: conn,
+    inChannTimeout: inTimeout,
+    outChannTimeout: outTimeout,
+    oid: genOid(),
+    maxChannCount: maxChannCount)
 
 method newStream*(m: Mplex,
                   name: string = "",
                   lazy: bool = false): Future[Connection] {.async, gcsafe.} =
-  let channel = await m.newStreamInternal(lazy = lazy)
+  let channel = m.newStreamInternal(timeout = m.inChannTimeout)
+
   if not lazy:
     await channel.open()
 
-  asyncCheck m.cleanupChann(channel)
   return Connection(channel)
 
 method close*(m: Mplex) {.async, gcsafe.} =
   if m.isClosed:
+    trace "Already closed", m
     return
+  m.isClosed = true
 
-  defer:
-    m.remote.clear()
-    m.local.clear()
-    m.isClosed = true
+  trace "Closing mplex", m
 
-  trace "closing mplex muxer", oid = $m.oid
-  let channs = toSeq(m.remote.values) &
-    toSeq(m.local.values)
+  var channs = toSeq(m.channels[false].values) & toSeq(m.channels[true].values)
+
+  for chann in channs:
+    await chann.close()
+
+  await m.connection.close()
+
+  # TODO while we're resetting, new channels may be created that will not be
+  #      closed properly
+
+  channs = toSeq(m.channels[false].values) & toSeq(m.channels[true].values)
 
   for chann in channs:
     await chann.reset()
-    await m.cleanupChann(chann)
 
-  await m.connection.close()
+  m.channels[false].clear()
+  m.channels[true].clear()
+
+  trace "Closed mplex", m

libp2p/muxers/mplex/types.nim

@@ -1,28 +0,0 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
-
-import chronos
-
-# https://github.com/libp2p/specs/tree/master/mplex#writing-to-a-stream
-const MaxMsgSize* = 1 shl 20 # 1mb
-const MaxChannels* = 1000
-const MplexCodec* = "/mplex/6.7.0"
-const MaxReadWriteTime* = 5.seconds
-
-type
-  MplexNoSuchChannel* = object of CatchableError
-
-  MessageType* {.pure.} = enum
-    New,
-    MsgIn,
-    MsgOut,
-    CloseIn,
-    CloseOut,
-    ResetIn,
-    ResetOut

libp2p/muxers/muxer.nim

@@ -1,31 +1,41 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 import chronos, chronicles
 import ../protocols/protocol,
        ../stream/connection,
-       ../peerinfo,
        ../errors
 
 logScope:
-  topics = "muxer"
+  topics = "libp2p muxer"
+
+const
+  DefaultChanTimeout* = 5.minutes
 
 type
-  StreamHandler* = proc(conn: Connection): Future[void] {.gcsafe.}
-  MuxerHandler* = proc(muxer: Muxer): Future[void] {.gcsafe.}
+  MuxerError* = object of LPError
+  TooManyChannels* = object of MuxerError
+
+  StreamHandler* = proc(conn: Connection): Future[void] {.gcsafe, raises: [Defect].}
+  MuxerHandler* = proc(muxer: Muxer): Future[void] {.gcsafe, raises: [Defect].}
 
   Muxer* = ref object of RootObj
     streamHandler*: StreamHandler
     connection*: Connection
 
   # user provider proc that returns a constructed Muxer
-  MuxerConstructor* = proc(conn: Connection): Muxer {.gcsafe, closure.}
+  MuxerConstructor* = proc(conn: Connection): Muxer {.gcsafe, closure, raises: [Defect].}
 
   # this wraps a creator proc that knows how to make muxers
   MuxerProvider* = ref object of LPProtocol
@@ -33,21 +43,28 @@ type
     streamHandler*: StreamHandler # triggered every time there is a new stream, called for any muxer instance
     muxerHandler*: MuxerHandler # triggered every time there is a new muxed connection created
 
+func shortLog*(m: Muxer): auto = shortLog(m.connection)
+chronicles.formatIt(Muxer): shortLog(it)
+
 # muxer interface
 method newStream*(m: Muxer, name: string = "", lazy: bool = false):
   Future[Connection] {.base, async, gcsafe.} = discard
 method close*(m: Muxer) {.base, async, gcsafe.} = discard
 method handle*(m: Muxer): Future[void] {.base, async, gcsafe.} = discard
 
-proc newMuxerProvider*(creator: MuxerConstructor, codec: string): MuxerProvider {.gcsafe.} =
-  new result
-  result.newMuxer = creator
-  result.codec = codec
-  result.init()
+proc new*(
+  T: typedesc[MuxerProvider],
+  creator: MuxerConstructor,
+  codec: string): T {.gcsafe.} =
+
+  let muxerProvider = T(newMuxer: creator)
+  muxerProvider.codec = codec
+  muxerProvider.init()
+  muxerProvider
 
 method init(c: MuxerProvider) =
   proc handler(conn: Connection, proto: string) {.async, gcsafe, closure.} =
-    trace "starting muxer handler", proto=proto, peer = $conn
+    trace "starting muxer handler", proto=proto, conn
     try:
       let
         muxer = c.newMuxer(conn)
@@ -60,17 +77,16 @@ method init(c: MuxerProvider) =
 
       # finally await both the futures
       if not isNil(c.muxerHandler):
-        futs &= c.muxerHandler(muxer)
+        await c.muxerHandler(muxer)
+        when defined(libp2p_agents_metrics):
+          conn.shortAgent = muxer.connection.shortAgent
 
       checkFutures(await allFinished(futs))
     except CancelledError as exc:
       raise exc
     except CatchableError as exc:
-      trace "exception in muxer handler", exc = exc.msg, peer = $conn, proto=proto
+      trace "exception in muxer handler", exc = exc.msg, conn, proto
     finally:
       await conn.close()
 
   c.handler = handler
-
-proc `$`*(m: Muxer): string =
-  $m.connection

libp2p/muxers/yamux/yamux.nim

@@ -0,0 +1,526 @@
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import sequtils, std/[tables]
+import chronos, chronicles, metrics, stew/[endians2, byteutils, objects]
+import ../muxer,
+       ../../stream/connection
+
+export muxer
+
+logScope:
+  topics = "libp2p yamux"
+
+const
+  YamuxCodec* = "/yamux/1.0.0"
+  YamuxVersion = 0.uint8
+  DefaultWindowSize = 256000
+  MaxChannelCount = 200
+
+when defined(libp2p_yamux_metrics):
+  declareGauge(libp2p_yamux_channels, "yamux channels", labels = ["initiator", "peer"])
+  declareHistogram libp2p_yamux_send_queue, "message send queue length (in byte)",
+    buckets = [0.0, 100.0, 250.0, 1000.0, 2000.0, 1600.0, 6400.0, 25600.0, 256000.0]
+  declareHistogram libp2p_yamux_recv_queue, "message recv queue length (in byte)",
+    buckets = [0.0, 100.0, 250.0, 1000.0, 2000.0, 1600.0, 6400.0, 25600.0, 256000.0]
+
+type
+  YamuxError* = object of CatchableError
+
+  MsgType = enum
+    Data = 0x0
+    WindowUpdate = 0x1
+    Ping = 0x2
+    GoAway = 0x3
+
+  MsgFlags {.size: 2.} = enum
+    Syn
+    Ack
+    Fin
+    Rst
+
+  GoAwayStatus = enum
+    NormalTermination = 0x0,
+    ProtocolError = 0x1,
+    InternalError = 0x2,
+
+  YamuxHeader = object
+    version: uint8
+    msgType: MsgType
+    flags: set[MsgFlags]
+    streamId: uint32
+    length: uint32
+
+proc readHeader(conn: LPStream): Future[YamuxHeader] {.async, gcsafe.} =
+  var buffer: array[12, byte]
+  await conn.readExactly(addr buffer[0], 12)
+
+  result.version = buffer[0]
+  let flags = fromBytesBE(uint16, buffer[2..3])
+  if not result.msgType.checkedEnumAssign(buffer[1]) or flags notin 0'u16..15'u16:
+     raise newException(YamuxError, "Wrong header")
+  result.flags = cast[set[MsgFlags]](flags)
+  result.streamId = fromBytesBE(uint32, buffer[4..7])
+  result.length = fromBytesBE(uint32, buffer[8..11])
+  return result
+
+proc `$`(header: YamuxHeader): string =
+  result = "{" & $header.msgType & ", "
+  result &= "{" & header.flags.foldl(if a != "": a & ", " & $b else: $b, "") & "}, "
+  result &= "streamId: " & $header.streamId & ", "
+  result &= "length: " & $header.length & "}"
+
+proc encode(header: YamuxHeader): array[12, byte] =
+  result[0] = header.version
+  result[1] = uint8(header.msgType)
+  result[2..3] = toBytesBE(cast[uint16](header.flags))
+  result[4..7] = toBytesBE(header.streamId)
+  result[8..11] = toBytesBE(header.length)
+
+proc write(conn: LPStream, header: YamuxHeader): Future[void] {.gcsafe.} =
+  trace "write directly on stream", h = $header
+  var buffer = header.encode()
+  return conn.write(@buffer)
+
+proc ping(T: type[YamuxHeader], flag: MsgFlags, pingData: uint32): T =
+  T(
+    version: YamuxVersion,
+    msgType: MsgType.Ping,
+    flags: {flag},
+    length: pingData
+  )
+
+proc goAway(T: type[YamuxHeader], status: GoAwayStatus): T =
+  T(
+    version: YamuxVersion,
+    msgType: MsgType.GoAway,
+    length: uint32(status)
+  )
+
+proc data(
+  T: type[YamuxHeader],
+  streamId: uint32,
+  length: uint32 = 0,
+  flags: set[MsgFlags] = {},
+  ): T =
+  T(
+    version: YamuxVersion,
+    msgType: MsgType.Data,
+    length: length,
+    flags: flags,
+    streamId: streamId
+  )
+
+proc windowUpdate(
+  T: type[YamuxHeader],
+  streamId: uint32,
+  delta: uint32,
+  flags: set[MsgFlags] = {},
+  ): T =
+  T(
+    version: YamuxVersion,
+    msgType: MsgType.WindowUpdate,
+    length: delta,
+    flags: flags,
+    streamId: streamId
+  )
+
+type
+  ToSend = tuple
+    data: seq[byte]
+    sent: int
+    fut: Future[void]
+  YamuxChannel* = ref object of Connection
+    id: uint32
+    recvWindow: int
+    sendWindow: int
+    maxRecvWindow: int
+    conn: Connection
+    isSrc: bool
+    opened: bool
+    isSending: bool
+    sendQueue: seq[ToSend]
+    recvQueue: seq[byte]
+    isReset: bool
+    remoteReset: bool
+    closedRemotely: Future[void]
+    closedLocally: bool
+    receivedData: AsyncEvent
+    returnedEof: bool
+
+proc `$`(channel: YamuxChannel): string =
+  result = if channel.conn.dir == Out: "=> " else: "<= "
+  result &= $channel.id
+  var s: seq[string] = @[]
+  if channel.closedRemotely.done():
+    s.add("ClosedRemotely")
+  if channel.closedLocally:
+    s.add("ClosedLocally")
+  if channel.isReset:
+    s.add("Reset")
+  if s.len > 0:
+    result &= " {" & s.foldl(if a != "": a & ", " & b else: b, "") & "}"
+
+proc sendQueueBytes(channel: YamuxChannel, limit: bool = false): int =
+  for (elem, sent, _) in channel.sendQueue:
+    result.inc(min(elem.len - sent, if limit: channel.maxRecvWindow div 3 else: elem.len - sent))
+
+proc actuallyClose(channel: YamuxChannel) {.async.} =
+  if channel.closedLocally and channel.sendQueue.len == 0 and
+     channel.closedRemotely.done():
+    await procCall Connection(channel).closeImpl()
+
+proc remoteClosed(channel: YamuxChannel) {.async.} =
+  if not channel.closedRemotely.done():
+    channel.closedRemotely.complete()
+    await channel.actuallyClose()
+
+method closeImpl*(channel: YamuxChannel) {.async, gcsafe.} =
+  if not channel.closedLocally:
+    channel.closedLocally = true
+
+    if channel.isReset == false and channel.sendQueue.len == 0:
+      await channel.conn.write(YamuxHeader.data(channel.id, 0, {Fin}))
+    await channel.actuallyClose()
+
+proc reset(channel: YamuxChannel, isLocal: bool = false) {.async.} =
+  if channel.isReset:
+    return
+  trace "Reset channel"
+  channel.isReset = true
+  channel.remoteReset = not isLocal
+  for (d, s, fut) in channel.sendQueue:
+    fut.fail(newLPStreamEOFError())
+  channel.sendQueue = @[]
+  channel.recvQueue = @[]
+  channel.sendWindow = 0
+  if not channel.closedLocally:
+    if isLocal:
+      try: await channel.conn.write(YamuxHeader.data(channel.id, 0, {Rst}))
+      except LPStreamEOFError as exc: discard
+      except LPStreamClosedError as exc: discard
+    await channel.close()
+  if not channel.closedRemotely.done():
+    await channel.remoteClosed()
+  channel.receivedData.fire()
+  if not isLocal:
+    # If we reset locally, we want to flush up to a maximum of recvWindow
+    # bytes. We use the recvWindow in the proc cleanupChann.
+    channel.recvWindow = 0
+
+proc updateRecvWindow(channel: YamuxChannel) {.async.} =
+  let inWindow = channel.recvWindow + channel.recvQueue.len
+  if inWindow > channel.maxRecvWindow div 2:
+    return
+
+  let delta = channel.maxRecvWindow - inWindow
+  channel.recvWindow.inc(delta)
+  await channel.conn.write(YamuxHeader.windowUpdate(
+    channel.id,
+    delta.uint32
+  ))
+  trace "increasing the recvWindow", delta
+
+method readOnce*(
+  channel: YamuxChannel,
+  pbytes: pointer,
+  nbytes: int):
+  Future[int] {.async.} =
+
+  if channel.isReset:
+    raise if channel.remoteReset:
+        newLPStreamResetError()
+      elif channel.closedLocally:
+        newLPStreamClosedError()
+      else:
+        newLPStreamConnDownError()
+  if channel.returnedEof:
+    raise newLPStreamRemoteClosedError()
+  if channel.recvQueue.len == 0:
+    channel.receivedData.clear()
+    await channel.closedRemotely or channel.receivedData.wait()
+    if channel.closedRemotely.done() and channel.recvQueue.len == 0:
+      channel.returnedEof = true
+      return 0
+
+  let toRead = min(channel.recvQueue.len, nbytes)
+
+  var p = cast[ptr UncheckedArray[byte]](pbytes)
+  toOpenArray(p, 0, nbytes - 1)[0..<toRead] = channel.recvQueue.toOpenArray(0, toRead - 1)
+  channel.recvQueue = channel.recvQueue[toRead..^1]
+
+  # We made some room in the recv buffer let the peer know
+  await channel.updateRecvWindow()
+  channel.activity = true
+  return toRead
+
+proc gotDataFromRemote(channel: YamuxChannel, b: seq[byte]) {.async.} =
+  channel.recvWindow -= b.len
+  channel.recvQueue = channel.recvQueue.concat(b)
+  channel.receivedData.fire()
+  when defined(libp2p_yamux_metrics):
+    libp2p_yamux_recv_queue.observe(channel.recvQueue.len.int64)
+  await channel.updateRecvWindow()
+
+proc setMaxRecvWindow*(channel: YamuxChannel, maxRecvWindow: int) =
+  channel.maxRecvWindow = maxRecvWindow
+
+proc trySend(channel: YamuxChannel) {.async.} =
+  if channel.isSending:
+    return
+  channel.isSending = true
+  defer: channel.isSending = false
+  while channel.sendQueue.len != 0:
+    channel.sendQueue.keepItIf(not (it.fut.cancelled() and it.sent == 0))
+    if channel.sendWindow == 0:
+      trace "send window empty"
+      if channel.sendQueueBytes(true) > channel.maxRecvWindow:
+        debug "channel send queue too big, resetting", maxSendWindow=channel.maxRecvWindow,
+          currentQueueSize = channel.sendQueueBytes(true)
+        try:
+          await channel.reset(true)
+        except CatchableError as exc:
+          debug "failed to reset", msg=exc.msg
+      break
+
+    let
+      bytesAvailable = channel.sendQueueBytes()
+      toSend = min(channel.sendWindow, bytesAvailable)
+    var
+      sendBuffer = newSeqUninitialized[byte](toSend + 12)
+      header = YamuxHeader.data(channel.id, toSend.uint32)
+      inBuffer = 0
+
+    if toSend >= bytesAvailable and channel.closedLocally:
+      trace "last buffer we'll sent on this channel", toSend, bytesAvailable
+      header.flags.incl({Fin})
+
+    sendBuffer[0..<12] = header.encode()
+
+    var futures: seq[Future[void]]
+    while inBuffer < toSend:
+      let (data, sent, fut) = channel.sendQueue[0]
+      let bufferToSend = min(data.len - sent, toSend - inBuffer)
+      sendBuffer.toOpenArray(12, 12 + toSend - 1)[inBuffer..<(inBuffer+bufferToSend)] =
+        channel.sendQueue[0].data.toOpenArray(sent, sent + bufferToSend - 1)
+      channel.sendQueue[0].sent.inc(bufferToSend)
+      if channel.sendQueue[0].sent >= data.len:
+        futures.add(fut)
+        channel.sendQueue.delete(0)
+      inBuffer.inc(bufferToSend)
+
+    trace "build send buffer", h = $header, msg=string.fromBytes(sendBuffer[12..^1])
+    channel.sendWindow.dec(toSend)
+    try: await channel.conn.write(sendBuffer)
+    except CatchableError as exc:
+      let connDown = newLPStreamConnDownError(exc)
+      for fut in futures.items():
+        fut.fail(connDown)
+      await channel.reset()
+      break
+    for fut in futures.items():
+      fut.complete()
+    channel.activity = true
+
+method write*(channel: YamuxChannel, msg: seq[byte]): Future[void] =
+  result = newFuture[void]("Yamux Send")
+  if channel.remoteReset:
+    result.fail(newLPStreamResetError())
+    return result
+  if channel.closedLocally or channel.isReset:
+    result.fail(newLPStreamClosedError())
+    return result
+  if msg.len == 0:
+    result.complete()
+    return result
+  channel.sendQueue.add((msg, 0, result))
+  when defined(libp2p_yamux_metrics):
+    libp2p_yamux_recv_queue.observe(channel.sendQueueBytes().int64)
+  asyncSpawn channel.trySend()
+
+proc open*(channel: YamuxChannel) {.async, gcsafe.} =
+  if channel.opened:
+    trace "Try to open channel twice"
+    return
+  channel.opened = true
+  await channel.conn.write(YamuxHeader.data(channel.id, 0, {if channel.isSrc: Syn else: Ack}))
+
+type
+  Yamux* = ref object of Muxer
+    channels: Table[uint32, YamuxChannel]
+    flushed: Table[uint32, int]
+    currentId: uint32
+    isClosed: bool
+    maxChannCount: int
+
+proc lenBySrc(m: Yamux, isSrc: bool): int =
+  for v in m.channels.values():
+    if v.isSrc == isSrc: result += 1
+
+proc cleanupChann(m: Yamux, channel: YamuxChannel) {.async.} =
+  await channel.join()
+  m.channels.del(channel.id)
+  when defined(libp2p_yamux_metrics):
+    libp2p_yamux_channels.set(m.lenBySrc(channel.isSrc).int64, [$channel.isSrc, $channel.peerId])
+  if channel.isReset and channel.recvWindow > 0:
+    m.flushed[channel.id] = channel.recvWindow
+
+proc createStream(m: Yamux, id: uint32, isSrc: bool): YamuxChannel =
+  result = YamuxChannel(
+    id: id,
+    maxRecvWindow: DefaultWindowSize,
+    recvWindow: DefaultWindowSize,
+    sendWindow: DefaultWindowSize,
+    isSrc: isSrc,
+    conn: m.connection,
+    receivedData: newAsyncEvent(),
+    closedRemotely: newFuture[void]()
+  )
+  result.objName = "YamuxStream"
+  result.dir = if isSrc: Direction.Out else: Direction.In
+  result.timeoutHandler = proc(): Future[void] {.gcsafe.} =
+    trace "Idle timeout expired, resetting YamuxChannel"
+    result.reset()
+  result.initStream()
+  result.peerId = m.connection.peerId
+  result.observedAddr = m.connection.observedAddr
+  result.transportDir = m.connection.transportDir
+  when defined(libp2p_agents_metrics):
+    result.shortAgent = m.connection.shortAgent
+  m.channels[id] = result
+  asyncSpawn m.cleanupChann(result)
+  trace "created channel", id, pid=m.connection.peerId
+  when defined(libp2p_yamux_metrics):
+    libp2p_yamux_channels.set(m.lenBySrc(isSrc).int64, [$isSrc, $result.peerId])
+
+method close*(m: Yamux) {.async.} =
+  if m.isClosed == true:
+    trace "Already closed"
+    return
+  m.isClosed = true
+
+  trace "Closing yamux"
+  let channels = toSeq(m.channels.values())
+  for channel in channels:
+    await channel.reset(true)
+  await m.connection.write(YamuxHeader.goAway(NormalTermination))
+  await m.connection.close()
+  trace "Closed yamux"
+
+proc handleStream(m: Yamux, channel: YamuxChannel) {.async.} =
+  ## call the muxer stream handler for this channel
+  ##
+  try:
+    await m.streamHandler(channel)
+    trace "finished handling stream"
+    doAssert(channel.isClosed, "connection not closed by handler!")
+  except CatchableError as exc:
+    trace "Exception in yamux stream handler", msg = exc.msg
+    await channel.reset()
+
+method handle*(m: Yamux) {.async, gcsafe.} =
+  trace "Starting yamux handler", pid=m.connection.peerId
+  try:
+    while not m.connection.atEof:
+      trace "waiting for header"
+      let header = await m.connection.readHeader()
+      trace "got message", h = $header
+
+      case header.msgType:
+      of Ping:
+        if MsgFlags.Syn in header.flags:
+          await m.connection.write(YamuxHeader.ping(MsgFlags.Ack, header.length))
+      of GoAway:
+        var status: GoAwayStatus
+        if status.checkedEnumAssign(header.length): trace "Received go away", status
+        else: trace "Received unexpected error go away"
+        break
+      of Data, WindowUpdate:
+        if MsgFlags.Syn in header.flags:
+          if header.streamId in m.channels:
+            debug "Trying to create an existing channel, skipping", id=header.streamId
+          else:
+            if header.streamId in m.flushed:
+              m.flushed.del(header.streamId)
+            if header.streamId mod 2 == m.currentId mod 2:
+              raise newException(YamuxError, "Peer used our reserved stream id")
+            let newStream = m.createStream(header.streamId, false)
+            if m.channels.len >= m.maxChannCount:
+              await newStream.reset()
+              continue
+            await newStream.open()
+            asyncSpawn m.handleStream(newStream)
+        elif header.streamId notin m.channels:
+          if header.streamId notin m.flushed:
+            raise newException(YamuxError, "Unknown stream ID: " & $header.streamId)
+          elif header.msgType == Data:
+            # Flush the data
+            m.flushed[header.streamId].dec(int(header.length))
+            if m.flushed[header.streamId] < 0:
+              raise newException(YamuxError, "Peer exhausted the recvWindow after reset")
+            if header.length > 0:
+              var buffer = newSeqUninitialized[byte](header.length)
+              await m.connection.readExactly(addr buffer[0], int(header.length))
+          continue
+
+        let channel = m.channels[header.streamId]
+
+        if header.msgType == WindowUpdate:
+          channel.sendWindow += int(header.length)
+          await channel.trySend()
+        else:
+          if header.length.int > channel.recvWindow.int:
+            # check before allocating the buffer
+            raise newException(YamuxError, "Peer exhausted the recvWindow")
+
+          if header.length > 0:
+            var buffer = newSeqUninitialized[byte](header.length)
+            await m.connection.readExactly(addr buffer[0], int(header.length))
+            trace "Msg Rcv", msg=string.fromBytes(buffer)
+            await channel.gotDataFromRemote(buffer)
+
+        if MsgFlags.Fin in header.flags:
+          trace "remote closed channel"
+          await channel.remoteClosed()
+        if MsgFlags.Rst in header.flags:
+          trace "remote reset channel"
+          await channel.reset()
+  except LPStreamEOFError as exc:
+    trace "Stream EOF", msg = exc.msg
+  except YamuxError as exc:
+    trace "Closing yamux connection", error=exc.msg
+    await m.connection.write(YamuxHeader.goAway(ProtocolError))
+  finally:
+    await m.close()
+  trace "Stopped yamux handler"
+
+method newStream*(
+  m: Yamux,
+  name: string = "",
+  lazy: bool = false): Future[Connection] {.async, gcsafe.} =
+
+  if m.channels.len > m.maxChannCount - 1:
+    raise newException(TooManyChannels, "max allowed channel count exceeded")
+  let stream = m.createStream(m.currentId, true)
+  m.currentId += 2
+  if not lazy:
+    await stream.open()
+  return stream
+
+proc new*(T: type[Yamux], conn: Connection, maxChannCount: int = MaxChannelCount): T =
+  T(
+    connection: conn,
+    currentId: if conn.dir == Out: 1 else: 2,
+    maxChannCount: maxChannCount
+  )

libp2p/nameresolving/dnsresolver.nim

@@ -0,0 +1,176 @@
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import
+  std/[streams, strutils, sets, sequtils],
+  chronos, chronicles, stew/byteutils,
+  dnsclientpkg/[protocol, types]
+
+import
+  nameresolver
+
+logScope:
+  topics = "libp2p dnsresolver"
+
+type
+  DnsResolver* = ref object of NameResolver
+    nameServers*: seq[TransportAddress]
+
+proc questionToBuf(address: string, kind: QKind): seq[byte] =
+  try:
+    var
+      header = initHeader()
+      question = initQuestion(address, kind)
+
+      requestStream = header.toStream()
+    question.toStream(requestStream)
+
+    let dataLen = requestStream.getPosition()
+    requestStream.setPosition(0)
+
+    var buf = newSeq[byte](dataLen)
+    discard requestStream.readData(addr buf[0], dataLen)
+    return buf
+  except CatchableError as exc:
+    info "Failed to created DNS buffer", msg = exc.msg
+    return newSeq[byte](0)
+
+proc getDnsResponse(
+  dnsServer: TransportAddress,
+  address: string,
+  kind: QKind): Future[Response] {.async.} =
+
+  var sendBuf = questionToBuf(address, kind)
+
+  if sendBuf.len == 0:
+    raise newException(ValueError, "Incorrect DNS query")
+
+  let receivedDataFuture = newFuture[void]()
+
+  proc datagramDataReceived(transp: DatagramTransport,
+                  raddr: TransportAddress): Future[void] {.async, closure.} =
+      receivedDataFuture.complete()
+
+  let sock =
+    if dnsServer.family == AddressFamily.IPv6:
+      newDatagramTransport6(datagramDataReceived)
+    else:
+      newDatagramTransport(datagramDataReceived)
+
+  try:
+    await sock.sendTo(dnsServer, addr sendBuf[0], sendBuf.len)
+
+    await receivedDataFuture or sleepAsync(5.seconds) #unix default
+
+    if not receivedDataFuture.finished:
+      raise newException(IOError, "DNS server timeout")
+
+    let rawResponse = sock.getMessage()
+    # parseResponse can has a raises: [Exception, ..] because of
+    # https://github.com/nim-lang/Nim/commit/035134de429b5d99c5607c5fae912762bebb6008
+    # it can't actually raise though
+    return parseResponse(string.fromBytes(rawResponse))
+  except CatchableError as exc: raise exc
+  except Exception as exc: raiseAssert exc.msg
+  finally:
+    await sock.closeWait()
+
+method resolveIp*(
+  self: DnsResolver,
+  address: string,
+  port: Port,
+  domain: Domain = Domain.AF_UNSPEC): Future[seq[TransportAddress]] {.async.} =
+
+  trace "Resolving IP using DNS", address, servers = self.nameServers.mapIt($it), domain
+  for _ in 0 ..< self.nameServers.len:
+    let server = self.nameServers[0]
+    var responseFutures: seq[Future[Response]]
+    if domain == Domain.AF_INET or domain == Domain.AF_UNSPEC:
+      responseFutures.add(getDnsResponse(server, address, A))
+
+    if domain == Domain.AF_INET6 or domain == Domain.AF_UNSPEC:
+      let fut = getDnsResponse(server, address, AAAA)
+      if server.family == AddressFamily.IPv6:
+        trace "IPv6 DNS server, puting AAAA records first", server = $server
+        responseFutures.insert(fut)
+      else:
+        responseFutures.add(fut)
+
+    var
+      resolvedAddresses: OrderedSet[string]
+      resolveFailed = false
+    for fut in responseFutures:
+      try:
+        let resp = await fut
+        for answer in resp.answers:
+          # toString can has a raises: [Exception, ..] because of
+          # https://github.com/nim-lang/Nim/commit/035134de429b5d99c5607c5fae912762bebb6008
+          # it can't actually raise though
+          resolvedAddresses.incl(
+            try: answer.toString()
+            except CatchableError as exc: raise exc
+            except Exception as exc: raiseAssert exc.msg
+          )
+      except CancelledError as e:
+        raise e
+      except ValueError as e:
+        info "Invalid DNS query", address, error=e.msg
+        return @[]
+      except CatchableError as e:
+        info "Failed to query DNS", address, error=e.msg
+        resolveFailed = true
+        break
+
+    if resolveFailed:
+      self.nameServers.add(self.nameServers[0])
+      self.nameServers.delete(0)
+      continue
+
+    trace "Got IPs from DNS server", resolvedAddresses, server = $server
+    return resolvedAddresses.toSeq().mapIt(initTAddress(it, port))
+
+  debug "Failed to resolve address, returning empty set"
+  return @[]
+
+method resolveTxt*(
+  self: DnsResolver,
+  address: string): Future[seq[string]] {.async.} =
+
+  trace "Resolving TXT using DNS", address, servers = self.nameServers.mapIt($it)
+  for _ in 0 ..< self.nameServers.len:
+    let server = self.nameServers[0]
+    try:
+      let response = await getDnsResponse(server, address, TXT)
+      trace "Got TXT response", server = $server, answer=response.answers.mapIt(it.toString())
+      return response.answers.mapIt(it.toString())
+    except CancelledError as e:
+      raise e
+    except CatchableError as e:
+      info "Failed to query DNS", address, error=e.msg
+      self.nameServers.add(self.nameServers[0])
+      self.nameServers.delete(0)
+      continue
+    except Exception as e:
+      # toString can has a raises: [Exception, ..] because of
+      # https://github.com/nim-lang/Nim/commit/035134de429b5d99c5607c5fae912762bebb6008
+      # it can't actually raise though
+      raiseAssert e.msg
+
+  debug "Failed to resolve TXT, returning empty set"
+  return @[]
+
+proc new*(
+  T: typedesc[DnsResolver],
+  nameServers: seq[TransportAddress]): T =
+  T(nameServers: nameServers)

libp2p/nameresolving/mockresolver.nim

@@ -0,0 +1,49 @@
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import
+  std/[streams, strutils, tables],
+  chronos, chronicles
+
+import nameresolver
+
+export tables
+
+logScope:
+  topics = "libp2p mockresolver"
+
+type MockResolver* = ref object of NameResolver
+  txtResponses*: Table[string, seq[string]]
+  # key: address, isipv6?
+  ipResponses*: Table[(string, bool), seq[string]]
+
+method resolveIp*(
+  self: MockResolver,
+  address: string,
+  port: Port,
+  domain: Domain = Domain.AF_UNSPEC): Future[seq[TransportAddress]] {.async.} =
+  if domain == Domain.AF_INET or domain == Domain.AF_UNSPEC:
+    for resp in self.ipResponses.getOrDefault((address, false)):
+      result.add(initTAddress(resp, port))
+
+  if domain == Domain.AF_INET6 or domain == Domain.AF_UNSPEC:
+    for resp in self.ipResponses.getOrDefault((address, true)):
+      result.add(initTAddress(resp, port))
+
+method resolveTxt*(
+  self: MockResolver,
+  address: string): Future[seq[string]] {.async.} =
+  return self.txtResponses.getOrDefault(address)
+
+proc new*(T: typedesc[MockResolver]): T = T()

libp2p/nameresolving/nameresolver.nim

@@ -0,0 +1,139 @@
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import std/[sugar, sets, sequtils, strutils]
+import
+  chronos,
+  chronicles,
+  stew/[endians2, byteutils]
+import ".."/[multiaddress, multicodec]
+
+logScope:
+  topics = "libp2p nameresolver"
+
+type
+  NameResolver* = ref object of RootObj
+
+method resolveTxt*(
+  self: NameResolver,
+  address: string): Future[seq[string]] {.async, base.} =
+  ## Get TXT record
+  ##
+
+  doAssert(false, "Not implemented!")
+
+method resolveIp*(
+  self: NameResolver,
+  address: string,
+  port: Port,
+  domain: Domain = Domain.AF_UNSPEC): Future[seq[TransportAddress]] {.async, base.} =
+  ## Resolve the specified address
+  ##
+
+  doAssert(false, "Not implemented!")
+
+proc getHostname*(ma: MultiAddress): string =
+  let
+    firstPart = ma[0].valueOr: return ""
+    fpSplitted = ($firstPart).split('/', 2)
+  if fpSplitted.len > 2: fpSplitted[2]
+  else: ""
+
+proc resolveOneAddress(
+  self: NameResolver,
+  ma: MultiAddress,
+  domain: Domain = Domain.AF_UNSPEC,
+  prefix = ""): Future[seq[MultiAddress]]
+  {.async, raises: [Defect, MaError, TransportAddressError].} =
+  #Resolve a single address
+  var pbuf: array[2, byte]
+
+  var dnsval = getHostname(ma)
+
+  if ma[1].tryGet().protoArgument(pbuf).tryGet() == 0:
+    raise newException(MaError, "Incorrect port number")
+  let
+    port = Port(fromBytesBE(uint16, pbuf))
+    resolvedAddresses = await self.resolveIp(prefix & dnsval, port, domain)
+
+  return collect(newSeqOfCap(4)):
+    for address in resolvedAddresses:
+      var createdAddress = MultiAddress.init(address).tryGet()[0].tryGet()
+      for part in ma:
+        if DNS.match(part.tryGet()): continue
+        createdAddress &= part.tryGet()
+      createdAddress
+
+proc resolveDnsAddr*(
+  self: NameResolver,
+  ma: MultiAddress,
+  depth: int = 0): Future[seq[MultiAddress]] {.async.} =
+
+  if not DNSADDR.matchPartial(ma):
+    return @[ma]
+
+  trace "Resolving dnsaddr", ma
+  if depth > 6:
+    info "Stopping DNSADDR recursion, probably malicious", ma
+    return @[]
+
+  var dnsval = getHostname(ma)
+
+  let txt = await self.resolveTxt("_dnsaddr." & dnsval)
+
+  trace "txt entries", txt
+
+  var result: seq[MultiAddress]
+  for entry in txt:
+    if not entry.startsWith("dnsaddr="): continue
+    let entryValue = MultiAddress.init(entry[8..^1]).tryGet()
+
+    if entryValue.contains(multiCodec("p2p")).tryGet() and ma.contains(multiCodec("p2p")).tryGet():
+      if entryValue[multiCodec("p2p")] != ma[multiCodec("p2p")]:
+        continue
+
+    let resolved = await self.resolveDnsAddr(entryValue, depth + 1)
+    for r in resolved:
+      result.add(r)
+
+  if result.len == 0:
+    debug "Failed to resolve a DNSADDR", ma
+    return @[]
+  return result
+
+
+proc resolveMAddress*(
+  self: NameResolver,
+  address: MultiAddress): Future[seq[MultiAddress]] {.async.} =
+  var res = initOrderedSet[MultiAddress]()
+
+  if not DNS.matchPartial(address):
+    res.incl(address)
+  else:
+    let code = address[0].get().protoCode().get()
+    let seq = case code:
+      of multiCodec("dns"):
+        await self.resolveOneAddress(address)
+      of multiCodec("dns4"):
+        await self.resolveOneAddress(address, Domain.AF_INET)
+      of multiCodec("dns6"):
+        await self.resolveOneAddress(address, Domain.AF_INET6)
+      of multiCodec("dnsaddr"):
+        await self.resolveDnsAddr(address)
+      else:
+        doAssert false
+        @[address]
+    for ad in seq:
+      res.incl(ad)
+  return res.toSeq

libp2p/peerid.nim

@@ -1,60 +1,79 @@
-## Nim-LibP2P
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module implementes API for libp2p peer.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+{.push public.}
 
-import hashes
-import nimcrypto/utils, stew/base58
-import crypto/crypto, multicodec, multihash, vbuffer
-import protobuf/minprotobuf
-import stew/results
-export results
+import
+  std/[hashes, strutils],
+  stew/[base58, results],
+  chronicles,
+  nimcrypto/utils,
+  utility,
+  ./crypto/crypto, ./multicodec, ./multihash, ./vbuffer,
+  ./protobuf/minprotobuf
+
+export results, utility
 
 const
   maxInlineKeyLength* = 42
 
-# TODO: add proper on disc serialization
-# using peer-id protobuf format
 type
-  PeerID* = object
+  PeerId* = object
     data*: seq[byte]
 
-  PeerIDError* = object of CatchableError
-
-proc pretty*(pid: PeerID): string {.inline.} =
+func `$`*(pid: PeerId): string =
   ## Return base58 encoded ``pid`` representation.
-  result = Base58.encode(pid.data)
+  # This unusual call syntax is used to avoid a strange Nim compilation error
+  base58.encode(Base58, pid.data)
 
-proc toBytes*(pid: PeerID, data: var openarray[byte]): int =
-  ## Store PeerID ``pid`` to array of bytes ``data``.
+func shortLog*(pid: PeerId): string =
+  ## Returns compact string representation of ``pid``.
+  var spid = $pid
+  if len(spid) > 10:
+    spid[3] = '*'
+
+    when (NimMajor, NimMinor) > (1, 4):
+      spid.delete(4 .. spid.high - 6)
+    else:
+      spid.delete(4, spid.high - 6)
+
+  spid
+
+chronicles.formatIt(PeerId): shortLog(it)
+
+func toBytes*(pid: PeerId, data: var openArray[byte]): int =
+  ## Store PeerId ``pid`` to array of bytes ``data``.
   ##
   ## Returns number of bytes needed to store ``pid``.
   result = len(pid.data)
   if len(data) >= result and result > 0:
     copyMem(addr data[0], unsafeAddr pid.data[0], result)
 
-proc getBytes*(pid: PeerID): seq[byte] {.inline.} =
-  ## Return PeerID ``pid`` as array of bytes.
-  result = pid.data
+template getBytes*(pid: PeerId): seq[byte] =
+  ## Return PeerId ``pid`` as array of bytes.
+  pid.data
 
-proc hex*(pid: PeerID): string {.inline.} =
+func hex*(pid: PeerId): string =
   ## Returns hexadecimal string representation of ``pid``.
-  if len(pid.data) > 0:
-    result = toHex(pid.data)
+  toHex(pid.data)
 
-proc len*(pid: PeerID): int {.inline.} =
+template len*(pid: PeerId): int =
   ## Returns length of ``pid`` binary representation.
-  result = len(pid.data)
+  len(pid.data)
 
-proc cmp*(a, b: PeerID): int =
+func cmp*(a, b: PeerId): int =
   ## Compares two peer ids ``a`` and ``b``.
   ## Returns:
   ##
@@ -69,30 +88,29 @@ proc cmp*(a, b: PeerID): int =
     inc(i)
   result = len(a.data) - len(b.data)
 
-proc `<=`*(a, b: PeerID): bool {.inline.} =
+template `<=`*(a, b: PeerId): bool =
   (cmp(a, b) <= 0)
 
-proc `<`*(a, b: PeerID): bool {.inline.} =
+template `<`*(a, b: PeerId): bool =
   (cmp(a, b) < 0)
 
-proc `>=`*(a, b: PeerID): bool {.inline.} =
+template `>=`*(a, b: PeerId): bool =
   (cmp(a, b) >= 0)
 
-proc `>`*(a, b: PeerID): bool {.inline.} =
+template `>`*(a, b: PeerId): bool =
   (cmp(a, b) > 0)
 
-proc `==`*(a, b: PeerID): bool {.inline.} =
+template `==`*(a, b: PeerId): bool =
   (cmp(a, b) == 0)
 
-proc hash*(pid: PeerID): Hash {.inline.} =
-  result = hash(pid.data)
+template hash*(pid: PeerId): Hash =
+  hash(pid.data)
 
-proc validate*(pid: PeerID): bool =
+func validate*(pid: PeerId): bool =
   ## Validate check if ``pid`` is empty or not.
-  if len(pid.data) > 0:
-    result = MultiHash.validate(pid.data)
+  len(pid.data) > 0 and MultiHash.validate(pid.data)
 
-proc hasPublicKey*(pid: PeerID): bool =
+func hasPublicKey*(pid: PeerId): bool =
   ## Returns ``true`` if ``pid`` is small enough to hold public key inside.
   if len(pid.data) > 0:
     var mh: MultiHash
@@ -100,41 +118,28 @@ proc hasPublicKey*(pid: PeerID): bool =
       if mh.mcodec == multiCodec("identity"):
         result = true
 
-proc extractPublicKey*(pid: PeerID, pubkey: var PublicKey): bool =
-  ## Returns ``true`` if public key was successfully decoded from PeerID
+func extractPublicKey*(pid: PeerId, pubkey: var PublicKey): bool =
+  ## Returns ``true`` if public key was successfully decoded from PeerId
   ## ``pid``and stored to ``pubkey``.
   ##
   ## Returns ``false`` otherwise.
-  var mh: MultiHash
   if len(pid.data) > 0:
+    var mh: MultiHash
     if MultiHash.decode(pid.data, mh).isOk:
       if mh.mcodec == multiCodec("identity"):
         let length = len(mh.data.buffer)
         result = pubkey.init(mh.data.buffer.toOpenArray(mh.dpos, length - 1))
 
-proc `$`*(pid: PeerID): string =
-  ## Returns compact string representation of ``pid``.
-  var spid = pid.pretty()
-  if len(spid) <= 10:
-    result = spid
-  else:
-    result = newStringOfCap(10)
-    for i in 0..<2:
-      result.add(spid[i])
-    result.add("*")
-    for i in (len(spid) - 6)..spid.high:
-      result.add(spid[i])
-
-proc init*(pid: var PeerID, data: openarray[byte]): bool =
+func init*(pid: var PeerId, data: openArray[byte]): bool =
   ## Initialize peer id from raw binary representation ``data``.
   ##
   ## Returns ``true`` if peer was successfully initialiazed.
-  var p = PeerID(data: @data)
+  var p = PeerId(data: @data)
   if p.validate():
     pid = p
     result = true
 
-proc init*(pid: var PeerID, data: string): bool =
+func init*(pid: var PeerId, data: string): bool =
   ## Initialize peer id from base58 encoded string representation.
   ##
   ## Returns ``true`` if peer was successfully initialiazed.
@@ -142,53 +147,54 @@ proc init*(pid: var PeerID, data: string): bool =
   var length = 0
   if Base58.decode(data, p, length) == Base58Status.Success:
     p.setLen(length)
-    var opid: PeerID
+    var opid: PeerId
     shallowCopy(opid.data, p)
     if opid.validate():
       pid = opid
       result = true
 
-proc init*(t: typedesc[PeerID], data: openarray[byte]): Result[PeerID, cstring] {.inline.} =
+func init*(t: typedesc[PeerId], data: openArray[byte]): Result[PeerId, cstring] =
   ## Create new peer id from raw binary representation ``data``.
-  var res: PeerID
+  var res: PeerId
   if not init(res, data):
-    err("peerid: incorrect PeerID binary form")
+    err("peerid: incorrect PeerId binary form")
   else:
     ok(res)
 
-proc init*(t: typedesc[PeerID], data: string): Result[PeerID, cstring] {.inline.} =
+func init*(t: typedesc[PeerId], data: string): Result[PeerId, cstring] =
   ## Create new peer id from base58 encoded string representation ``data``.
-  var res: PeerID
+  var res: PeerId
   if not init(res, data):
-    err("peerid: incorrect PeerID string")
+    err("peerid: incorrect PeerId string")
   else:
     ok(res)
 
-proc init*(t: typedesc[PeerID], pubkey: PublicKey): Result[PeerID, cstring] =
+func init*(t: typedesc[PeerId], pubkey: PublicKey): Result[PeerId, cstring] =
   ## Create new peer id from public key ``pubkey``.
-  var pubraw = ? pubkey.getBytes().orError("peerid: failed to get bytes from given key")
+  var pubraw = ? pubkey.getBytes().orError(
+    cstring("peerid: failed to get bytes from given key"))
   var mh: MultiHash
   if len(pubraw) <= maxInlineKeyLength:
     mh = ? MultiHash.digest("identity", pubraw)
   else:
     mh = ? MultiHash.digest("sha2-256", pubraw)
-  ok(PeerID(data: mh.data.buffer))
+  ok(PeerId(data: mh.data.buffer))
 
-proc init*(t: typedesc[PeerID], seckey: PrivateKey): Result[PeerID, cstring] {.inline.} =
+func init*(t: typedesc[PeerId], seckey: PrivateKey): Result[PeerId, cstring] =
   ## Create new peer id from private key ``seckey``.
-  PeerID.init(? seckey.getKey().orError("invalid private key"))
+  PeerId.init(? seckey.getPublicKey().orError(cstring("invalid private key")))
 
-proc match*(pid: PeerID, pubkey: PublicKey): bool {.inline.} =
+func match*(pid: PeerId, pubkey: PublicKey): bool =
   ## Returns ``true`` if ``pid`` matches public key ``pubkey``.
-  let p = PeerID.init(pubkey)
+  let p = PeerId.init(pubkey)
   if p.isErr:
     false
   else:
     pid == p.get()
 
-proc match*(pid: PeerID, seckey: PrivateKey): bool {.inline.} =
+func match*(pid: PeerId, seckey: PrivateKey): bool =
   ## Returns ``true`` if ``pid`` matches private key ``seckey``.
-  let p = PeerID.init(seckey)
+  let p = PeerId.init(seckey)
   if p.isErr:
     false
   else:
@@ -196,39 +202,25 @@ proc match*(pid: PeerID, seckey: PrivateKey): bool {.inline.} =
 
 ## Serialization/Deserialization helpers
 
-proc write*(vb: var VBuffer, pid: PeerID) {.inline.} =
-  ## Write PeerID value ``peerid`` to buffer ``vb``.
+func write*(vb: var VBuffer, pid: PeerId) =
+  ## Write PeerId value ``peerid`` to buffer ``vb``.
   vb.writeSeq(pid.data)
 
-proc initProtoField*(index: int, pid: PeerID): ProtoField {.deprecated.} =
-  ## Initialize ProtoField with PeerID ``value``.
-  result = initProtoField(index, pid.data)
-
-proc getValue*(data: var ProtoBuffer, field: int, value: var PeerID): int {.
-     deprecated.} =
-  ## Read ``PeerID`` from ProtoBuf's message and validate it.
-  var pid: PeerID
-  result = getLengthValue(data, field, pid.data)
-  if result > 0:
-    if not pid.validate():
-      result = -1
-    else:
-      value = pid
-
-proc write*(pb: var ProtoBuffer, field: int, pid: PeerID) =
-  ## Write PeerID value ``peerid`` to object ``pb`` using ProtoBuf's encoding.
+func write*(pb: var ProtoBuffer, field: int, pid: PeerId) =
+  ## Write PeerId value ``peerid`` to object ``pb`` using ProtoBuf's encoding.
   write(pb, field, pid.data)
 
-proc getField*(pb: ProtoBuffer, field: int, pid: var PeerID): bool =
-  ## Read ``PeerID`` from ProtoBuf's message and validate it
+func getField*(pb: ProtoBuffer, field: int,
+               pid: var PeerId): ProtoResult[bool] {.inline.} =
+  ## Read ``PeerId`` from ProtoBuf's message and validate it
   var buffer: seq[byte]
-  var peerId: PeerID
-  if not(getField(pb, field, buffer)):
-    return false
-  if len(buffer) == 0:
-    return false
-  if peerId.init(buffer):
-    pid = peerId
-    true
+  let res = ? pb.getField(field, buffer)
+  if not(res):
+    ok(false)
   else:
-    false
+    var peerId: PeerId
+    if peerId.init(buffer):
+      pid = peerId
+      ok(true)
+    else:
+      err(ProtoError.IncorrectBlob)

libp2p/peerinfo.nim

@@ -1,135 +1,86 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-import options, sequtils
-import chronos, chronicles
-import peerid, multiaddress, crypto/crypto
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+{.push public.}
 
-export peerid, multiaddress, crypto
+import std/[options, sequtils]
+import pkg/[chronos, chronicles, stew/results]
+import peerid, multiaddress, crypto/crypto, routing_record, errors, utility
 
-## A peer can be constructed in one of tree ways:
-## 1) A local peer with a private key
-## 2) A remote peer with a PeerID and it's public key stored
-## in the ``id`` itself
-## 3) A remote peer with a standalone public key, that isn't
-## encoded in the ``id``
-##
+export peerid, multiaddress, crypto, routing_record, errors, results
+
+## Our local peer info
 
 type
-  KeyType* = enum
-    HasPrivate,
-    HasPublic
+  PeerInfoError* = LPError
 
-  PeerInfo* = ref object of RootObj
-    peerId*: PeerID
+  PeerInfo* {.public.} = ref object
+    peerId*: PeerId
     addrs*: seq[MultiAddress]
     protocols*: seq[string]
-    lifefut: Future[void]
     protoVersion*: string
     agentVersion*: string
-    secure*: string
-    case keyType*: KeyType:
-    of HasPrivate:
-      privateKey*: PrivateKey
-    of HasPublic:
-      key: Option[PublicKey]
+    privateKey*: PrivateKey
+    publicKey*: PublicKey
+    signedPeerRecord*: SignedPeerRecord
 
-proc id*(p: PeerInfo): string =
-  if not(isNil(p)):
-    return p.peerId.pretty()
-
-proc `$`*(p: PeerInfo): string = p.id
-
-proc shortLog*(p: PeerInfo): auto =
+func shortLog*(p: PeerInfo): auto =
   (
-    id: p.id(),
+    peerId: $p.peerId,
     addrs: mapIt(p.addrs, $it),
     protocols: mapIt(p.protocols, $it),
     protoVersion: p.protoVersion,
     agentVersion: p.agentVersion,
   )
+chronicles.formatIt(PeerInfo): shortLog(it)
 
-template postInit(peerinfo: PeerInfo,
-                  addrs: openarray[MultiAddress],
-                  protocols: openarray[string]) =
-  if len(addrs) > 0:
-    peerinfo.addrs = @addrs
-  if len(protocols) > 0:
-    peerinfo.protocols = @protocols
-  peerinfo.lifefut = newFuture[void]("libp2p.peerinfo.lifetime")
-
-proc init*(p: typedesc[PeerInfo],
-           key: PrivateKey,
-           addrs: openarray[MultiAddress] = [],
-           protocols: openarray[string] = []): PeerInfo {.inline.} =
-  result = PeerInfo(keyType: HasPrivate, peerId: PeerID.init(key).tryGet(),
-                    privateKey: key)
-  result.postInit(addrs, protocols)
-
-proc init*(p: typedesc[PeerInfo],
-           peerId: PeerID,
-           addrs: openarray[MultiAddress] = [],
-           protocols: openarray[string] = []): PeerInfo {.inline.} =
-  result = PeerInfo(keyType: HasPublic, peerId: peerId)
-  result.postInit(addrs, protocols)
-
-proc init*(p: typedesc[PeerInfo],
-           peerId: string,
-           addrs: openarray[MultiAddress] = [],
-           protocols: openarray[string] = []): PeerInfo {.inline.} =
-  result = PeerInfo(keyType: HasPublic, peerId: PeerID.init(peerId).tryGet())
-  result.postInit(addrs, protocols)
-
-proc init*(p: typedesc[PeerInfo],
-           key: PublicKey,
-           addrs: openarray[MultiAddress] = [],
-           protocols: openarray[string] = []): PeerInfo {.inline.} =
-  result = PeerInfo(keyType: HasPublic,
-                    peerId: PeerID.init(key).tryGet(),
-                    key: some(key))
-
-  result.postInit(addrs, protocols)
-
-proc close*(p: PeerInfo) {.inline.} =
-  if not p.lifefut.finished:
-    p.lifefut.complete()
+proc update*(p: PeerInfo) =
+  let sprRes = SignedPeerRecord.init(
+    p.privateKey,
+    PeerRecord.init(p.peerId, p.addrs)
+  )
+  if sprRes.isOk:
+    p.signedPeerRecord = sprRes.get()
   else:
-    # TODO this should ideally not happen
-    notice "Closing closed peer", peer = p.id
+    discard
+    #info "Can't update the signed peer record"
 
-proc join*(p: PeerInfo): Future[void] {.inline.} =
-  var retFuture = newFuture[void]()
-  proc continuation(udata: pointer) {.gcsafe.} =
-    if not(retFuture.finished()):
-      retFuture.complete()
-  proc cancellation(udata: pointer) {.gcsafe.} =
-    p.lifefut.removeCallback(continuation)
-  if p.lifefut.finished:
-    retFuture.complete()
-  else:
-    p.lifefut.addCallback(continuation)
-    retFuture.cancelCallback = cancellation
-  return retFuture
+proc new*(
+  p: typedesc[PeerInfo],
+  key: PrivateKey,
+  addrs: openArray[MultiAddress] = [],
+  protocols: openArray[string] = [],
+  protoVersion: string = "",
+  agentVersion: string = ""): PeerInfo
+  {.raises: [Defect, PeerInfoError].} =
 
-proc isClosed*(p: PeerInfo): bool {.inline.} =
-  result = p.lifefut.finished()
+  let pubkey = try:
+      key.getPublicKey().tryGet()
+    except CatchableError:
+      raise newException(PeerInfoError, "invalid private key")
+  
+  let peerId = PeerId.init(key).tryGet()
 
-proc lifeFuture*(p: PeerInfo): Future[void] {.inline.} =
-  result = p.lifefut
+  let peerInfo = PeerInfo(
+    peerId: peerId,
+    publicKey: pubkey,
+    privateKey: key,
+    protoVersion: protoVersion,
+    agentVersion: agentVersion,
+    addrs: @addrs,
+    protocols: @protocols,
+  )
 
-proc publicKey*(p: PeerInfo): Option[PublicKey] {.inline.} =
-  if p.keyType == HasPublic:
-    if p.peerId.hasPublicKey():
-      var pubKey: PublicKey
-      if p.peerId.extractPublicKey(pubKey):
-        result = some(pubKey)
-    elif p.key.isSome:
-      result = p.key
-  else:
-    result = some(p.privateKey.getKey().tryGet())
+  peerInfo.update()
+
+  return peerInfo

libp2p/peerstore.nim

@@ -0,0 +1,186 @@
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+## Stores generic informations about peers.
+runnableExamples:
+  # Will keep info of all connected peers +
+  # last 50 disconnected peers
+  let peerStore = PeerStore.new(capacity = 50)
+
+  # Create a custom book type
+  type MoodBook = ref object of PeerBook[string]
+
+  var somePeerId: PeerId
+  discard somePeerId.init("")
+
+  peerStore[MoodBook][somePeerId] = "Happy"
+  doAssert peerStore[MoodBook][somePeerId] == "Happy"
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import
+  std/[tables, sets, options, macros],
+  ./crypto/crypto,
+  ./protocols/identify,
+  ./peerid, ./peerinfo,
+  ./routing_record,
+  ./multiaddress,
+  utility
+
+type
+  #################
+  # Handler types #
+  #################
+
+  PeerBookChangeHandler* = proc(peerId: PeerId) {.gcsafe, raises: [Defect].}
+
+  #########
+  # Books #
+  #########
+
+  # Each book contains a book (map) and event handler(s)
+  BasePeerBook = ref object of RootObj
+    changeHandlers: seq[PeerBookChangeHandler]
+    deletor: PeerBookChangeHandler
+
+  PeerBook*[T] {.public.} = ref object of BasePeerBook
+    book*: Table[PeerId, T]
+
+  SeqPeerBook*[T] = ref object of PeerBook[seq[T]]
+
+  AddressBook* {.public.} = ref object of SeqPeerBook[MultiAddress]
+  ProtoBook* {.public.} = ref object of SeqPeerBook[string]
+  KeyBook* {.public.} = ref object of PeerBook[PublicKey]
+
+  AgentBook* {.public.} = ref object of PeerBook[string]
+  ProtoVersionBook* {.public.} = ref object of PeerBook[string]
+  SPRBook* {.public.} = ref object of PeerBook[Envelope]
+
+  ####################
+  # Peer store types #
+  ####################
+
+  PeerStore* {.public.} = ref object
+    books: Table[string, BasePeerBook]
+    capacity*: int
+    toClean*: seq[PeerId]
+
+proc new*(T: type PeerStore, capacity = 1000): PeerStore {.public.} =
+  T(capacity: capacity)
+
+#########################
+# Generic Peer Book API #
+#########################
+
+proc `[]`*[T](peerBook: PeerBook[T],
+             peerId: PeerId): T {.public.} =
+  ## Get all known metadata of a provided peer, or default(T) if missing
+  peerBook.book.getOrDefault(peerId)
+
+proc `[]=`*[T](peerBook: PeerBook[T],
+             peerId: PeerId,
+             entry: T) {.public.} =
+  ## Set metadata for a given peerId.
+
+  peerBook.book[peerId] = entry
+
+  # Notify clients
+  for handler in peerBook.changeHandlers:
+    handler(peerId)
+
+proc del*[T](peerBook: PeerBook[T],
+                peerId: PeerId): bool {.public.} =
+  ## Delete the provided peer from the book. Returns whether the peer was in the book
+
+  if peerId notin peerBook.book:
+    return false
+  else:
+    peerBook.book.del(peerId)
+    # Notify clients
+    for handler in peerBook.changeHandlers:
+      handler(peerId)
+    return true
+
+proc contains*[T](peerBook: PeerBook[T], peerId: PeerId): bool {.public.} =
+  peerId in peerBook.book
+
+proc addHandler*[T](peerBook: PeerBook[T], handler: PeerBookChangeHandler) {.public.} =
+  ## Adds a callback that will be called everytime the book changes
+  peerBook.changeHandlers.add(handler)
+
+proc len*[T](peerBook: PeerBook[T]): int {.public.} = peerBook.book.len
+
+##################
+# Peer Store API #
+##################
+macro getTypeName(t: type): untyped =
+  # Generate unique name in form of Module.Type
+  let typ = getTypeImpl(t)[1]
+  newLit(repr(typ.owner()) & "." & repr(typ))
+
+proc `[]`*[T](p: PeerStore, typ: type[T]): T {.public.} =
+  ## Get a book from the PeerStore (ex: peerStore[AddressBook])
+  let name = getTypeName(T)
+  result = T(p.books.getOrDefault(name))
+  if result.isNil:
+    result = T.new()
+    result.deletor = proc(pid: PeerId) =
+      # Manual method because generic method
+      # don't work
+      discard T(p.books.getOrDefault(name)).del(pid)
+    p.books[name] = result
+  return result
+
+proc del*(peerStore: PeerStore,
+             peerId: PeerId) {.public.} =
+  ## Delete the provided peer from every book.
+  for _, book in peerStore.books:
+    book.deletor(peerId)
+
+proc updatePeerInfo*(
+  peerStore: PeerStore,
+  info: IdentifyInfo) =
+
+  if info.addrs.len > 0:
+    peerStore[AddressBook][info.peerId] = info.addrs
+
+  if info.agentVersion.isSome:
+    peerStore[AgentBook][info.peerId] = info.agentVersion.get().string
+
+  if info.protoVersion.isSome:
+    peerStore[ProtoVersionBook][info.peerId] = info.protoVersion.get().string
+
+  if info.protos.len > 0:
+    peerStore[ProtoBook][info.peerId] = info.protos
+
+  if info.signedPeerRecord.isSome:
+    peerStore[SPRBook][info.peerId] = info.signedPeerRecord.get()
+
+  let cleanupPos = peerStore.toClean.find(info.peerId)
+  if cleanupPos >= 0:
+    peerStore.toClean.delete(cleanupPos)
+
+proc cleanup*(
+  peerStore: PeerStore,
+  peerId: PeerId) =
+
+  if peerStore.capacity == 0:
+    peerStore.del(peerId)
+    return
+  elif peerStore.capacity < 0:
+    #infinite capacity
+    return
+
+  peerStore.toClean.add(peerId)
+  while peerStore.toClean.len > peerStore.capacity:
+    peerStore.del(peerStore.toClean[0])
+    peerStore.toClean.delete(0)

libp2p/protobuf/minprotobuf.nim

@@ -1,17 +1,23 @@
-## Nim-Libp2p
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module implements minimal Google's ProtoBuf primitives.
 
-{.push raises: [Defect].}
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
-import ../varint, stew/endians2
+import ../varint, ../utility, stew/[endians2, results]
+export results, utility
+
+{.push public.}
 
 const
   MaxMessageSize* = 1'u shl 22
@@ -51,12 +57,16 @@ type
     of StartGroup, EndGroup:
       discard
 
-  ProtoResult {.pure.} = enum
-    VarintDecodeError,
-    MessageIncompleteError,
-    BufferOverflowError,
-    MessageSizeTooBigError,
-    NoError
+  ProtoError* {.pure.} = enum
+    VarintDecode,
+    MessageIncomplete,
+    BufferOverflow,
+    MessageTooBig,
+    BadWireType,
+    IncorrectBlob,
+    RequiredFieldMissing
+
+  ProtoResult*[T] = Result[T, ProtoError]
 
   ProtoScalar* = uint | uint32 | uint64 | zint | zint32 | zint64 |
                  hint | hint32 | hint64 | float32 | float64
@@ -111,43 +121,6 @@ proc vsizeof*(field: ProtoField): int {.inline.} =
   else:
     0
 
-proc initProtoField*(index: int, value: SomeVarint): ProtoField {.deprecated.} =
-  ## Initialize ProtoField with integer value.
-  result = ProtoField(kind: Varint, index: index)
-  when type(value) is uint64:
-    result.vint = value
-  else:
-    result.vint = cast[uint64](value)
-
-proc initProtoField*(index: int, value: bool): ProtoField {.deprecated.} =
-  ## Initialize ProtoField with integer value.
-  result = ProtoField(kind: Varint, index: index)
-  result.vint = byte(value)
-
-proc initProtoField*(index: int,
-                     value: openarray[byte]): ProtoField {.deprecated.} =
-  ## Initialize ProtoField with bytes array.
-  result = ProtoField(kind: Length, index: index)
-  if len(value) > 0:
-    result.vbuffer = newSeq[byte](len(value))
-    copyMem(addr result.vbuffer[0], unsafeAddr value[0], len(value))
-
-proc initProtoField*(index: int, value: string): ProtoField {.deprecated.} =
-  ## Initialize ProtoField with string.
-  result = ProtoField(kind: Length, index: index)
-  if len(value) > 0:
-    result.vbuffer = newSeq[byte](len(value))
-    copyMem(addr result.vbuffer[0], unsafeAddr value[0], len(value))
-
-proc initProtoField*(index: int,
-                     value: ProtoBuffer): ProtoField {.deprecated, inline.} =
-  ## Initialize ProtoField with nested message stored in ``value``.
-  ##
-  ## Note: This procedure performs shallow copy of ``value`` sequence.
-  result = ProtoField(kind: Length, index: index)
-  if len(value.buffer) > 0:
-    shallowCopy(result.vbuffer, value.buffer)
-
 proc initProtoBuffer*(data: seq[byte], offset = 0,
                       options: set[ProtoFlags] = {}): ProtoBuffer =
   ## Initialize ProtoBuffer with shallow copy of ``data``.
@@ -155,7 +128,7 @@ proc initProtoBuffer*(data: seq[byte], offset = 0,
   result.offset = offset
   result.options = options
 
-proc initProtoBuffer*(data: openarray[byte], offset = 0,
+proc initProtoBuffer*(data: openArray[byte], offset = 0,
                       options: set[ProtoFlags] = {}): ProtoBuffer =
   ## Initialize ProtoBuffer with copy of ``data``.
   result.buffer = @data
@@ -164,7 +137,7 @@ proc initProtoBuffer*(data: openarray[byte], offset = 0,
 
 proc initProtoBuffer*(options: set[ProtoFlags] = {}): ProtoBuffer =
   ## Initialize ProtoBuffer with new sequence of capacity ``cap``.
-  result.buffer = newSeqOfCap[byte](128)
+  result.buffer = newSeq[byte]()
   result.options = options
   if WithVarintLength in options:
     # Our buffer will start from position 10, so we can store length of buffer
@@ -223,7 +196,7 @@ proc write*[T: ProtoScalar](pb: var ProtoBuffer,
     pb.offset += sizeof(T)
 
 proc writePacked*[T: ProtoScalar](pb: var ProtoBuffer, field: int,
-                                  value: openarray[T]) =
+                                  value: openArray[T]) =
   checkFieldNumber(field)
   var length = 0
   let dlength =
@@ -271,7 +244,7 @@ proc writePacked*[T: ProtoScalar](pb: var ProtoBuffer, field: int,
       pb.offset += sizeof(T)
 
 proc write*[T: byte|char](pb: var ProtoBuffer, field: int,
-                          value: openarray[T]) =
+                          value: openArray[T]) =
   checkFieldNumber(field)
   var length = 0
   let flength = vsizeof(getProtoHeader(field, ProtoFieldKind.Length)) +
@@ -295,55 +268,10 @@ proc write*(pb: var ProtoBuffer, field: int, value: ProtoBuffer) {.inline.} =
   ## ``pb`` with field number ``field``.
   write(pb, field, value.buffer)
 
-proc write*(pb: var ProtoBuffer, field: ProtoField) {.deprecated.} =
-  ## Encode protobuf's field ``field`` and store it to protobuf's buffer ``pb``.
-  var length = 0
-  var res: VarintResult[void]
-  pb.buffer.setLen(len(pb.buffer) + vsizeof(field))
-  res = PB.putUVarint(pb.toOpenArray(), length, getProtoHeader(field))
-  doAssert(res.isOk())
-  pb.offset += length
-  case field.kind
-  of ProtoFieldKind.Varint:
-    res = PB.putUVarint(pb.toOpenArray(), length, field.vint)
-    doAssert(res.isOk())
-    pb.offset += length
-  of ProtoFieldKind.Fixed64:
-    doAssert(pb.isEnough(8))
-    var value = cast[uint64](field.vfloat64)
-    pb.buffer[pb.offset] = byte(value and 0xFF'u32)
-    pb.buffer[pb.offset + 1] = byte((value shr 8) and 0xFF'u64)
-    pb.buffer[pb.offset + 2] = byte((value shr 16) and 0xFF'u64)
-    pb.buffer[pb.offset + 3] = byte((value shr 24) and 0xFF'u64)
-    pb.buffer[pb.offset + 4] = byte((value shr 32) and 0xFF'u64)
-    pb.buffer[pb.offset + 5] = byte((value shr 40) and 0xFF'u64)
-    pb.buffer[pb.offset + 6] = byte((value shr 48) and 0xFF'u64)
-    pb.buffer[pb.offset + 7] = byte((value shr 56) and 0xFF'u64)
-    pb.offset += 8
-  of ProtoFieldKind.Fixed32:
-    doAssert(pb.isEnough(4))
-    var value = cast[uint32](field.vfloat32)
-    pb.buffer[pb.offset] = byte(value and 0xFF'u32)
-    pb.buffer[pb.offset + 1] = byte((value shr 8) and 0xFF'u32)
-    pb.buffer[pb.offset + 2] = byte((value shr 16) and 0xFF'u32)
-    pb.buffer[pb.offset + 3] = byte((value shr 24) and 0xFF'u32)
-    pb.offset += 4
-  of ProtoFieldKind.Length:
-    res = PB.putUVarint(pb.toOpenArray(), length, uint(len(field.vbuffer)))
-    doAssert(res.isOk())
-    pb.offset += length
-    doAssert(pb.isEnough(len(field.vbuffer)))
-    if len(field.vbuffer) > 0:
-      copyMem(addr pb.buffer[pb.offset], unsafeAddr field.vbuffer[0],
-              len(field.vbuffer))
-      pb.offset += len(field.vbuffer)
-  else:
-    discard
-
 proc finish*(pb: var ProtoBuffer) =
   ## Prepare protobuf's buffer ``pb`` for writing to stream.
-  doAssert(len(pb.buffer) > 0)
   if WithVarintLength in pb.options:
+    doAssert(len(pb.buffer) >= 10)
     let size = uint(len(pb.buffer) - 10)
     let pos = 10 - vsizeof(size)
     var usedBytes = 0
@@ -351,17 +279,21 @@ proc finish*(pb: var ProtoBuffer) =
     doAssert(res.isOk())
     pb.offset = pos
   elif WithUint32BeLength in pb.options:
+    doAssert(len(pb.buffer) >= 4)
     let size = uint(len(pb.buffer) - 4)
     pb.buffer[0 ..< 4] = toBytesBE(uint32(size))
     pb.offset = 4
   elif WithUint32LeLength in pb.options:
+    doAssert(len(pb.buffer) >= 4)
     let size = uint(len(pb.buffer) - 4)
     pb.buffer[0 ..< 4] = toBytesLE(uint32(size))
     pb.offset = 4
   else:
+    doAssert(len(pb.buffer) > 0)
     pb.offset = 0
 
-proc getHeader(data: var ProtoBuffer, header: var ProtoHeader): bool =
+proc getHeader(data: var ProtoBuffer,
+               header: var ProtoHeader): ProtoResult[void] =
   var length = 0
   var hdr = 0'u64
   if PB.getUVarint(data.toOpenArray(), length, hdr).isOk():
@@ -370,34 +302,34 @@ proc getHeader(data: var ProtoBuffer, header: var ProtoHeader): bool =
     if wire in SupportedWireTypes:
       data.offset += length
       header = ProtoHeader(index: index, wire: cast[ProtoFieldKind](wire))
-      true
+      ok()
     else:
-      false
+      err(ProtoError.BadWireType)
   else:
-    false
+    err(ProtoError.VarintDecode)
 
-proc skipValue(data: var ProtoBuffer, header: ProtoHeader): bool =
+proc skipValue(data: var ProtoBuffer, header: ProtoHeader): ProtoResult[void] =
   case header.wire
   of ProtoFieldKind.Varint:
     var length = 0
     var value = 0'u64
     if PB.getUVarint(data.toOpenArray(), length, value).isOk():
       data.offset += length
-      true
+      ok()
     else:
-      false
+      err(ProtoError.VarintDecode)
   of ProtoFieldKind.Fixed32:
     if data.isEnough(sizeof(uint32)):
       data.offset += sizeof(uint32)
-      true
+      ok()
     else:
-      false
+      err(ProtoError.VarintDecode)
   of ProtoFieldKind.Fixed64:
     if data.isEnough(sizeof(uint64)):
       data.offset += sizeof(uint64)
-      true
+      ok()
     else:
-      false
+      err(ProtoError.VarintDecode)
   of ProtoFieldKind.Length:
     var length = 0
     var bsize = 0'u64
@@ -406,19 +338,19 @@ proc skipValue(data: var ProtoBuffer, header: ProtoHeader): bool =
       if bsize <= uint64(MaxMessageSize):
         if data.isEnough(int(bsize)):
           data.offset += int(bsize)
-          true
+          ok()
         else:
-          false
+          err(ProtoError.MessageIncomplete)
       else:
-        false
+        err(ProtoError.MessageTooBig)
     else:
-      false
+      err(ProtoError.VarintDecode)
   of ProtoFieldKind.StartGroup, ProtoFieldKind.EndGroup:
-    false
+    err(ProtoError.BadWireType)
 
 proc getValue[T: ProtoScalar](data: var ProtoBuffer,
                               header: ProtoHeader,
-                              outval: var T): ProtoResult =
+                              outval: var T): ProtoResult[void] =
   when (T is uint64) or (T is uint32) or (T is uint):
     doAssert(header.wire == ProtoFieldKind.Varint)
     var length = 0
@@ -426,9 +358,9 @@ proc getValue[T: ProtoScalar](data: var ProtoBuffer,
     if PB.getUVarint(data.toOpenArray(), length, value).isOk():
       data.offset += length
       outval = value
-      ProtoResult.NoError
+      ok()
     else:
-      ProtoResult.VarintDecodeError
+      err(ProtoError.VarintDecode)
   elif (T is zint64) or (T is zint32) or (T is zint) or
        (T is hint64) or (T is hint32) or (T is hint):
     doAssert(header.wire == ProtoFieldKind.Varint)
@@ -437,29 +369,29 @@ proc getValue[T: ProtoScalar](data: var ProtoBuffer,
     if getSVarint(data.toOpenArray(), length, value).isOk():
       data.offset += length
       outval = value
-      ProtoResult.NoError
+      ok()
     else:
-      ProtoResult.VarintDecodeError
+      err(ProtoError.VarintDecode)
   elif T is float32:
     doAssert(header.wire == ProtoFieldKind.Fixed32)
     if data.isEnough(sizeof(float32)):
       outval = cast[float32](fromBytesLE(uint32, data.toOpenArray()))
       data.offset += sizeof(float32)
-      ProtoResult.NoError
+      ok()
     else:
-      ProtoResult.MessageIncompleteError
+      err(ProtoError.MessageIncomplete)
   elif T is float64:
     doAssert(header.wire == ProtoFieldKind.Fixed64)
     if data.isEnough(sizeof(float64)):
       outval = cast[float64](fromBytesLE(uint64, data.toOpenArray()))
       data.offset += sizeof(float64)
-      ProtoResult.NoError
+      ok()
     else:
-      ProtoResult.MessageIncompleteError
+      err(ProtoError.MessageIncomplete)
 
 proc getValue[T:byte|char](data: var ProtoBuffer, header: ProtoHeader,
-                           outBytes: var openarray[T],
-                           outLength: var int): ProtoResult =
+                           outBytes: var openArray[T],
+                           outLength: var int): ProtoResult[void] =
   doAssert(header.wire == ProtoFieldKind.Length)
   var length = 0
   var bsize = 0'u64
@@ -474,20 +406,20 @@ proc getValue[T:byte|char](data: var ProtoBuffer, header: ProtoHeader,
           if bsize > 0'u64:
             copyMem(addr outBytes[0], addr data.buffer[data.offset], int(bsize))
           data.offset += int(bsize)
-          ProtoResult.NoError
+          ok()
         else:
           # Buffer overflow should not be critical failure
           data.offset += int(bsize)
-          ProtoResult.BufferOverflowError
+          err(ProtoError.BufferOverflow)
       else:
-        ProtoResult.MessageIncompleteError
+        err(ProtoError.MessageIncomplete)
     else:
-      ProtoResult.MessageSizeTooBigError
+      err(ProtoError.MessageTooBig)
   else:
-    ProtoResult.VarintDecodeError
+    err(ProtoError.VarintDecode)
 
 proc getValue[T:seq[byte]|string](data: var ProtoBuffer, header: ProtoHeader,
-                                  outBytes: var T): ProtoResult =
+                                  outBytes: var T): ProtoResult[void] =
   doAssert(header.wire == ProtoFieldKind.Length)
   var length = 0
   var bsize = 0'u64
@@ -501,27 +433,24 @@ proc getValue[T:seq[byte]|string](data: var ProtoBuffer, header: ProtoHeader,
         if bsize > 0'u64:
           copyMem(addr outBytes[0], addr data.buffer[data.offset], int(bsize))
         data.offset += int(bsize)
-        ProtoResult.NoError
+        ok()
       else:
-        ProtoResult.MessageIncompleteError
+        err(ProtoError.MessageIncomplete)
     else:
-      ProtoResult.MessageSizeTooBigError
+      err(ProtoError.MessageTooBig)
   else:
-    ProtoResult.VarintDecodeError
+    err(ProtoError.VarintDecode)
 
 proc getField*[T: ProtoScalar](data: ProtoBuffer, field: int,
-                               output: var T): bool =
+                               output: var T): ProtoResult[bool] =
   checkFieldNumber(field)
-  var value: T
+  var current: T
   var res = false
   var pb = data
-  output = T(0)
 
   while not(pb.isEmpty()):
     var header: ProtoHeader
-    if not(pb.getHeader(header)):
-      output = T(0)
-      return false
+    ? pb.getHeader(header)
     let wireCheck =
       when (T is uint64) or (T is uint32) or (T is uint) or
            (T is zint64) or (T is zint32) or (T is zint) or
@@ -533,28 +462,29 @@ proc getField*[T: ProtoScalar](data: ProtoBuffer, field: int,
         header.wire == ProtoFieldKind.Fixed64
     if header.index == uint64(field):
       if wireCheck:
-        let r = getValue(pb, header, value)
-        case r
-        of ProtoResult.NoError:
+        var value: T
+        let vres = pb.getValue(header, value)
+        if vres.isOk():
           res = true
-          output = value
+          current = value
         else:
-          return false
+          return err(vres.error)
       else:
         # We are ignoring wire types different from what we expect, because it
         # is how `protoc` is working.
-        if not(skipValue(pb, header)):
-          output = T(0)
-          return false
+        ? pb.skipValue(header)
     else:
-      if not(skipValue(pb, header)):
-        output = T(0)
-        return false
-  res
+      ? pb.skipValue(header)
+
+  if res:
+    output = current
+    ok(true)
+  else:
+    ok(false)
 
 proc getField*[T: byte|char](data: ProtoBuffer, field: int,
-                             output: var openarray[T],
-                             outlen: var int): bool =
+                             output: var openArray[T],
+                             outlen: var int): ProtoResult[bool] =
   checkFieldNumber(field)
   var pb = data
   var res = false
@@ -563,182 +493,213 @@ proc getField*[T: byte|char](data: ProtoBuffer, field: int,
 
   while not(pb.isEmpty()):
     var header: ProtoHeader
-    if not(pb.getHeader(header)):
+    let hres = pb.getHeader(header)
+    if hres.isErr():
       if len(output) > 0:
         zeroMem(addr output[0], len(output))
       outlen = 0
-      return false
-
+      return err(hres.error)
     if header.index == uint64(field):
       if header.wire == ProtoFieldKind.Length:
-        let r = getValue(pb, header, output, outlen)
-        case r
-        of ProtoResult.NoError:
+        let vres = pb.getValue(header, output, outlen)
+        if vres.isOk():
           res = true
-        of ProtoResult.BufferOverflowError:
+        else:
           # Buffer overflow error is not critical error, we still can get
           # field values with proper size.
-          discard
-        else:
-          if len(output) > 0:
-            zeroMem(addr output[0], len(output))
-          return false
+          if vres.error != ProtoError.BufferOverflow:
+            if len(output) > 0:
+              zeroMem(addr output[0], len(output))
+            outlen = 0
+            return err(vres.error)
       else:
         # We are ignoring wire types different from ProtoFieldKind.Length,
         # because it is how `protoc` is working.
-        if not(skipValue(pb, header)):
+        let sres = pb.skipValue(header)
+        if sres.isErr():
           if len(output) > 0:
             zeroMem(addr output[0], len(output))
           outlen = 0
-          return false
+          return err(sres.error)
     else:
-      if not(skipValue(pb, header)):
+      let sres = pb.skipValue(header)
+      if sres.isErr():
         if len(output) > 0:
           zeroMem(addr output[0], len(output))
         outlen = 0
-        return false
+        return err(sres.error)
 
-  res
+  if res:
+    ok(true)
+  else:
+    ok(false)
 
 proc getField*[T: seq[byte]|string](data: ProtoBuffer, field: int,
-                                    output: var T): bool =
+                                    output: var T): ProtoResult[bool] =
   checkFieldNumber(field)
   var res = false
   var pb = data
 
   while not(pb.isEmpty()):
     var header: ProtoHeader
-    if not(pb.getHeader(header)):
+    let hres = pb.getHeader(header)
+    if hres.isErr():
       output.setLen(0)
-      return false
-
+      return err(hres.error)
     if header.index == uint64(field):
       if header.wire == ProtoFieldKind.Length:
-        let r = getValue(pb, header, output)
-        case r
-        of ProtoResult.NoError:
+        let vres = pb.getValue(header, output)
+        if vres.isOk():
           res = true
-        of ProtoResult.BufferOverflowError:
-          # Buffer overflow error is not critical error, we still can get
-          # field values with proper size.
-          discard
         else:
           output.setLen(0)
-          return false
+          return err(vres.error)
       else:
         # We are ignoring wire types different from ProtoFieldKind.Length,
         # because it is how `protoc` is working.
-        if not(skipValue(pb, header)):
+        let sres = pb.skipValue(header)
+        if sres.isErr():
           output.setLen(0)
-          return false
+          return err(sres.error)
     else:
-      if not(skipValue(pb, header)):
+      let sres = pb.skipValue(header)
+      if sres.isErr():
         output.setLen(0)
-        return false
-
-  res
-
-proc getField*(pb: ProtoBuffer, field: int, output: var ProtoBuffer): bool {.
-     inline.} =
-  var buffer: seq[byte]
-  if pb.getField(field, buffer):
-    output = initProtoBuffer(buffer)
-    true
+        return err(sres.error)
+  if res:
+    ok(true)
   else:
-    false
+    ok(false)
+
+proc getField*(pb: ProtoBuffer, field: int,
+               output: var ProtoBuffer): ProtoResult[bool] {.inline.} =
+  var buffer: seq[byte]
+  let res = pb.getField(field, buffer)
+  if res.isOk():
+    if res.get():
+      output = initProtoBuffer(buffer)
+      ok(true)
+    else:
+      ok(false)
+  else:
+    err(res.error)
+
+proc getRequiredField*[T](pb: ProtoBuffer, field: int,
+               output: var T): ProtoResult[void] {.inline.} =
+  let res = pb.getField(field, output)
+  if res.isOk():
+    if res.get():
+      ok()
+    else:
+      err(RequiredFieldMissing)
+  else:
+    err(res.error)
 
 proc getRepeatedField*[T: seq[byte]|string](data: ProtoBuffer, field: int,
-                                            output: var seq[T]): bool =
+                                        output: var seq[T]): ProtoResult[bool] =
   checkFieldNumber(field)
   var pb = data
   output.setLen(0)
 
   while not(pb.isEmpty()):
     var header: ProtoHeader
-    if not(pb.getHeader(header)):
+    let hres = pb.getHeader(header)
+    if hres.isErr():
       output.setLen(0)
-      return false
-
+      return err(hres.error)
     if header.index == uint64(field):
       if header.wire == ProtoFieldKind.Length:
         var item: T
-        let r = getValue(pb, header, item)
-        case r
-        of ProtoResult.NoError:
+        let vres = pb.getValue(header, item)
+        if vres.isOk():
           output.add(item)
         else:
           output.setLen(0)
-          return false
+          return err(vres.error)
       else:
-        if not(skipValue(pb, header)):
+        let sres = pb.skipValue(header)
+        if sres.isErr():
           output.setLen(0)
-          return false
+          return err(sres.error)
     else:
-      if not(skipValue(pb, header)):
+      let sres = pb.skipValue(header)
+      if sres.isErr():
         output.setLen(0)
-        return false
+        return err(sres.error)
 
   if len(output) > 0:
-    true
+    ok(true)
   else:
-    false
+    ok(false)
 
-proc getRepeatedField*[T: uint64|float32|float64](data: ProtoBuffer,
-                                                  field: int,
-                                                  output: var seq[T]): bool =
+proc getRepeatedField*[T: ProtoScalar](data: ProtoBuffer, field: int,
+                                       output: var seq[T]): ProtoResult[bool] =
   checkFieldNumber(field)
   var pb = data
   output.setLen(0)
 
   while not(pb.isEmpty()):
     var header: ProtoHeader
-    if not(pb.getHeader(header)):
+    let hres = pb.getHeader(header)
+    if hres.isErr():
       output.setLen(0)
-      return false
+      return err(hres.error)
 
     if header.index == uint64(field):
       if header.wire in {ProtoFieldKind.Varint, ProtoFieldKind.Fixed32,
                          ProtoFieldKind.Fixed64}:
         var item: T
-        let r = getValue(pb, header, item)
-        case r
-        of ProtoResult.NoError:
+        let vres = getValue(pb, header, item)
+        if vres.isOk():
           output.add(item)
         else:
           output.setLen(0)
-          return false
+          return err(vres.error)
       else:
-        if not(skipValue(pb, header)):
+        let sres = skipValue(pb, header)
+        if sres.isErr():
           output.setLen(0)
-          return false
+          return err(sres.error)
     else:
-      if not(skipValue(pb, header)):
+      let sres = skipValue(pb, header)
+      if sres.isErr():
         output.setLen(0)
-        return false
+        return err(sres.error)
 
   if len(output) > 0:
-    true
+    ok(true)
   else:
-    false
+    ok(false)
+
+proc getRequiredRepeatedField*[T](pb: ProtoBuffer, field: int,
+               output: var seq[T]): ProtoResult[void] {.inline.} =
+  let res = pb.getRepeatedField(field, output)
+  if res.isOk():
+    if res.get():
+      ok()
+    else:
+      err(RequiredFieldMissing)
+  else:
+    err(res.error)
 
 proc getPackedRepeatedField*[T: ProtoScalar](data: ProtoBuffer, field: int,
-                                             output: var seq[T]): bool =
+                                        output: var seq[T]): ProtoResult[bool] =
   checkFieldNumber(field)
   var pb = data
   output.setLen(0)
 
   while not(pb.isEmpty()):
     var header: ProtoHeader
-    if not(pb.getHeader(header)):
+    let hres = pb.getHeader(header)
+    if hres.isErr():
       output.setLen(0)
-      return false
+      return err(hres.error)
 
     if header.index == uint64(field):
       if header.wire == ProtoFieldKind.Length:
         var arritem: seq[byte]
-        let rarr = getValue(pb, header, arritem)
-        case rarr
-        of ProtoResult.NoError:
+        let ares = getValue(pb, header, arritem)
+        if ares.isOk():
           var pbarr = initProtoBuffer(arritem)
           let itemHeader =
             when (T is uint64) or (T is uint32) or (T is uint) or
@@ -751,116 +712,27 @@ proc getPackedRepeatedField*[T: ProtoScalar](data: ProtoBuffer, field: int,
               ProtoHeader(wire: ProtoFieldKind.Fixed64)
           while not(pbarr.isEmpty()):
             var item: T
-            let res = getValue(pbarr, itemHeader, item)
-            case res
-            of ProtoResult.NoError:
+            let vres = getValue(pbarr, itemHeader, item)
+            if vres.isOk():
               output.add(item)
             else:
               output.setLen(0)
-              return false
+              return err(vres.error)
         else:
           output.setLen(0)
-          return false
+          return err(ares.error)
       else:
-        if not(skipValue(pb, header)):
+        let sres = skipValue(pb, header)
+        if sres.isErr():
           output.setLen(0)
-          return false
+          return err(sres.error)
     else:
-      if not(skipValue(pb, header)):
+      let sres = skipValue(pb, header)
+      if sres.isErr():
         output.setLen(0)
-        return false
+        return err(sres.error)
 
   if len(output) > 0:
-    true
+    ok(true)
   else:
-    false
-
-proc getVarintValue*(data: var ProtoBuffer, field: int,
-                     value: var SomeVarint): int {.deprecated.} =
-  ## Get value of `Varint` type.
-  var length = 0
-  var header = 0'u64
-  var soffset = data.offset
-
-  if not data.isEmpty() and PB.getUVarint(data.toOpenArray(),
-                                          length, header).isOk():
-    data.offset += length
-    if header == getProtoHeader(field, Varint):
-      if not data.isEmpty():
-        when type(value) is int32 or type(value) is int64 or type(value) is int:
-          let res = getSVarint(data.toOpenArray(), length, value)
-        else:
-          let res = PB.getUVarint(data.toOpenArray(), length, value)
-        if res.isOk():
-          data.offset += length
-          result = length
-          return
-  # Restore offset on error
-  data.offset = soffset
-
-proc getLengthValue*[T: string|seq[byte]](data: var ProtoBuffer, field: int,
-                                          buffer: var T): int {.deprecated.} =
-  ## Get value of `Length` type.
-  var length = 0
-  var header = 0'u64
-  var ssize = 0'u64
-  var soffset = data.offset
-  result = -1
-  buffer.setLen(0)
-  if not data.isEmpty() and PB.getUVarint(data.toOpenArray(),
-                                          length, header).isOk():
-    data.offset += length
-    if header == getProtoHeader(field, Length):
-      if not data.isEmpty() and PB.getUVarint(data.toOpenArray(),
-                                              length, ssize).isOk():
-        data.offset += length
-        if ssize <= MaxMessageSize and data.isEnough(int(ssize)):
-          buffer.setLen(ssize)
-          # Protobuf allow zero-length values.
-          if ssize > 0'u64:
-            copyMem(addr buffer[0], addr data.buffer[data.offset], ssize)
-          result = int(ssize)
-          data.offset += int(ssize)
-          return
-  # Restore offset on error
-  data.offset = soffset
-
-proc getBytes*(data: var ProtoBuffer, field: int,
-               buffer: var seq[byte]): int {.deprecated, inline.} =
-  ## Get value of `Length` type as bytes.
-  result = getLengthValue(data, field, buffer)
-
-proc getString*(data: var ProtoBuffer, field: int,
-                buffer: var string): int {.deprecated, inline.} =
-  ## Get value of `Length` type as string.
-  result = getLengthValue(data, field, buffer)
-
-proc enterSubmessage*(pb: var ProtoBuffer): int {.deprecated.} =
-  ## Processes protobuf's sub-message and adjust internal offset to enter
-  ## inside of sub-message. Returns field index of sub-message field or
-  ## ``0`` on error.
-  var length = 0
-  var header = 0'u64
-  var msize = 0'u64
-  var soffset = pb.offset
-
-  if not pb.isEmpty() and PB.getUVarint(pb.toOpenArray(),
-                                        length, header).isOk():
-    pb.offset += length
-    if (header and 0x07'u64) == cast[uint64](ProtoFieldKind.Length):
-      if not pb.isEmpty() and PB.getUVarint(pb.toOpenArray(),
-                                            length, msize).isOk():
-        pb.offset += length
-        if msize <= MaxMessageSize and pb.isEnough(int(msize)):
-          pb.length = int(msize)
-          result = int(header shr 3)
-          return
-  # Restore offset on error
-  pb.offset = soffset
-
-proc skipSubmessage*(pb: var ProtoBuffer) {.deprecated.} =
-  ## Skip current protobuf's sub-message and adjust internal offset to the
-  ## end of sub-message.
-  doAssert(pb.length != 0)
-  pb.offset += pb.length
-  pb.length = 0
+    ok(false)

libp2p/protocols/connectivity/autonat.nim

@@ -0,0 +1,309 @@
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import std/[options, sets, sequtils]
+import stew/results
+import chronos, chronicles, stew/objects
+import ../protocol,
+       ../../switch,
+       ../../multiaddress,
+       ../../multicodec,
+       ../../peerid,
+       ../../utils/semaphore,
+       ../../errors
+
+logScope:
+  topics = "libp2p autonat"
+
+const
+  AutonatCodec* = "/libp2p/autonat/1.0.0"
+  AddressLimit = 8
+
+type
+  AutonatError* = object of LPError
+
+  MsgType* = enum
+    Dial = 0
+    DialResponse = 1
+
+  ResponseStatus* = enum
+    Ok = 0
+    DialError = 100
+    DialRefused = 101
+    BadRequest = 200
+    InternalError = 300
+
+  AutonatPeerInfo* = object
+    id: Option[PeerId]
+    addrs: seq[MultiAddress]
+
+  AutonatDial* = object
+    peerInfo: Option[AutonatPeerInfo]
+
+  AutonatDialResponse* = object
+    status*: ResponseStatus
+    text*: Option[string]
+    ma*: Option[MultiAddress]
+
+  AutonatMsg = object
+    msgType: MsgType
+    dial: Option[AutonatDial]
+    response: Option[AutonatDialResponse]
+
+proc encode*(msg: AutonatMsg): ProtoBuffer =
+  result = initProtoBuffer()
+  result.write(1, msg.msgType.uint)
+  if msg.dial.isSome():
+    var dial = initProtoBuffer()
+    if msg.dial.get().peerInfo.isSome():
+      var bufferPeerInfo = initProtoBuffer()
+      let peerInfo = msg.dial.get().peerInfo.get()
+      if peerInfo.id.isSome():
+        bufferPeerInfo.write(1, peerInfo.id.get())
+      for ma in peerInfo.addrs:
+        bufferPeerInfo.write(2, ma.data.buffer)
+      bufferPeerInfo.finish()
+      dial.write(1, bufferPeerInfo.buffer)
+    dial.finish()
+    result.write(2, dial.buffer)
+  if msg.response.isSome():
+    var bufferResponse = initProtoBuffer()
+    let response = msg.response.get()
+    bufferResponse.write(1, response.status.uint)
+    if response.text.isSome():
+      bufferResponse.write(2, response.text.get())
+    if response.ma.isSome():
+      bufferResponse.write(3, response.ma.get())
+    bufferResponse.finish()
+    result.write(3, bufferResponse.buffer)
+  result.finish()
+
+proc encode*(d: AutonatDial): ProtoBuffer =
+  result = initProtoBuffer()
+  result.write(1, MsgType.Dial.uint)
+  var dial = initProtoBuffer()
+  if d.peerInfo.isSome():
+    var bufferPeerInfo = initProtoBuffer()
+    let peerInfo = d.peerInfo.get()
+    if peerInfo.id.isSome():
+      bufferPeerInfo.write(1, peerInfo.id.get())
+    for ma in peerInfo.addrs:
+      bufferPeerInfo.write(2, ma.data.buffer)
+    bufferPeerInfo.finish()
+    dial.write(1, bufferPeerInfo.buffer)
+  dial.finish()
+  result.write(2, dial.buffer)
+  result.finish()
+
+proc encode*(r: AutonatDialResponse): ProtoBuffer =
+  result = initProtoBuffer()
+  result.write(1, MsgType.DialResponse.uint)
+  var bufferResponse = initProtoBuffer()
+  bufferResponse.write(1, r.status.uint)
+  if r.text.isSome():
+    bufferResponse.write(2, r.text.get())
+  if r.ma.isSome():
+    bufferResponse.write(3, r.ma.get())
+  bufferResponse.finish()
+  result.write(3, bufferResponse.buffer)
+  result.finish()
+
+proc decode(_: typedesc[AutonatMsg], buf: seq[byte]): Option[AutonatMsg] =
+  var
+    msgTypeOrd: uint32
+    pbDial: ProtoBuffer
+    pbResponse: ProtoBuffer
+    msg: AutonatMsg
+
+  let
+    pb = initProtoBuffer(buf)
+    r1 = pb.getField(1, msgTypeOrd)
+    r2 = pb.getField(2, pbDial)
+    r3 = pb.getField(3, pbResponse)
+  if r1.isErr() or r2.isErr() or r3.isErr(): return none(AutonatMsg)
+
+  if r1.get() and not checkedEnumAssign(msg.msgType, msgTypeOrd):
+    return none(AutonatMsg)
+  if r2.get():
+    var
+      pbPeerInfo: ProtoBuffer
+      dial: AutonatDial
+    let
+      r4 = pbDial.getField(1, pbPeerInfo)
+    if r4.isErr(): return none(AutonatMsg)
+
+    var peerInfo: AutonatPeerInfo
+    if r4.get():
+      var pid: PeerId
+      let
+        r5 = pbPeerInfo.getField(1, pid)
+        r6 = pbPeerInfo.getRepeatedField(2, peerInfo.addrs)
+      if r5.isErr() or r6.isErr(): return none(AutonatMsg)
+      if r5.get(): peerInfo.id = some(pid)
+      dial.peerInfo = some(peerInfo)
+    msg.dial = some(dial)
+
+  if r3.get():
+    var
+      statusOrd: uint
+      text: string
+      ma: MultiAddress
+      response: AutonatDialResponse
+
+    let
+      r4 = pbResponse.getField(1, statusOrd)
+      r5 = pbResponse.getField(2, text)
+      r6 = pbResponse.getField(3, ma)
+
+    if r4.isErr() or r5.isErr() or r6.isErr() or
+       (r4.get() and not checkedEnumAssign(response.status, statusOrd)):
+      return none(AutonatMsg)
+    if r5.get(): response.text = some(text)
+    if r6.get(): response.ma = some(ma)
+    msg.response = some(response)
+
+  return some(msg)
+
+proc sendDial(conn: Connection, pid: PeerId, addrs: seq[MultiAddress]) {.async.} =
+  let pb = AutonatDial(peerInfo: some(AutonatPeerInfo(
+                         id: some(pid),
+                         addrs: addrs
+                       ))).encode()
+  await conn.writeLp(pb.buffer)
+
+proc sendResponseError(conn: Connection, status: ResponseStatus, text: string = "") {.async.} =
+  let pb = AutonatDialResponse(
+             status: status,
+             text: if text == "": none(string) else: some(text),
+             ma: none(MultiAddress)
+           ).encode()
+  await conn.writeLp(pb.buffer)
+
+proc sendResponseOk(conn: Connection, ma: MultiAddress) {.async.} =
+  let pb = AutonatDialResponse(
+             status: ResponseStatus.Ok,
+             text: some("Ok"),
+             ma: some(ma)
+           ).encode()
+  await conn.writeLp(pb.buffer)
+
+type
+  Autonat* = ref object of LPProtocol
+    sem: AsyncSemaphore
+    switch*: Switch
+
+proc dialMe*(a: Autonat, pid: PeerId, ma: MultiAddress|seq[MultiAddress]):
+    Future[MultiAddress] {.async.} =
+  let addrs = when ma is MultiAddress: @[ma] else: ma
+  let conn = await a.switch.dial(pid, addrs, AutonatCodec)
+  defer: await conn.close()
+  await conn.sendDial(a.switch.peerInfo.peerId, a.switch.peerInfo.addrs)
+  let msgOpt = AutonatMsg.decode(await conn.readLp(1024))
+  if msgOpt.isNone() or
+     msgOpt.get().msgType != DialResponse or
+     msgOpt.get().response.isNone():
+    raise newException(AutonatError, "Unexpected response")
+  let response = msgOpt.get().response.get()
+  if response.status != ResponseStatus.Ok:
+    raise newException(AutonatError, "Bad status " &
+                                      $response.status & " " &
+                                      response.text.get(""))
+  if response.ma.isNone():
+    raise newException(AutonatError, "Missing address")
+  return response.ma.get()
+
+proc tryDial(a: Autonat, conn: Connection, addrs: seq[MultiAddress]) {.async.} =
+  try:
+    await a.sem.acquire()
+    let ma = await a.switch.dialer.tryDial(conn.peerId, addrs)
+    if ma.isSome:
+      await conn.sendResponseOk(ma.get())
+    else:
+      await conn.sendResponseError(DialError, "Missing observed address")
+  except CancelledError as exc:
+    raise exc
+  except CatchableError as exc:
+    await conn.sendResponseError(DialError, exc.msg)
+  finally:
+    a.sem.release()
+
+proc handleDial(a: Autonat, conn: Connection, msg: AutonatMsg): Future[void] =
+  if msg.dial.isNone() or msg.dial.get().peerInfo.isNone():
+    return conn.sendResponseError(BadRequest, "Missing Peer Info")
+  let peerInfo = msg.dial.get().peerInfo.get()
+  if peerInfo.id.isSome() and peerInfo.id.get() != conn.peerId:
+    return conn.sendResponseError(BadRequest, "PeerId mismatch")
+
+  if conn.observedAddr.isNone:
+    return conn.sendResponseError(BadRequest, "Missing observed address")
+  let observedAddr = conn.observedAddr.get()
+
+  var isRelayed = observedAddr.contains(multiCodec("p2p-circuit"))
+  if isRelayed.isErr() or isRelayed.get():
+    return conn.sendResponseError(DialRefused, "Refused to dial a relayed observed address")
+  let hostIp = observedAddr[0]
+  if hostIp.isErr() or not IP.match(hostIp.get()):
+    trace "wrong observed address", address=observedAddr
+    return conn.sendResponseError(InternalError, "Expected an IP address")
+  var addrs = initHashSet[MultiAddress]()
+  addrs.incl(observedAddr)
+  for ma in peerInfo.addrs:
+    isRelayed = ma.contains(multiCodec("p2p-circuit"))
+    if isRelayed.isErr() or isRelayed.get():
+      continue
+    let maFirst = ma[0]
+    if maFirst.isErr() or not IP.match(maFirst.get()):
+      continue
+
+    try:
+      addrs.incl(
+        if maFirst.get() == hostIp.get():
+          ma
+        else:
+          let maEnd = ma[1..^1]
+          if maEnd.isErr(): continue
+          hostIp.get() & maEnd.get()
+      )
+    except LPError as exc:
+      continue
+    if len(addrs) >= AddressLimit:
+      break
+
+  if len(addrs) == 0:
+    return conn.sendResponseError(DialRefused, "No dialable address")
+  return a.tryDial(conn, toSeq(addrs))
+
+proc new*(T: typedesc[Autonat], switch: Switch, semSize: int = 1): T =
+  let autonat = T(switch: switch, sem: newAsyncSemaphore(semSize))
+  autonat.init()
+  autonat
+
+method init*(a: Autonat) =
+  proc handleStream(conn: Connection, proto: string) {.async, gcsafe.} =
+    try:
+      let msgOpt = AutonatMsg.decode(await conn.readLp(1024))
+      if msgOpt.isNone() or msgOpt.get().msgType != MsgType.Dial:
+        raise newException(AutonatError, "Received malformed message")
+      let msg = msgOpt.get()
+      await a.handleDial(conn, msg)
+    except CancelledError as exc:
+      raise exc
+    except CatchableError as exc:
+      trace "exception in autonat handler", exc = exc.msg, conn
+    finally:
+      trace "exiting autonat handler", conn
+      await conn.close()
+
+  a.handler = handleStream
+  a.codec = AutonatCodec

libp2p/protocols/connectivity/relay/client.nim

@@ -0,0 +1,294 @@
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import times, options
+
+import chronos, chronicles
+
+import ./relay,
+       ./messages,
+       ./rconn,
+       ./utils,
+       ../../../peerinfo,
+       ../../../switch,
+       ../../../multiaddress,
+       ../../../stream/connection
+
+
+logScope:
+  topics = "libp2p relay relay-client"
+
+const RelayClientMsgSize = 4096
+
+type
+  RelayClientError* = object of LPError
+  ReservationError* = object of RelayClientError
+  RelayV1DialError* = object of RelayClientError
+  RelayV2DialError* = object of RelayClientError
+  RelayClientAddConn* = proc(conn: Connection,
+                        duration: uint32,
+                        data: uint64): Future[void] {.gcsafe, raises: [Defect].}
+  RelayClient* = ref object of Relay
+    onNewConnection*: RelayClientAddConn
+    canHop: bool
+
+  Rsvp* = object
+    expire*: uint64           # required, Unix expiration time (UTC)
+    addrs*: seq[MultiAddress] # relay address for reserving peer
+    voucher*: Option[Voucher] # optional, reservation voucher
+    limitDuration*: uint32    # seconds
+    limitData*: uint64        # bytes
+
+proc sendStopError(conn: Connection, code: StatusV2) {.async.} =
+  trace "send stop status", status = $code & " (" & $ord(code) & ")"
+  let msg = StopMessage(msgType: StopMessageType.Status, status: some(code))
+  await conn.writeLp(encode(msg).buffer)
+
+proc handleRelayedConnect(cl: RelayClient, conn: Connection, msg: StopMessage) {.async.} =
+  if msg.peer.isNone():
+    await sendStopError(conn, MalformedMessage)
+    return
+  let
+    # TODO: check the go version to see in which way this could fail
+    # it's unclear in the spec
+    src = msg.peer.get()
+    limitDuration = msg.limit.duration
+    limitData = msg.limit.data
+    msg = StopMessage(
+      msgType: StopMessageType.Status,
+      status: some(Ok))
+    pb = encode(msg)
+
+  trace "incoming relay connection", src
+
+  if cl.onNewConnection == nil:
+    await sendStopError(conn, StatusV2.ConnectionFailed)
+    await conn.close()
+    return
+  await conn.writeLp(pb.buffer)
+  # This sound redundant but the callback could, in theory, be set to nil during
+  # conn.writeLp so it's safer to double check
+  if cl.onNewConnection != nil: await cl.onNewConnection(conn, limitDuration, limitData)
+  else: await conn.close()
+
+proc reserve*(cl: RelayClient,
+              peerId: PeerId,
+              addrs: seq[MultiAddress] = @[]): Future[Rsvp] {.async.} =
+  let conn = await cl.switch.dial(peerId, addrs, RelayV2HopCodec)
+  defer: await conn.close()
+  let
+    pb = encode(HopMessage(msgType: HopMessageType.Reserve))
+    msg = try:
+      await conn.writeLp(pb.buffer)
+      HopMessage.decode(await conn.readLp(RelayClientMsgSize)).get()
+    except CancelledError as exc:
+      raise exc
+    except CatchableError as exc:
+      trace "error writing or reading reservation message", exc=exc.msg
+      raise newException(ReservationError, exc.msg)
+
+  if msg.msgType != HopMessageType.Status:
+    raise newException(ReservationError, "Unexpected relay response type")
+  if msg.status.get(UnexpectedMessage) != Ok:
+    raise newException(ReservationError, "Reservation failed")
+  if msg.reservation.isNone():
+    raise newException(ReservationError, "Missing reservation information")
+
+  let reservation = msg.reservation.get()
+  if reservation.expire > int64.high().uint64 or
+     now().utc > reservation.expire.int64.fromUnix.utc:
+    raise newException(ReservationError, "Bad expiration date")
+  result.expire = reservation.expire
+  result.addrs = reservation.addrs
+
+  if reservation.svoucher.isSome():
+    let svoucher = SignedVoucher.decode(reservation.svoucher.get())
+    if svoucher.isErr() or svoucher.get().data.relayPeerId != peerId:
+      raise newException(ReservationError, "Invalid voucher")
+    result.voucher = some(svoucher.get().data)
+
+  result.limitDuration = msg.limit.duration
+  result.limitData = msg.limit.data
+
+proc dialPeerV1*(
+    cl: RelayClient,
+    conn: Connection,
+    dstPeerId: PeerId,
+    dstAddrs: seq[MultiAddress]): Future[Connection] {.async.} =
+  var
+    msg = RelayMessage(
+      msgType: some(RelayType.Hop),
+      srcPeer: some(RelayPeer(peerId: cl.switch.peerInfo.peerId, addrs: cl.switch.peerInfo.addrs)),
+      dstPeer: some(RelayPeer(peerId: dstPeerId, addrs: dstAddrs)))
+    pb = encode(msg)
+
+  trace "Dial peer", msgSend=msg
+
+  try:
+    await conn.writeLp(pb.buffer)
+  except CancelledError as exc:
+    raise exc
+  except CatchableError as exc:
+    trace "error writing hop request", exc=exc.msg
+    raise exc
+
+  let msgRcvFromRelayOpt = try:
+    RelayMessage.decode(await conn.readLp(RelayClientMsgSize))
+  except CancelledError as exc:
+    raise exc
+  except CatchableError as exc:
+    trace "error reading stop response", exc=exc.msg
+    await sendStatus(conn, StatusV1.HopCantOpenDstStream)
+    raise exc
+
+  try:
+    if msgRcvFromRelayOpt.isNone:
+      raise newException(RelayV1DialError, "Hop can't open destination stream")
+    let msgRcvFromRelay = msgRcvFromRelayOpt.get()
+    if msgRcvFromRelay.msgType.isNone or msgRcvFromRelay.msgType.get() != RelayType.Status:
+      raise newException(RelayV1DialError, "Hop can't open destination stream: wrong message type")
+    if msgRcvFromRelay.status.isNone or msgRcvFromRelay.status.get() != StatusV1.Success:
+      raise newException(RelayV1DialError, "Hop can't open destination stream: status failed")
+  except RelayV1DialError as exc:
+    await sendStatus(conn, StatusV1.HopCantOpenDstStream)
+    raise exc
+  result = conn
+
+proc dialPeerV2*(
+    cl: RelayClient,
+    conn: RelayConnection,
+    dstPeerId: PeerId,
+    dstAddrs: seq[MultiAddress]): Future[Connection] {.async.} =
+  let
+    p = Peer(peerId: dstPeerId, addrs: dstAddrs)
+    pb = encode(HopMessage(msgType: HopMessageType.Connect, peer: some(p)))
+
+  trace "Dial peer", p
+
+  let msgRcvFromRelay = try:
+    await conn.writeLp(pb.buffer)
+    HopMessage.decode(await conn.readLp(RelayClientMsgSize)).get()
+  except CancelledError as exc:
+    raise exc
+  except CatchableError as exc:
+    trace "error reading stop response", exc=exc.msg
+    raise newException(RelayV2DialError, exc.msg)
+
+  if msgRcvFromRelay.msgType != HopMessageType.Status:
+    raise newException(RelayV2DialError, "Unexpected stop response")
+  if msgRcvFromRelay.status.get(UnexpectedMessage) != Ok:
+    trace "Relay stop failed", msg = msgRcvFromRelay.status.get()
+    raise newException(RelayV2DialError, "Relay stop failure")
+  conn.limitDuration = msgRcvFromRelay.limit.duration
+  conn.limitData = msgRcvFromRelay.limit.data
+  return conn
+
+proc handleStopStreamV2(cl: RelayClient, conn: Connection) {.async, gcsafe.} =
+  let msgOpt = StopMessage.decode(await conn.readLp(RelayClientMsgSize))
+  if msgOpt.isNone():
+    await sendHopStatus(conn, MalformedMessage)
+    return
+  trace "client circuit relay v2 handle stream", msg = msgOpt.get()
+  let msg = msgOpt.get()
+
+  if msg.msgType == StopMessageType.Connect:
+    await cl.handleRelayedConnect(conn, msg)
+  else:
+    trace "Unexpected client / relayv2 handshake", msgType=msg.msgType
+    await sendStopError(conn, MalformedMessage)
+
+proc handleStop(cl: RelayClient, conn: Connection, msg: RelayMessage) {.async, gcsafe.} =
+  if msg.srcPeer.isNone:
+    await sendStatus(conn, StatusV1.StopSrcMultiaddrInvalid)
+    return
+  let src = msg.srcPeer.get()
+
+  if msg.dstPeer.isNone:
+    await sendStatus(conn, StatusV1.StopDstMultiaddrInvalid)
+    return
+
+  let dst = msg.dstPeer.get()
+  if dst.peerId != cl.switch.peerInfo.peerId:
+    await sendStatus(conn, StatusV1.StopDstMultiaddrInvalid)
+    return
+
+  trace "get a relay connection", src, conn
+
+  if cl.onNewConnection == nil:
+    await sendStatus(conn, StatusV1.StopRelayRefused)
+    await conn.close()
+    return
+  await sendStatus(conn, StatusV1.Success)
+  # This sound redundant but the callback could, in theory, be set to nil during
+  # sendStatus(Success) so it's safer to double check
+  if cl.onNewConnection != nil: await cl.onNewConnection(conn, 0, 0)
+  else: await conn.close()
+
+proc handleStreamV1(cl: RelayClient, conn: Connection) {.async, gcsafe.} =
+  let msgOpt = RelayMessage.decode(await conn.readLp(RelayClientMsgSize))
+  if msgOpt.isNone:
+    await sendStatus(conn, StatusV1.MalformedMessage)
+    return
+  trace "client circuit relay v1 handle stream", msg = msgOpt.get()
+  let msg = msgOpt.get()
+  case msg.msgType.get:
+    of RelayType.Hop:
+      if cl.canHop: await cl.handleHop(conn, msg)
+      else: await sendStatus(conn, StatusV1.HopCantSpeakRelay)
+    of RelayType.Stop: await cl.handleStop(conn, msg)
+    of RelayType.CanHop:
+      if cl.canHop: await sendStatus(conn, StatusV1.Success)
+      else: await sendStatus(conn, StatusV1.HopCantSpeakRelay)
+    else:
+      trace "Unexpected relay handshake", msgType=msg.msgType
+      await sendStatus(conn, StatusV1.MalformedMessage)
+
+proc new*(T: typedesc[RelayClient], canHop: bool = false,
+     reservationTTL: times.Duration = DefaultReservationTTL,
+     limitDuration: uint32 = DefaultLimitDuration,
+     limitData: uint64 = DefaultLimitData,
+     heartbeatSleepTime: uint32 = DefaultHeartbeatSleepTime,
+     maxCircuit: int = MaxCircuit,
+     maxCircuitPerPeer: int = MaxCircuitPerPeer,
+     msgSize: int = RelayClientMsgSize,
+     circuitRelayV1: bool = false): T =
+
+  let cl = T(canHop: canHop,
+             reservationTTL: reservationTTL,
+             limit: Limit(duration: limitDuration, data: limitData),
+             heartbeatSleepTime: heartbeatSleepTime,
+             maxCircuit: maxCircuit,
+             maxCircuitPerPeer: maxCircuitPerPeer,
+             msgSize: msgSize,
+             isCircuitRelayV1: circuitRelayV1)
+  proc handleStream(conn: Connection, proto: string) {.async, gcsafe.} =
+    try:
+      case proto:
+      of RelayV1Codec: await cl.handleStreamV1(conn)
+      of RelayV2StopCodec: await cl.handleStopStreamV2(conn)
+      of RelayV2HopCodec: await cl.handleHopStreamV2(conn)
+    except CancelledError as exc:
+      raise exc
+    except CatchableError as exc:
+      trace "exception in client handler", exc = exc.msg, conn
+    finally:
+      trace "exiting client handler", conn
+      await conn.close()
+
+  cl.handler = handleStream
+  cl.codecs = if cl.canHop:
+                @[RelayV1Codec, RelayV2HopCodec, RelayV2StopCodec]
+              else:
+                @[RelayV1Codec, RelayV2StopCodec]
+  cl

libp2p/protocols/connectivity/relay/messages.nim

@@ -0,0 +1,370 @@
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import options, macros, sequtils
+import stew/objects
+import ../../../peerinfo,
+       ../../../signed_envelope
+
+# Circuit Relay V1 Message
+
+type
+  RelayType* {.pure.} = enum
+    Hop = 1
+    Stop = 2
+    Status = 3
+    CanHop = 4
+
+  StatusV1* {.pure.} = enum
+    Success = 100
+    HopSrcAddrTooLong = 220
+    HopDstAddrTooLong = 221
+    HopSrcMultiaddrInvalid = 250
+    HopDstMultiaddrInvalid = 251
+    HopNoConnToDst = 260
+    HopCantDialDst = 261
+    HopCantOpenDstStream = 262
+    HopCantSpeakRelay = 270
+    HopCantRelayToSelf = 280
+    StopSrcAddrTooLong = 320
+    StopDstAddrTooLong = 321
+    StopSrcMultiaddrInvalid = 350
+    StopDstMultiaddrInvalid = 351
+    StopRelayRefused = 390
+    MalformedMessage = 400
+
+  RelayPeer* = object
+    peerId*: PeerId
+    addrs*: seq[MultiAddress]
+
+  RelayMessage* = object
+    msgType*: Option[RelayType]
+    srcPeer*: Option[RelayPeer]
+    dstPeer*: Option[RelayPeer]
+    status*: Option[StatusV1]
+
+proc encode*(msg: RelayMessage): ProtoBuffer =
+  result = initProtoBuffer()
+
+  if isSome(msg.msgType):
+    result.write(1, msg.msgType.get().ord.uint)
+  if isSome(msg.srcPeer):
+    var peer = initProtoBuffer()
+    peer.write(1, msg.srcPeer.get().peerId)
+    for ma in msg.srcPeer.get().addrs:
+      peer.write(2, ma.data.buffer)
+    peer.finish()
+    result.write(2, peer.buffer)
+  if isSome(msg.dstPeer):
+    var peer = initProtoBuffer()
+    peer.write(1, msg.dstPeer.get().peerId)
+    for ma in msg.dstPeer.get().addrs:
+      peer.write(2, ma.data.buffer)
+    peer.finish()
+    result.write(3, peer.buffer)
+  if isSome(msg.status):
+    result.write(4, msg.status.get().ord.uint)
+
+  result.finish()
+
+proc decode*(_: typedesc[RelayMessage], buf: seq[byte]): Option[RelayMessage] =
+  var
+    rMsg: RelayMessage
+    msgTypeOrd: uint32
+    src: RelayPeer
+    dst: RelayPeer
+    statusOrd: uint32
+    pbSrc: ProtoBuffer
+    pbDst: ProtoBuffer
+
+  let
+    pb = initProtoBuffer(buf)
+    r1 = pb.getField(1, msgTypeOrd)
+    r2 = pb.getField(2, pbSrc)
+    r3 = pb.getField(3, pbDst)
+    r4 = pb.getField(4, statusOrd)
+
+  if r1.isErr() or r2.isErr() or r3.isErr() or r4.isErr():
+    return none(RelayMessage)
+
+  if r2.get() and
+     (pbSrc.getField(1, src.peerId).isErr() or
+      pbSrc.getRepeatedField(2, src.addrs).isErr()):
+    return none(RelayMessage)
+
+  if r3.get() and
+     (pbDst.getField(1, dst.peerId).isErr() or
+      pbDst.getRepeatedField(2, dst.addrs).isErr()):
+    return none(RelayMessage)
+
+  if r1.get():
+    if msgTypeOrd.int notin RelayType:
+      return none(RelayMessage)
+    rMsg.msgType = some(RelayType(msgTypeOrd))
+  if r2.get(): rMsg.srcPeer = some(src)
+  if r3.get(): rMsg.dstPeer = some(dst)
+  if r4.get():
+    if statusOrd.int notin StatusV1:
+      return none(RelayMessage)
+    rMsg.status = some(StatusV1(statusOrd))
+  some(rMsg)
+
+# Voucher
+
+type
+  Voucher* = object
+    relayPeerId*: PeerId     # peer ID of the relay
+    reservingPeerId*: PeerId # peer ID of the reserving peer
+    expiration*: uint64      # UNIX UTC expiration time for the reservation
+
+proc decode*(_: typedesc[Voucher], buf: seq[byte]): Result[Voucher, ProtoError] =
+  let pb = initProtoBuffer(buf)
+  var v = Voucher()
+
+  ? pb.getRequiredField(1, v.relayPeerId)
+  ? pb.getRequiredField(2, v.reservingPeerId)
+  ? pb.getRequiredField(3, v.expiration)
+
+  ok(v)
+
+proc encode*(v: Voucher): seq[byte] =
+  var pb = initProtoBuffer()
+
+  pb.write(1, v.relayPeerId)
+  pb.write(2, v.reservingPeerId)
+  pb.write(3, v.expiration)
+
+  pb.finish()
+  pb.buffer
+
+proc init*(T: typedesc[Voucher],
+          relayPeerId: PeerId,
+          reservingPeerId: PeerId,
+          expiration: uint64): T =
+  T(
+    relayPeerId = relayPeerId,
+    reservingPeerId = reservingPeerId,
+    expiration: expiration
+  )
+
+type SignedVoucher* = SignedPayload[Voucher]
+
+proc payloadDomain*(_: typedesc[Voucher]): string = "libp2p-relay-rsvp"
+proc payloadType*(_: typedesc[Voucher]): seq[byte] = @[ (byte)0x03, (byte)0x02 ]
+
+proc checkValid*(spr: SignedVoucher): Result[void, EnvelopeError] =
+  if not spr.data.relayPeerId.match(spr.envelope.publicKey):
+    err(EnvelopeInvalidSignature)
+  else:
+    ok()
+
+# Circuit Relay V2 Hop Message
+
+type
+  Peer* = object
+    peerId*: PeerId
+    addrs*: seq[MultiAddress]
+  Reservation* = object
+    expire*: uint64              # required, Unix expiration time (UTC)
+    addrs*: seq[MultiAddress]    # relay address for reserving peer
+    svoucher*: Option[seq[byte]] # optional, reservation voucher
+  Limit* = object
+    duration*: uint32            # seconds
+    data*: uint64                # bytes
+
+  StatusV2* = enum
+    Ok = 100
+    ReservationRefused = 200
+    ResourceLimitExceeded = 201
+    PermissionDenied = 202
+    ConnectionFailed = 203
+    NoReservation = 204
+    MalformedMessage = 400
+    UnexpectedMessage = 401
+  HopMessageType* {.pure.} = enum
+    Reserve = 0
+    Connect = 1
+    Status = 2
+  HopMessage* = object
+    msgType*: HopMessageType
+    peer*: Option[Peer]
+    reservation*: Option[Reservation]
+    limit*: Limit
+    status*: Option[StatusV2]
+
+proc encode*(msg: HopMessage): ProtoBuffer =
+  var pb = initProtoBuffer()
+
+  pb.write(1, msg.msgType.ord.uint)
+  if msg.peer.isSome():
+    var ppb = initProtoBuffer()
+    ppb.write(1, msg.peer.get().peerId)
+    for ma in msg.peer.get().addrs:
+      ppb.write(2, ma.data.buffer)
+    ppb.finish()
+    pb.write(2, ppb.buffer)
+  if msg.reservation.isSome():
+    let rsrv = msg.reservation.get()
+    var rpb = initProtoBuffer()
+    rpb.write(1, rsrv.expire)
+    for ma in rsrv.addrs:
+      rpb.write(2, ma.data.buffer)
+    if rsrv.svoucher.isSome():
+      rpb.write(3, rsrv.svoucher.get())
+    rpb.finish()
+    pb.write(3, rpb.buffer)
+  if msg.limit.duration > 0 or msg.limit.data > 0:
+    var lpb = initProtoBuffer()
+    if msg.limit.duration > 0: lpb.write(1, msg.limit.duration)
+    if msg.limit.data > 0: lpb.write(2, msg.limit.data)
+    lpb.finish()
+    pb.write(4, lpb.buffer)
+  if msg.status.isSome():
+    pb.write(5, msg.status.get().ord.uint)
+
+  pb.finish()
+  pb
+
+proc decode*(_: typedesc[HopMessage], buf: seq[byte]): Option[HopMessage] =
+  var
+    msg: HopMessage
+    msgTypeOrd: uint32
+    pbPeer: ProtoBuffer
+    pbReservation: ProtoBuffer
+    pbLimit: ProtoBuffer
+    statusOrd: uint32
+    peer: Peer
+    reservation: Reservation
+    limit: Limit
+    res: bool
+
+  let
+    pb = initProtoBuffer(buf)
+    r1 = pb.getRequiredField(1, msgTypeOrd)
+    r2 = pb.getField(2, pbPeer)
+    r3 = pb.getField(3, pbReservation)
+    r4 = pb.getField(4, pbLimit)
+    r5 = pb.getField(5, statusOrd)
+
+  if r1.isErr() or r2.isErr() or r3.isErr() or r4.isErr() or r5.isErr():
+    return none(HopMessage)
+
+  if r2.get() and
+     (pbPeer.getRequiredField(1, peer.peerId).isErr() or
+      pbPeer.getRepeatedField(2, peer.addrs).isErr()):
+    return none(HopMessage)
+
+  if r3.get():
+    var svoucher: seq[byte]
+    let rSVoucher = pbReservation.getField(3, svoucher)
+    if pbReservation.getRequiredField(1, reservation.expire).isErr() or
+       pbReservation.getRepeatedField(2, reservation.addrs).isErr() or
+       rSVoucher.isErr():
+      return none(HopMessage)
+    if rSVoucher.get(): reservation.svoucher = some(svoucher)
+
+  if r4.get() and
+     (pbLimit.getField(1, limit.duration).isErr() or
+      pbLimit.getField(2, limit.data).isErr()):
+    return none(HopMessage)
+
+  if not checkedEnumAssign(msg.msgType, msgTypeOrd):
+    return none(HopMessage)
+  if r2.get(): msg.peer = some(peer)
+  if r3.get(): msg.reservation = some(reservation)
+  if r4.get(): msg.limit = limit
+  if r5.get():
+    if statusOrd.int notin StatusV2:
+      return none(HopMessage)
+    msg.status = some(StatusV2(statusOrd))
+  some(msg)
+
+# Circuit Relay V2 Stop Message
+
+type
+  StopMessageType* {.pure.} = enum
+    Connect = 0
+    Status = 1
+  StopMessage* = object
+    msgType*: StopMessageType
+    peer*: Option[Peer]
+    limit*: Limit
+    status*: Option[StatusV2]
+
+
+proc encode*(msg: StopMessage): ProtoBuffer =
+  var pb = initProtoBuffer()
+
+  pb.write(1, msg.msgType.ord.uint)
+  if msg.peer.isSome():
+    var ppb = initProtoBuffer()
+    ppb.write(1, msg.peer.get().peerId)
+    for ma in msg.peer.get().addrs:
+      ppb.write(2, ma.data.buffer)
+    ppb.finish()
+    pb.write(2, ppb.buffer)
+  if msg.limit.duration > 0 or msg.limit.data > 0:
+    var lpb = initProtoBuffer()
+    if msg.limit.duration > 0: lpb.write(1, msg.limit.duration)
+    if msg.limit.data > 0: lpb.write(2, msg.limit.data)
+    lpb.finish()
+    pb.write(3, lpb.buffer)
+  if msg.status.isSome():
+    pb.write(4, msg.status.get().ord.uint)
+
+  pb.finish()
+  pb
+
+proc decode*(_: typedesc[StopMessage], buf: seq[byte]): Option[StopMessage] =
+  var
+    msg: StopMessage
+    msgTypeOrd: uint32
+    pbPeer: ProtoBuffer
+    pbLimit: ProtoBuffer
+    statusOrd: uint32
+    peer: Peer
+    limit: Limit
+    rVoucher: ProtoResult[bool]
+    res: bool
+
+  let
+    pb = initProtoBuffer(buf)
+    r1 = pb.getRequiredField(1, msgTypeOrd)
+    r2 = pb.getField(2, pbPeer)
+    r3 = pb.getField(3, pbLimit)
+    r4 = pb.getField(4, statusOrd)
+
+  if r1.isErr() or r2.isErr() or r3.isErr() or r4.isErr():
+    return none(StopMessage)
+
+  if r2.get() and
+     (pbPeer.getRequiredField(1, peer.peerId).isErr() or
+      pbPeer.getRepeatedField(2, peer.addrs).isErr()):
+    return none(StopMessage)
+
+  if r3.get() and
+     (pbLimit.getField(1, limit.duration).isErr() or
+      pbLimit.getField(2, limit.data).isErr()):
+    return none(StopMessage)
+
+  if msgTypeOrd.int notin StopMessageType.low.ord .. StopMessageType.high.ord:
+    return none(StopMessage)
+  msg.msgType = StopMessageType(msgTypeOrd)
+  if r2.get(): msg.peer = some(peer)
+  if r3.get(): msg.limit = limit
+  if r4.get():
+    if statusOrd.int notin StatusV2:
+      return none(StopMessage)
+    msg.status = some(StatusV2(statusOrd))
+  some(msg)

libp2p/protocols/connectivity/relay/rconn.nim

@@ -0,0 +1,61 @@
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import chronos
+
+import ../../../stream/connection
+
+type
+  RelayConnection* = ref object of Connection
+    conn*: Connection
+    limitDuration*: uint32
+    limitData*: uint64
+    dataSent*: uint64
+
+method readOnce*(
+    self: RelayConnection,
+    pbytes: pointer,
+    nbytes: int): Future[int] {.async.} =
+  self.activity = true
+  return await self.conn.readOnce(pbytes, nbytes)
+
+method write*(self: RelayConnection, msg: seq[byte]): Future[void] {.async.} =
+  self.dataSent.inc(msg.len)
+  if self.limitData != 0 and self.dataSent > self.limitData:
+    await self.close()
+    return
+  self.activity = true
+  await self.conn.write(msg)
+
+method closeImpl*(self: RelayConnection): Future[void] {.async.} =
+  await self.conn.close()
+  await procCall Connection(self).closeImpl()
+
+method getWrapped*(self: RelayConnection): Connection = self.conn
+
+proc new*(
+  T: typedesc[RelayConnection],
+  conn: Connection,
+  limitDuration: uint32,
+  limitData: uint64): T =
+  let rc = T(conn: conn, limitDuration: limitDuration, limitData: limitData)
+  rc.initStream()
+  if limitDuration > 0:
+    proc checkDurationConnection() {.async.} =
+      let sleep = sleepAsync(limitDuration.seconds())
+      await sleep or conn.join()
+      if sleep.finished: await conn.close()
+      else: sleep.cancel()
+    asyncSpawn checkDurationConnection()
+  return rc

libp2p/protocols/connectivity/relay/relay.nim

@@ -0,0 +1,386 @@
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import options, sequtils, tables, sugar
+
+import chronos, chronicles
+
+import ./messages,
+       ./rconn,
+       ./utils,
+       ../../../peerinfo,
+       ../../../switch,
+       ../../../multiaddress,
+       ../../../multicodec,
+       ../../../stream/connection,
+       ../../../protocols/protocol,
+       ../../../transports/transport,
+       ../../../errors,
+       ../../../utils/heartbeat,
+       ../../../signed_envelope
+
+# TODO:
+# * Eventually replace std/times by chronos/timer. Currently chronos/timer
+#   doesn't offer the possibility to get a datetime in UNIX UTC
+# * Eventually add an access control list in the handleReserve, handleConnect
+#   and handleHop
+# * Better reservation management ie find a way to re-reserve when the end is nigh
+
+import std/times
+export chronicles
+
+const
+  RelayMsgSize* = 4096
+  DefaultReservationTTL* = initDuration(hours = 1)
+  DefaultLimitDuration* = 120
+  DefaultLimitData* = 1 shl 17
+  DefaultHeartbeatSleepTime* = 1
+  MaxCircuit* = 128
+  MaxCircuitPerPeer* = 16
+
+logScope:
+  topics = "libp2p relay"
+
+type
+  RelayV2Error* = object of LPError
+  SendStopError = object of RelayV2Error
+
+# Relay Side
+
+type
+  Relay* = ref object of LPProtocol
+    switch*: Switch
+    peerCount: CountTable[PeerId]
+
+    # number of reservation (relayv2) + number of connection (relayv1)
+    maxCircuit*: int
+
+    maxCircuitPerPeer*: int
+    msgSize*: int
+    # RelayV1
+    isCircuitRelayV1*: bool
+    streamCount: int
+    # RelayV2
+    rsvp: Table[PeerId, DateTime]
+    reservationLoop: Future[void]
+    reservationTTL*: times.Duration
+    heartbeatSleepTime*: uint32
+    limit*: Limit
+
+# Relay V2
+
+proc createReserveResponse(
+    r: Relay,
+    pid: PeerId,
+    expire: DateTime): Result[HopMessage, CryptoError] =
+  let
+    expireUnix = expire.toTime.toUnix.uint64
+    v = Voucher(relayPeerId: r.switch.peerInfo.peerId,
+                  reservingPeerId: pid,
+                  expiration: expireUnix)
+    sv = ? SignedVoucher.init(r.switch.peerInfo.privateKey, v)
+    ma = ? MultiAddress.init("/p2p/" & $r.switch.peerInfo.peerId).orErr(CryptoError.KeyError)
+    rsrv = Reservation(expire: expireUnix,
+                       addrs: r.switch.peerInfo.addrs.mapIt(
+                         ? it.concat(ma).orErr(CryptoError.KeyError)),
+                       svoucher: some(? sv.encode))
+    msg = HopMessage(msgType: HopMessageType.Status,
+                     reservation: some(rsrv),
+                     limit: r.limit,
+                     status: some(Ok))
+  return ok(msg)
+
+proc isRelayed(conn: Connection): bool =
+  var wrappedConn = conn
+  while not isNil(wrappedConn):
+    if wrappedConn of RelayConnection:
+      return true
+    wrappedConn = wrappedConn.getWrapped()
+  return false
+
+proc handleReserve(r: Relay, conn: Connection) {.async, gcsafe.} =
+  if conn.isRelayed():
+    trace "reservation attempt over relay connection", pid = conn.peerId
+    await sendHopStatus(conn, PermissionDenied)
+    return
+
+  if r.peerCount[conn.peerId] + r.rsvp.len() >= r.maxCircuit:
+    trace "Too many reservations", pid = conn.peerId
+    await sendHopStatus(conn, ReservationRefused)
+    return
+  let
+    pid = conn.peerId
+    expire = now().utc + r.reservationTTL
+    msg = r.createReserveResponse(pid, expire)
+
+  trace "reserving relay slot for", pid
+  if msg.isErr():
+    trace "error signing the voucher", error = error(msg), pid
+    return
+  r.rsvp[pid] = expire
+  await conn.writeLp(encode(msg.get()).buffer)
+
+proc handleConnect(r: Relay,
+                   connSrc: Connection,
+                   msg: HopMessage) {.async, gcsafe.} =
+  if connSrc.isRelayed():
+    trace "connection attempt over relay connection"
+    await sendHopStatus(connSrc, PermissionDenied)
+    return
+  if msg.peer.isNone():
+    await sendHopStatus(connSrc, MalformedMessage)
+    return
+
+  let
+    src = connSrc.peerId
+    dst = msg.peer.get().peerId
+  if dst notin r.rsvp:
+    trace "refusing connection, no reservation", src, dst
+    await sendHopStatus(connSrc, NoReservation)
+    return
+
+  r.peerCount.inc(src)
+  r.peerCount.inc(dst)
+  defer:
+    r.peerCount.inc(src, -1)
+    r.peerCount.inc(dst, -1)
+
+  if r.peerCount[src] > r.maxCircuitPerPeer or
+     r.peerCount[dst] > r.maxCircuitPerPeer:
+    trace "too many connections", src = r.peerCount[src],
+                                  dst = r.peerCount[dst],
+                                  max = r.maxCircuitPerPeer
+    await sendHopStatus(connSrc, ResourceLimitExceeded)
+    return
+
+  let connDst = try:
+    await r.switch.dial(dst, RelayV2StopCodec)
+  except CancelledError as exc:
+    raise exc
+  except CatchableError as exc:
+    trace "error opening relay stream", dst, exc=exc.msg
+    await sendHopStatus(connSrc, ConnectionFailed)
+    return
+  defer:
+    await connDst.close()
+
+  proc sendStopMsg() {.async.} =
+    let stopMsg = StopMessage(msgType: StopMessageType.Connect,
+                            peer: some(Peer(peerId: src, addrs: @[])),
+                            limit: r.limit)
+    await connDst.writeLp(encode(stopMsg).buffer)
+    let msg = StopMessage.decode(await connDst.readLp(r.msgSize)).get()
+    if msg.msgType != StopMessageType.Status:
+      raise newException(SendStopError, "Unexpected stop response, not a status message")
+    if msg.status.get(UnexpectedMessage) != Ok:
+      raise newException(SendStopError, "Relay stop failure")
+    await connSrc.writeLp(encode(HopMessage(msgType: HopMessageType.Status,
+                                          status: some(Ok))).buffer)
+  try:
+    await sendStopMsg()
+  except CancelledError as exc:
+    raise exc
+  except CatchableError as exc:
+    trace "error sending stop message", msg = exc.msg
+    await sendHopStatus(connSrc, ConnectionFailed)
+    return
+
+  trace "relaying connection", src, dst
+  let
+    rconnSrc = RelayConnection.new(connSrc, r.limit.duration, r.limit.data)
+    rconnDst = RelayConnection.new(connDst, r.limit.duration, r.limit.data)
+  defer:
+    await rconnSrc.close()
+    await rconnDst.close()
+  await bridge(rconnSrc, rconnDst)
+
+proc handleHopStreamV2*(r: Relay, conn: Connection) {.async, gcsafe.} =
+  let msgOpt = HopMessage.decode(await conn.readLp(r.msgSize))
+  if msgOpt.isNone():
+    await sendHopStatus(conn, MalformedMessage)
+    return
+  trace "relayv2 handle stream", msg = msgOpt.get()
+  let msg = msgOpt.get()
+  case msg.msgType:
+  of HopMessageType.Reserve: await r.handleReserve(conn)
+  of HopMessageType.Connect: await r.handleConnect(conn, msg)
+  else:
+    trace "Unexpected relayv2 handshake", msgType=msg.msgType
+    await sendHopStatus(conn, MalformedMessage)
+
+# Relay V1
+
+proc handleHop*(r: Relay, connSrc: Connection, msg: RelayMessage) {.async, gcsafe.} =
+  r.streamCount.inc()
+  defer: r.streamCount.dec()
+  if r.streamCount + r.rsvp.len() >= r.maxCircuit:
+    trace "refusing connection; too many active circuit", streamCount = r.streamCount, rsvp = r.rsvp.len()
+    await sendStatus(connSrc, StatusV1.HopCantSpeakRelay)
+    return
+
+  proc checkMsg(): Result[RelayMessage, StatusV1] =
+    if msg.srcPeer.isNone:
+      return err(StatusV1.HopSrcMultiaddrInvalid)
+    let src = msg.srcPeer.get()
+    if src.peerId != connSrc.peerId:
+      return err(StatusV1.HopSrcMultiaddrInvalid)
+    if msg.dstPeer.isNone:
+      return err(StatusV1.HopDstMultiaddrInvalid)
+    let dst = msg.dstPeer.get()
+    if dst.peerId == r.switch.peerInfo.peerId:
+      return err(StatusV1.HopCantRelayToSelf)
+    if not r.switch.isConnected(dst.peerId):
+      trace "relay not connected to dst", dst
+      return err(StatusV1.HopNoConnToDst)
+    ok(msg)
+  let check = checkMsg()
+  if check.isErr:
+    await sendStatus(connSrc, check.error())
+    return
+
+  let
+    src = msg.srcPeer.get()
+    dst = msg.dstPeer.get()
+  if r.peerCount[src.peerId] >= r.maxCircuitPerPeer or
+     r.peerCount[dst.peerId] >= r.maxCircuitPerPeer:
+    trace "refusing connection; too many connection from src or to dst", src, dst
+    await sendStatus(connSrc, StatusV1.HopCantSpeakRelay)
+    return
+  r.peerCount.inc(src.peerId)
+  r.peerCount.inc(dst.peerId)
+  defer:
+    r.peerCount.inc(src.peerId, -1)
+    r.peerCount.inc(dst.peerId, -1)
+
+  let connDst = try:
+    await r.switch.dial(dst.peerId, RelayV1Codec)
+  except CancelledError as exc:
+    raise exc
+  except CatchableError as exc:
+    trace "error opening relay stream", dst, exc=exc.msg
+    await sendStatus(connSrc, StatusV1.HopCantDialDst)
+    return
+  defer:
+    await connDst.close()
+
+  let msgToSend = RelayMessage(
+    msgType: some(RelayType.Stop),
+    srcPeer: some(src),
+    dstPeer: some(dst))
+
+  let msgRcvFromDstOpt = try:
+    await connDst.writeLp(encode(msgToSend).buffer)
+    RelayMessage.decode(await connDst.readLp(r.msgSize))
+  except CancelledError as exc:
+    raise exc
+  except CatchableError as exc:
+    trace "error writing stop handshake or reading stop response", exc=exc.msg
+    await sendStatus(connSrc, StatusV1.HopCantOpenDstStream)
+    return
+
+  if msgRcvFromDstOpt.isNone:
+    trace "error reading stop response", msg = msgRcvFromDstOpt
+    await sendStatus(connSrc, StatusV1.HopCantOpenDstStream)
+    return
+
+  let msgRcvFromDst = msgRcvFromDstOpt.get()
+  if msgRcvFromDst.msgType.get(RelayType.Stop) != RelayType.Status or
+     msgRcvFromDst.status.get(StatusV1.StopRelayRefused) != StatusV1.Success:
+    trace "unexcepted relay stop response", msgRcvFromDst
+    await sendStatus(connSrc, StatusV1.HopCantOpenDstStream)
+    return
+
+  await sendStatus(connSrc, StatusV1.Success)
+  trace "relaying connection", src, dst
+  await bridge(connSrc, connDst)
+
+proc handleStreamV1(r: Relay, conn: Connection) {.async, gcsafe.} =
+  let msgOpt = RelayMessage.decode(await conn.readLp(r.msgSize))
+  if msgOpt.isNone:
+    await sendStatus(conn, StatusV1.MalformedMessage)
+    return
+  trace "relay handle stream", msg = msgOpt.get()
+  let msg = msgOpt.get()
+  case msg.msgType.get:
+    of RelayType.Hop: await r.handleHop(conn, msg)
+    of RelayType.Stop: await sendStatus(conn, StatusV1.StopRelayRefused)
+    of RelayType.CanHop: await sendStatus(conn, StatusV1.Success)
+    else:
+      trace "Unexpected relay handshake", msgType=msg.msgType
+      await sendStatus(conn, StatusV1.MalformedMessage)
+
+proc setup*(r: Relay, switch: Switch) =
+  r.switch = switch
+  r.switch.addPeerEventHandler(
+    proc (peerId: PeerId, event: PeerEvent) {.async.} =
+      r.rsvp.del(peerId),
+    Left)
+
+proc new*(T: typedesc[Relay],
+     reservationTTL: times.Duration = DefaultReservationTTL,
+     limitDuration: uint32 = DefaultLimitDuration,
+     limitData: uint64 = DefaultLimitData,
+     heartbeatSleepTime: uint32 = DefaultHeartbeatSleepTime,
+     maxCircuit: int = MaxCircuit,
+     maxCircuitPerPeer: int = MaxCircuitPerPeer,
+     msgSize: int = RelayMsgSize,
+     circuitRelayV1: bool = false): T =
+
+  let r = T(reservationTTL: reservationTTL,
+            limit: Limit(duration: limitDuration, data: limitData),
+            heartbeatSleepTime: heartbeatSleepTime,
+            maxCircuit: maxCircuit,
+            maxCircuitPerPeer: maxCircuitPerPeer,
+            msgSize: msgSize,
+            isCircuitRelayV1: circuitRelayV1)
+
+  proc handleStream(conn: Connection, proto: string) {.async, gcsafe.} =
+    try:
+      case proto:
+      of RelayV2HopCodec: await r.handleHopStreamV2(conn)
+      of RelayV1Codec: await r.handleStreamV1(conn)
+    except CancelledError as exc:
+      raise exc
+    except CatchableError as exc:
+      debug "exception in relayv2 handler", exc = exc.msg, conn
+    finally:
+      trace "exiting relayv2 handler", conn
+      await conn.close()
+
+  r.codecs = if r.isCircuitRelayV1: @[RelayV1Codec]
+             else: @[RelayV2HopCodec, RelayV1Codec]
+  r.handler = handleStream
+  r
+
+proc deletesReservation(r: Relay) {.async.} =
+  heartbeat "Reservation timeout", r.heartbeatSleepTime.seconds():
+    let n = now().utc
+    for k in toSeq(r.rsvp.keys):
+      if n > r.rsvp[k]:
+        r.rsvp.del(k)
+
+method start*(r: Relay) {.async.} =
+  if not r.reservationLoop.isNil:
+    warn "Starting relay twice"
+    return
+  r.reservationLoop = r.deletesReservation()
+  r.started = true
+
+method stop*(r: Relay) {.async.} =
+  if r.reservationLoop.isNil:
+    warn "Stopping relay without starting it"
+    return
+  r.started = false
+  r.reservationLoop.cancel()
+  r.reservationLoop = nil

libp2p/protocols/connectivity/relay/rtransport.nim

@@ -0,0 +1,109 @@
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import sequtils, strutils
+
+import chronos, chronicles
+
+import ./client,
+       ./rconn,
+       ./utils,
+       ../../../switch,
+       ../../../stream/connection,
+       ../../../transports/transport
+
+logScope:
+  topics = "libp2p relay relay-transport"
+
+type
+  RelayTransport* = ref object of Transport
+    client*: RelayClient
+    queue: AsyncQueue[Connection]
+    selfRunning: bool
+
+method start*(self: RelayTransport, ma: seq[MultiAddress]) {.async.} =
+  if self.selfRunning:
+    trace "Relay transport already running"
+    return
+
+  self.client.onNewConnection = proc(
+    conn: Connection,
+    duration: uint32 = 0,
+    data: uint64 = 0) {.async, gcsafe, raises: [Defect].} =
+      await self.queue.addLast(RelayConnection.new(conn, duration, data))
+      await conn.join()
+  self.selfRunning = true
+  await procCall Transport(self).start(ma)
+  trace "Starting Relay transport"
+
+method stop*(self: RelayTransport) {.async, gcsafe.} =
+  self.running = false
+  self.selfRunning = false
+  self.client.onNewConnection = nil
+  while not self.queue.empty():
+    await self.queue.popFirstNoWait().close()
+
+method accept*(self: RelayTransport): Future[Connection] {.async, gcsafe.} =
+  result = await self.queue.popFirst()
+
+proc dial*(self: RelayTransport, ma: MultiAddress): Future[Connection] {.async, gcsafe.} =
+  let
+    sma = toSeq(ma.items())
+    relayAddrs = sma[0..sma.len-4].mapIt(it.tryGet()).foldl(a & b)
+  var
+    relayPeerId: PeerId
+    dstPeerId: PeerId
+  if not relayPeerId.init(($(sma[^3].get())).split('/')[2]):
+    raise newException(RelayV2DialError, "Relay doesn't exist")
+  if not dstPeerId.init(($(sma[^1].get())).split('/')[2]):
+    raise newException(RelayV2DialError, "Destination doesn't exist")
+  trace "Dial", relayPeerId, dstPeerId
+
+  let conn = await self.client.switch.dial(
+    relayPeerId,
+    @[ relayAddrs ],
+    @[ RelayV2HopCodec, RelayV1Codec ])
+  conn.dir = Direction.Out
+  var rc: RelayConnection
+  try:
+    case conn.protocol:
+    of RelayV1Codec:
+      return await self.client.dialPeerV1(conn, dstPeerId, @[])
+    of RelayV2HopCodec:
+      rc = RelayConnection.new(conn, 0, 0)
+      return await self.client.dialPeerV2(rc, dstPeerId, @[])
+  except CancelledError as exc:
+    raise exc
+  except CatchableError as exc:
+    if not rc.isNil: await rc.close()
+    raise exc
+
+method dial*(
+  self: RelayTransport,
+  hostname: string,
+  address: MultiAddress): Future[Connection] {.async, gcsafe.} =
+  result = await self.dial(address)
+
+method handles*(self: RelayTransport, ma: MultiAddress): bool {.gcsafe} =
+  if ma.protocols.isOk():
+    let sma = toSeq(ma.items())
+    if sma.len >= 3:
+      result = CircuitRelay.match(sma[^2].get()) and
+               P2PPattern.match(sma[^1].get())
+  trace "Handles return", ma, result
+
+proc new*(T: typedesc[RelayTransport], cl: RelayClient, upgrader: Upgrade): T =
+  result = T(client: cl, upgrader: upgrader)
+  result.running = true
+  result.queue = newAsyncQueue[Connection](0)

libp2p/protocols/connectivity/relay/utils.nim

@@ -0,0 +1,89 @@
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import options
+
+import chronos, chronicles
+
+import ./messages,
+       ../../../stream/connection
+
+logScope:
+  topics = "libp2p relay relay-utils"
+
+const
+  RelayV1Codec* = "/libp2p/circuit/relay/0.1.0"
+  RelayV2HopCodec* = "/libp2p/circuit/relay/0.2.0/hop"
+  RelayV2StopCodec* = "/libp2p/circuit/relay/0.2.0/stop"
+
+proc sendStatus*(conn: Connection, code: StatusV1) {.async, gcsafe.} =
+  trace "send relay/v1 status", status = $code & "(" & $ord(code) & ")"
+  let
+    msg = RelayMessage(msgType: some(RelayType.Status), status: some(code))
+    pb = encode(msg)
+  await conn.writeLp(pb.buffer)
+
+proc sendHopStatus*(conn: Connection, code: StatusV2) {.async, gcsafe.} =
+  trace "send hop relay/v2 status", status = $code & "(" & $ord(code) & ")"
+  let
+    msg = HopMessage(msgType: HopMessageType.Status, status: some(code))
+    pb = encode(msg)
+  await conn.writeLp(pb.buffer)
+
+proc sendStopStatus*(conn: Connection, code: StatusV2) {.async.} =
+  trace "send stop relay/v2 status", status = $code & " (" & $ord(code) & ")"
+  let
+    msg = StopMessage(msgType: StopMessageType.Status, status: some(code))
+    pb = encode(msg)
+  await conn.writeLp(pb.buffer)
+
+proc bridge*(connSrc: Connection, connDst: Connection) {.async.} =
+  const bufferSize = 4096
+  var
+    bufSrcToDst: array[bufferSize, byte]
+    bufDstToSrc: array[bufferSize, byte]
+    futSrc = connSrc.readOnce(addr bufSrcToDst[0], bufSrcToDst.high + 1)
+    futDst = connDst.readOnce(addr bufDstToSrc[0], bufDstToSrc.high + 1)
+    bytesSendFromSrcToDst = 0
+    bytesSendFromDstToSrc = 0
+    bufRead: int
+
+  try:
+    while not connSrc.closed() and not connDst.closed():
+      await futSrc or futDst
+      if futSrc.finished():
+        bufRead = await futSrc
+        if bufRead > 0:
+          bytesSendFromSrcToDst.inc(bufRead)
+          await connDst.write(@bufSrcToDst[0..<bufRead])
+          zeroMem(addr(bufSrcToDst), bufSrcToDst.high + 1)
+        futSrc = connSrc.readOnce(addr bufSrcToDst[0], bufSrcToDst.high + 1)
+      if futDst.finished():
+        bufRead = await futDst
+        if bufRead > 0:
+          bytesSendFromDstToSrc += bufRead
+          await connSrc.write(bufDstToSrc[0..<bufRead])
+          zeroMem(addr(bufDstToSrc), bufDstToSrc.high + 1)
+        futDst = connDst.readOnce(addr bufDstToSrc[0], bufDstToSrc.high + 1)
+  except CancelledError as exc:
+    raise exc
+  except CatchableError as exc:
+    if connSrc.closed() or connSrc.atEof():
+      trace "relay src closed connection", src = connSrc.peerId
+    if connDst.closed() or connDst.atEof():
+      trace "relay dst closed connection", dst = connDst.peerId
+    trace "relay error", exc=exc.msg
+  trace "end relaying", bytesSendFromSrcToDst, bytesSendFromDstToSrc
+  await futSrc.cancelAndWait()
+  await futDst.cancelAndWait()

libp2p/protocols/identify.nim

@@ -1,13 +1,22 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-import options
+## `Identify <https://docs.libp2p.io/concepts/protocols/#identify>`_ and
+## `Push Identify <https://docs.libp2p.io/concepts/protocols/#identify-push>`_ implementation
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import std/[sequtils, options, strutils, sugar]
+import stew/results
 import chronos, chronicles
 import ../protobuf/minprotobuf,
        ../peerinfo,
@@ -16,10 +25,11 @@ import ../protobuf/minprotobuf,
        ../crypto/crypto,
        ../multiaddress,
        ../protocols/protocol,
-       ../utility
+       ../utility,
+       ../errors
 
 logScope:
-  topics = "identify"
+  topics = "libp2p identify"
 
 const
   IdentifyCodec* = "/ipfs/id/1.0.0"
@@ -27,123 +37,220 @@ const
   ProtoVersion* = "ipfs/0.1.0"
   AgentVersion* = "nim-libp2p/0.0.1"
 
-#TODO: implement push identify, leaving out for now as it is not essential
-
 type
-  IdentityNoMatchError* = object of CatchableError
-  IdentityInvalidMsgError* = object of CatchableError
+  IdentifyError* = object of LPError
+  IdentityNoMatchError* = object of IdentifyError
+  IdentityInvalidMsgError* = object of IdentifyError
+  IdentifyNoPubKeyError* = object of IdentifyError
 
-  IdentifyInfo* = object
-    pubKey*: Option[PublicKey]
+  IdentifyInfo* {.public.} = object
+    pubkey*: Option[PublicKey]
+    peerId*: PeerId
     addrs*: seq[MultiAddress]
     observedAddr*: Option[MultiAddress]
     protoVersion*: Option[string]
     agentVersion*: Option[string]
     protos*: seq[string]
+    signedPeerRecord*: Option[Envelope]
 
   Identify* = ref object of LPProtocol
     peerInfo*: PeerInfo
+    sendSignedPeerRecord*: bool
 
-proc encodeMsg*(peerInfo: PeerInfo, observedAddr: Multiaddress): ProtoBuffer =
+  IdentifyPushHandler* = proc (
+    peer: PeerId,
+    newInfo: IdentifyInfo):
+    Future[void]
+    {.gcsafe, raises: [Defect], public.}
+
+  IdentifyPush* = ref object of LPProtocol
+    identifyHandler: IdentifyPushHandler
+
+chronicles.expandIt(IdentifyInfo):
+  pubkey = ($it.pubkey).shortLog
+  addresses = it.addrs.map(x => $x).join(",")
+  protocols = it.protos.map(x => $x).join(",")
+  observable_address =
+    if it.observedAddr.isSome(): $it.observedAddr.get()
+    else: "None"
+  proto_version = it.protoVersion.get("None")
+  agent_version = it.agentVersion.get("None")
+  signedPeerRecord =
+    # The SPR contains the same data as the identify message
+    # would be cumbersome to log
+    if iinfo.signedPeerRecord.isSome(): "Some"
+    else: "None"
+
+proc encodeMsg(peerInfo: PeerInfo, observedAddr: Opt[MultiAddress], sendSpr: bool): ProtoBuffer
+  {.raises: [Defect].} =
   result = initProtoBuffer()
 
-  result.write(1, peerInfo.publicKey.get().getBytes().tryGet())
+  let pkey = peerInfo.publicKey
 
+  result.write(1, pkey.getBytes().get())
   for ma in peerInfo.addrs:
     result.write(2, ma.data.buffer)
-
   for proto in peerInfo.protocols:
     result.write(3, proto)
-
-  result.write(4, observedAddr.data.buffer)
-
+  if observedAddr.isSome:
+    result.write(4, observedAddr.get().data.buffer)
   let protoVersion = ProtoVersion
   result.write(5, protoVersion)
-
-  let agentVersion = AgentVersion
+  let agentVersion = if peerInfo.agentVersion.len <= 0:
+    AgentVersion
+  else:
+    peerInfo.agentVersion
   result.write(6, agentVersion)
+
+  ## Optionally populate signedPeerRecord field.
+  ## See https://github.com/libp2p/go-libp2p/blob/ddf96ce1cfa9e19564feb9bd3e8269958bbc0aba/p2p/protocol/identify/pb/identify.proto for reference.
+  if sendSpr:
+    let sprBuff = peerInfo.signedPeerRecord.envelope.encode()
+    if sprBuff.isOk():
+      result.write(8, sprBuff.get())
+
   result.finish()
 
-proc decodeMsg*(buf: seq[byte]): IdentifyInfo =
+proc decodeMsg*(buf: seq[byte]): Option[IdentifyInfo] =
+  var
+    iinfo: IdentifyInfo
+    pubkey: PublicKey
+    oaddr: MultiAddress
+    protoVersion: string
+    agentVersion: string
+    signedPeerRecord: SignedPeerRecord
+
   var pb = initProtoBuffer(buf)
 
-  var pubKey: PublicKey
-  if pb.getField(1, pubKey):
-    trace "read public key from message", pubKey = ($pubKey).shortLog
-    result.pubKey = some(pubKey)
+  let r1 = pb.getField(1, pubkey)
+  let r2 = pb.getRepeatedField(2, iinfo.addrs)
+  let r3 = pb.getRepeatedField(3, iinfo.protos)
+  let r4 = pb.getField(4, oaddr)
+  let r5 = pb.getField(5, protoVersion)
+  let r6 = pb.getField(6, agentVersion)
 
-  if pb.getRepeatedField(2, result.addrs):
-    trace "read addresses from message", addresses = result.addrs
+  let r8 = pb.getField(8, signedPeerRecord)
 
-  if pb.getRepeatedField(3, result.protos):
-    trace "read protos from message", protocols = result.protos
+  let res = r1.isOk() and r2.isOk() and r3.isOk() and
+            r4.isOk() and r5.isOk() and r6.isOk() and
+            r8.isOk()
 
-  var observableAddr: MultiAddress
-  if pb.getField(4, observableAddr):
-    trace "read observableAddr from message", address = observableAddr
-    result.observedAddr = some(observableAddr)
+  if res:
+    if r1.get():
+      iinfo.pubkey = some(pubkey)
+    if r4.get():
+      iinfo.observedAddr = some(oaddr)
+    if r5.get():
+      iinfo.protoVersion = some(protoVersion)
+    if r6.get():
+      iinfo.agentVersion = some(agentVersion)
+    if r8.get() and r1.get():
+      if iinfo.pubkey.get() == signedPeerRecord.envelope.publicKey:
+        iinfo.signedPeerRecord = some(signedPeerRecord.envelope)
+    debug "decodeMsg: decoded identify", iinfo
+    some(iinfo)
+  else:
+    trace "decodeMsg: failed to decode received message"
+    none[IdentifyInfo]()
 
-  var protoVersion = ""
-  if pb.getField(5, protoVersion):
-    trace "read protoVersion from message", protoVersion = protoVersion
-    result.protoVersion = some(protoVersion)
-
-  var agentVersion = ""
-  if pb.getField(6, agentVersion):
-    trace "read agentVersion from message", agentVersion = agentVersion
-    result.agentVersion = some(agentVersion)
-
-proc newIdentify*(peerInfo: PeerInfo): Identify =
-  new result
-  result.peerInfo = peerInfo
-  result.init()
+proc new*(
+  T: typedesc[Identify],
+  peerInfo: PeerInfo,
+  sendSignedPeerRecord = false
+  ): T =
+  let identify = T(
+    peerInfo: peerInfo,
+    sendSignedPeerRecord: sendSignedPeerRecord
+  )
+  identify.init()
+  identify
 
 method init*(p: Identify) =
   proc handle(conn: Connection, proto: string) {.async, gcsafe, closure.} =
     try:
-      defer:
-        trace "exiting identify handler", oid = conn.oid
-        await conn.close()
-
-      trace "handling identify request", oid = conn.oid
-      var pb = encodeMsg(p.peerInfo, conn.observedAddr)
+      trace "handling identify request", conn
+      var pb = encodeMsg(p.peerInfo, conn.observedAddr, p.sendSignedPeerRecord)
       await conn.writeLp(pb.buffer)
     except CancelledError as exc:
       raise exc
     except CatchableError as exc:
-      trace "exception in identify handler", exc = exc.msg
+      trace "exception in identify handler", exc = exc.msg, conn
+    finally:
+      trace "exiting identify handler", conn
+      await conn.closeWithEOF()
 
   p.handler = handle
   p.codec = IdentifyCodec
 
 proc identify*(p: Identify,
                conn: Connection,
-               remotePeerInfo: PeerInfo): Future[IdentifyInfo] {.async, gcsafe.} =
-  trace "initiating identify", peer = $conn
+               remotePeerId: PeerId): Future[IdentifyInfo] {.async, gcsafe.} =
+  trace "initiating identify", conn
   var message = await conn.readLp(64*1024)
   if len(message) == 0:
-    trace "identify: Invalid or empty message received!"
-    raise newException(IdentityInvalidMsgError,
-      "Invalid or empty message received!")
+    trace "identify: Empty message received!", conn
+    raise newException(IdentityInvalidMsgError, "Empty message received!")
 
-  result = decodeMsg(message)
+  let infoOpt = decodeMsg(message)
+  if infoOpt.isNone():
+    raise newException(IdentityInvalidMsgError, "Incorrect message received!")
+  result = infoOpt.get()
 
-  if not isNil(remotePeerInfo) and result.pubKey.isSome:
-    let peer = PeerID.init(result.pubKey.get())
+  if result.pubkey.isSome:
+    let peer = PeerId.init(result.pubkey.get())
     if peer.isErr:
       raise newException(IdentityInvalidMsgError, $peer.error)
     else:
-      # do a string comaprison of the ids,
-      # because that is the only thing we
-      # have in most cases
-      if peer.get() != remotePeerInfo.peerId:
+      result.peerId = peer.get()
+      if peer.get() != remotePeerId:
         trace "Peer ids don't match",
-              remote = peer.get().pretty(),
-              local = remotePeerInfo.id
+              remote = peer,
+              local = remotePeerId
 
         raise newException(IdentityNoMatchError, "Peer ids don't match")
+  else:
+    raise newException(IdentityInvalidMsgError, "No pubkey in identify")
 
-proc push*(p: Identify, conn: Connection) {.async.} =
-  await conn.write(IdentifyPushCodec)
-  var pb = encodeMsg(p.peerInfo, conn.observedAddr)
+proc new*(T: typedesc[IdentifyPush], handler: IdentifyPushHandler = nil): T {.public.} =
+  ## Create a IdentifyPush protocol. `handler` will be called every time
+  ## a peer sends us new `PeerInfo`
+  let identifypush = T(identifyHandler: handler)
+  identifypush.init()
+  identifypush
+
+proc init*(p: IdentifyPush) =
+  proc handle(conn: Connection, proto: string) {.async, gcsafe, closure.} =
+    trace "handling identify push", conn
+    try:
+      var message = await conn.readLp(64*1024)
+
+      let infoOpt = decodeMsg(message)
+      if infoOpt.isNone():
+        raise newException(IdentityInvalidMsgError, "Incorrect message received!")
+
+      var indentInfo = infoOpt.get()
+
+      if indentInfo.pubkey.isSome:
+        let receivedPeerId = PeerId.init(indentInfo.pubkey.get()).tryGet()
+        if receivedPeerId != conn.peerId:
+          raise newException(IdentityNoMatchError, "Peer ids don't match")
+        indentInfo.peerId = receivedPeerId
+
+      trace "triggering peer event", peerInfo = conn.peerId
+      if not isNil(p.identifyHandler):
+        await p.identifyHandler(conn.peerId, indentInfo)
+    except CancelledError as exc:
+      raise exc
+    except CatchableError as exc:
+      info "exception in identify push handler", exc = exc.msg, conn
+    finally:
+      trace "exiting identify push handler", conn
+      await conn.closeWithEOF()
+
+  p.handler = handle
+  p.codec = IdentifyPushCodec
+
+proc push*(p: IdentifyPush, peerInfo: PeerInfo, conn: Connection) {.async, public.} =
+  ## Send new `peerInfo`s to a connection
+  var pb = encodeMsg(peerInfo, conn.observedAddr, true)
   await conn.writeLp(pb.buffer)

libp2p/protocols/ping.nim

@@ -0,0 +1,104 @@
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+## `Ping <https://docs.libp2p.io/concepts/protocols/#ping>`_ protocol implementation
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import chronos, chronicles
+import bearssl/[rand, hash]
+import ../protobuf/minprotobuf,
+       ../peerinfo,
+       ../stream/connection,
+       ../peerid,
+       ../crypto/crypto,
+       ../multiaddress,
+       ../protocols/protocol,
+       ../utility,
+       ../errors
+
+export chronicles, rand, connection
+
+logScope:
+  topics = "libp2p ping"
+
+const
+  PingCodec* = "/ipfs/ping/1.0.0"
+  PingSize = 32
+
+type
+  PingError* = object of LPError
+  WrongPingAckError* = object of PingError
+
+  PingHandler* {.public.} = proc (
+    peer: PeerId):
+    Future[void]
+    {.gcsafe, raises: [Defect].}
+
+  Ping* = ref object of LPProtocol
+    pingHandler*: PingHandler
+    rng: ref HmacDrbgContext
+
+proc new*(T: typedesc[Ping], handler: PingHandler = nil, rng: ref HmacDrbgContext = newRng()): T {.public.} =
+  let ping = Ping(pinghandler: handler, rng: rng)
+  ping.init()
+  ping
+
+method init*(p: Ping) =
+  proc handle(conn: Connection, proto: string) {.async, gcsafe, closure.} =
+    try:
+      trace "handling ping", conn
+      var buf: array[PingSize, byte]
+      await conn.readExactly(addr buf[0], PingSize)
+      trace "echoing ping", conn
+      await conn.write(@buf)
+      if not isNil(p.pingHandler):
+        await p.pingHandler(conn.peerId)
+    except CancelledError as exc:
+      raise exc
+    except CatchableError as exc:
+      trace "exception in ping handler", exc = exc.msg, conn
+
+  p.handler = handle
+  p.codec = PingCodec
+
+proc ping*(
+  p: Ping,
+  conn: Connection,
+  ): Future[Duration] {.async, gcsafe, public.} =
+  ## Sends ping to `conn`, returns the delay
+
+  trace "initiating ping", conn
+
+  var
+    randomBuf: array[PingSize, byte]
+    resultBuf: array[PingSize, byte]
+
+  hmacDrbgGenerate(p.rng[], randomBuf)
+
+  let startTime = Moment.now()
+
+  trace "sending ping", conn
+  await conn.write(@randomBuf)
+
+  await conn.readExactly(addr resultBuf[0], PingSize)
+
+  let responseDur = Moment.now() - startTime
+
+  trace "got ping response", conn, responseDur
+
+  for i in 0..<randomBuf.len:
+    if randomBuf[i] != resultBuf[i]:
+      raise newException(WrongPingAckError, "Incorrect ping data from peer!")
+
+  trace "valid ping response", conn
+  return responseDur

libp2p/protocols/protocol.nim

@@ -1,22 +1,42 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
 
 import chronos
 import ../stream/connection
 
 type
-  LPProtoHandler* = proc (conn: Connection,
-                          proto: string):
-                          Future[void] {.gcsafe, closure.}
+  LPProtoHandler* = proc (
+    conn: Connection,
+    proto: string):
+    Future[void]
+    {.gcsafe, raises: [Defect].}
 
   LPProtocol* = ref object of RootObj
-    codec*: string
+    codecs*: seq[string]
     handler*: LPProtoHandler ## this handler gets invoked by the protocol negotiator
+    started*: bool
 
 method init*(p: LPProtocol) {.base, gcsafe.} = discard
+method start*(p: LPProtocol) {.async, base.} = p.started = true
+method stop*(p: LPProtocol) {.async, base.} = p.started = false
+
+
+func codec*(p: LPProtocol): string =
+  assert(p.codecs.len > 0, "Codecs sequence was empty!")
+  p.codecs[0]
+
+func `codec=`*(p: LPProtocol, codec: string) =
+  # always insert as first codec
+  # if we use this abstraction
+  p.codecs.insert(codec, 0)

libp2p/protocols/pubsub.nim

@@ -0,0 +1,8 @@
+import ./pubsub/[pubsub, floodsub, gossipsub]
+
+## Home of PubSub & it's implementations:
+## | **pubsub**: base interface for pubsub implementations
+## | **floodsub**: simple flood-based publishing
+## | **gossipsub**: more sophisticated gossip based publishing
+
+export pubsub, floodsub, gossipsub

libp2p/protocols/pubsub/errors.nim

@@ -0,0 +1,8 @@
+# this module will be further extended in PR
+# https://github.com/status-im/nim-libp2p/pull/107/
+
+import ../../utility
+
+type
+  ValidationResult* {.pure, public.} = enum
+    Accept, Reject, Ignore

libp2p/protocols/pubsub/floodsub.nim

@@ -1,110 +1,160 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-import sequtils, tables, sets, strutils
+when (NimMajor, NimMinor) < (1, 4):
+  {.push raises: [Defect].}
+else:
+  {.push raises: [].}
+
+import std/[sequtils, sets, hashes, tables]
 import chronos, chronicles, metrics
-import pubsub,
-       pubsubpeer,
-       timedcache,
-       rpc/[messages, message],
+import ./pubsub,
+       ./pubsubpeer,
+       ./timedcache,
+       ./peertable,
+       ./rpc/[message, messages],
+       ../../crypto/crypto,
        ../../stream/connection,
        ../../peerid,
        ../../peerinfo,
-       ../../utility,
-       ../../errors
+       ../../utility
+
+## Simple flood-based publishing.
 
 logScope:
-  topics = "floodsub"
+  topics = "libp2p floodsub"
 
 const FloodSubCodec* = "/floodsub/1.0.0"
 
 type
-  FloodSub* = ref object of PubSub
-    floodsub*: PeerTable # topic to remote peer map
-    seen*: TimedCache[string]                 # list of messages forwarded to peers
+  FloodSub* {.public.} = ref object of PubSub
+    floodsub*: PeerTable      # topic to remote peer map
+    seen*: TimedCache[MessageId] # message id:s already seen on the network
+    seenSalt*: seq[byte]
 
-method subscribeTopic*(f: FloodSub,
-                       topic: string,
-                       subscribe: bool,
-                       peerId: string) {.gcsafe, async.} =
-  await procCall PubSub(f).subscribeTopic(topic, subscribe, peerId)
+proc hasSeen*(f: FloodSub, msgId: MessageId): bool =
+  f.seenSalt & msgId in f.seen
 
-  let peer = f.peers.getOrDefault(peerId)
-  if peer == nil:
-    debug "subscribeTopic on a nil peer!"
+proc addSeen*(f: FloodSub, msgId: MessageId): bool =
+  # Salting the seen hash helps avoid attacks against the hash function used
+  # in the nim hash table
+  # Return true if the message has already been seen
+  f.seen.put(f.seenSalt & msgId)
+
+proc firstSeen*(f: FloodSub, msgId: MessageId): Moment =
+  f.seen.addedAt(f.seenSalt & msgId)
+
+proc handleSubscribe*(f: FloodSub,
+                      peer: PubSubPeer,
+                      topic: string,
+                      subscribe: bool) =
+  logScope:
+    peer
+    topic
+
+  # this is a workaround for a race condition
+  # that can happen if we disconnect the peer very early
+  # in the future we might use this as a test case
+  # and eventually remove this workaround
+  if subscribe and peer.peerId notin f.peers:
+    trace "ignoring unknown peer"
     return
 
-  if topic notin f.floodsub:
-    f.floodsub[topic] = initHashSet[PubSubPeer]()
+  if subscribe and not(isNil(f.subscriptionValidator)) and not(f.subscriptionValidator(topic)):
+    # this is a violation, so warn should be in order
+    warn "ignoring invalid topic subscription", topic, peer
+    return
 
   if subscribe:
-    trace "adding subscription for topic", peer = peer.id, name = topic
+    trace "adding subscription for topic", peer, topic
+
     # subscribe the peer to the topic
-    f.floodsub[topic].incl(peer)
+    f.floodsub.mgetOrPut(topic, HashSet[PubSubPeer]()).incl(peer)
   else:
-    trace "removing subscription for topic", peer = peer.id, name = topic
-    # unsubscribe the peer from the topic
-    f.floodsub[topic].excl(peer)
+    f.floodsub.withValue(topic, peers):
+      trace "removing subscription for topic", peer, topic
 
-method handleDisconnect*(f: FloodSub, peer: PubSubPeer) =
+      # unsubscribe the peer from the topic
+      peers[].excl(peer)
+
+method unsubscribePeer*(f: FloodSub, peer: PeerId) =
   ## handle peer disconnects
-  for t in toSeq(f.floodsub.keys):
-    if t in f.floodsub:
-      f.floodsub[t].excl(peer)
+  ##
+  trace "unsubscribing floodsub peer", peer
+  let pubSubPeer = f.peers.getOrDefault(peer)
+  if pubSubPeer.isNil:
+    return
 
-  procCall PubSub(f).handleDisconnect(peer)
+  for _, v in f.floodsub.mpairs():
+    v.excl(pubSubPeer)
+
+  procCall PubSub(f).unsubscribePeer(peer)
 
 method rpcHandler*(f: FloodSub,
                    peer: PubSubPeer,
-                   rpcMsgs: seq[RPCMsg]) {.async.} =
-  await procCall PubSub(f).rpcHandler(peer, rpcMsgs)
+                   rpcMsg: RPCMsg) {.async.} =
+  for i in 0..<min(f.topicsHigh, rpcMsg.subscriptions.len):
+    template sub: untyped = rpcMsg.subscriptions[i]
+    f.handleSubscribe(peer, sub.topic, sub.subscribe)
 
-  for m in rpcMsgs:                                  # for all RPC messages
-    if m.messages.len > 0:                           # if there are any messages
-      var toSendPeers = initHashSet[PubSubPeer]()
-      for msg in m.messages:                         # for every message
-        let msgId = f.msgIdProvider(msg)
-        logScope: msgId
+  for msg in rpcMsg.messages:                         # for every message
+    let msgIdResult = f.msgIdProvider(msg)
+    if msgIdResult.isErr:
+      debug "Dropping message due to failed message id generation",
+        error = msgIdResult.error
+      # TODO: descore peers due to error during message validation (malicious?)
+      continue
 
-        if msgId notin f.seen:
-          f.seen.put(msgId)                          # add the message to the seen cache
+    let msgId = msgIdResult.get
 
-          if f.verifySignature and not msg.verify(peer.peerInfo):
-            trace "dropping message due to failed signature verification"
-            continue
+    if f.addSeen(msgId):
+      trace "Dropping already-seen message", msgId, peer
+      continue
 
-          if not (await f.validate(msg)):
-            trace "dropping message due to failed validation"
-            continue
+    if (msg.signature.len > 0 or f.verifySignature) and not msg.verify():
+      # always validate if signature is present or required
+      debug "Dropping message due to failed signature verification", msgId, peer
+      continue
 
-          for t in msg.topicIDs:                     # for every topic in the message
-            if t in f.floodsub:
-              toSendPeers.incl(f.floodsub[t])        # get all the peers interested in this topic
-            if t in f.topics:                        # check that we're subscribed to it
-              for h in f.topics[t].handler:
-                trace "calling handler for message", topicId = t,
-                                                     localPeer = f.peerInfo.id,
-                                                     fromPeer = msg.fromPeer.pretty
+    if msg.seqno.len > 0 and msg.seqno.len != 8:
+      # if we have seqno should be 8 bytes long
+      debug "Dropping message due to invalid seqno length", msgId, peer
+      continue
 
-                try:
-                  await h(t, msg.data)                 # trigger user provided handler
-                except CatchableError as exc:
-                  trace "exception in message handler", exc = exc.msg
+    # g.anonymize needs no evaluation when receiving messages
+    # as we have a "lax" policy and allow signed messages
 
-        # forward the message to all peers interested in it
-        let (published, failed) = await f.sendHelper(toSendPeers, m.messages)
-        for p in failed:
-          let peer = f.peers.getOrDefault(p)
-          if not(isNil(peer)):
-            f.handleDisconnect(peer) # cleanup failed peers
+    let validation = await f.validate(msg)
+    case validation
+    of ValidationResult.Reject:
+      debug "Dropping message after validation, reason: reject", msgId, peer
+      continue
+    of ValidationResult.Ignore:
+      debug "Dropping message after validation, reason: ignore", msgId, peer
+      continue
+    of ValidationResult.Accept:
+      discard
 
-        trace "forwared message to peers", peers = published.len
+    var toSendPeers = initHashSet[PubSubPeer]()
+    for t in msg.topicIds:                     # for every topic in the message
+      if t notin f.topics:
+        continue
+      f.floodsub.withValue(t, peers): toSendPeers.incl(peers[])
+
+      await handleData(f, t, msg.data)
+
+    # In theory, if topics are the same in all messages, we could batch - we'd
+    # also have to be careful to only include validated messages
+    f.broadcast(toSendPeers, RPCMsg(messages: @[msg]))
+    trace "Forwared message to peers", peers = toSendPeers.len
+
+  f.updateMetrics(rpcMsg)
 
 method init*(f: FloodSub) =
   proc handler(conn: Connection, proto: string) {.async.} =
@@ -112,57 +162,71 @@ method init*(f: FloodSub) =
     ## connection for a protocol string
     ## e.g. ``/floodsub/1.0.0``, etc...
     ##
-
-    await f.handleConn(conn, proto)
+    try:
+      await f.handleConn(conn, proto)
+    except CancelledError:
+      # This is top-level procedure which will work as separate task, so it
+      # do not need to propagate CancelledError.
+      trace "Unexpected cancellation in floodsub handler", conn
+    except CatchableError as exc:
+      trace "FloodSub handler leaks an error", exc = exc.msg, conn
 
   f.handler = handler
   f.codec = FloodSubCodec
 
-method subscribePeer*(p: FloodSub,
-                      conn: Connection) =
-  procCall PubSub(p).subscribePeer(conn)
-  asyncCheck p.handleConn(conn, FloodSubCodec)
-
 method publish*(f: FloodSub,
                 topic: string,
                 data: seq[byte]): Future[int] {.async.} =
   # base returns always 0
   discard await procCall PubSub(f).publish(topic, data)
 
-  if data.len <= 0 or topic.len <= 0:
-    trace "topic or data missing, skipping publish"
-    return
+  trace "Publishing message on topic", data = data.shortLog, topic
 
-  if topic notin f.floodsub:
-    trace "missing peers for topic, skipping publish"
-    return
+  if topic.len <= 0: # data could be 0/empty
+    debug "Empty topic, skipping publish", topic
+    return 0
 
-  trace "publishing on topic", name = topic
-  let msg = Message.init(f.peerInfo, data, topic, f.sign)
-  # start the future but do not wait yet
-  let (published, failed) = await f.sendHelper(f.floodsub.getOrDefault(topic), @[msg])
-  for p in failed:
-    let peer = f.peers.getOrDefault(p)
-    if not isNil(peer):
-      f.handleDisconnect(peer) # cleanup failed peers
+  let peers = f.floodsub.getOrDefault(topic)
 
-  libp2p_pubsub_messages_published.inc(labelValues = [topic])
+  if peers.len == 0:
+    debug "No peers for topic, skipping publish", topic
+    return 0
 
-  trace "published message to peers", peers = published.len,
-                                      msg = msg.shortLog()
-  return published.len
+  let
+    msg =
+      if f.anonymize:
+        Message.init(none(PeerInfo), data, topic, none(uint64), false)
+      else:
+        inc f.msgSeqno
+        Message.init(some(f.peerInfo), data, topic, some(f.msgSeqno), f.sign)
+    msgId = f.msgIdProvider(msg).valueOr:
+      trace "Error generating message id, skipping publish",
+        error = error
+      return 0
 
-method unsubscribe*(f: FloodSub,
-                    topics: seq[TopicPair]) {.async.} =
-  await procCall PubSub(f).unsubscribe(topics)
+  trace "Created new message",
+    msg = shortLog(msg), peers = peers.len, topic, msgId
 
-  for p in f.peers.values:
-    await f.sendSubs(p, topics.mapIt(it.topic).deduplicate(), false)
+  if f.addSeen(msgId):
+    # custom msgid providers might cause this
+    trace "Dropping already-seen message", msgId, topic
+    return 0
 
-method initPubSub*(f: FloodSub) =
+  # Try to send to all peers that are known to be interested
+  f.broadcast(peers, RPCMsg(messages: @[msg]))
+
+  when defined(libp2p_expensive_metrics):
+    libp2p_pubsub_messages_published.inc(labelValues = [topic])
+
+  trace "Published message to peers", msgId, topic
+
+  return peers.len
+
+method initPubSub*(f: FloodSub)
+  {.raises: [Defect, InitializationError].} =
   procCall PubSub(f).initPubSub()
-  f.peers = initTable[string, PubSubPeer]()
-  f.topics = initTable[string, Topic]()
-  f.floodsub = initTable[string, HashSet[PubSubPeer]]()
-  f.seen = newTimedCache[string](2.minutes)
+  f.seen = TimedCache[MessageId].init(2.minutes)
+  f.seenSalt = newSeqUninitialized[byte](sizeof(Hash))
+  hmacDrbgGenerate(f.rng[], f.seenSalt)
+
   f.init()