Tunable max concurrent upgrades

Co-authored-by: Ludovic Chenut <ludovic@status.im>

.assets/small-logo.svg

@@ -0,0 +1,96 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:v="https://vecta.io/nano" xmlns:xlink="http://www.w3.org/1999/xlink" width="172.071" height="196.414" viewBox="0 0 45.527 51.968">
+   <g transform="matrix(.2822 0 0 .2822 -212.833275 -150.656248)">
+      <path d="M835.432 533.821l-12.483 9.783c-6.482-.207-19.197 1.251-26.086 3.769-6.346-4.04-11.923-8.5-11.923-8.5l-7.762 13.071c-4.444 2.375-8.906 5.046-12.883 8.58l-10.162-4.17c6.125 12.414 10.243 24.844 21.445 32.316 17.834-28.299 100.705-25.691 118.907-.16 11.764-6.165 16.339-19.429 20.965-31.674-.507.168-6.802 2.285-10.882 3.849-2.436-2.665-8.179-6.763-11.443-8.741-3.096-5.696-7.602-13.391-7.602-13.391s-5.337 3.988-11.523 8.34c-8.357-1.55-18.465-3.433-26.966-2.967-5.787-4.779-11.603-10.104-11.603-10.104z" fill="#f3d400" />
+      <g opacity=".9" transform="matrix(.9375 0 0 .9375 765.1166 550.13225)">
+         <path d="M99.952 106.898l.215-.107 24.755-14.248-24.97-14.535-24.97 14.374z" fill="#cc2a65" />
+         <use xlink:href="#B" fill="#a21d4c" />
+         <path d="M124.922 92.542l-24.755 14.248-.215.107v28.89l24.97-14.356z" fill="#b62454" />
+         <path d="M50.012 106.737l.215-.125 24.755-14.248-24.97-14.517-24.97 14.356z" fill="#c8d92b" />
+         <path d="M50.012 135.609v-28.872l-24.97-14.535v28.89h.018z" fill="#c2d02f" />
+         <path d="M74.982 92.381l-24.755 14.23-.215.125v28.872.018h.018l24.952-14.356v-.018z" fill="#b9be33" />
+         <path d="M74.982 121.253l.215-.107 24.755-14.248-24.97-14.535-24.97 14.374z" fill="#cc2a65" />
+         <use xlink:href="#B" x="-24.97" y="14.356" fill="#a21d4c" />
+         <path d="M99.952 106.898l-24.755 14.248-.215.107v28.89H75l24.952-14.356z" fill="#b62454" />
+         <path d="M124.905 121.415l.215-.125 24.737-14.23-24.952-14.535-24.97 14.356z" fill="#a159a2" />
+         <path d="M124.905 150.305v-28.89l-24.97-14.535v28.89h.018z" fill="#772a86" />
+         <path d="M149.875 107.059l-24.755 14.23-.215.125v28.89l24.97-14.356z" fill="#8e3b95" />
+         <path d="M74.982 92.345l.215-.125 24.737-14.248-24.952-14.517-24.97 14.356z" fill="#bec831" />
+         <path d="M74.982 121.217V92.345L50.012 77.81v28.89h.018z" fill="#a1a938" />
+         <path d="M99.952 77.989l-24.755 14.23-.215.125v28.872.018l24.97-14.356v-.018z" fill="#999b37" />
+         <path d="M75 60.645l.197-.125 24.755-14.23L75 31.755 50.029 46.11l24.952 14.535z" fill="#bec831" />
+         <path d="M74.982 89.535L75 60.645 50.029 46.11 50.012 75h.018z" fill="#a1a938" />
+         <path d="M99.97 46.307L75.197 60.52l-.197.125h-.018v28.89l24.97-14.338v-.018z" fill="#999b37" />
+         <path d="M99.952 75.179l.215-.107 24.755-14.23L99.97 46.306 75 60.644z" fill="#ee539a" />
+         <path d="M99.952 104.069v-28.89L75 60.644l-.018 28.89H75z" fill="#d01b68" />
+         <path d="M124.922 60.841l-24.755 14.23-.215.107v28.89l24.97-14.338.018-28.89z" fill="#ec0f68" />
+         <path d="M124.923 89.731l.215-.125 24.755-14.23-24.952-14.535h-.018l-24.97 14.338z" fill="#a159a2" />
+         <path d="M124.905 118.622l.018-28.89-24.97-14.535v28.872.018z" fill="#772a86" />
+         <path d="M149.893 75.376l-24.755 14.23-.215.125-.018 28.89h.018l24.97-14.356z" fill="#8e3b95" />
+         <path d="M50.03 75l.197-.125 24.755-14.23L50.03 46.109 25.06 60.447l24.952 14.535z" fill="#c8d92b" />
+         <path d="M50.012 103.872L50.03 75 25.06 60.447l-.018 28.89h.018z" fill="#c2d02f" />
+         <path d="M75 60.644l-24.773 14.23-.197.125-.018 28.872 24.97-14.338z" fill="#b9be33" />
+         <path d="M74.982 89.534l.215-.125 24.755-14.23L75 60.644l-24.97 14.338z" fill="#f7af19" />
+         <path d="M74.982 118.425v-.018.018-28.89L50.029 75l-.018 28.872.018.018z" fill="#f2901f" />
+         <path d="M99.952 75.179l-24.755 14.23-.215.125v28.89l24.97-14.356.018-28.89z" fill="#f9a120" />
+         <path d="M99.934 135.769l.215-.125 24.684-14.356-25.042-14.409L74.91 121.36z" fill="#833593" />
+         <path d="M100.077 164.66l-.143-28.89-25.042-14.409.143 28.89h.018z" fill="#652977" />
+         <path d="M124.833 121.288l-24.684 14.356-.215.125.143 28.89 24.899-14.481z" fill="#4d1f5b" />
+         <path d="M99.952 104.069l.215-.107 24.755-14.23L99.97 75.179h-.018l-24.97 14.356z" fill="#a159a2" />
+         <path d="M99.934 132.959l.018-28.89-24.97-14.535v28.89z" fill="#772a86" />
+         <path d="M124.922 89.732l-24.755 14.23-.215.107-.018 28.89h.018l24.97-14.338z" fill="#8e3b95" />
+         <path d="M25.042 121.074l.197-.125 24.755-14.248-24.952-14.517h-.018L.071 106.54l24.952 14.535z" fill="#f6dd03" />
+         <path d="M25.024 149.947h.018v-28.872L.071 106.54v28.89z" fill="#f9bb1d" />
+         <path d="M49.994 106.719l-24.755 14.23-.197.125v28.872.018l24.952-14.356h.018v-.018-28.872z" fill="#e9ae20" />
+         <path d="M25.06 89.338l.197-.125 24.755-14.23L25.06 60.447.089 74.803l24.952 14.535z" fill="#f6dd03" />
+         <path d="M25.042 118.228l.018-28.89L.089 74.803.072 103.675l.018.018z" fill="#f9bb1d" />
+         <path d="M50.03 75L25.257 89.212l-.197.125-.018 28.89 24.97-14.356v.018-.018z" fill="#e9ae20" />
+         <path d="M50.012 135.59l.215-.107 24.737-14.248L50.012 106.7l-24.97 14.374z" fill="#f7af19" />
+         <path d="M50.012 164.481v-28.89l-24.97-14.517v28.872.018z" fill="#f2901f" />
+         <path d="M74.964 121.235l-24.755 14.248-.197.107v28.89l24.97-14.356v-28.89z" fill="#f9a120" />
+         <path d="M50.012 103.872l.215-.107 24.755-14.23L50.03 74.982 25.06 89.338z" fill="#f7af19" />
+         <path d="M50.012 132.763v-28.89L25.06 89.338l-.018 28.89h.018z" fill="#f2901f" />
+         <path d="M74.982 89.535l-24.755 14.23-.215.107v28.89l24.97-14.338.018-28.89z" fill="#f9a120" />
+         <path d="M74.982 150.125l.197-.125 24.755-14.23-24.952-14.535h-.018l-24.952 14.356 24.952 14.535z" fill="#f7af19" />
+         <path d="M74.964 179.015h.018v-28.89l-24.97-14.517v28.872.018z" fill="#f2901f" />
+         <path d="M99.934 135.77L75.179 150l-.197.125v28.89l24.97-14.356v-28.89z" fill="#f9a120" />
+         <path d="M74.982 118.425l.215-.125 24.755-14.23L75 89.535h-.018l-24.97 14.338z" fill="#31838b" />
+         <path d="M74.964 147.297l.018-28.872-24.97-14.535v-.018 28.89.018z" fill="#22626c" />
+         <path d="M99.952 104.069L75.197 118.3l-.215.125-.018 28.872v.018h.018l24.97-14.356z" fill="#1b4b56" />
+         <path d="M74.982 28.962l.215-.125 24.737-14.248L74.982.072l-24.97 14.356 24.97 14.517z" fill="#bec831" />
+         <path d="M74.982 57.834V28.962l-24.97-14.535v28.89h.018z" fill="#a1a938" />
+         <path d="M99.952 14.606l-24.755 14.23-.215.125v28.872l24.97-14.356z" fill="#999b37" />
+         <path d="M74.964 28.944l.215-.125 24.755-14.23L74.982.054h-.018l-24.97 14.338z" fill="#a159a2" />
+         <path d="M74.946 57.835l.018-28.89-24.97-14.535v28.872.018z" fill="#772a86" />
+         <path d="M99.934 14.589l-24.755 14.23-.215.125-.018 28.89h.018l24.97-14.356z" fill="#8e3b95" />
+         <path d="M99.952 43.479l.215-.107 24.755-14.248-24.97-14.535-24.97 14.356z" fill="#ee539a" />
+         <use xlink:href="#B" y="-63.419" fill="#d01b68" />
+         <path d="M124.922 29.123l-24.755 14.248-.215.107v28.89l24.97-14.356z" fill="#ec0f68" />
+         <path d="M50.03 43.317l.215-.125L75 28.961 50.048 14.427h-.018L25.06 28.765z" fill="#31838b" />
+         <path d="M50.012 72.189l.018-28.872-24.97-14.535v-.018 28.89.018z" fill="#22626c" />
+         <path d="M75 28.961l-24.755 14.23-.215.125-.018 28.872v.018h.018L75 57.852z" fill="#1b4b56" />
+         <path d="M124.923 58.013l.215-.125 24.737-14.23-24.952-14.535-24.97 14.356z" fill="#cc2a65" />
+         <use xlink:href="#B" x="24.971" y="-48.884" fill="#a21d4c" />
+         <path d="M149.893 43.658l-24.755 14.23-.215.125v28.89l24.97-14.356z" fill="#b62454" />
+         <path d="M74.982 57.835l.215-.107 24.755-14.248-24.97-14.535L50.012 43.3z" fill="#c8d92b" />
+         <path d="M74.982 86.725v-28.89l-24.97-14.517V72.19l.018.018z" fill="#c2d02f" />
+         <path d="M99.952 43.479L75.197 57.727l-.215.107v28.89H75l24.952-14.356z" fill="#b9be33" />
+         <path d="M99.952 72.369l.215-.125 24.755-14.23-24.97-14.535-24.97 14.356z" fill="#33b4d7" />
+         <use xlink:href="#B" y="-34.529" fill="#209ac5" />
+         <path d="M124.922 58.014l-24.755 14.23-.215.125v28.89h.018l24.952-14.356z" fill="#0f8cae" />
+         <path d="M25.06 57.673l.197-.125L50.012 43.3 25.06 28.783h-.018L.089 43.139l24.952 14.535z" fill="#94d6e3" />
+         <path d="M25.042 86.546h.018V57.673L.089 43.139v28.89z" fill="#73ccdd" />
+         <path d="M50.012 43.318l-24.755 14.23-.197.125v28.872.018l24.952-14.356h.018v-.018-28.872z" fill="#3bafbb" />
+         <path d="M50.03 72.19l.215-.107 24.737-14.248L50.03 43.318 25.06 57.674z" fill="#94d6e3" />
+         <path d="M50.03 101.08V72.208v-.018L25.06 57.674v28.872.018z" fill="#73ccdd" />
+         <path d="M74.982 57.835L50.227 72.083l-.197.107v28.89L75 86.725v-28.89z" fill="#3bafbb" />
+         <path d="M75 86.724l.197-.107 24.755-14.248L75 57.834h-.018L50.029 72.189l24.952 14.535z" fill="#33b4d7" />
+         <path d="M74.982 115.614H75v-28.89l-24.97-14.517v28.872.018z" fill="#209ac5" />
+         <path d="M99.952 72.368L75.197 86.617l-.197.107v28.89l24.97-14.356v-28.89z" fill="#0f8cae" />
+      </g>
+      <path d="M759.126 567.007s10.273 21.02 16.364 35.698c25.549 33.869 90.792 36.224 119.235.656 9.484-17.619 16.733-36.357 16.733-36.357-7.297 10.862-20.094 18.056-27.408 22.095-5.197 2.861-17.189 4.59-17.189 4.59l-31.482-16.393-31.663 16.065s-11.832-1.91-17.189-4.426c-10.811-5.799-19.735-12.549-27.401-21.928z" fill="#ffe953" />
+   </g>
+   <defs>
+      <path id="B" d="M99.952 135.788v-28.89l-24.97-14.517v28.872l.018.018z" />
+   </defs>
+</svg>

.github/workflows/codecov.yml

@@ -25,6 +25,8 @@ jobs:
         ]
     steps:
       - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
       - name: Run
         run: |
           sudo apt-get update
@@ -63,6 +65,8 @@ jobs:
         ]
     steps:
       - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
       - name: Run
         run: |
           sudo apt-get update
@@ -104,6 +108,8 @@ jobs:
         ]
     steps:
       - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
       - name: Run
         run: |
           sudo apt-get update

.github/workflows/doc.yml

@@ -0,0 +1,63 @@
+name: Docgen
+on:
+  push:
+  workflow_dispatch:
+
+
+jobs:
+  build:
+    timeout-minutes: 20
+
+    name: 'Generate & upload documentation'
+    runs-on: 'ubuntu-20.04'
+    continue-on-error: true
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          submodules: true
+
+      - uses: jiro4989/setup-nim-action@v1
+        with:
+          nim-version: 'stable'
+
+      - name: Generate doc
+        run: |
+          nim --version
+          nimble --version
+          nimble install_pinned
+          # nim doc can "fail", but the doc is still generated
+          nim doc --git.url:https://github.com/status-im/nim-libp2p --git.commit:${GITHUB_REF##*/} --outdir:${GITHUB_REF##*/} --project libp2p || true
+
+          # check that the folder exists
+          ls ${GITHUB_REF##*/}
+
+      - name: Clone the gh-pages branch
+        uses: actions/checkout@v2
+        with:
+          repository: status-im/nim-libp2p
+          ref: gh-pages
+          path: subdoc
+          submodules: true
+          fetch-depth: 0
+
+      - name: Commit & push
+        run: |
+          cd subdoc
+
+          # Delete merged branches doc's
+          for branch in $(git branch -vv | grep ': gone]' | awk '{print $1}'); do rm -rf $branch; done
+
+          # Update / create this branch doc
+          rm -rf ${GITHUB_REF##*/}
+          mv ../${GITHUB_REF##*/} .
+
+          # Remove .idx files
+          # NOTE: git also uses idx files in his
+          # internal folder, hence the `*` instead of `.`
+          find * -name "*.idx" -delete
+          git add .
+          git config --global user.email "${{ github.actor }}@users.noreply.github.com"
+          git config --global user.name = "${{ github.actor }}"
+          git commit -a -m "update docs for ${GITHUB_REF##*/}"
+          git push origin gh-pages

.gitignore

@@ -13,3 +13,5 @@ build/
 .vscode/
 .DS_Store
 tests/pubsub/testgossipsub
+nimble.develop
+nimble.paths

.pinned

@@ -1,5 +1,5 @@
 asynctest;https://github.com/markspanbroek/asynctest@#5347c59b4b057443a014722aa40800cd8bb95c69
-bearssl;https://github.com/status-im/nim-bearssl@#0ebb1d7a4af5f4b4d4756a9b6dbfe5d411fa55d9
+bearssl;https://github.com/status-im/nim-bearssl@#c4aec8b664709356060b71b7e495be62ca446a65
 chronicles;https://github.com/status-im/nim-chronicles@#2a2681b60289aaf7895b7056f22616081eb1a882
 chronos;https://github.com/status-im/nim-chronos@#875d7d8e6ef0803ae1c331dbf76b1981b0caeb15
 dnsclient;https://github.com/ba0f3/dnsclient.nim@#fbb76f8af8a33ab818184a7d4406d9fee20993be
@@ -14,4 +14,4 @@ stew;https://github.com/status-im/nim-stew@#cdb1f213d073fd2ecbdaf35a866417657da9
 testutils;https://github.com/status-im/nim-testutils@#aa6e5216f4b4ab5aa971cdcdd70e1ec1203cedf2
 unittest2;https://github.com/status-im/nim-unittest2@#4e2893eacb916c7678fdc4935ff7420f13bf3a9c
 websock;https://github.com/status-im/nim-websock@#8927db93f6ca96abaacfea39f8ca50ce9d41bcdb
-zlib;https://github.com/status-im/nim-zlib@#74cdeb54b21bededb5a515d36f608bc1850555a2
+zlib;https://github.com/status-im/nim-zlib@#74cdeb54b21bededb5a515d36f608bc1850555a2

README.md

@@ -1,5 +1,5 @@
 <h1 align="center">
-  <a href="https://libp2p.io"><img width="250" src="https://github.com/libp2p/libp2p/blob/master/logo/black-bg-2.png?raw=true" alt="libp2p hex logo" /></a>
+  <a href="https://libp2p.io"><img width="250" src="./.assets/full-logo.svg?raw=true" alt="nim-libp2p logo" /></a>
 </h1>
 
 <h3 align="center">The Nim implementation of the libp2p Networking Stack.</h3>
@@ -18,28 +18,16 @@
 
 ## Introduction
 
-An implementation of [libp2p](https://libp2p.io/) in Nim.
-
-## Project Status
-libp2p is used in production by a few projects at [Status](https://github.com/status-im), including [Nimbus](https://github.com/status-im/nimbus-eth2).
-
-While far from complete, currently available components are stable.
-
-Check our [examples folder](/examples) to get started!
+An implementation of [libp2p](https://libp2p.io/) in [Nim](https://nim-lang.org/).
 
 # Table of Contents
 - [Background](#background)
 - [Install](#install)
-  - [Prerequisite](#prerequisite)
-- [Usage](#usage)
-  - [API](#api)
-  - [Getting Started](#getting-started)
-  - [Tutorials and Examples](#tutorials-and-examples)
-  - [Using the Go Daemon](#using-the-go-daemon)
+- [Getting Started](#getting-started)
+- [Modules](#modules)
+- [Users](#users)
 - [Development](#development)
-  - [Tests](#tests)
-  - [Packages](#packages)
-- [Contribute](#contribute)
+  - [Contribute](#contribute)
   - [Core Developers](#core-developers)
 - [License](#license)
 
@@ -54,33 +42,63 @@ libp2p grew out of IPFS, but it is built so that lots of people can use it, for
 - [Here](https://github.com/libp2p/libp2p#description) is an overview of libp2p and its implementations in other programming languages.
 
 ## Install
+**Prerequisite**
+- [Nim](https://nim-lang.org/install.html)
 ```
 nimble install libp2p
 ```
-### Prerequisite
-- [Nim](https://nim-lang.org/install.html)
 
-## Usage
+## Getting Started
+You'll find some tutorials [here](examples/tutorial_1_connect.md), or some examples:
+- [hello world (ping)](examples/helloworld.nim)
+- [direct chat](examples/directchat.nim)
 
-### API
-The specification is available in the [docs/api](docs/api) folder.
+**Go Daemon:**
+Please find the installation and usage intructions in [daemonapi.md](examples/go-daemon/daemonapi.md).
 
-### Getting Started
-Please read the [GETTING_STARTED.md](docs/GETTING_STARTED.md) guide.
+## Modules
 
-### Tutorials and Examples
-Example code can be found in the [examples folder](/examples).
+List of packages modules implemented in nim-libp2p:
 
-#### Direct Chat Tutorial
-- [Part I](https://our.status.im/nim-libp2p-tutorial-a-peer-to-peer-chat-example-1/): Set up the main function and use multi-thread for processing IO.
-- [Part II](https://our.status.im/nim-libp2p-tutorial-a-peer-to-peer-chat-example-2/): Dial remote peer and allow customized user input commands.
-- [Part III](https://our.status.im/nim-libp2p-tutorial-a-peer-to-peer-chat-example-3/): Configure and establish a libp2p node.
+| Name                                                       | Description                                                                                                      |
+| ---------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------- |
+| **Libp2p**                                                 |                                                                                                                  |
+| [libp2p](libp2p/switch.nim)                                | The core of the project                                                                                          |
+| [connmanager](libp2p/connmanager.nim)                      | Connection manager                                                                                               |
+| [identify / push identify](libp2p/protocols/identify.nim)  | [Identify](https://docs.libp2p.io/concepts/protocols/#identify) protocol                                         |
+| [ping](libp2p/protocols/ping.nim)                          | [Ping](https://docs.libp2p.io/concepts/protocols/#ping) protocol                                                 |
+| [libp2p-daemon-client](libp2p/daemon/daemonapi.nim)        | [go-daemon](https://github.com/libp2p/go-libp2p-daemon) nim wrapper                                              |
+| [interop-libp2p](tests/testinterop.nim)                    | Interop tests                                                                                                    |
+| **Transports**                                             |                                                                                                                  |
+| [libp2p-tcp](libp2p/transports/tcptransport.nim)           | TCP transport                                                                                                    |
+| [libp2p-ws](libp2p/transports/wstransport.nim)             | WebSocket & WebSocket Secure transport                                                                           |
+| **Secure Channels**                                        |                                                                                                                  |
+| [libp2p-secio](libp2p/protocols/secure/secio.nim)          | [Secio](https://docs.libp2p.io/concepts/protocols/#secio) secure channel                                         |
+| [libp2p-noise](libp2p/protocols/secure/noise.nim)          | [Noise](https://github.com/libp2p/specs/tree/master/noise) secure channel                                        |
+| [libp2p-plaintext](libp2p/protocols/secure/plaintext.nim)  | [Plain Text](https://github.com/libp2p/specs/tree/master/plaintext) for development purposes                     |
+| **Stream Multiplexers**                                    |                                                                                                                  |
+| [libp2p-mplex](libp2p/muxers/mplex/mplex.nim)              | [MPlex](https://github.com/libp2p/specs/tree/master/mplex) multiplexer                                           |
+| **Data Types**                                             |                                                                                                                  |
+| [peer-id](libp2p/peerid.nim)                               | [Cryptographic identifiers](https://docs.libp2p.io/concepts/peer-id/)                                            |
+| [peer-store](libp2p/peerstore.nim)                         | ["Phone book" of known peers](https://docs.libp2p.io/concepts/peer-id/#peerinfo)                                 |
+| [multiaddress](libp2p/multiaddress.nim)                    | [Composable network addresses](https://github.com/multiformats/multiaddr)                                        |
+| [signed envelope](libp2p/signed_envelope.nim)              | [Signed generic data container](https://github.com/libp2p/specs/blob/master/RFC/0002-signed-envelopes.md)        |
+| [routing record](libp2p/routing_record.nim)                | [Signed peer dialing informations](https://github.com/libp2p/specs/blob/master/RFC/0003-routing-records.md)      |
+| **Utilities**                                              |                                                                                                                  |
+| [libp2p-crypto](libp2p/crypto)                             | Cryptographic backend                                                                                            |
+| [libp2p-crypto-secp256k1](libp2p/crypto/secp.nim)          |                                                                                                                  |
+| **Pubsub**                                                 |                                                                                                                  |
+| [libp2p-pubsub](libp2p/protocols/pubsub/pubsub.nim)        | Pub-Sub generic interface                                                                                        |
+| [libp2p-floodsub](libp2p/protocols/pubsub/floodsub.nim)    | FloodSub implementation                                                                                          |
+| [libp2p-gossipsub](libp2p/protocols/pubsub/gossipsub.nim)  | [GossipSub](https://docs.libp2p.io/concepts/publish-subscribe/) implementation                                   |
 
+## Users
 
-### Using the Go Daemon
-Please find the installation and usage intructions in [daemonapi.md](docs/api/libp2p/daemonapi.md).
-
-Examples can be found in the [examples/go-daemon folder](https://github.com/status-im/nim-libp2p/tree/readme/examples/go-daemon);
+nim-libp2p is used by:
+- [Nimbus](https://github.com/status-im/nimbus-eth2), an Ethereum client
+- [nwaku](https://github.com/status-im/nwaku), a decentralized messaging application
+- [nim-codex](https://github.com/status-im/nim-codex), a decentralized storage application
+- (open a pull request if you want to be included here)
 
 ## Development
 **Clone and Install dependencies:**
@@ -90,96 +108,47 @@ git clone https://github.com/status-im/nim-libp2p
 cd nim-libp2p
 nimble install
 ```
-#### Run unit tests
+
+**Run unit tests**
 ```sh
 # run all the unit tests
 nimble test
 ```
 The code follows the [Status Nim Style Guide](https://status-im.github.io/nim-style-guide/).
 
-### Packages
 
-List of packages currently in existence for nim-libp2p:
-
-#### Libp2p
-- [libp2p](https://github.com/status-im/nim-libp2p)
-- [libp2p-daemon-client](https://github.com/status-im/nim-libp2p/blob/master/libp2p/daemon/daemonapi.nim)
-- [interop-libp2p](https://github.com/status-im/nim-libp2p/blob/master/tests/testinterop.nim)
-
-#### Transports
-- [libp2p-tcp](https://github.com/status-im/nim-libp2p/blob/master/libp2p/transports/tcptransport.nim)
-- [libp2p-ws](https://github.com/status-im/nim-libp2p/blob/master/libp2p/transports/wstransport.nim)
-
-#### Secure Channels
-- [libp2p-secio](https://github.com/status-im/nim-libp2p/blob/master/libp2p/protocols/secure/secio.nim)
-- [libp2p-noise](https://github.com/status-im/nim-libp2p/blob/master/libp2p/protocols/secure/noise.nim)
-- [libp2p-plaintext](https://github.com/status-im/nim-libp2p/blob/master/libp2p/protocols/secure/plaintext.nim)
-
-#### Stream Multiplexers
-- [libp2p-mplex](https://github.com/status-im/nim-libp2p/blob/master/libp2p/muxers/mplex/mplex.nim)
-
-#### Utilities
-- [libp2p-crypto](https://github.com/status-im/nim-libp2p/tree/master/libp2p/crypto)
-- [libp2p-crypto-secp256k1](https://github.com/status-im/nim-libp2p/blob/master/libp2p/crypto/secp.nim)
-
-#### Data Types
-- [peer-id](https://github.com/status-im/nim-libp2p/blob/master/libp2p/peer.nim)
-- [peer-info](https://github.com/status-im/nim-libp2p/blob/master/libp2p/peerinfo.nim)
-
-#### Pubsub
-- [libp2p-pubsub](https://github.com/status-im/nim-libp2p/blob/master/libp2p/protocols/pubsub/pubsub.nim)
-- [libp2p-floodsub](https://github.com/status-im/nim-libp2p/blob/master/libp2p/protocols/pubsub/floodsub.nim)
-- [libp2p-gossipsub](https://github.com/status-im/nim-libp2p/blob/master/libp2p/protocols/pubsub/gossipsub.nim)
-
-
-Packages that exist in the original libp2p specs and are under active development:
-- libp2p-daemon
-- libp2p-webrtc-direct
-- libp2p-webrtc-star
-- libp2p-spdy
-- libp2p-bootstrap
-- libp2p-kad-dht
-- libp2p-mdns
-- libp2p-webrtc-star
-- libp2p-delegated-content-routing
-- libp2p-delegated-peer-routing
-- libp2p-nat-mgnr
-- libp2p-utils
-
-** Note that the current stack reflects the minimal requirements for the upcoming Eth2 implementation.
-
-### Tips and tricks
-
-#### enable expensive metrics:
-
-```bash
-nim c -d:libp2p_expensive_metrics some_file.nim
-```
-
-#### use identify metrics
-
-```bash
-nim c -d:libp2p_agents_metrics -d:KnownLibP2PAgents=nimbus,lighthouse,prysm,teku some_file.nim
-```
-
-### specify gossipsub specific topics to measure
-
-```bash
-nim c -d:KnownLibP2PTopics=topic1,topic2,topic3 some_file.nim
-```
-
-## Contribute
+### Contribute
 
 The libp2p implementation in Nim is a work in progress. We welcome contributors to help out! Specifically, you can:
 - Go through the modules and **check out existing issues**. This would be especially useful for modules in active development. Some knowledge of IPFS/libp2p may be required, as well as the infrastructure behind it.
 - **Perform code reviews**. Feel free to let us know if you found anything that can a) speed up the project development b) ensure better quality and c) reduce possible future bugs.
-- **Add tests**. Help nim-libp2p to be more robust by adding more tests to the [tests folder](https://github.com/status-im/nim-libp2p/tree/master/tests).
+- **Add tests**. Help nim-libp2p to be more robust by adding more tests to the [tests folder](tests/).
 
 The code follows the [Status Nim Style Guide](https://status-im.github.io/nim-style-guide/).
 
 ### Core Developers
 [@cheatfate](https://github.com/cheatfate), [Dmitriy Ryajov](https://github.com/dryajov), [Tanguy](https://github.com/Menduist), [Zahary Karadjov](https://github.com/zah)
 
+### Tips and tricks
+
+**enable expensive metrics:**
+
+```bash
+nim c -d:libp2p_expensive_metrics some_file.nim
+```
+
+**use identify metrics**
+
+```bash
+nim c -d:libp2p_agents_metrics -d:KnownLibP2PAgents=nimbus,lighthouse,prysm,teku some_file.nim
+```
+
+**specify gossipsub specific topics to measure**
+
+```bash
+nim c -d:KnownLibP2PTopics=topic1,topic2,topic3 some_file.nim
+```
+
 ## License
 
 Licensed and distributed under either of
@@ -191,4 +160,3 @@ or
 * Apache License, Version 2.0, ([LICENSE-APACHEv2](LICENSE-APACHEv2) or http://www.apache.org/licenses/LICENSE-2.0)
 
 at your option. These files may not be copied, modified, or distributed except according to those terms.
-

config.nims

@@ -3,4 +3,8 @@ if dirExists("nimbledeps/pkgs"):
   switch("NimblePath", "nimbledeps/pkgs")
 
 when (NimMajor, NimMinor) > (1, 2):
-  switch("hint", "XCannotRaiseY:off")
+  switch("hint", "XCannotRaiseY:off")
+# begin Nimble config (version 1)
+when fileExists("nimble.paths"):
+  include "nimble.paths"
+# end Nimble config

docs/API.md

@@ -1,3 +0,0 @@
-# API
-
-Coming Soon... 

docs/GETTING_STARTED.md

@@ -1,7 +0,0 @@
-# Getting Started
-Welcome to nim-libp2p!
-
-
-To get started, please look at the [tutorials](../examples/tutorial_1_connect.md)
-
-For more concrete examples, you can look at the [hello world example](../examples/helloworld.nim) or the [direct chat](../examples/directchat.nim)

docs/README.md

@@ -1,29 +0,0 @@
-# Introduction
-This folder contains the documentation for each nim-libp2p module and the sample code for the tutorials. 
-
-# Table of Contents 
-### [Getting Started](GETTING_STARTED.md)
-### Tutorials
-- P2P Chat Example
-    - [part I](tutorial/directchat/start.nim)
-    - [part II](tutorial/directchat/second.nim)
-### API Specifications
-- libp2p
-    - [libp2p-daemon-client](api/libp2p/daemonapi.md)
-    - [interop-libp2p](api/libp2p/interop.md)
-- transports
-    - [libp2p-tcp](api/transports/tcptransport.md)
-- secure channels
-    - [libp2p-secio](api/secure_channels/secio.md)
-- stream multiplexers
-    - [libp2p-mplex](api/stream_multiplexers/mplex.md)
-- utilities
-    - [libp2p-crypto](api/utilities/crypto.md)
-    - [libp2p-crypto-secp256k1](api/utilities/secp256k1.md)
-- data types
-    - [peer-id](api/data_types/peer.md)
-    - [peer-info](api/data_types/peerinfo.md)
-- pubsub
-    - [libp2p-pubsub](api/pubsub/pubsub.md)
-    - [libp2p-floodsub](api/pubsub/floodsub.md)
-    - [libp2p-gossipsub](api/pubsub/gossipsub.md)

docs/tutorial/directchat/second.nim

@@ -1,149 +0,0 @@
-when not(compileOption("threads")):
-  {.fatal: "Please, compile this program with the --threads:on option!".}
-
-import tables, strformat, strutils
-import chronos
-import ../libp2p/[switch,
-                  multistream,
-                  crypto/crypto,
-                  protocols/identify,
-                  connection,
-                  transports/transport,
-                  transports/tcptransport,
-                  multiaddress,
-                  peerinfo,
-                  peerid,
-                  protocols/protocol,
-                  protocols/secure/secure,
-                  protocols/secure/secio,
-                  muxers/muxer,
-                  muxers/mplex/mplex]
-
-const ChatCodec = "/nim-libp2p/chat/1.0.0"
-const DefaultAddr = "/ip4/127.0.0.1/tcp/55505"
-
-const Help = """
-  Commands: /[?|hep|connect|disconnect|exit]
-  help: Prints this help
-  connect: dials a remote peer
-  disconnect: ends current session
-  exit: closes the chat
-"""
-
-type ChatProto = ref object of LPProtocol
-  switch: Switch          # a single entry point for dialing and listening to peer
-  transp: StreamTransport # transport streams between read & write file descriptor
-  conn: Connection        # create and close read & write stream
-  connected: bool         # if the node is connected to another peer
-  started: bool           # if the node has started
-
-# copied from https://github.com/status-im/nimbus-eth2/blob/0ed657e953740a92458f23033d47483ffa17ccb0/beacon_chain/eth2_network.nim#L109-L115
-proc initAddress(T: type MultiAddress, str: string): T =
-  let address = MultiAddress.init(str)
-  if IPFS.match(address) and matchPartial(multiaddress.TCP, address):
-    result = address
-  else:
-    raise newException(MultiAddressError,
-                         "Invalid bootstrap node multi-address")
-
-proc dialPeer(p: ChatProto, address: string) {.async.} =
-  let multiAddr = MultiAddress.initAddress(address);
-  let parts = address.split("/")
-  let remotePeer = PeerInfo.init(parts[^1],
-                                 [multiAddr])
-
-  echo &"dialing peer: {multiAddr}"
-  p.conn = await p.switch.dial(remotePeer, ChatCodec)
-  p.connected = true
-
-proc readAndPrint(p: ChatProto) {.async.} =
-  while true:
-    while p.connected:
-      echo cast[string](await p.conn.readLp(1024))
-    await sleepAsync(100.millis)
-
-proc writeAndPrint(p: ChatProto) {.async.} =
-  while true:
-    if not p.connected:
-      echo "type an address or wait for a connection:"
-      echo "type /[help|?] for help"
-
-    let line = await p.transp.readLine()
-    if line.startsWith("/help") or line.startsWith("/?") or not p.started:
-      echo Help
-      continue
-
-    if line.startsWith("/disconnect"):
-      echo "Ending current session"
-      if p.connected and p.conn.closed.not:
-        await p.conn.close()
-      p.connected = false
-    elif line.startsWith("/connect"):
-      if p.connected:
-        var yesno = "N"
-        echo "a session is already in progress, do you want end it [y/N]?"
-        yesno = await p.transp.readLine()
-        if yesno.cmpIgnoreCase("y") == 0:
-          await p.conn.close()
-          p.connected = false
-        elif yesno.cmpIgnoreCase("n") == 0:
-          continue
-        else:
-          echo "unrecognized response"
-          continue
-
-      echo "enter address of remote peer"
-      let address = await p.transp.readLine()
-      if address.len > 0:
-        await p.dialPeer(address)
-
-    elif line.startsWith("/exit"):
-      if p.connected and p.conn.closed.not:
-        await p.conn.close()
-        p.connected = false
-
-      await p.switch.stop()
-      echo "quitting..."
-      quit(0)
-    else:
-      if p.connected:
-        await p.conn.writeLp(line)
-      else:
-        try:
-          if line.startsWith("/") and "ipfs" in line:
-            await p.dialPeer(line)
-        except:
-          echo &"unable to dial remote peer {line}"
-          echo getCurrentExceptionMsg()
-
-proc readWriteLoop(p: ChatProto) {.async.} =
-  asyncSpawn p.writeAndPrint() # execute the async function but does not block
-  asyncSpawn p.readAndPrint()
-
-proc processInput(rfd: AsyncFD) {.async.} =
-  let transp = fromPipe(rfd)
-  while true:
-    let a = await transp.readLine()
-    echo "You just entered: " & a
-
-proc readInput(wfd: AsyncFD) {.thread.} =
-  ## This procedure performs reading from `stdin` and sends data over
-  ## pipe to main thread.
-  let transp = fromPipe(wfd)
-
-  while true:
-    let line = stdin.readLine()
-    discard waitFor transp.write(line & "\r\n")
-
-proc main() {.async.} =
-  let (rfd, wfd) = createAsyncPipe()
-  if rfd == asyncInvalidPipe or wfd == asyncInvalidPipe:
-    raise newException(ValueError, "Could not initialize pipe!")
-
-  var thread: Thread[AsyncFD]
-  thread.createThread(readInput, wfd)
-
-  await processInput(rfd)
-
-when isMainModule:      # isMainModule = true when the module is compiled as the main file
-  waitFor(main())

docs/tutorial/directchat/start.nim

@@ -1,39 +0,0 @@
-when not(compileOption("threads")):
-  {.fatal: "Please, compile this program with the --threads:on option!".}
-  
-import chronos          # an efficient library for async
-
-proc processInput(rfd: AsyncFD) {.async.} =
-  echo "Type something below to see if the multithread IO works:\nType 'exit' to exit."
-
-  let transp = fromPipe(rfd)
-  while true:
-    let a = await transp.readLine()
-
-    if a == "exit":
-      quit(0);
-
-    echo "You just entered: " & a
-
-proc readInput(wfd: AsyncFD) {.thread.} =
-  ## This procedure performs reading from `stdin` and sends data over
-  ## pipe to main thread.
-  let transp = fromPipe(wfd)
-
-  while true:
-    let line = stdin.readLine()
-    discard waitFor transp.write(line & "\r\n")
-
-proc main() {.async.} =
-  let (rfd, wfd) = createAsyncPipe()
-  if rfd == asyncInvalidPipe or wfd == asyncInvalidPipe:
-    raise newException(ValueError, "Could not initialize pipe!")
-  
-  var thread: Thread[AsyncFD]
-  thread.createThread(readInput, wfd)
-  
-  await processInput(rfd)
-
-when isMainModule:      # isMainModule = true when the module is compiled as the main file
-  waitFor(main())
- 

docs/tutorial/second.nim

@@ -1,205 +0,0 @@
-when not(compileOption("threads")):
-  {.fatal: "Please, compile this program with the --threads:on option!".}
-
-import tables, strformat, strutils, bearssl
-import chronos                              # an efficient library for async
-import ../libp2p/[switch,                   # manage transports, a single entry point for dialing and listening
-                  builders,                 # helper to build the switch object
-                  multistream,              # tag stream with short header to identify it
-                  multicodec,               # multicodec utilities
-                  crypto/crypto,            # cryptographic functions
-                  errors,                   # error handling utilities
-                  protocols/identify,       # identify the peer info of a peer
-                  stream/connection,        # create and close stream read / write connections
-                  transports/transport,     # listen and dial to other peers using p2p protocol
-                  transports/tcptransport,  # listen and dial to other peers using client-server protocol
-                  multiaddress,             # encode different addressing schemes. For example, /ip4/7.7.7.7/tcp/6543 means it is using IPv4 protocol and TCP
-                  peerinfo,                 # manage the information of a peer, such as peer ID and public / private key
-                  peerid,                   # Implement how peers interact
-                  protocols/protocol,       # define the protocol base type
-                  protocols/secure/secure,  # define the protocol of secure connection
-                  protocols/secure/secio,   # define the protocol of secure input / output, allows encrypted communication that uses public keys to validate signed messages instead of a certificate authority like in TLS
-                  muxers/muxer,             # define an interface for stream multiplexing, allowing peers to offer many protocols over a single connection
-                  muxers/mplex/mplex]       # define some contants and message types for stream multiplexing
-
-const ChatCodec = "/nim-libp2p/chat/1.0.0"
-const DefaultAddr = "/ip4/127.0.0.1/tcp/55505"
-
-const Help = """
-  Commands: /[?|hep|connect|disconnect|exit]
-  help: Prints this help
-  connect: dials a remote peer
-  disconnect: ends current session
-  exit: closes the chat
-"""
-
-type ChatProto = ref object of LPProtocol
-  switch: Switch          # a single entry point for dialing and listening to peer
-  transp: StreamTransport # transport streams between read & write file descriptor
-  conn: Connection        # create and close read & write stream
-  connected: bool         # if the node is connected to another peer
-  started: bool           # if the node has started
-
-proc readAndPrint(p: ChatProto) {.async.} =
-  while true:
-    var strData = await p.conn.readLp(1024)
-    strData &= '\0'.uint8
-    var str = cast[cstring](addr strdata[0])
-    echo $p.switch.peerInfo.peerId & ": " & $str
-    await sleepAsync(100.millis)
-
-proc dialPeer(p: ChatProto, address: string) {.async.} =
-  let
-    multiAddr = MultiAddress.init(address).tryGet()
-    # split the peerId part /p2p/...
-    peerIdBytes = multiAddr[multiCodec("p2p")]
-      .tryGet()
-      .protoAddress()
-      .tryGet()
-    remotePeer = PeerId.init(peerIdBytes).tryGet()
-    # split the wire address
-    ip4Addr = multiAddr[multiCodec("ip4")].tryGet()
-    tcpAddr = multiAddr[multiCodec("tcp")].tryGet()
-    wireAddr = ip4Addr & tcpAddr
-
-  echo &"dialing peer: {multiAddr}"
-  p.conn = await p.switch.dial(remotePeer, @[wireAddr], ChatCodec)
-  p.connected = true
-  asyncSpawn p.readAndPrint()
-
-proc writeAndPrint(p: ChatProto) {.async.} =
-  while true:
-    if not p.connected:
-      echo "type an address or wait for a connection:"
-      echo "type /[help|?] for help"
-
-    let line = await p.transp.readLine()
-    if line.startsWith("/help") or line.startsWith("/?") or not p.started:
-      echo Help
-      continue
-
-    if line.startsWith("/disconnect"):
-      echo "Ending current session"
-      if p.connected and p.conn.closed.not:
-        await p.conn.close()
-      p.connected = false
-    elif line.startsWith("/connect"):
-      if p.connected:
-        var yesno = "N"
-        echo "a session is already in progress, do you want end it [y/N]?"
-        yesno = await p.transp.readLine()
-        if yesno.cmpIgnoreCase("y") == 0:
-          await p.conn.close()
-          p.connected = false
-        elif yesno.cmpIgnoreCase("n") == 0:
-          continue
-        else:
-          echo "unrecognized response"
-          continue
-
-      echo "enter address of remote peer"
-      let address = await p.transp.readLine()
-      if address.len > 0:
-        await p.dialPeer(address)
-
-    elif line.startsWith("/exit"):
-      if p.connected and p.conn.closed.not:
-        await p.conn.close()
-        p.connected = false
-
-      await p.switch.stop()
-      echo "quitting..."
-      quit(0)
-    else:
-      if p.connected:
-        await p.conn.writeLp(line)
-      else:
-        try:
-          if line.startsWith("/") and "p2p" in line:
-            await p.dialPeer(line)
-        except:
-          echo &"unable to dial remote peer {line}"
-          echo getCurrentExceptionMsg()
-
-proc readWriteLoop(p: ChatProto) {.async.} =
-  await p.writeAndPrint()
-
-proc newChatProto(switch: Switch, transp: StreamTransport): ChatProto =
-  var chatproto = ChatProto(switch: switch, transp: transp, codecs: @[ChatCodec])
-
-  # create handler for incoming connection
-  proc handle(stream: Connection, proto: string) {.async.} =
-    if chatproto.connected and not chatproto.conn.closed:
-      echo "a chat session is already in progress - disconnecting!"
-      await stream.close()
-    else:
-      chatproto.conn = stream
-      chatproto.connected = true
-      await chatproto.readAndPrint()
-
-  # assign the new handler
-  chatproto.handler = handle
-  return chatproto
-
-proc readInput(wfd: AsyncFD) {.thread.} =
-  ## This procedure performs reading from `stdin` and sends data over
-  ## pipe to main thread.
-  let transp = fromPipe(wfd)
-
-  while true:
-    let line = stdin.readLine()
-    discard waitFor transp.write(line & "\r\n")
-
-proc processInput(rfd: AsyncFD, rng: ref BrHmacDrbgContext) {.async.} =
-  let transp = fromPipe(rfd)
-
-  let seckey = PrivateKey.random(RSA, rng[]).get()
-  var localAddress = DefaultAddr
-  while true:
-    echo &"Type an address to bind to or Enter to use the default {DefaultAddr}"
-    let a = await transp.readLine()
-    try:
-      if a.len > 0:
-        localAddress = a
-        break
-      # uise default
-      break
-    except:
-      echo "invalid address"
-      localAddress = DefaultAddr
-      continue
-
-  var switch = SwitchBuilder
-    .init()
-    .withRng(rng)
-    .withPrivateKey(seckey)
-    .withAddress(MultiAddress.init(localAddress).tryGet())
-    .build()
-
-  let chatProto = newChatProto(switch, transp)
-  switch.mount(chatProto)
-  let libp2pFuts = await switch.start()
-  chatProto.started = true
-
-  let id = $switch.peerInfo.peerId
-  echo "PeerId: " & id
-  echo "listening on: "
-  for a in switch.peerInfo.addrs:
-    echo &"{a}/p2p/{id}"
-
-  await chatProto.readWriteLoop()
-  await allFuturesThrowing(libp2pFuts)
-
-proc main() {.async.} =
-  let rng = newRng() # Singe random number source for the whole application
-  let (rfd, wfd) = createAsyncPipe()
-  if rfd == asyncInvalidPipe or wfd == asyncInvalidPipe:
-    raise newException(ValueError, "Could not initialize pipe!")
-
-  var thread: Thread[AsyncFD]
-  thread.createThread(readInput, wfd)
-
-  await processInput(rfd, rng)
-
-when isMainModule: # isMainModule = true when the module is compiled as the main file
-  waitFor(main())

docs/tutorial/start.nim

@@ -1,39 +0,0 @@
-when not(compileOption("threads")):
-  {.fatal: "Please, compile this program with the --threads:on option!".}
-  
-import chronos          # an efficient library for async
-
-proc processInput(rfd: AsyncFD) {.async.} =
-  echo "Type something below to see if the multithread IO works:\nType 'exit' to exit."
-
-  let transp = fromPipe(rfd)
-  while true:
-    let a = await transp.readLine()
-
-    if a == "exit":
-      quit(0);
-
-    echo "You just entered: " & a
-
-proc readInput(wfd: AsyncFD) {.thread.} =
-  ## This procedure performs reading from `stdin` and sends data over
-  ## pipe to main thread.
-  let transp = fromPipe(wfd)
-
-  while true:
-    let line = stdin.readLine()
-    discard waitFor transp.write(line & "\r\n")
-
-proc main() {.async.} =
-  let (rfd, wfd) = createAsyncPipe()
-  if rfd == asyncInvalidPipe or wfd == asyncInvalidPipe:
-    raise newException(ValueError, "Could not initialize pipe!")
-  
-  var thread: Thread[AsyncFD]
-  thread.createThread(readInput, wfd)
-  
-  await processInput(rfd)
-
-when isMainModule:      # isMainModule = true when the module is compiled as the main file
-  waitFor(main())
- 

examples/directchat.nim

@@ -2,7 +2,7 @@ when not(compileOption("threads")):
   {.fatal: "Please, compile this program with the --threads:on option!".}
 
 import
-  strformat, strutils, bearssl,
+  strformat, strutils,
   stew/byteutils,
   chronos,
   ../libp2p

examples/helloworld.nim

@@ -1,4 +1,3 @@
-import bearssl
 import chronos         # an efficient library for async
 import stew/byteutils  # various utils
 import ../libp2p       # when installed through nimble, just use `import libp2p`
@@ -26,7 +25,7 @@ proc new(T: typedesc[TestProto]): T =
 ##
 # Helper to create a switch/node
 ##
-proc createSwitch(ma: MultiAddress, rng: ref BrHmacDrbgContext): Switch =
+proc createSwitch(ma: MultiAddress, rng: ref HmacDrbgContext): Switch =
   var switch = SwitchBuilder
     .new()
     .withRng(rng)       # Give the application RNG

examples/tutorial_1_connect.md

@@ -24,18 +24,16 @@ _TIP: You can extract the code from this tutorial by running `nim c -r tools/mar
 
 Let's create a `part1.nim`, and import our dependencies:
 ```nim
-import bearssl
 import chronos
 
 import libp2p
 import libp2p/protocols/ping
 ```
-[bearssl](https://github.com/status-im/nim-bearssl) is used as a [cryptographic pseudorandom number generator](https://en.wikipedia.org/wiki/Cryptographically-secure_pseudorandom_number_generator)  
 [chronos](https://github.com/status-im/nim-chronos) the asynchronous framework used by `nim-libp2p`
 
 Next, we'll create an helper procedure to create our switches. A switch needs a bit of configuration, and it will be easier to do this configuration only once:
 ```nim
-proc createSwitch(ma: MultiAddress, rng: ref BrHmacDrbgContext): Switch =
+proc createSwitch(ma: MultiAddress, rng: ref HmacDrbgContext): Switch =
   var switch = SwitchBuilder
     .new()
     .withRng(rng)       # Give the application RNG

examples/tutorial_2_customproto.md

@@ -5,7 +5,6 @@ We'll now look at how to create a custom protocol inside the libp2p
 # Custom protocol in libp2p
 Let's create a `part2.nim`, and import our dependencies:
 ```nim
-import bearssl
 import chronos
 import stew/byteutils
 

libp2p.nim

@@ -1,40 +1,72 @@
-## Nim-LibP2P
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
-import
-  libp2p/[protobuf/minprotobuf,
-          muxers/muxer,
-          muxers/mplex/mplex,
-          stream/lpstream,
-          stream/bufferstream,
-          stream/connection,
-          transports/transport,
-          transports/tcptransport,
-          protocols/secure/noise,
-          cid,
-          multihash,
-          multibase,
-          multicodec,
-          errors,
-          switch,
-          peerid,
-          peerinfo,
-          multiaddress,
-          builders,
-          crypto/crypto,
-          protocols/pubsub]
+when defined(nimdoc):
+  ## Welcome to the nim-libp2p documentation!
+  ##
+  ## On the left, you'll find a switch that allows you to see private
+  ## procedures. By default, you'll only see the public one (marked with `{.public.}`)
+  ##
+  ## The difference between public and private procedures is that public procedure
+  ## stay backward compatible during the Major version, whereas private ones can
+  ## change at each new Minor version.
+  ##
+  ## If you're new to nim-libp2p, you can find a tutorial `here<https://github.com/status-im/nim-libp2p/blob/master/examples/tutorial_1_connect.md>`_
+  ## that can help you get started.
 
-import bearssl
+  # Import stuff for doc
+  import libp2p/[
+      protobuf/minprotobuf,
+      switch,
+      stream/lpstream,
+      builders,
+      transports/tcptransport,
+      transports/wstransport,
+      protocols/ping,
+      protocols/pubsub,
+      peerid,
+      peerinfo,
+      peerstore,
+      multiaddress]
 
-export
-  minprotobuf, switch, peerid, peerinfo,
-  connection, multiaddress, crypto, lpstream,
-  bufferstream, bearssl, muxer, mplex, transport,
-  tcptransport, noise, errors, cid, multihash,
-  multicodec, builders, pubsub
+  proc dummyPrivateProc*() =
+    ## A private proc example
+    discard
+else:
+  import
+    libp2p/[protobuf/minprotobuf,
+            muxers/muxer,
+            muxers/mplex/mplex,
+            stream/lpstream,
+            stream/bufferstream,
+            stream/connection,
+            transports/transport,
+            transports/tcptransport,
+            transports/wstransport,
+            protocols/secure/noise,
+            protocols/ping,
+            cid,
+            multihash,
+            multibase,
+            multicodec,
+            errors,
+            switch,
+            peerid,
+            peerinfo,
+            multiaddress,
+            builders,
+            crypto/crypto,
+            protocols/pubsub]
+
+  export
+    minprotobuf, switch, peerid, peerinfo,
+    connection, multiaddress, crypto, lpstream,
+    bufferstream, muxer, mplex, transport,
+    tcptransport, noise, errors, cid, multihash,
+    multicodec, builders, pubsub

libp2p.nimble

@@ -28,16 +28,14 @@ const nimflags =
 proc runTest(filename: string, verify: bool = true, sign: bool = true,
              moreoptions: string = "") =
   var excstr = "nim c --opt:speed -d:debug -d:libp2p_agents_metrics -d:libp2p_protobuf_metrics -d:libp2p_network_protocols_metrics -d:libp2p_mplex_metrics "
+  excstr.add(" -d:chronicles_sinks=textlines[stdout],json[dynamic] -d:chronicles_log_level=TRACE ")
+  excstr.add(" -d:chronicles_runtime_filtering=TRUE ")
   excstr.add(" " & getEnv("NIMFLAGS") & " ")
   excstr.add(" " & nimflags & " ")
   excstr.add(" -d:libp2p_pubsub_sign=" & $sign)
   excstr.add(" -d:libp2p_pubsub_verify=" & $verify)
   excstr.add(" " & moreoptions & " ")
-  if verify and sign:
-    # build it with TRACE and JSON logs
-    exec excstr & " -d:chronicles_log_level=TRACE -d:chronicles_sinks:json" & " tests/" & filename
-  # build it again, to run it with less verbose logs
-  exec excstr & " -d:chronicles_log_level=INFO -r" & " tests/" & filename
+  exec excstr & " -r " & " tests/" & filename
   rmFile "tests/" & filename.toExe
 
 proc buildSample(filename: string, run = false) =
@@ -102,6 +100,23 @@ task examples_build, "Build the samples":
   buildTutorial("examples/tutorial_1_connect.md")
   buildTutorial("examples/tutorial_2_customproto.md")
 
+proc tutorialToHtml(source, output: string) =
+  var html = gorge("./nimbledeps/bin/markdown < " & source)
+  html &= """
+<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/water.css@2/out/water.css">
+<link rel="stylesheet" href="https://unpkg.com/@highlightjs/cdn-assets@11.5.1/styles/default.min.css">
+<script src="https://unpkg.com/@highlightjs/cdn-assets@11.5.1/highlight.min.js"></script>
+<script src="https://unpkg.com/@highlightjs/cdn-assets@11.5.1/languages/nim.min.js"></script>
+<script>hljs.highlightAll();</script>
+  """
+  writeFile(output, html)
+
+
+task markdown_to_html, "Build the tutorials HTML":
+  exec "nimble install -y markdown"
+  tutorialToHtml("examples/tutorial_1_connect.md", "tuto1.html")
+  tutorialToHtml("examples/tutorial_2_customproto.md", "tuto2.html")
+
 # pin system
 # while nimble lockfile
 # isn't available

libp2p/builders.nim

@@ -1,46 +1,56 @@
-## Nim-Libp2p
-## Copyright (c) 2020 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+## This module contains a Switch Building helper.
+runnableExamples:
+  let switch =
+   SwitchBuilder.new()
+   .withRng(rng)
+   .withAddresses(multiaddress)
+   # etc
+   .build()
 
 {.push raises: [Defect].}
 
 import
-  options, tables, chronos, chronicles, bearssl,
+  options, tables, chronos, chronicles,
   switch, peerid, peerinfo, stream/connection, multiaddress,
   crypto/crypto, transports/[transport, tcptransport],
-  muxers/[muxer, mplex/mplex],
+  muxers/[muxer, mplex/mplex, yamux/yamux],
   protocols/[identify, secure/secure, secure/noise, relay],
   connmanager, upgrademngrs/muxedupgrade,
   nameresolving/nameresolver,
-  errors
+  errors, utility
 
 export
   switch, peerid, peerinfo, connection, multiaddress, crypto, errors
 
 type
-  TransportProvider* = proc(upgr: Upgrade): Transport {.gcsafe, raises: [Defect].}
+  TransportProvider* {.public.} = proc(upgr: Upgrade): Transport {.gcsafe, raises: [Defect].}
 
   SecureProtocol* {.pure.} = enum
     Noise,
     Secio {.deprecated.}
 
-  MplexOpts = object
-    enable: bool
+  MuxerBuilder = object
+    codec: string
     newMuxer: MuxerConstructor
 
   SwitchBuilder* = ref object
     privKey: Option[PrivateKey]
     addresses: seq[MultiAddress]
     secureManagers: seq[SecureProtocol]
-    mplexOpts: MplexOpts
+    muxers: seq[MuxerBuilder]
     transports: seq[TransportProvider]
-    rng: ref BrHmacDrbgContext
+    rng: ref HmacDrbgContext
     maxConnections: int
+    maxConcurrentUpgrades: int
     maxIn: int
     sendSignedPeerRecord: bool
     maxOut: int
@@ -52,7 +62,8 @@ type
     isCircuitRelay: bool
     circuitRelayCanHop: bool
 
-proc new*(T: type[SwitchBuilder]): T =
+proc new*(T: type[SwitchBuilder]): T {.public.} =
+  ## Creates a SwitchBuilder
 
   let address = MultiAddress
   .init("/ip4/127.0.0.1/tcp/0")
@@ -63,6 +74,7 @@ proc new*(T: type[SwitchBuilder]): T =
     addresses: @[address],
     secureManagers: @[],
     maxConnections: MaxConnections,
+    maxConcurrentUpgrades: ConcurrentUpgrades,
     maxIn: -1,
     maxOut: -1,
     maxConnsPerPeer: MaxConnectionsPerPeer,
@@ -70,19 +82,28 @@ proc new*(T: type[SwitchBuilder]): T =
     agentVersion: AgentVersion,
     isCircuitRelay: false)
 
-proc withPrivateKey*(b: SwitchBuilder, privateKey: PrivateKey): SwitchBuilder =
+proc withPrivateKey*(b: SwitchBuilder, privateKey: PrivateKey): SwitchBuilder {.public.} =
+  ## Set the private key of the switch. Will be used to
+  ## generate a PeerId
+
   b.privKey = some(privateKey)
   b
 
-proc withAddress*(b: SwitchBuilder, address: MultiAddress): SwitchBuilder =
+proc withAddress*(b: SwitchBuilder, address: MultiAddress): SwitchBuilder {.public.} =
+  ## | Set the listening address of the switch
+  ## | Calling it multiple time will override the value
+
   b.addresses = @[address]
   b
 
-proc withAddresses*(b: SwitchBuilder, addresses: seq[MultiAddress]): SwitchBuilder =
+proc withAddresses*(b: SwitchBuilder, addresses: seq[MultiAddress]): SwitchBuilder {.public.} =
+  ## | Set the listening addresses of the switch
+  ## | Calling it multiple time will override the value
+
   b.addresses = addresses
   b
 
-proc withSignedPeerRecord*(b: SwitchBuilder, sendIt = true): SwitchBuilder =
+proc withSignedPeerRecord*(b: SwitchBuilder, sendIt = true): SwitchBuilder {.public.} =
   b.sendSignedPeerRecord = sendIt
   b
 
@@ -90,7 +111,9 @@ proc withMplex*(
     b: SwitchBuilder,
     inTimeout = 5.minutes,
     outTimeout = 5.minutes,
-    maxChannCount = 200): SwitchBuilder =
+    maxChannCount = 200): SwitchBuilder {.public.} =
+  ## | Uses `Mplex <https://docs.libp2p.io/concepts/stream-multiplexing/#mplex>`_ as a multiplexer
+  ## | `Timeout` is the duration after which a inactive connection will be closed
   proc newMuxer(conn: Connection): Muxer =
     Mplex.new(
       conn,
@@ -98,57 +121,74 @@ proc withMplex*(
       outTimeout,
       maxChannCount)
 
-  b.mplexOpts = MplexOpts(
-    enable: true,
-    newMuxer: newMuxer,
-  )
-
+  b.muxers.add(MuxerBuilder(codec: MplexCodec, newMuxer: newMuxer))
   b
 
-proc withNoise*(b: SwitchBuilder): SwitchBuilder =
+proc withYamux*(b: SwitchBuilder): SwitchBuilder =
+  proc newMuxer(conn: Connection): Muxer = Yamux.new(conn)
+
+  b.muxers.add(MuxerBuilder(codec: YamuxCodec, newMuxer: newMuxer))
+  b
+
+proc withNoise*(b: SwitchBuilder): SwitchBuilder {.public.} =
   b.secureManagers.add(SecureProtocol.Noise)
   b
 
-proc withTransport*(b: SwitchBuilder, prov: TransportProvider): SwitchBuilder =
+proc withTransport*(b: SwitchBuilder, prov: TransportProvider): SwitchBuilder {.public.} =
+  ## Use a custom transport
+  runnableExamples:
+    let switch =
+      SwitchBuilder.new()
+      .withTransport(proc(upgr: Upgrade): Transport = TcpTransport.new(flags, upgr))
+      .build()
   b.transports.add(prov)
   b
 
-proc withTcpTransport*(b: SwitchBuilder, flags: set[ServerFlags] = {}): SwitchBuilder =
+proc withTcpTransport*(b: SwitchBuilder, flags: set[ServerFlags] = {}): SwitchBuilder {.public.} =
   b.withTransport(proc(upgr: Upgrade): Transport = TcpTransport.new(flags, upgr))
 
-proc withRng*(b: SwitchBuilder, rng: ref BrHmacDrbgContext): SwitchBuilder =
+proc withRng*(b: SwitchBuilder, rng: ref HmacDrbgContext): SwitchBuilder {.public.} =
   b.rng = rng
   b
 
-proc withMaxConnections*(b: SwitchBuilder, maxConnections: int): SwitchBuilder =
+proc withMaxConnections*(b: SwitchBuilder, maxConnections: int): SwitchBuilder {.public.} =
+  ## Maximum concurrent connections of the switch. You should either use this, or
+  ## `withMaxIn <#withMaxIn,SwitchBuilder,int>`_ & `withMaxOut<#withMaxOut,SwitchBuilder,int>`_
   b.maxConnections = maxConnections
   b
 
-proc withMaxIn*(b: SwitchBuilder, maxIn: int): SwitchBuilder =
+proc withMaxConcurrentUpgrades*(b: SwitchBuilder, maxConcurrentUpgrades: int): SwitchBuilder {.public.} =
+  ## Maximum concurrent upgrades on the switch. Default to 4
+  b.maxConcurrentUpgrades = maxConcurrentUpgrades
+  b
+
+proc withMaxIn*(b: SwitchBuilder, maxIn: int): SwitchBuilder {.public.} =
+  ## Maximum concurrent incoming connections. Should be used with `withMaxOut<#withMaxOut,SwitchBuilder,int>`_
   b.maxIn = maxIn
   b
 
-proc withMaxOut*(b: SwitchBuilder, maxOut: int): SwitchBuilder =
+proc withMaxOut*(b: SwitchBuilder, maxOut: int): SwitchBuilder {.public.} =
+  ## Maximum concurrent outgoing connections. Should be used with `withMaxIn<#withMaxIn,SwitchBuilder,int>`_
   b.maxOut = maxOut
   b
 
-proc withMaxConnsPerPeer*(b: SwitchBuilder, maxConnsPerPeer: int): SwitchBuilder =
+proc withMaxConnsPerPeer*(b: SwitchBuilder, maxConnsPerPeer: int): SwitchBuilder {.public.} =
   b.maxConnsPerPeer = maxConnsPerPeer
   b
 
-proc withPeerStore*(b: SwitchBuilder, capacity: int): SwitchBuilder =
+proc withPeerStore*(b: SwitchBuilder, capacity: int): SwitchBuilder {.public.} =
   b.peerStoreCapacity = some(capacity)
   b
 
-proc withProtoVersion*(b: SwitchBuilder, protoVersion: string): SwitchBuilder =
+proc withProtoVersion*(b: SwitchBuilder, protoVersion: string): SwitchBuilder {.public.} =
   b.protoVersion = protoVersion
   b
 
-proc withAgentVersion*(b: SwitchBuilder, agentVersion: string): SwitchBuilder =
+proc withAgentVersion*(b: SwitchBuilder, agentVersion: string): SwitchBuilder {.public.} =
   b.agentVersion = agentVersion
   b
 
-proc withNameResolver*(b: SwitchBuilder, nameResolver: NameResolver): SwitchBuilder =
+proc withNameResolver*(b: SwitchBuilder, nameResolver: NameResolver): SwitchBuilder {.public.} =
   b.nameResolver = nameResolver
   b
 
@@ -158,7 +198,7 @@ proc withRelayTransport*(b: SwitchBuilder, canHop: bool): SwitchBuilder =
   b
 
 proc build*(b: SwitchBuilder): Switch
-  {.raises: [Defect, LPError].} =
+  {.raises: [Defect, LPError], public.} =
 
   if b.rng == nil: # newRng could fail
     raise newException(Defect, "Cannot initialize RNG")
@@ -182,8 +222,8 @@ proc build*(b: SwitchBuilder): Switch
   let
     muxers = block:
       var muxers: Table[string, MuxerProvider]
-      if b.mplexOpts.enable:
-        muxers[MplexCodec] = MuxerProvider.new(b.mplexOpts.newMuxer, MplexCodec)
+      for m in b.muxers:
+        muxers[m.codec] = MuxerProvider.new(m.newMuxer, m.codec)
       muxers
 
   let
@@ -220,6 +260,7 @@ proc build*(b: SwitchBuilder): Switch
     connManager = connManager,
     ms = ms,
     nameResolver = b.nameResolver,
+    concurrentUpgrades = b.maxConcurrentUpgrades,
     peerStore = peerStore)
 
   if b.isCircuitRelay:
@@ -246,7 +287,9 @@ proc newStandardSwitch*(
   nameResolver: NameResolver = nil,
   sendSignedPeerRecord = false,
   peerStoreCapacity = 1000): Switch
-  {.raises: [Defect, LPError].} =
+  {.raises: [Defect, LPError], public.} =
+  ## Helper for common switch configurations.
+
   if SecureProtocol.Secio in secureManagers:
       quit("Secio is deprecated!") # use of secio is unsafe
 

libp2p/cid.nim

@@ -1,11 +1,11 @@
-## Nim-LibP2P
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module implementes CID (Content IDentifier).
 

libp2p/connmanager.nim

@@ -1,11 +1,11 @@
-## Nim-LibP2P
-## Copyright (c) 2020 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 {.push raises: [Defect].}
 

libp2p/crypto/chacha20poly1305.nim

@@ -1,11 +1,11 @@
-## Nim-Libp2p
-## Copyright (c) 2020-2022 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module integrates BearSSL ChaCha20+Poly1305
 ##
@@ -17,17 +17,10 @@
 
 {.push raises: [Defect].}
 
-import bearssl
+import bearssl/blockx
 from stew/assign2 import assign
 from stew/ranges/ptr_arith import baseAddr
 
-# have to do this due to a nim bug and raises[] on callbacks
-# https://github.com/nim-lang/Nim/issues/13905
-proc ourPoly1305CtmulRun*(key: pointer; iv: pointer; data: pointer; len: int;
-                      aad: pointer; aadLen: int; tag: pointer; ichacha: pointer;
-                      encrypt: cint) {.cdecl, importc: "br_poly1305_ctmul_run",
-                                     header: "bearssl_block.h".}
-
 const
   ChaChaPolyKeySize = 32
   ChaChaPolyNonceSize = 12
@@ -67,15 +60,16 @@ proc encrypt*(_: type[ChaChaPoly],
          else:
            nil
 
-  ourPoly1305CtmulRun(
+  poly1305CtmulRun(
     unsafeAddr key[0],
     unsafeAddr nonce[0],
     baseAddr(data),
-    data.len,
+    uint(data.len),
     ad,
-    aad.len,
+    uint(aad.len),
     baseAddr(tag),
-    chacha20CtRun,
+    # cast is required to workaround https://github.com/nim-lang/Nim/issues/13905
+    cast[Chacha20Run](chacha20CtRun),
     #[encrypt]# 1.cint)
 
 proc decrypt*(_: type[ChaChaPoly],
@@ -90,13 +84,14 @@ proc decrypt*(_: type[ChaChaPoly],
          else:
            nil
 
-  ourPoly1305CtmulRun(
+  poly1305CtmulRun(
     unsafeAddr key[0],
     unsafeAddr nonce[0],
     baseAddr(data),
-    data.len,
+    uint(data.len),
     ad,
-    aad.len,
+    uint(aad.len),
     baseAddr(tag),
-    chacha20CtRun,
+    # cast is required to workaround https://github.com/nim-lang/Nim/issues/13905
+    cast[Chacha20Run](chacha20CtRun),
     #[decrypt]# 0.cint)

libp2p/crypto/crypto.nim

@@ -1,11 +1,11 @@
-## Nim-Libp2p
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module implements Public Key and Private Key interface for libp2p.
 {.push raises: [Defect].}
@@ -69,17 +69,17 @@ when supported(PKScheme.Secp256k1):
 # We are still importing `ecnist` because, it is used for SECIO handshake,
 # but it will be impossible to create ECNIST keys or import ECNIST keys.
 
-import ecnist, bearssl
+import ecnist, bearssl/rand, bearssl/hash as bhash
 import ../protobuf/minprotobuf, ../vbuffer, ../multihash, ../multicodec
 import nimcrypto/[rijndael, twofish, sha2, hash, hmac]
 # We use `ncrutils` for constant-time hexadecimal encoding/decoding procedures.
 import nimcrypto/utils as ncrutils
 import ../utility
 import stew/results
-export results
+export results, utility
 
 # This is workaround for Nim's `import` bug
-export rijndael, twofish, sha2, hash, hmac, ncrutils
+export rijndael, twofish, sha2, hash, hmac, ncrutils, rand
 
 type
   DigestSheme* = enum
@@ -158,26 +158,28 @@ type
 template orError*(exp: untyped, err: untyped): untyped =
   (exp.mapErr do (_: auto) -> auto: err)
 
-proc newRng*(): ref BrHmacDrbgContext =
+proc newRng*(): ref HmacDrbgContext =
   # You should only create one instance of the RNG per application / library
   # Ref is used so that it can be shared between components
   # TODO consider moving to bearssl
-  var seeder = brPrngSeederSystem(nil)
+  var seeder = prngSeederSystem(nil)
   if seeder == nil:
     return nil
 
-  var rng = (ref BrHmacDrbgContext)()
-  brHmacDrbgInit(addr rng[], addr sha256Vtable, nil, 0)
+  var rng = (ref HmacDrbgContext)()
+  hmacDrbgInit(rng[], addr sha256Vtable, nil, 0)
   if seeder(addr rng.vtable) == 0:
     return nil
   rng
 
 proc shuffle*[T](
-  rng: ref BrHmacDrbgContext,
+  rng: ref HmacDrbgContext,
   x: var openArray[T]) =
 
+  if x.len == 0: return
+
   var randValues = newSeqUninitialized[byte](len(x) * 2)
-  brHmacDrbgGenerate(rng[], randValues)
+  hmacDrbgGenerate(rng[], randValues)
 
   for i in countdown(x.high, 1):
     let
@@ -186,7 +188,7 @@ proc shuffle*[T](
     swap(x[i], x[y])
 
 proc random*(T: typedesc[PrivateKey], scheme: PKScheme,
-             rng: var BrHmacDrbgContext,
+             rng: var HmacDrbgContext,
              bits = RsaDefaultKeySize): CryptoResult[PrivateKey] =
   ## Generate random private key for scheme ``scheme``.
   ##
@@ -218,7 +220,7 @@ proc random*(T: typedesc[PrivateKey], scheme: PKScheme,
     else:
       err(SchemeError)
 
-proc random*(T: typedesc[PrivateKey], rng: var BrHmacDrbgContext,
+proc random*(T: typedesc[PrivateKey], rng: var HmacDrbgContext,
              bits = RsaDefaultKeySize): CryptoResult[PrivateKey] =
   ## Generate random private key using default public-key cryptography scheme.
   ##
@@ -242,7 +244,7 @@ proc random*(T: typedesc[PrivateKey], rng: var BrHmacDrbgContext,
     err(SchemeError)
 
 proc random*(T: typedesc[KeyPair], scheme: PKScheme,
-             rng: var BrHmacDrbgContext,
+             rng: var HmacDrbgContext,
              bits = RsaDefaultKeySize): CryptoResult[KeyPair] =
   ## Generate random key pair for scheme ``scheme``.
   ##
@@ -282,7 +284,7 @@ proc random*(T: typedesc[KeyPair], scheme: PKScheme,
     else:
       err(SchemeError)
 
-proc random*(T: typedesc[KeyPair], rng: var BrHmacDrbgContext,
+proc random*(T: typedesc[KeyPair], rng: var HmacDrbgContext,
              bits = RsaDefaultKeySize): CryptoResult[KeyPair] =
   ## Generate random private pair of keys using default public-key cryptography
   ## scheme.
@@ -870,7 +872,7 @@ proc mac*(secret: Secret, id: int): seq[byte] {.inline.} =
 
 proc ephemeral*(
     scheme: ECDHEScheme,
-    rng: var BrHmacDrbgContext): CryptoResult[EcKeyPair] =
+    rng: var HmacDrbgContext): CryptoResult[EcKeyPair] =
   ## Generate ephemeral keys used to perform ECDHE.
   var keypair: EcKeyPair
   if scheme == Secp256r1:
@@ -882,7 +884,7 @@ proc ephemeral*(
   ok(keypair)
 
 proc ephemeral*(
-    scheme: string, rng: var BrHmacDrbgContext): CryptoResult[EcKeyPair] =
+    scheme: string, rng: var HmacDrbgContext): CryptoResult[EcKeyPair] =
   ## Generate ephemeral keys used to perform ECDHE using string encoding.
   ##
   ## Currently supported encoding strings are P-256, P-384, P-521, if encoding

libp2p/crypto/curve25519.nim

@@ -1,11 +1,11 @@
-## Nim-Libp2p
-## Copyright (c) 2020-2022 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022-2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module integrates BearSSL Cyrve25519 mul and mulgen
 ##
@@ -17,7 +17,7 @@
 
 {.push raises: [Defect].}
 
-import bearssl
+import bearssl/[ec, rand, hash]
 import stew/results
 from stew/assign2 import assign
 export results
@@ -46,7 +46,7 @@ proc byteswap(buf: var Curve25519Key) {.inline.} =
     buf[31 - i] = x
 
 proc mul*(_: type[Curve25519], point: var Curve25519Key, multiplier: Curve25519Key) =
-  let defaultBrEc = brEcGetDefault()
+  let defaultBrEc = ecGetDefault()
 
   # multiplier needs to be big-endian
   var
@@ -54,15 +54,15 @@ proc mul*(_: type[Curve25519], point: var Curve25519Key, multiplier: Curve25519K
   multiplierBs.byteswap()
   let
     res = defaultBrEc.mul(
-      cast[pcuchar](addr point[0]),
+      addr point[0],
       Curve25519KeySize,
-      cast[pcuchar](addr multiplierBs[0]),
+      addr multiplierBs[0],
       Curve25519KeySize,
       EC_curve25519)
   assert res == 1
 
 proc mulgen(_: type[Curve25519], dst: var Curve25519Key, point: Curve25519Key) =
-  let defaultBrEc = brEcGetDefault()
+  let defaultBrEc = ecGetDefault()
 
   var
     rpoint = point
@@ -70,8 +70,8 @@ proc mulgen(_: type[Curve25519], dst: var Curve25519Key, point: Curve25519Key) =
 
   let
     size = defaultBrEc.mulgen(
-      cast[pcuchar](addr dst[0]),
-      cast[pcuchar](addr rpoint[0]),
+      addr dst[0],
+      addr rpoint[0],
       Curve25519KeySize,
       EC_curve25519)
   
@@ -80,10 +80,10 @@ proc mulgen(_: type[Curve25519], dst: var Curve25519Key, point: Curve25519Key) =
 proc public*(private: Curve25519Key): Curve25519Key =
   Curve25519.mulgen(result, private)
 
-proc random*(_: type[Curve25519Key], rng: var BrHmacDrbgContext): Curve25519Key =
+proc random*(_: type[Curve25519Key], rng: var HmacDrbgContext): Curve25519Key =
   var res: Curve25519Key
-  let defaultBrEc = brEcGetDefault()
-  let len = brEcKeygen(
+  let defaultBrEc = ecGetDefault()
+  let len = ecKeygen(
     addr rng.vtable, defaultBrEc, nil, addr res[0], EC_curve25519)
   # Per bearssl documentation, the keygen only fails if the curve is
   # unrecognised -

libp2p/crypto/ecnist.nim

@@ -1,11 +1,11 @@
-## Nim-Libp2p
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module implements constant-time ECDSA and ECDHE for NIST elliptic
 ## curves secp256r1, secp384r1 and secp521r1.
@@ -16,7 +16,7 @@
 
 {.push raises: [Defect].}
 
-import bearssl
+import bearssl/[ec, rand, hash]
 # We use `ncrutils` for constant-time hexadecimal encoding/decoding procedures.
 import nimcrypto/utils as ncrutils
 import minasn1
@@ -40,12 +40,12 @@ const
 
 type
   EcPrivateKey* = ref object
-    buffer*: array[BR_EC_KBUF_PRIV_MAX_SIZE, byte]
-    key*: BrEcPrivateKey
+    buffer*: array[EC_KBUF_PRIV_MAX_SIZE, byte]
+    key*: ec.EcPrivateKey
 
   EcPublicKey* = ref object
-    buffer*: array[BR_EC_KBUF_PUB_MAX_SIZE, byte]
-    key*: BrEcPublicKey
+    buffer*: array[EC_KBUF_PUB_MAX_SIZE, byte]
+    key*: ec.EcPublicKey
 
   EcKeyPair* = object
     seckey*: EcPrivateKey
@@ -55,9 +55,9 @@ type
     buffer*: seq[byte]
 
   EcCurveKind* = enum
-    Secp256r1 = BR_EC_SECP256R1,
-    Secp384r1 = BR_EC_SECP384R1,
-    Secp521r1 = BR_EC_SECP521R1
+    Secp256r1 = EC_SECP256R1,
+    Secp384r1 = EC_SECP384R1,
+    Secp521r1 = EC_SECP521R1
 
   EcPKI* = EcPrivateKey | EcPublicKey | EcSignature
 
@@ -101,15 +101,15 @@ proc checkScalar(scalar: openArray[byte], curve: cint): uint32 =
   ##   - ``scalar`` is lower than the curve ``order``.
   ##
   ## Otherwise, return ``0``.
-  var impl = brEcGetDefault()
-  var orderlen = 0
-  var order = cast[ptr UncheckedArray[byte]](impl.order(curve, addr orderlen))
+  var impl = ecGetDefault()
+  var orderlen: uint = 0
+  var order = cast[ptr UncheckedArray[byte]](impl.order(curve, orderlen))
 
   var z = 0'u32
   var c = 0'i32
   for u in scalar:
     z = z or u
-  if len(scalar) == orderlen:
+  if len(scalar) == int(orderlen):
     for i in 0..<len(scalar):
       c = c or (-(cast[int32](EQ0(c))) and CMP(scalar[i], order[i]))
   else:
@@ -119,12 +119,12 @@ proc checkScalar(scalar: openArray[byte], curve: cint): uint32 =
 proc checkPublic(key: openArray[byte], curve: cint): uint32 =
   ## Return ``1`` if public key ``key`` is on curve.
   var ckey = @key
-  var x = [0x00'u8, 0x01'u8]
-  var impl = brEcGetDefault()
-  var orderlen = 0
-  discard impl.order(curve, addr orderlen)
-  result = impl.mul(cast[ptr char](unsafeAddr ckey[0]), len(ckey),
-                    cast[ptr char](addr x[0]), len(x), curve)
+  var x = [byte 0x00, 0x01]
+  var impl = ecGetDefault()
+  var orderlen: uint = 0
+  discard impl.order(curve, orderlen)
+  result = impl.mul(unsafeAddr ckey[0], uint(len(ckey)),
+                    addr x[0], uint(len(x)), curve)
 
 proc getOffset(pubkey: EcPublicKey): int {.inline.} =
   let o = cast[uint](pubkey.key.q) - cast[uint](unsafeAddr pubkey.buffer[0])
@@ -174,7 +174,7 @@ proc copy*[T: EcPKI](dst: var T, src: T): bool =
           dst.buffer = src.buffer
           dst.key.curve = src.key.curve
           dst.key.xlen = length
-          dst.key.x = cast[ptr char](addr dst.buffer[offset])
+          dst.key.x = addr dst.buffer[offset]
           result = true
     elif T is EcPublicKey:
       let length = src.key.qlen
@@ -184,7 +184,7 @@ proc copy*[T: EcPKI](dst: var T, src: T): bool =
           dst.buffer = src.buffer
           dst.key.curve = src.key.curve
           dst.key.qlen = length
-          dst.key.q = cast[ptr char](addr dst.buffer[offset])
+          dst.key.q = addr dst.buffer[offset]
           result = true
     else:
       let length = len(src.buffer)
@@ -230,15 +230,15 @@ proc clear*[T: EcPKI|EcKeyPair](pki: var T) =
 
 proc random*(
     T: typedesc[EcPrivateKey], kind: EcCurveKind,
-    rng: var BrHmacDrbgContext): EcResult[EcPrivateKey] =
+    rng: var HmacDrbgContext): EcResult[EcPrivateKey] =
   ## Generate new random EC private key using BearSSL's HMAC-SHA256-DRBG
   ## algorithm.
   ##
   ## ``kind`` elliptic curve kind of your choice (secp256r1, secp384r1 or
   ## secp521r1).
-  var ecimp = brEcGetDefault()
+  var ecimp = ecGetDefault()
   var res = new EcPrivateKey
-  if brEcKeygen(addr rng.vtable, ecimp,
+  if ecKeygen(addr rng.vtable, ecimp,
                 addr res.key, addr res.buffer[0],
                 cast[cint](kind)) == 0:
     err(EcKeyGenError)
@@ -250,12 +250,12 @@ proc getPublicKey*(seckey: EcPrivateKey): EcResult[EcPublicKey] =
   if isNil(seckey):
     return err(EcKeyIncorrectError)
 
-  var ecimp = brEcGetDefault()
+  var ecimp = ecGetDefault()
   if seckey.key.curve in EcSupportedCurvesCint:
     var res = new EcPublicKey
     assert res.buffer.len > getPublicKeyLength(cast[EcCurveKind](seckey.key.curve))
-    if brEcComputePublicKey(ecimp, addr res.key,
-                            addr res.buffer[0], unsafeAddr seckey.key) == 0:
+    if ecComputePub(ecimp, addr res.key,
+                    addr res.buffer[0], unsafeAddr seckey.key) == 0:
       err(EcKeyIncorrectError)
     else:
       ok(res)
@@ -264,7 +264,7 @@ proc getPublicKey*(seckey: EcPrivateKey): EcResult[EcPublicKey] =
 
 proc random*(
     T: typedesc[EcKeyPair], kind: EcCurveKind,
-    rng: var BrHmacDrbgContext): EcResult[T] =
+    rng: var HmacDrbgContext): EcResult[T] =
   ## Generate new random EC private and public keypair using BearSSL's
   ## HMAC-SHA256-DRBG algorithm.
   ##
@@ -373,24 +373,24 @@ proc toBytes*(seckey: EcPrivateKey, data: var openArray[byte]): EcResult[int] =
     var p = Asn1Composite.init(Asn1Tag.Sequence)
     var c0 = Asn1Composite.init(0)
     var c1 = Asn1Composite.init(1)
-    if seckey.key.curve == BR_EC_SECP256R1:
+    if seckey.key.curve == EC_SECP256R1:
       c0.write(Asn1Tag.Oid, Asn1OidSecp256r1)
-    elif seckey.key.curve == BR_EC_SECP384R1:
+    elif seckey.key.curve == EC_SECP384R1:
       c0.write(Asn1Tag.Oid, Asn1OidSecp384r1)
-    elif seckey.key.curve == BR_EC_SECP521R1:
+    elif seckey.key.curve == EC_SECP521R1:
       c0.write(Asn1Tag.Oid, Asn1OidSecp521r1)
     c0.finish()
     offset = pubkey.getOffset()
     if offset < 0:
       return err(EcKeyIncorrectError)
-    length = pubkey.key.qlen
+    length = int(pubkey.key.qlen)
     c1.write(Asn1Tag.BitString,
              pubkey.buffer.toOpenArray(offset, offset + length - 1))
     c1.finish()
     offset = seckey.getOffset()
     if offset < 0:
       return err(EcKeyIncorrectError)
-    length = seckey.key.xlen
+    length = int(seckey.key.xlen)
     p.write(1'u64)
     p.write(Asn1Tag.OctetString,
             seckey.buffer.toOpenArray(offset, offset + length - 1))
@@ -421,18 +421,18 @@ proc toBytes*(pubkey: EcPublicKey, data: var openArray[byte]): EcResult[int] =
     var p = Asn1Composite.init(Asn1Tag.Sequence)
     var c = Asn1Composite.init(Asn1Tag.Sequence)
     c.write(Asn1Tag.Oid, Asn1OidEcPublicKey)
-    if pubkey.key.curve == BR_EC_SECP256R1:
+    if pubkey.key.curve == EC_SECP256R1:
       c.write(Asn1Tag.Oid, Asn1OidSecp256r1)
-    elif pubkey.key.curve == BR_EC_SECP384R1:
+    elif pubkey.key.curve == EC_SECP384R1:
       c.write(Asn1Tag.Oid, Asn1OidSecp384r1)
-    elif pubkey.key.curve == BR_EC_SECP521R1:
+    elif pubkey.key.curve == EC_SECP521R1:
       c.write(Asn1Tag.Oid, Asn1OidSecp521r1)
     c.finish()
     p.write(c)
     let offset = getOffset(pubkey)
     if offset < 0:
       return err(EcKeyIncorrectError)
-    let length = pubkey.key.qlen
+    let length = int(pubkey.key.qlen)
     p.write(Asn1Tag.BitString,
             pubkey.buffer.toOpenArray(offset, offset + length - 1))
     p.finish()
@@ -638,8 +638,8 @@ proc init*(key: var EcPrivateKey, data: openArray[byte]): Result[void, Asn1Error
   if checkScalar(raw.toOpenArray(), curve) == 1'u32:
     key = new EcPrivateKey
     copyMem(addr key.buffer[0], addr raw.buffer[raw.offset], raw.length)
-    key.key.x = cast[ptr char](addr key.buffer[0])
-    key.key.xlen = raw.length
+    key.key.x = addr key.buffer[0]
+    key.key.xlen = uint(raw.length)
     key.key.curve = curve
     ok()
   else:
@@ -697,8 +697,8 @@ proc init*(pubkey: var EcPublicKey, data: openArray[byte]): Result[void, Asn1Err
   if checkPublic(raw.toOpenArray(), curve) != 0:
     pubkey = new EcPublicKey
     copyMem(addr pubkey.buffer[0], addr raw.buffer[raw.offset], raw.length)
-    pubkey.key.q = cast[ptr char](addr pubkey.buffer[0])
-    pubkey.key.qlen = raw.length
+    pubkey.key.q = addr pubkey.buffer[0]
+    pubkey.key.qlen = uint(raw.length)
     pubkey.key.curve = curve
     ok()
   else:
@@ -785,8 +785,8 @@ proc initRaw*(key: var EcPrivateKey, data: openArray[byte]): bool =
       let length = len(data)
       key = new EcPrivateKey
       copyMem(addr key.buffer[0], unsafeAddr data[0], length)
-      key.key.x = cast[ptr char](addr key.buffer[0])
-      key.key.xlen = length
+      key.key.x = addr key.buffer[0]
+      key.key.xlen = uint(length)
       key.key.curve = curve
       result = true
 
@@ -816,8 +816,8 @@ proc initRaw*(pubkey: var EcPublicKey, data: openArray[byte]): bool =
       let length = len(data)
       pubkey = new EcPublicKey
       copyMem(addr pubkey.buffer[0], unsafeAddr data[0], length)
-      pubkey.key.q = cast[ptr char](addr pubkey.buffer[0])
-      pubkey.key.qlen = length
+      pubkey.key.q = addr pubkey.buffer[0]
+      pubkey.key.qlen = uint(length)
       pubkey.key.curve = curve
       result = true
 
@@ -883,7 +883,7 @@ proc scalarMul*(pub: EcPublicKey, sec: EcPrivateKey): EcPublicKey =
   ##
   ## Returns point in curve as ``pub * sec`` or ``nil`` otherwise.
   doAssert((not isNil(pub)) and (not isNil(sec)))
-  var impl = brEcGetDefault()
+  var impl = ecGetDefault()
   if sec.key.curve in EcSupportedCurvesCint:
     if pub.key.curve == sec.key.curve:
       var key = new EcPublicKey
@@ -891,9 +891,9 @@ proc scalarMul*(pub: EcPublicKey, sec: EcPrivateKey): EcPublicKey =
         let poffset = key.getOffset()
         let soffset = sec.getOffset()
         if poffset >= 0 and soffset >= 0:
-          let res = impl.mul(cast[ptr char](addr key.buffer[poffset]),
+          let res = impl.mul(addr key.buffer[poffset],
                              key.key.qlen,
-                             cast[ptr char](unsafeAddr sec.buffer[soffset]),
+                             unsafeAddr sec.buffer[soffset],
                              sec.key.xlen,
                              key.key.curve)
           if res != 0:
@@ -913,11 +913,11 @@ proc toSecret*(pubkey: EcPublicKey, seckey: EcPrivateKey,
   doAssert((not isNil(pubkey)) and (not isNil(seckey)))
   var mult = scalarMul(pubkey, seckey)
   if not isNil(mult):
-    if seckey.key.curve == BR_EC_SECP256R1:
+    if seckey.key.curve == EC_SECP256R1:
       result = Secret256Length
-    elif seckey.key.curve == BR_EC_SECP384R1:
+    elif seckey.key.curve == EC_SECP384R1:
       result = Secret384Length
-    elif seckey.key.curve == BR_EC_SECP521R1:
+    elif seckey.key.curve == EC_SECP521R1:
       result = Secret521Length
     if len(data) >= result:
       var qplus1 = cast[pointer](cast[uint](mult.key.q) + 1'u)
@@ -941,20 +941,20 @@ proc sign*[T: byte|char](seckey: EcPrivateKey,
   ## Get ECDSA signature of data ``message`` using private key ``seckey``.
   if isNil(seckey):
     return err(EcKeyIncorrectError)
-  var hc: BrHashCompatContext
+  var hc: HashCompatContext
   var hash: array[32, byte]
-  var impl = brEcGetDefault()
+  var impl = ecGetDefault()
   if seckey.key.curve in EcSupportedCurvesCint:
     var sig = new EcSignature
     sig.buffer = newSeq[byte](256)
     var kv = addr sha256Vtable
     kv.init(addr hc.vtable)
     if len(message) > 0:
-      kv.update(addr hc.vtable, unsafeAddr message[0], len(message))
+      kv.update(addr hc.vtable, unsafeAddr message[0], uint(len(message)))
     else:
       kv.update(addr hc.vtable, nil, 0)
-    kv.output(addr hc.vtable, addr hash[0])
-    let res = brEcdsaSignAsn1(impl, kv, addr hash[0], addr seckey.key,
+    kv.out(addr hc.vtable, addr hash[0])
+    let res = ecdsaI31SignAsn1(impl, kv, addr hash[0], addr seckey.key,
                               addr sig.buffer[0])
     # Clear context with initial value
     kv.init(addr hc.vtable)
@@ -974,20 +974,20 @@ proc verify*[T: byte|char](sig: EcSignature, message: openArray[T],
   ## Return ``true`` if message verification succeeded, ``false`` if
   ## verification failed.
   doAssert((not isNil(sig)) and (not isNil(pubkey)))
-  var hc: BrHashCompatContext
+  var hc: HashCompatContext
   var hash: array[32, byte]
-  var impl = brEcGetDefault()
+  var impl = ecGetDefault()
   if pubkey.key.curve in EcSupportedCurvesCint:
     var kv = addr sha256Vtable
     kv.init(addr hc.vtable)
     if len(message) > 0:
-      kv.update(addr hc.vtable, unsafeAddr message[0], len(message))
+      kv.update(addr hc.vtable, unsafeAddr message[0], uint(len(message)))
     else:
       kv.update(addr hc.vtable, nil, 0)
-    kv.output(addr hc.vtable, addr hash[0])
-    let res = brEcdsaVerifyAsn1(impl, addr hash[0], len(hash),
-                                unsafeAddr pubkey.key,
-                                addr sig.buffer[0], len(sig.buffer))
+    kv.out(addr hc.vtable, addr hash[0])
+    let res = ecdsaI31VrfyAsn1(impl, addr hash[0], uint(len(hash)),
+                               unsafeAddr pubkey.key,
+                               addr sig.buffer[0], uint(len(sig.buffer)))
     # Clear context with initial value
     kv.init(addr hc.vtable)
     result = (res == 1)

libp2p/crypto/ed25519/constants.nim

@@ -1,11 +1,11 @@
-## Nim-Libp2p
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module implements ED25519.
 ## This is pure nim implementation of ED25519 ref10.

libp2p/crypto/ed25519/ed25519.nim

@@ -1,11 +1,11 @@
-## Nim-Libp2p
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module implements ED25519.
 ## This code is a port of the public domain, "ref10" implementation of ed25519
@@ -13,7 +13,8 @@
 
 {.push raises: Defect.}
 
-import constants, bearssl
+import bearssl/rand
+import constants
 import nimcrypto/[hash, sha2]
 # We use `ncrutils` for constant-time hexadecimal encoding/decoding procedures.
 import nimcrypto/utils as ncrutils
@@ -21,7 +22,7 @@ import stew/[results, ctops]
 export results
 
 # This workaround needed because of some bugs in Nim Static[T].
-export hash, sha2
+export hash, sha2, rand
 
 const
   EdPrivateKeySize* = 64
@@ -1644,14 +1645,14 @@ proc checkScalar*(scalar: openArray[byte]): uint32 =
     c = -1
   result = NEQ(z, 0'u32) and LT0(c)
 
-proc random*(t: typedesc[EdPrivateKey], rng: var BrHmacDrbgContext): EdPrivateKey =
+proc random*(t: typedesc[EdPrivateKey], rng: var HmacDrbgContext): EdPrivateKey =
   ## Generate new random ED25519 private key using the given random number generator
   var
     point: GeP3
     pk: array[EdPublicKeySize, byte]
     res: EdPrivateKey
 
-  brHmacDrbgGenerate(addr rng, addr res.data[0], 32)
+  hmacDrbgGenerate(rng, res.data.toOpenArray(0, 31))
 
   var hh = sha512.digest(res.data.toOpenArray(0, 31))
   hh.data[0] = hh.data[0] and 0xF8'u8
@@ -1663,14 +1664,14 @@ proc random*(t: typedesc[EdPrivateKey], rng: var BrHmacDrbgContext): EdPrivateKe
 
   res
 
-proc random*(t: typedesc[EdKeyPair], rng: var BrHmacDrbgContext): EdKeyPair =
+proc random*(t: typedesc[EdKeyPair], rng: var HmacDrbgContext): EdKeyPair =
   ## Generate new random ED25519 private and public keypair using OS specific
   ## CSPRNG.
   var
     point: GeP3
     res: EdKeyPair
 
-  brHmacDrbgGenerate(addr rng, addr res.seckey.data[0], 32)
+  hmacDrbgGenerate(rng, res.seckey.data.toOpenArray(0, 31))
 
   var hh = sha512.digest(res.seckey.data.toOpenArray(0, 31))
   hh.data[0] = hh.data[0] and 0xF8'u8

libp2p/crypto/hkdf.nim

@@ -1,40 +1,33 @@
-## Nim-LibP2P
-## Copyright (c) 2020 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 # https://tools.ietf.org/html/rfc5869
 
 {.push raises: [Defect].}
 
 import nimcrypto
-import bearssl
+import bearssl/[kdf, rand, hash]
 
-type
-  BearHKDFContext {.importc: "br_hkdf_context", header: "bearssl_kdf.h".} = object
-  HKDFResult*[len: static int] = array[len, byte]
+type HkdfResult*[len: static int] = array[len, byte]
 
-proc br_hkdf_init(ctx: ptr BearHKDFContext; hashClass: ptr HashClass; salt: pointer; len: csize_t) {.importc: "br_hkdf_init", header: "bearssl_kdf.h", raises: [].}
-proc br_hkdf_inject(ctx: ptr BearHKDFContext; ikm: pointer; len: csize_t) {.importc: "br_hkdf_inject", header: "bearssl_kdf.h", raises: [].}
-proc br_hkdf_flip(ctx: ptr BearHKDFContext) {.importc: "br_hkdf_flip", header: "bearssl_kdf.h", raises: [].}
-proc br_hkdf_produce(ctx: ptr BearHKDFContext; info: pointer; infoLen: csize_t; output: pointer; outputLen: csize_t) {.importc: "br_hkdf_produce", header: "bearssl_kdf.h", raises: [].}
-
-proc hkdf*[T: sha256; len: static int](_: type[T]; salt, ikm, info: openArray[byte]; outputs: var openArray[HKDFResult[len]]) =
+proc hkdf*[T: sha256; len: static int](_: type[T]; salt, ikm, info: openArray[byte]; outputs: var openArray[HkdfResult[len]]) =
   var
-    ctx: BearHKDFContext
-  br_hkdf_init(
-    addr ctx, addr sha256Vtable,
+    ctx: HkdfContext
+  hkdfInit(
+    ctx, addr sha256Vtable,
     if salt.len > 0: unsafeAddr salt[0] else: nil, csize_t(salt.len))
-  br_hkdf_inject(
-    addr ctx, if ikm.len > 0: unsafeAddr ikm[0] else: nil, csize_t(ikm.len))
-  br_hkdf_flip(addr ctx)
+  hkdfInject(
+    ctx, if ikm.len > 0: unsafeAddr ikm[0] else: nil, csize_t(ikm.len))
+  hkdfFlip(ctx)
   for i in 0..outputs.high:
-    br_hkdf_produce(
-      addr ctx,
+    discard hkdfProduce(
+      ctx,
       if info.len > 0: unsafeAddr info[0]
       else: nil, csize_t(info.len),
       addr outputs[i][0], csize_t(outputs[i].len))

libp2p/crypto/minasn1.nim

@@ -1,11 +1,11 @@
-## Nim-Libp2p
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module implements minimal ASN.1 encoding/decoding primitives.
 

libp2p/crypto/rsa.nim

@@ -1,11 +1,11 @@
-## Nim-Libp2p
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module implements constant-time RSA PKCS#1.5 DSA.
 ##
@@ -14,7 +14,7 @@
 ## Copyright(C) 2018 Thomas Pornin <pornin@bolet.org>.
 
 {.push raises: Defect.}
-import bearssl
+import bearssl/[rsa, rand, hash]
 import minasn1
 import stew/[results, ctops]
 # We use `ncrutils` for constant-time hexadecimal encoding/decoding procedures.
@@ -33,41 +33,41 @@ const
     ## Default RSA key size in bits.
 
   RsaOidSha1* = [
-    0x05'u8, 0x2B'u8, 0x0E'u8, 0x03'u8, 0x02'u8, 0x1A'u8
+    byte 0x05, 0x2B, 0x0E, 0x03, 0x02, 0x1A
   ]
     ## RSA PKCS#1.5 SHA-1 hash object identifier.
   RsaOidSha224* = [
-    0x09'u8, 0x60'u8, 0x86'u8, 0x48'u8, 0x01'u8, 0x65'u8, 0x03'u8, 0x04'u8,
-    0x02'u8, 0x04'u8
+    byte 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04,
+    0x02, 0x04
   ]
     ## RSA PKCS#1.5 SHA-224 hash object identifier.
   RsaOidSha256* = [
-    0x09'u8, 0x60'u8, 0x86'u8, 0x48'u8, 0x01'u8, 0x65'u8, 0x03'u8, 0x04'u8,
-    0x02'u8, 0x01'u8
+    byte 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04,
+    0x02, 0x01
   ]
     ## RSA PKCS#1.5 SHA-256 hash object identifier.
   RsaOidSha384* = [
-    0x09'u8, 0x60'u8, 0x86'u8, 0x48'u8, 0x01'u8, 0x65'u8, 0x03'u8, 0x04'u8,
-    0x02'u8, 0x02'u8
+    byte 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04,
+    0x02, 0x02
   ]
     ## RSA PKCS#1.5 SHA-384 hash object identifier.
   RsaOidSha512* = [
-    0x09'u8, 0x60'u8, 0x86'u8, 0x48'u8, 0x01'u8, 0x65'u8, 0x03'u8, 0x04'u8,
-    0x02'u8, 0x03'u8
+    byte 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04,
+    0x02, 0x03
   ]
     ## RSA PKCS#1.5 SHA-512 hash object identifier.
 
 type
   RsaPrivateKey* = ref object
     buffer*: seq[byte]
-    seck*: BrRsaPrivateKey
-    pubk*: BrRsaPublicKey
-    pexp*: ptr char
-    pexplen*: int
+    seck*: rsa.RsaPrivateKey
+    pubk*: rsa.RsaPublicKey
+    pexp*: ptr byte
+    pexplen*: uint
 
   RsaPublicKey* = ref object
     buffer*: seq[byte]
-    key*: BrRsaPublicKey
+    key*: rsa.RsaPublicKey
 
   RsaKeyPair* = RsaPrivateKey
 
@@ -99,8 +99,8 @@ template getFinish(bs, os, ls: untyped): untyped =
   var eo = -1
   if p >= s:
     let so = cast[int](p - s)
-    if so + ls <= len(bs):
-      eo = so + ls - 1
+    if so + int(ls) <= len(bs):
+      eo = so + int(ls) - 1
   eo
 
 template getArray*(bs, os, ls: untyped): untyped =
@@ -109,12 +109,12 @@ template getArray*(bs, os, ls: untyped): untyped =
 template trimZeroes(b: seq[byte], pt, ptlen: untyped) =
   var length = ptlen
   for i in 0..<length:
-    if pt[] != cast[char](0x00'u8):
+    if pt[] != byte(0x00):
       break
-    pt = cast[ptr char](cast[uint](pt) + 1)
+    pt = cast[ptr byte](cast[uint](pt) + 1)
     ptlen -= 1
 
-proc random*[T: RsaKP](t: typedesc[T], rng: var BrHmacDrbgContext,
+proc random*[T: RsaKP](t: typedesc[T], rng: var HmacDrbgContext,
                        bits = DefaultKeySize,
                        pubexp = DefaultPublicExponent): RsaResult[T] =
   ## Generate new random RSA private key using BearSSL's HMAC-SHA256-DRBG
@@ -129,14 +129,14 @@ proc random*[T: RsaKP](t: typedesc[T], rng: var BrHmacDrbgContext,
 
   let
     sko = 0
-    pko = brRsaPrivateKeyBufferSize(bits)
-    eko = pko + brRsaPublicKeyBufferSize(bits)
+    pko = rsaKbufPrivSize(bits)
+    eko = pko + rsaKbufPubSize(bits)
     length = eko + ((bits + 7) shr 3)
 
   let res = new T
   res.buffer = newSeq[byte](length)
 
-  var keygen = brRsaKeygenGetDefault()
+  var keygen = rsaKeygenGetDefault()
 
   if keygen(addr rng.vtable,
             addr res.seck, addr res.buffer[sko],
@@ -145,12 +145,12 @@ proc random*[T: RsaKP](t: typedesc[T], rng: var BrHmacDrbgContext,
     return err(RsaGenError)
 
   let
-    compute = brRsaComputePrivexpGetDefault()
+    compute = rsaComputePrivexpGetDefault()
     computed = compute(addr res.buffer[eko], addr res.seck, pubexp)
   if computed == 0:
     return err(RsaGenError)
 
-  res.pexp = cast[ptr char](addr res.buffer[eko])
+  res.pexp = addr res.buffer[eko]
   res.pexplen = computed
 
   trimZeroes(res.buffer, res.seck.p, res.seck.plen)
@@ -169,12 +169,12 @@ proc copy*[T: RsaPKI](key: T): T =
   doAssert(not isNil(key))
   when T is RsaPrivateKey:
     if len(key.buffer) > 0:
-      let length = key.seck.plen + key.seck.qlen + key.seck.dplen +
-                   key.seck.dqlen + key.seck.iqlen + key.pubk.nlen +
-                   key.pubk.elen + key.pexplen
+      let length = key.seck.plen.uint + key.seck.qlen.uint + key.seck.dplen.uint +
+                   key.seck.dqlen.uint + key.seck.iqlen.uint + key.pubk.nlen.uint +
+                   key.pubk.elen.uint + key.pexplen.uint
       result = new RsaPrivateKey
       result.buffer = newSeq[byte](length)
-      let po = 0
+      let po: uint = 0
       let qo = po + key.seck.plen
       let dpo = qo + key.seck.qlen
       let dqo = dpo + key.seck.dplen
@@ -190,14 +190,14 @@ proc copy*[T: RsaPKI](key: T): T =
       copyMem(addr result.buffer[no], key.pubk.n, key.pubk.nlen)
       copyMem(addr result.buffer[eo], key.pubk.e, key.pubk.elen)
       copyMem(addr result.buffer[peo], key.pexp, key.pexplen)
-      result.seck.p = cast[ptr char](addr result.buffer[po])
-      result.seck.q = cast[ptr char](addr result.buffer[qo])
-      result.seck.dp = cast[ptr char](addr result.buffer[dpo])
-      result.seck.dq = cast[ptr char](addr result.buffer[dqo])
-      result.seck.iq = cast[ptr char](addr result.buffer[iqo])
-      result.pubk.n = cast[ptr char](addr result.buffer[no])
-      result.pubk.e = cast[ptr char](addr result.buffer[eo])
-      result.pexp = cast[ptr char](addr result.buffer[peo])
+      result.seck.p = addr result.buffer[po]
+      result.seck.q = addr result.buffer[qo]
+      result.seck.dp = addr result.buffer[dpo]
+      result.seck.dq = addr result.buffer[dqo]
+      result.seck.iq = addr result.buffer[iqo]
+      result.pubk.n = addr result.buffer[no]
+      result.pubk.e = addr result.buffer[eo]
+      result.pexp = addr result.buffer[peo]
       result.seck.plen = key.seck.plen
       result.seck.qlen = key.seck.qlen
       result.seck.dplen = key.seck.dplen
@@ -231,8 +231,8 @@ proc getPublicKey*(key: RsaPrivateKey): RsaPublicKey =
   let length = key.pubk.nlen + key.pubk.elen
   result = new RsaPublicKey
   result.buffer = newSeq[byte](length)
-  result.key.n = cast[ptr char](addr result.buffer[0])
-  result.key.e = cast[ptr char](addr result.buffer[key.pubk.nlen])
+  result.key.n = addr result.buffer[0]
+  result.key.e = addr result.buffer[key.pubk.nlen]
   copyMem(addr result.buffer[0], cast[pointer](key.pubk.n), key.pubk.nlen)
   copyMem(addr result.buffer[key.pubk.nlen], cast[pointer](key.pubk.e),
           key.pubk.elen)
@@ -472,22 +472,22 @@ proc init*(key: var RsaPrivateKey, data: openArray[byte]): Result[void, Asn1Erro
      len(rawdp) > 0 and len(rawdq) > 0 and len(rawiq) > 0:
     key = new RsaPrivateKey
     key.buffer = @data
-    key.pubk.n = cast[ptr char](addr key.buffer[rawn.offset])
-    key.pubk.e = cast[ptr char](addr key.buffer[rawpube.offset])
-    key.seck.p = cast[ptr char](addr key.buffer[rawp.offset])
-    key.seck.q = cast[ptr char](addr key.buffer[rawq.offset])
-    key.seck.dp = cast[ptr char](addr key.buffer[rawdp.offset])
-    key.seck.dq = cast[ptr char](addr key.buffer[rawdq.offset])
-    key.seck.iq = cast[ptr char](addr key.buffer[rawiq.offset])
-    key.pexp = cast[ptr char](addr key.buffer[rawprie.offset])
-    key.pubk.nlen = len(rawn)
-    key.pubk.elen = len(rawpube)
-    key.seck.plen = len(rawp)
-    key.seck.qlen = len(rawq)
-    key.seck.dplen = len(rawdp)
-    key.seck.dqlen = len(rawdq)
-    key.seck.iqlen = len(rawiq)
-    key.pexplen = len(rawprie)
+    key.pubk.n = addr key.buffer[rawn.offset]
+    key.pubk.e = addr key.buffer[rawpube.offset]
+    key.seck.p = addr key.buffer[rawp.offset]
+    key.seck.q = addr key.buffer[rawq.offset]
+    key.seck.dp = addr key.buffer[rawdp.offset]
+    key.seck.dq = addr key.buffer[rawdq.offset]
+    key.seck.iq = addr key.buffer[rawiq.offset]
+    key.pexp = addr key.buffer[rawprie.offset]
+    key.pubk.nlen = uint(len(rawn))
+    key.pubk.elen = uint(len(rawpube))
+    key.seck.plen = uint(len(rawp))
+    key.seck.qlen = uint(len(rawq))
+    key.seck.dplen = uint(len(rawdp))
+    key.seck.dqlen = uint(len(rawdq))
+    key.seck.iqlen = uint(len(rawiq))
+    key.pexplen = uint(len(rawprie))
     key.seck.nBitlen = cast[uint32](len(rawn) shl 3)
     ok()
   else:
@@ -554,10 +554,10 @@ proc init*(key: var RsaPublicKey, data: openArray[byte]): Result[void, Asn1Error
   if len(rawn) >= (MinKeySize shr 3) and len(rawe) > 0:
     key = new RsaPublicKey
     key.buffer = @data
-    key.key.n = cast[ptr char](addr key.buffer[rawn.offset])
-    key.key.e = cast[ptr char](addr key.buffer[rawe.offset])
-    key.key.nlen = len(rawn)
-    key.key.elen = len(rawe)
+    key.key.n = addr key.buffer[rawn.offset]
+    key.key.e = addr key.buffer[rawe.offset]
+    key.key.nlen = uint(len(rawn))
+    key.key.elen = uint(len(rawe))
     ok()
   else:
     err(Asn1Error.Incorrect)
@@ -749,22 +749,22 @@ proc sign*[T: byte|char](key: RsaPrivateKey,
   if isNil(key):
     return err(RsaKeyIncorrectError)
 
-  var hc: BrHashCompatContext
+  var hc: HashCompatContext
   var hash: array[32, byte]
-  let impl = BrRsaPkcs1SignGetDefault()
+  let impl = rsaPkcs1SignGetDefault()
   var res = new RsaSignature
   res.buffer = newSeq[byte]((key.seck.nBitlen + 7) shr 3)
   var kv = addr sha256Vtable
   kv.init(addr hc.vtable)
   if len(message) > 0:
-    kv.update(addr hc.vtable, unsafeAddr message[0], len(message))
+    kv.update(addr hc.vtable, unsafeAddr message[0], uint(len(message)))
   else:
     kv.update(addr hc.vtable, nil, 0)
-  kv.output(addr hc.vtable, addr hash[0])
+  kv.out(addr hc.vtable, addr hash[0])
   var oid = RsaOidSha256
-  let implRes = impl(cast[ptr char](addr oid[0]),
-                 cast[ptr char](addr hash[0]), len(hash),
-                 addr key.seck, cast[ptr char](addr res.buffer[0]))
+  let implRes = impl(addr oid[0],
+                 addr hash[0], uint(len(hash)),
+                 addr key.seck, addr res.buffer[0])
   if implRes == 0:
     err(RsaSignatureError)
   else:
@@ -779,20 +779,20 @@ proc verify*[T: byte|char](sig: RsaSignature, message: openArray[T],
   ## verification failed.
   doAssert((not isNil(sig)) and (not isNil(pubkey)))
   if len(sig.buffer) > 0:
-    var hc: BrHashCompatContext
+    var hc: HashCompatContext
     var hash: array[32, byte]
     var check: array[32, byte]
-    var impl = BrRsaPkcs1VrfyGetDefault()
+    var impl = rsaPkcs1VrfyGetDefault()
     var kv = addr sha256Vtable
     kv.init(addr hc.vtable)
     if len(message) > 0:
-      kv.update(addr hc.vtable, unsafeAddr message[0], len(message))
+      kv.update(addr hc.vtable, unsafeAddr message[0], uint(len(message)))
     else:
       kv.update(addr hc.vtable, nil, 0)
-    kv.output(addr hc.vtable, addr hash[0])
+    kv.out(addr hc.vtable, addr hash[0])
     var oid = RsaOidSha256
-    let res = impl(cast[ptr char](addr sig.buffer[0]), len(sig.buffer),
-                   cast[ptr char](addr oid[0]),
-                   len(check), addr pubkey.key, cast[ptr char](addr check[0]))
+    let res = impl(addr sig.buffer[0], uint(len(sig.buffer)),
+                   addr oid[0],
+                   uint(len(check)), addr pubkey.key, addr check[0])
     if res == 1:
       result = equalMem(addr check[0], addr hash[0], len(hash))

libp2p/crypto/secp.nim

@@ -1,20 +1,21 @@
-## Nim-Libp2p
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 {.push raises: [Defect].}
 
+import bearssl/rand
 import
-  secp256k1, bearssl,
+  secp256k1,
   stew/[byteutils, results],
   nimcrypto/[hash, sha2]
 
-export sha2, results
+export sha2, results, rand
 
 const
   SkRawPrivateKeySize* = 256 div 8
@@ -34,17 +35,18 @@ type
 template pubkey*(v: SkKeyPair): SkPublicKey = SkPublicKey(secp256k1.SkKeyPair(v).pubkey)
 template seckey*(v: SkKeyPair): SkPrivateKey = SkPrivateKey(secp256k1.SkKeyPair(v).seckey)
 
-proc random*(t: typedesc[SkPrivateKey], rng: var BrHmacDrbgContext): SkPrivateKey =
-  let rngPtr = unsafeAddr rng # doesn't escape
+proc random*(t: typedesc[SkPrivateKey], rng: var HmacDrbgContext): SkPrivateKey =
+  #TODO is there a better way?
+  var rngPtr = addr rng
   proc callRng(data: var openArray[byte]) =
-    brHmacDrbgGenerate(rngPtr[], data)
+    hmacDrbgGenerate(rngPtr[], data)
 
   SkPrivateKey(SkSecretKey.random(callRng))
 
-proc random*(t: typedesc[SkKeyPair], rng: var BrHmacDrbgContext): SkKeyPair =
-  let rngPtr = unsafeAddr rng # doesn't escape
+proc random*(t: typedesc[SkKeyPair], rng: var HmacDrbgContext): SkKeyPair =
+  let rngPtr = addr rng
   proc callRng(data: var openArray[byte]) =
-    brHmacDrbgGenerate(rngPtr[], data)
+    hmacDrbgGenerate(rngPtr[], data)
 
   SkKeyPair(secp256k1.SkKeyPair.random(callRng))
 

libp2p/daemon/daemonapi.nim

@@ -1,11 +1,11 @@
-## Nim-LibP2P
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 {.push raises: [Defect].}
 

libp2p/daemon/transpool.nim

@@ -1,11 +1,11 @@
-## Nim-Libp2p
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 {.push raises: [Defect].}
 

libp2p/debugutils.nim

@@ -1,11 +1,11 @@
-## Nim-LibP2P
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## To enable dump of all incoming and outgoing unencrypted messages you need
 ## to compile project with ``-d:libp2p_dump`` compile-time option. When this

libp2p/dial.nim

@@ -1,11 +1,11 @@
-## Nim-LibP2P
-## Copyright (c) 2021 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 {.push raises: [Defect].}
 

libp2p/dialer.nim

@@ -1,11 +1,11 @@
-## Nim-LibP2P
-## Copyright (c) 2021 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 import std/[sugar, tables]
 

libp2p/multiaddress.nim

@@ -1,23 +1,24 @@
-## Nim-Libp2p
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module implements MultiAddress.
 
 {.push raises: [Defect].}
+{.push public.}
 
 import pkg/chronos
 import std/[nativesockets, hashes]
 import tables, strutils, sets, stew/shims/net
 import multicodec, multihash, multibase, transcoder, vbuffer, peerid,
-       protobuf/minprotobuf, errors
+       protobuf/minprotobuf, errors, utility
 import stew/[base58, base32, endians2, results]
-export results, minprotobuf, vbuffer
+export results, minprotobuf, vbuffer, utility
 
 type
   MAKind* = enum

libp2p/multibase.nim

@@ -1,11 +1,11 @@
-## Nim-Libp2p
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module implements MultiBase.
 ##

libp2p/multicodec.nim

@@ -1,11 +1,11 @@
-## Nim-Libp2p
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not BE copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not BE copied, modified, or distributed except according to
+# those terms.
 
 ## This module implements MultiCodec.
 

libp2p/multihash.nim

@@ -1,11 +1,11 @@
-## Nim-Libp2p
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module implements MultiHash.
 ## Supported hashes are:

libp2p/multistream.nim

@@ -1,15 +1,15 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 {.push raises: [Defect].}
 
-import std/[strutils]
+import std/[strutils, sequtils]
 import chronos, chronicles, stew/byteutils
 import stream/connection,
        protocols/protocol
@@ -209,3 +209,9 @@ proc addHandler*(m: MultistreamSelect,
   m.handlers.add(HandlerHolder(protos: @[codec],
                                protocol: protocol,
                                match: matcher))
+
+proc start*(m: MultistreamSelect) {.async.} =
+  await allFutures(m.handlers.mapIt(it.protocol.start()))
+
+proc stop*(m: MultistreamSelect) {.async.} =
+  await allFutures(m.handlers.mapIt(it.protocol.stop()))

libp2p/muxers/mplex/coder.nim

@@ -1,11 +1,11 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 {.push raises: [Defect].}
 

libp2p/muxers/mplex/lpchannel.nim

@@ -1,11 +1,11 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 {.push raises: [Defect].}
 
@@ -78,6 +78,8 @@ proc open*(s: LPChannel) {.async, gcsafe.} =
   try:
     await s.conn.writeMsg(s.id, MessageType.New, s.name)
     s.isOpen = true
+  except CancelledError as exc:
+    raise exc
   except CatchableError as exc:
     await s.conn.close()
     raise exc
@@ -221,6 +223,11 @@ proc completeWrite(
         libp2p_protocols_bytes.inc(msgLen.int64, labelValues=[s.tag, "out"])
 
     s.activity = true
+  except CancelledError as exc:
+    # Chronos may still send the data
+    raise exc
+  except LPStreamClosedError as exc:
+    raise exc
   except CatchableError as exc:
     trace "exception in lpchannel write handler", s, msg = exc.msg
     await s.reset()

libp2p/muxers/mplex/mplex.nim

@@ -1,11 +1,11 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 {.push raises: [Defect].}
 

libp2p/muxers/muxer.nim

@@ -1,11 +1,11 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 {.push raises: [Defect].}
 

libp2p/muxers/yamux/yamux.nim

@@ -0,0 +1,468 @@
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+{.push raises: [Defect].}
+
+import sequtils, std/[tables]
+import chronos, chronicles, stew/[endians2, byteutils, objects]
+import ../muxer,
+       ../../stream/connection
+
+export muxer
+
+logScope:
+  topics = "libp2p yamux"
+
+const
+  YamuxCodec* = "/yamux/1.0.0"
+  YamuxVersion = 0.uint8
+  DefaultWindowSize = 256000
+
+type
+  YamuxError* = object of CatchableError
+
+  MsgType = enum
+    Data = 0x0
+    WindowUpdate = 0x1
+    Ping = 0x2
+    GoAway = 0x3
+
+  MsgFlags {.size: 2.} = enum
+    Syn
+    Ack
+    Fin
+    Rst
+
+  GoAwayStatus = enum
+    NormalTermination = 0x0,
+    ProtocolError = 0x1,
+    InternalError = 0x2,
+
+  YamuxHeader = object
+    version: uint8
+    msgType: MsgType
+    flags: set[MsgFlags]
+    streamId: uint32
+    length: uint32
+
+proc readHeader(conn: LPStream): Future[YamuxHeader] {.async, gcsafe.} =
+  var buffer: array[12, byte]
+  await conn.readExactly(addr buffer[0], 12)
+
+  result.version = buffer[0]
+  let flags = fromBytesBE(uint16, buffer[2..3])
+  if not result.msgType.checkedEnumAssign(buffer[1]) or flags notin 0'u16..15'u16:
+     raise newException(YamuxError, "Wrong header")
+  result.flags = cast[set[MsgFlags]](flags)
+  result.streamId = fromBytesBE(uint32, buffer[4..7])
+  result.length = fromBytesBE(uint32, buffer[8..11])
+  return result
+
+proc `$`(header: YamuxHeader): string =
+  result = "{" & $header.msgType & ", "
+  result &= "{" & header.flags.foldl(if a != "": a & ", " & $b else: $b, "") & "}, "
+  result &= "streamId: " & $header.streamId & ", "
+  result &= "length: " & $header.length & "}"
+
+proc encode(header: YamuxHeader): array[12, byte] =
+  result[0] = header.version
+  result[1] = uint8(header.msgType)
+  result[2..3] = toBytesBE(cast[uint16](header.flags))
+  result[4..7] = toBytesBE(header.streamId)
+  result[8..11] = toBytesBE(header.length)
+
+proc write(conn: LPStream, header: YamuxHeader): Future[void] {.gcsafe.} =
+  trace "write directly on stream", h = $header
+  var buffer = header.encode()
+  return conn.write(@buffer)
+
+proc ping(T: type[YamuxHeader], flag: MsgFlags, pingData: uint32): T =
+  T(
+    version: YamuxVersion,
+    msgType: MsgType.Ping,
+    flags: {flag},
+    length: pingData
+  )
+
+proc goAway(T: type[YamuxHeader], status: GoAwayStatus): T =
+  T(
+    version: YamuxVersion,
+    msgType: MsgType.GoAway,
+    length: uint32(status)
+  )
+
+proc data(
+  T: type[YamuxHeader],
+  streamId: uint32,
+  length: uint32 = 0,
+  flags: set[MsgFlags] = {},
+  ): T =
+  T(
+    version: YamuxVersion,
+    msgType: MsgType.Data,
+    length: length,
+    flags: flags,
+    streamId: streamId
+  )
+
+proc windowUpdate(
+  T: type[YamuxHeader],
+  streamId: uint32,
+  delta: uint32,
+  flags: set[MsgFlags] = {},
+  ): T =
+  T(
+    version: YamuxVersion,
+    msgType: MsgType.WindowUpdate,
+    length: delta,
+    flags: flags,
+    streamId: streamId
+  )
+
+type
+  ToSend = tuple
+    data: seq[byte]
+    sent: int
+    fut: Future[void]
+  YamuxChannel* = ref object of Connection
+    id: uint32
+    recvWindow: int
+    sendWindow: int
+    maxRecvWindow: int
+    conn: Connection
+    isSrc: bool
+    opened: bool
+    isSending: bool
+    sendQueue: seq[ToSend]
+    recvQueue: seq[byte]
+    isReset: bool
+    closedRemotely: Future[void]
+    closedLocally: bool
+    receivedData: AsyncEvent
+    returnedEof: bool
+
+proc `$`(channel: YamuxChannel): string =
+  result = if channel.conn.dir == Out: "=> " else: "<= "
+  result &= $channel.id
+  var s: seq[string] = @[]
+  if channel.closedRemotely.done():
+    s.add("ClosedRemotely")
+  if channel.closedLocally:
+    s.add("ClosedLocally")
+  if channel.isReset:
+    s.add("Reset")
+  if s.len > 0:
+    result &= " {" & s.foldl(if a != "": a & ", " & b else: b, "") & "}"
+
+proc sendQueueBytes(channel: YamuxChannel, limit: bool = false): int =
+  for (elem, sent, _) in channel.sendQueue:
+    result.inc(min(elem.len - sent, if limit: channel.maxRecvWindow div 3 else: elem.len - sent))
+
+proc actuallyClose(channel: YamuxChannel) {.async.} =
+  if channel.closedLocally and channel.sendQueue.len == 0 and
+     channel.closedRemotely.done():
+    await procCall Connection(channel).closeImpl()
+
+proc remoteClosed(channel: YamuxChannel) {.async.} =
+  if not channel.closedRemotely.done():
+    channel.closedRemotely.complete()
+    await channel.actuallyClose()
+
+method closeImpl*(channel: YamuxChannel) {.async, gcsafe.} =
+  if not channel.closedLocally:
+    channel.closedLocally = true
+
+    if channel.isReset == false and channel.sendQueue.len == 0:
+      await channel.conn.write(YamuxHeader.data(channel.id, 0, {Fin}))
+    await channel.actuallyClose()
+
+proc reset(channel: YamuxChannel, isLocal: bool = false) {.async.} =
+  if not channel.isReset:
+    trace "Reset channel"
+    channel.isReset = true
+    for (d, s, fut) in channel.sendQueue:
+      fut.fail(newLPStreamEOFError())
+    channel.sendQueue = @[]
+    channel.recvQueue = @[]
+    channel.sendWindow = 0
+    if not channel.closedLocally:
+      if isLocal:
+        try: await channel.conn.write(YamuxHeader.data(channel.id, 0, {Rst}))
+        except LPStreamEOFError as exc: discard
+      await channel.close()
+    if not channel.closedRemotely.done():
+      await channel.remoteClosed()
+    channel.receivedData.fire()
+  if not isLocal:
+    # If we reset locally, we want to flush up to a maximum of recvWindow
+    # bytes. We use the recvWindow in the proc cleanupChann.
+    channel.recvWindow = 0
+
+proc updateRecvWindow(channel: YamuxChannel) {.async.} =
+  let inWindow = channel.recvWindow + channel.recvQueue.len
+  if inWindow > channel.maxRecvWindow div 2:
+    return
+
+  let delta = channel.maxRecvWindow - inWindow
+  channel.recvWindow.inc(delta)
+  await channel.conn.write(YamuxHeader.windowUpdate(
+    channel.id,
+    delta.uint32
+  ))
+  trace "increasing the recvWindow", delta
+
+method readOnce*(
+  channel: YamuxChannel,
+  pbytes: pointer,
+  nbytes: int):
+  Future[int] {.async.} =
+
+  if channel.returnedEof: raise newLPStreamEOFError()
+  if channel.recvQueue.len == 0:
+    channel.receivedData.clear()
+    await channel.closedRemotely or channel.receivedData.wait()
+    if channel.closedRemotely.done() and channel.recvQueue.len == 0:
+      channel.returnedEof = true
+      return 0
+
+  let toRead = min(channel.recvQueue.len, nbytes)
+
+  var p = cast[ptr UncheckedArray[byte]](pbytes)
+  toOpenArray(p, 0, nbytes - 1)[0..<toRead] = channel.recvQueue.toOpenArray(0, toRead - 1)
+  channel.recvQueue = channel.recvQueue[toRead..^1]
+
+  # We made some room in the recv buffer let the peer know
+  await channel.updateRecvWindow()
+  channel.activity = true
+  return toRead
+
+proc gotDataFromRemote(channel: YamuxChannel, b: seq[byte]) {.async.} =
+  channel.recvWindow -= b.len
+  channel.recvQueue = channel.recvQueue.concat(b)
+  channel.receivedData.fire()
+  await channel.updateRecvWindow()
+
+proc setMaxRecvWindow*(channel: YamuxChannel, maxRecvWindow: int) =
+  channel.maxRecvWindow = maxRecvWindow
+
+proc trySend(channel: YamuxChannel) {.async.} =
+  if channel.isSending:
+    return
+  channel.isSending = true
+  defer: channel.isSending = false
+  while channel.sendQueue.len != 0:
+    channel.sendQueue.keepItIf(not (it.fut.cancelled() and it.sent == 0))
+    if channel.sendWindow == 0:
+      trace "send window empty"
+      if channel.sendQueueBytes(true) > channel.maxRecvWindow:
+        await channel.reset(true)
+      break
+
+    let
+      bytesAvailable = channel.sendQueueBytes()
+      toSend = min(channel.sendWindow, bytesAvailable)
+    var
+      sendBuffer = newSeqUninitialized[byte](toSend + 12)
+      header = YamuxHeader.data(channel.id, toSend.uint32)
+      inBuffer = 0
+
+    if toSend >= bytesAvailable and channel.closedLocally:
+      trace "last buffer we'll sent on this channel", toSend, bytesAvailable
+      header.flags.incl({Fin})
+
+    sendBuffer[0..<12] = header.encode()
+
+    var futures: seq[Future[void]]
+    while inBuffer < toSend:
+      let (data, sent, fut) = channel.sendQueue[0]
+      let bufferToSend = min(data.len - sent, toSend - inBuffer)
+      sendBuffer.toOpenArray(12, 12 + toSend - 1)[inBuffer..<(inBuffer+bufferToSend)] =
+        channel.sendQueue[0].data.toOpenArray(sent, sent + bufferToSend - 1)
+      channel.sendQueue[0].sent.inc(bufferToSend)
+      if channel.sendQueue[0].sent >= data.len:
+        futures.add(fut)
+        channel.sendQueue.delete(0)
+      inBuffer.inc(bufferToSend)
+
+    trace "build send buffer", h = $header, msg=string.fromBytes(sendBuffer[12..^1])
+    channel.sendWindow.dec(toSend)
+    try: await channel.conn.write(sendBuffer)
+    except LPStreamEOFError as exc:
+      for fut in futures.items():
+        fut.fail(exc)
+      await channel.reset()
+      break
+    for fut in futures.items():
+      fut.complete()
+    channel.activity = true
+
+method write*(channel: YamuxChannel, msg: seq[byte]): Future[void] =
+  result = newFuture[void]("Yamux Send")
+  if channel.closedLocally or channel.isReset:
+    result.fail(newLPStreamEOFError())
+    return result
+  if msg.len == 0:
+    result.complete()
+    return result
+  channel.sendQueue.add((msg, 0, result))
+  asyncSpawn channel.trySend()
+
+proc open*(channel: YamuxChannel) {.async, gcsafe.} =
+  if channel.opened:
+    trace "Try to open channel twice"
+    return
+  channel.opened = true
+  await channel.conn.write(YamuxHeader.data(channel.id, 0, {if channel.isSrc: Syn else: Ack}))
+
+type
+  Yamux* = ref object of Muxer
+    channels: Table[uint32, YamuxChannel]
+    flushed: Table[uint32, int]
+    currentId: uint32
+    isClosed: bool
+
+proc cleanupChann(m: Yamux, channel: YamuxChannel) {.async.} =
+  await channel.join()
+  m.channels.del(channel.id)
+  if channel.isReset and channel.recvWindow > 0:
+    m.flushed[channel.id] = channel.recvWindow
+
+proc createStream(m: Yamux, id: uint32, isSrc: bool): YamuxChannel =
+  result = YamuxChannel(
+    id: id,
+    maxRecvWindow: DefaultWindowSize,
+    recvWindow: DefaultWindowSize,
+    sendWindow: DefaultWindowSize,
+    isSrc: isSrc,
+    conn: m.connection,
+    receivedData: newAsyncEvent(),
+    closedRemotely: newFuture[void]()
+  )
+  result.initStream()
+  result.peerId = m.connection.peerId
+  result.observedAddr = m.connection.observedAddr
+  result.transportDir = m.connection.transportDir
+  when defined(libp2p_agents_metrics):
+    result.shortAgent = m.connection.shortAgent
+  m.channels[id] = result
+  asyncSpawn m.cleanupChann(result)
+  trace "created channel", id, pid=m.connection.peerId
+
+method close*(m: Yamux) {.async.} =
+  if m.isClosed == true:
+    trace "Already closed"
+    return
+  m.isClosed = true
+
+  trace "Closing yamux"
+  for channel in m.channels.values:
+    await channel.reset()
+  await m.connection.write(YamuxHeader.goAway(NormalTermination))
+  await m.connection.close()
+  trace "Closed yamux"
+
+proc handleStream(m: Yamux, channel: YamuxChannel) {.async.} =
+  ## call the muxer stream handler for this channel
+  ##
+  try:
+    await m.streamHandler(channel)
+    trace "finished handling stream"
+    doAssert(channel.isClosed, "connection not closed by handler!")
+  except CatchableError as exc:
+    trace "Exception in yamux stream handler", msg = exc.msg
+    await channel.reset()
+
+method handle*(m: Yamux) {.async, gcsafe.} =
+  trace "Starting yamux handler", pid=m.connection.peerId
+  try:
+    while not m.connection.atEof:
+      trace "waiting for header"
+      let header = await m.connection.readHeader()
+      trace "got message", h = $header
+
+      case header.msgType:
+      of Ping:
+        if MsgFlags.Syn in header.flags:
+          await m.connection.write(YamuxHeader.ping(MsgFlags.Ack, header.length))
+      of GoAway:
+        var status: GoAwayStatus
+        if status.checkedEnumAssign(header.length): trace "Received go away", status
+        else: trace "Received unexpected error go away"
+        break
+      of Data, WindowUpdate:
+        if MsgFlags.Syn in header.flags:
+          if header.streamId in m.channels:
+            debug "Trying to create an existing channel, skipping", id=header.streamId
+          else:
+            if header.streamId in m.flushed:
+              m.flushed.del(header.streamId)
+            if header.streamId mod 2 == m.currentId mod 2:
+              raise newException(YamuxError, "Peer used our reserved stream id")
+            let newStream = m.createStream(header.streamId, false)
+            await newStream.open()
+            asyncSpawn m.handleStream(newStream)
+        elif header.streamId notin m.channels:
+          if header.streamId notin m.flushed:
+            raise newException(YamuxError, "Unknown stream ID: " & $header.streamId)
+          elif header.msgType == Data:
+            # Flush the data
+            m.flushed[header.streamId].dec(int(header.length))
+            if m.flushed[header.streamId] < 0:
+              raise newException(YamuxError, "Peer exhausted the recvWindow after reset")
+            var buffer = newSeqUninitialized[byte](header.length)
+            await m.connection.readExactly(addr buffer[0], int(header.length))
+          continue
+
+        let channel = m.channels[header.streamId]
+
+        if header.msgType == WindowUpdate:
+          channel.sendWindow += int(header.length)
+          await channel.trySend()
+        else:
+          if header.length.int > channel.recvWindow.int:
+            # check before allocating the buffer
+            raise newException(YamuxError, "Peer exhausted the recvWindow")
+
+          if header.length > 0:
+            var buffer = newSeqUninitialized[byte](header.length)
+            await m.connection.readExactly(addr buffer[0], int(header.length))
+            trace "Msg Rcv", msg=string.fromBytes(buffer)
+            await channel.gotDataFromRemote(buffer)
+
+        if MsgFlags.Fin in header.flags:
+          trace "remote closed channel"
+          await channel.remoteClosed()
+        if MsgFlags.Rst in header.flags:
+          trace "remote reset channel"
+          await channel.reset()
+  except LPStreamEOFError as exc:
+    trace "Stream EOF", msg = exc.msg
+  except YamuxError as exc:
+    trace "Closing yamux connection", error=exc.msg
+    await m.connection.write(YamuxHeader.goAway(ProtocolError))
+  finally:
+    await m.close()
+  trace "Stopped yamux handler"
+
+method newStream*(
+  m: Yamux,
+  name: string = "",
+  lazy: bool = false): Future[Connection] {.async, gcsafe.} =
+
+  let stream = m.createStream(m.currentId, true)
+  m.currentId += 2
+  if not lazy:
+    await stream.open()
+  return stream
+
+proc new*(T: type[Yamux], conn: Connection): T =
+  T(
+    connection: conn,
+    currentId: if conn.dir == Out: 1 else: 2
+  )

libp2p/nameresolving/dnsresolver.nim

@@ -1,11 +1,11 @@
-## Nim-LibP2P
-## Copyright (c) 2021 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 {.push raises: [Defect].}
 

libp2p/nameresolving/mockresolver.nim

@@ -1,11 +1,11 @@
-## Nim-LibP2P
-## Copyright (c) 2021 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 {.push raises: [Defect].}
 

libp2p/nameresolving/nameresolver.nim

@@ -1,11 +1,11 @@
-## Nim-LibP2P
-## Copyright (c) 2021 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 {.push raises: [Defect].}
 

libp2p/peerid.nim

@@ -1,25 +1,27 @@
-## Nim-LibP2P
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module implementes API for libp2p peer.
 
 {.push raises: [Defect].}
+{.push public.}
 
 import
   std/[hashes, strutils],
   stew/[base58, results],
   chronicles,
   nimcrypto/utils,
+  utility,
   ./crypto/crypto, ./multicodec, ./multihash, ./vbuffer,
   ./protobuf/minprotobuf
 
-export results
+export results, utility
 
 const
   maxInlineKeyLength* = 42

libp2p/peerinfo.nim

@@ -1,17 +1,18 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 {.push raises: [Defect].}
+{.push public.}
 
 import std/[options, sequtils]
 import pkg/[chronos, chronicles, stew/results]
-import peerid, multiaddress, crypto/crypto, routing_record, errors
+import peerid, multiaddress, crypto/crypto, routing_record, errors, utility
 
 export peerid, multiaddress, crypto, routing_record, errors, results
 
@@ -20,7 +21,7 @@ export peerid, multiaddress, crypto, routing_record, errors, results
 type
   PeerInfoError* = LPError
 
-  PeerInfo* = ref object
+  PeerInfo* {.public.} = ref object
     peerId*: PeerId
     addrs*: seq[MultiAddress]
     protocols*: seq[string]

libp2p/peerstore.nim

@@ -1,11 +1,26 @@
-## Nim-LibP2P
-## Copyright (c) 2021 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+## Stores generic informations about peers.
+runnableExamples:
+  # Will keep info of all connected peers +
+  # last 50 disconnected peers
+  let peerStore = PeerStore.new(capacity = 50)
+
+  # Create a custom book type
+  type MoodBook = ref object of PeerBook[string]
+
+  var somePeerId: PeerId
+  discard somePeerId.init("")
+
+  peerStore[MoodBook][somePeerId] = "Happy"
+  doAssert peerStore[MoodBook][somePeerId] == "Happy"
 
 {.push raises: [Defect].}
 
@@ -15,7 +30,8 @@ import
   ./protocols/identify,
   ./peerid, ./peerinfo,
   ./routing_record,
-  ./multiaddress
+  ./multiaddress,
+  utility
 
 type
   #################
@@ -23,7 +39,7 @@ type
   #################
 
   PeerBookChangeHandler* = proc(peerId: PeerId) {.gcsafe, raises: [Defect].}
-  
+
   #########
   # Books #
   #########
@@ -33,29 +49,29 @@ type
     changeHandlers: seq[PeerBookChangeHandler]
     deletor: PeerBookChangeHandler
 
-  PeerBook*[T] = ref object of BasePeerBook
+  PeerBook*[T] {.public.} = ref object of BasePeerBook
     book*: Table[PeerId, T]
 
   SeqPeerBook*[T] = ref object of PeerBook[seq[T]]
-  
-  AddressBook* = ref object of SeqPeerBook[MultiAddress]
-  ProtoBook* = ref object of SeqPeerBook[string]
-  KeyBook* = ref object of PeerBook[PublicKey]
 
-  AgentBook* = ref object of PeerBook[string]
-  ProtoVersionBook* = ref object of PeerBook[string]
-  SPRBook* = ref object of PeerBook[Envelope]
-  
+  AddressBook* {.public.} = ref object of SeqPeerBook[MultiAddress]
+  ProtoBook* {.public.} = ref object of SeqPeerBook[string]
+  KeyBook* {.public.} = ref object of PeerBook[PublicKey]
+
+  AgentBook* {.public.} = ref object of PeerBook[string]
+  ProtoVersionBook* {.public.} = ref object of PeerBook[string]
+  SPRBook* {.public.} = ref object of PeerBook[Envelope]
+
   ####################
   # Peer store types #
   ####################
 
-  PeerStore* = ref object
+  PeerStore* {.public.} = ref object
     books: Table[string, BasePeerBook]
     capacity*: int
     toClean*: seq[PeerId]
-  
-proc new*(T: type PeerStore, capacity = 1000): PeerStore =
+
+proc new*(T: type PeerStore, capacity = 1000): PeerStore {.public.} =
   T(capacity: capacity)
 
 #########################
@@ -63,16 +79,15 @@ proc new*(T: type PeerStore, capacity = 1000): PeerStore =
 #########################
 
 proc `[]`*[T](peerBook: PeerBook[T],
-             peerId: PeerId): T =
-  ## Get all the known metadata of a provided peer.
+             peerId: PeerId): T {.public.} =
+  ## Get all known metadata of a provided peer, or default(T) if missing
   peerBook.book.getOrDefault(peerId)
 
 proc `[]=`*[T](peerBook: PeerBook[T],
              peerId: PeerId,
-             entry: T) =
-  ## Set metadata for a given peerId. This will replace any
-  ## previously stored metadata.
-  
+             entry: T) {.public.} =
+  ## Set metadata for a given peerId.
+
   peerBook.book[peerId] = entry
 
   # Notify clients
@@ -80,9 +95,9 @@ proc `[]=`*[T](peerBook: PeerBook[T],
     handler(peerId)
 
 proc del*[T](peerBook: PeerBook[T],
-                peerId: PeerId): bool =
-  ## Delete the provided peer from the book.
-  
+                peerId: PeerId): bool {.public.} =
+  ## Delete the provided peer from the book. Returns whether the peer was in the book
+
   if peerId notin peerBook.book:
     return false
   else:
@@ -92,15 +107,16 @@ proc del*[T](peerBook: PeerBook[T],
       handler(peerId)
     return true
 
-proc contains*[T](peerBook: PeerBook[T], peerId: PeerId): bool =
+proc contains*[T](peerBook: PeerBook[T], peerId: PeerId): bool {.public.} =
   peerId in peerBook.book
 
-proc addHandler*[T](peerBook: PeerBook[T], handler: PeerBookChangeHandler) =
+proc addHandler*[T](peerBook: PeerBook[T], handler: PeerBookChangeHandler) {.public.} =
+  ## Adds a callback that will be called everytime the book changes
   peerBook.changeHandlers.add(handler)
 
-proc len*[T](peerBook: PeerBook[T]): int = peerBook.book.len
+proc len*[T](peerBook: PeerBook[T]): int {.public.} = peerBook.book.len
 
-##################  
+##################
 # Peer Store API #
 ##################
 macro getTypeName(t: type): untyped =
@@ -108,7 +124,8 @@ macro getTypeName(t: type): untyped =
   let typ = getTypeImpl(t)[1]
   newLit(repr(typ.owner()) & "." & repr(typ))
 
-proc `[]`*[T](p: PeerStore, typ: type[T]): T =
+proc `[]`*[T](p: PeerStore, typ: type[T]): T {.public.} =
+  ## Get a book from the PeerStore (ex: peerStore[AddressBook])
   let name = getTypeName(T)
   result = T(p.books.getOrDefault(name))
   if result.isNil:
@@ -121,7 +138,7 @@ proc `[]`*[T](p: PeerStore, typ: type[T]): T =
   return result
 
 proc del*(peerStore: PeerStore,
-             peerId: PeerId) =
+             peerId: PeerId) {.public.} =
   ## Delete the provided peer from every book.
   for _, book in peerStore.books:
     book.deletor(peerId)

libp2p/protobuf/minprotobuf.nim

@@ -1,18 +1,20 @@
-## Nim-Libp2p
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module implements minimal Google's ProtoBuf primitives.
 
 {.push raises: [Defect].}
 
-import ../varint, stew/[endians2, results]
-export results
+import ../varint, ../utility, stew/[endians2, results]
+export results, utility
+
+{.push public.}
 
 const
   MaxMessageSize* = 1'u shl 22

libp2p/protocols/identify.nim

@@ -1,11 +1,14 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+## `Identify <https://docs.libp2p.io/concepts/protocols/#identify>`_ and
+## `Push Identify <https://docs.libp2p.io/concepts/protocols/#identify-push>`_ implementation
 
 {.push raises: [Defect].}
 
@@ -36,7 +39,7 @@ type
   IdentityInvalidMsgError* = object of IdentifyError
   IdentifyNoPubKeyError* = object of IdentifyError
 
-  IdentifyInfo* = object
+  IdentifyInfo* {.public.} = object
     pubkey*: Option[PublicKey]
     peerId*: PeerId
     addrs*: seq[MultiAddress]
@@ -54,7 +57,7 @@ type
     peer: PeerId,
     newInfo: IdentifyInfo):
     Future[void]
-    {.gcsafe, raises: [Defect].}
+    {.gcsafe, raises: [Defect], public.}
 
   IdentifyPush* = ref object of LPProtocol
     identifyHandler: IdentifyPushHandler
@@ -203,7 +206,9 @@ proc identify*(p: Identify,
   else:
     raise newException(IdentityInvalidMsgError, "No pubkey in identify")
 
-proc new*(T: typedesc[IdentifyPush], handler: IdentifyPushHandler = nil): T =
+proc new*(T: typedesc[IdentifyPush], handler: IdentifyPushHandler = nil): T {.public.} =
+  ## Create a IdentifyPush protocol. `handler` will be called every time
+  ## a peer sends us new `PeerInfo`
   let identifypush = T(identifyHandler: handler)
   identifypush.init()
   identifypush
@@ -240,6 +245,7 @@ proc init*(p: IdentifyPush) =
   p.handler = handle
   p.codec = IdentifyPushCodec
 
-proc push*(p: IdentifyPush, peerInfo: PeerInfo, conn: Connection) {.async.} =
+proc push*(p: IdentifyPush, peerInfo: PeerInfo, conn: Connection) {.async, public.} =
+  ## Send new `peerInfo`s to a connection
   var pb = encodeMsg(peerInfo, conn.observedAddr, true)
   await conn.writeLp(pb.buffer)

libp2p/protocols/ping.nim

@@ -1,15 +1,18 @@
-## Nim-LibP2P
-## Copyright (c) 2021 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+## `Ping <https://docs.libp2p.io/concepts/protocols/#ping>`_ protocol implementation
 
 {.push raises: [Defect].}
 
-import chronos, chronicles, bearssl
+import chronos, chronicles
+import bearssl/[rand, hash]
 import ../protobuf/minprotobuf,
        ../peerinfo,
        ../stream/connection,
@@ -17,8 +20,11 @@ import ../protobuf/minprotobuf,
        ../crypto/crypto,
        ../multiaddress,
        ../protocols/protocol,
+       ../utility,
        ../errors
 
+export chronicles, rand, connection
+
 logScope:
   topics = "libp2p ping"
 
@@ -28,18 +34,18 @@ const
 
 type
   PingError* = object of LPError
-  WrongPingAckError* = object of LPError
+  WrongPingAckError* = object of PingError
 
-  PingHandler* = proc (
+  PingHandler* {.public.} = proc (
     peer: PeerId):
     Future[void]
     {.gcsafe, raises: [Defect].}
 
   Ping* = ref object of LPProtocol
     pingHandler*: PingHandler
-    rng: ref BrHmacDrbgContext
+    rng: ref HmacDrbgContext
 
-proc new*(T: typedesc[Ping], handler: PingHandler = nil, rng: ref BrHmacDrbgContext = newRng()): T =
+proc new*(T: typedesc[Ping], handler: PingHandler = nil, rng: ref HmacDrbgContext = newRng()): T {.public.} =
   let ping = Ping(pinghandler: handler, rng: rng)
   ping.init()
   ping
@@ -65,10 +71,8 @@ method init*(p: Ping) =
 proc ping*(
   p: Ping,
   conn: Connection,
-  ): Future[Duration] {.async, gcsafe.} =
-  ## Sends ping to `conn`
-  ## Returns the delay
-  ##
+  ): Future[Duration] {.async, gcsafe, public.} =
+  ## Sends ping to `conn`, returns the delay
 
   trace "initiating ping", conn
 
@@ -76,7 +80,7 @@ proc ping*(
     randomBuf: array[PingSize, byte]
     resultBuf: array[PingSize, byte]
 
-  p.rng[].brHmacDrbgGenerate(randomBuf)
+  hmacDrbgGenerate(p.rng[], randomBuf)
 
   let startTime = Moment.now()
 

libp2p/protocols/protocol.nim

@@ -1,11 +1,11 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 {.push raises: [Defect].}
 
@@ -22,8 +22,12 @@ type
   LPProtocol* = ref object of RootObj
     codecs*: seq[string]
     handler*: LPProtoHandler ## this handler gets invoked by the protocol negotiator
+    started*: bool
 
 method init*(p: LPProtocol) {.base, gcsafe.} = discard
+method start*(p: LPProtocol) {.async, base.} = p.started = true
+method stop*(p: LPProtocol) {.async, base.} = p.started = false
+
 
 func codec*(p: LPProtocol): string =
   assert(p.codecs.len > 0, "Codecs sequence was empty!")

libp2p/protocols/pubsub.nim

@@ -1,3 +1,8 @@
 import ./pubsub/[pubsub, floodsub, gossipsub]
 
+## Home of PubSub & it's implementations:
+## | **pubsub**: base interface for pubsub implementations
+## | **floodsub**: simple flood-based publishing
+## | **gossipsub**: more sophisticated gossip based publishing
+
 export pubsub, floodsub, gossipsub

libp2p/protocols/pubsub/errors.nim

@@ -1,6 +1,8 @@
 # this module will be further extended in PR
 # https://github.com/status-im/nim-libp2p/pull/107/
 
+import ../../utility
+
 type
-  ValidationResult* {.pure.} = enum
+  ValidationResult* {.pure, public.} = enum
     Accept, Reject, Ignore

libp2p/protocols/pubsub/floodsub.nim

@@ -1,16 +1,16 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 {.push raises: [Defect].}
 
 import std/[sequtils, sets, hashes, tables]
-import chronos, chronicles, metrics, bearssl
+import chronos, chronicles, metrics
 import ./pubsub,
        ./pubsubpeer,
        ./timedcache,
@@ -22,13 +22,15 @@ import ./pubsub,
        ../../peerinfo,
        ../../utility
 
+## Simple flood-based publishing.
+
 logScope:
   topics = "libp2p floodsub"
 
 const FloodSubCodec* = "/floodsub/1.0.0"
 
 type
-  FloodSub* = ref object of PubSub
+  FloodSub* {.public.} = ref object of PubSub
     floodsub*: PeerTable      # topic to remote peer map
     seen*: TimedCache[MessageID] # message id:s already seen on the network
     seenSalt*: seq[byte]
@@ -187,21 +189,17 @@ method publish*(f: FloodSub,
     debug "No peers for topic, skipping publish", topic
     return 0
 
-  inc f.msgSeqno
   let
     msg =
       if f.anonymize:
         Message.init(none(PeerInfo), data, topic, none(uint64), false)
       else:
+        inc f.msgSeqno
         Message.init(some(f.peerInfo), data, topic, some(f.msgSeqno), f.sign)
-    msgIdResult = f.msgIdProvider(msg)
-
-  if msgIdResult.isErr:
-    trace "Error generating message id, skipping publish",
-      error = msgIdResult.error
-    return 0
-
-  let msgId = msgIdResult.get
+    msgId = f.msgIdProvider(msg).valueOr:
+      trace "Error generating message id, skipping publish",
+        error = error
+      return 0
 
   trace "Created new message",
     msg = shortLog(msg), peers = peers.len, topic, msgId
@@ -226,6 +224,6 @@ method initPubSub*(f: FloodSub)
   procCall PubSub(f).initPubSub()
   f.seen = TimedCache[MessageID].init(2.minutes)
   f.seenSalt = newSeqUninitialized[byte](sizeof(Hash))
-  brHmacDrbgGenerate(f.rng[], f.seenSalt)
+  hmacDrbgGenerate(f.rng[], f.seenSalt)
 
   f.init()

libp2p/protocols/pubsub/gossipsub.nim

@@ -1,11 +1,13 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+## Gossip based publishing
 
 {.push raises: [Defect].}
 
@@ -506,7 +508,7 @@ method publish*(g: GossipSub,
     g.rng.shuffle(fanoutPeers)
     if fanoutPeers.len + peers.len > g.parameters.d:
       fanoutPeers.setLen(g.parameters.d - peers.len)
-    
+
     for fanPeer in fanoutPeers:
       peers.incl(fanPeer)
       if peers.len > g.parameters.d: break
@@ -520,29 +522,25 @@ method publish*(g: GossipSub,
 
   if peers.len == 0:
     let topicPeers = g.gossipsub.getOrDefault(topic).toSeq()
-    notice "No peers for topic, skipping publish",  peersOnTopic = topicPeers.len,
-                                                    connectedPeers = topicPeers.filterIt(it.connected).len,
-                                                    topic
+    debug "No peers for topic, skipping publish",  peersOnTopic = topicPeers.len,
+                                                   connectedPeers = topicPeers.filterIt(it.connected).len,
+                                                   topic
     # skipping topic as our metrics finds that heavy
     libp2p_gossipsub_failed_publish.inc()
     return 0
 
-  inc g.msgSeqno
   let
     msg =
       if g.anonymize:
         Message.init(none(PeerInfo), data, topic, none(uint64), false)
       else:
+        inc g.msgSeqno
         Message.init(some(g.peerInfo), data, topic, some(g.msgSeqno), g.sign)
-    msgIdResult = g.msgIdProvider(msg)
-
-  if msgIdResult.isErr:
-    trace "Error generating message id, skipping publish",
-      error = msgIdResult.error
-    libp2p_gossipsub_failed_publish.inc()
-    return 0
-
-  let msgId = msgIdResult.get
+    msgId = g.msgIdProvider(msg).valueOr:
+      trace "Error generating message id, skipping publish",
+        error = error
+      libp2p_gossipsub_failed_publish.inc()
+      return 0
 
   logScope: msgId = shortLog(msgId)
 
@@ -566,22 +564,28 @@ method publish*(g: GossipSub,
 
   return peers.len
 
+proc maintainDirectPeer(g: GossipSub, id: PeerId, addrs: seq[MultiAddress]) {.async.} =
+  let peer = g.peers.getOrDefault(id)
+  if isNil(peer):
+    trace "Attempting to dial a direct peer", peer = id
+    try:
+      await g.switch.connect(id, addrs)
+      # populate the peer after it's connected
+      discard g.getOrCreatePeer(id, g.codecs)
+    except CancelledError as exc:
+      trace "Direct peer dial canceled"
+      raise exc
+    except CatchableError as exc:
+      debug "Direct peer error dialing", msg = exc.msg
+
+proc addDirectPeer*(g: GossipSub, id: PeerId, addrs: seq[MultiAddress]) {.async.} =
+  g.parameters.directPeers[id] = addrs
+  await g.maintainDirectPeer(id, addrs)
+
 proc maintainDirectPeers(g: GossipSub) {.async.} =
   heartbeat "GossipSub DirectPeers", 1.minutes:
     for id, addrs in g.parameters.directPeers:
-      let peer = g.peers.getOrDefault(id)
-      if isNil(peer):
-        trace "Attempting to dial a direct peer", peer = id
-        try:
-          # dial, internally connection will be stored
-          let _ = await g.switch.dial(id, addrs, g.codecs)
-          # populate the peer after it's connected
-          discard g.getOrCreatePeer(id, g.codecs)
-        except CancelledError as exc:
-          trace "Direct peer dial canceled"
-          raise exc
-        except CatchableError as exc:
-          debug "Direct peer error dialing", msg = exc.msg
+      await g.addDirectPeer(id, addrs)
 
 method start*(g: GossipSub) {.async.} =
   trace "gossipsub start"
@@ -593,9 +597,11 @@ method start*(g: GossipSub) {.async.} =
   g.heartbeatFut = g.heartbeat()
   g.scoringHeartbeatFut = g.scoringHeartbeat()
   g.directPeersLoop = g.maintainDirectPeers()
+  g.started = true
 
 method stop*(g: GossipSub) {.async.} =
   trace "gossipsub stop"
+  g.started = false
   if g.heartbeatFut.isNil:
     warn "Stopping gossipsub without starting it"
     return

libp2p/protocols/pubsub/gossipsub/behavior.nim

@@ -1,11 +1,11 @@
-## Nim-LibP2P
-## Copyright (c) 2021 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 {.push raises: [Defect].}
 

libp2p/protocols/pubsub/gossipsub/scoring.nim

@@ -1,11 +1,11 @@
-## Nim-LibP2P
-## Copyright (c) 2021 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 {.push raises: [Defect].}
 

libp2p/protocols/pubsub/gossipsub/types.nim

@@ -1,11 +1,11 @@
-## Nim-LibP2P
-## Copyright (c) 2021 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 {.push raises: [Defect].}
 

libp2p/protocols/pubsub/mcache.nim

@@ -1,11 +1,11 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 {.push raises: [Defect].}
 

libp2p/protocols/pubsub/peertable.nim

@@ -1,11 +1,11 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 {.push raises: [Defect].}
 

libp2p/protocols/pubsub/pubsub.nim

@@ -1,16 +1,22 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+## Base interface for pubsub protocols
+##
+## You can `subscribe<#subscribe%2CPubSub%2Cstring%2CTopicHandler>`_ to a topic,
+## `publish<#publish.e%2CPubSub%2Cstring%2Cseq%5Bbyte%5D>`_ something on it,
+## and eventually `unsubscribe<#unsubscribe%2CPubSub%2Cstring%2CTopicHandler>`_ from it.
 
 {.push raises: [Defect].}
 
 import std/[tables, sequtils, sets, strutils]
-import chronos, chronicles, metrics, bearssl
+import chronos, chronicles, metrics
 import ./errors as pubsub_errors,
        ./pubsubpeer,
        ./rpc/[message, messages, protobuf],
@@ -75,37 +81,40 @@ declarePublicCounter(libp2p_pubsub_received_prune, "pubsub broadcast prune", lab
 type
   InitializationError* = object of LPError
 
-  TopicHandler* = proc(topic: string,
+  TopicHandler* {.public.} = proc(topic: string,
                        data: seq[byte]): Future[void] {.gcsafe, raises: [Defect].}
 
-  ValidatorHandler* = proc(topic: string,
+  ValidatorHandler* {.public.} = proc(topic: string,
                            message: Message): Future[ValidationResult] {.gcsafe, raises: [Defect].}
 
   TopicPair* = tuple[topic: string, handler: TopicHandler]
 
-  MsgIdProvider* =
+  MsgIdProvider* {.public.} =
     proc(m: Message): Result[MessageID, ValidationResult] {.noSideEffect, raises: [Defect], gcsafe.}
 
-  SubscriptionValidator* =
+  SubscriptionValidator* {.public.} =
     proc(topic: string): bool {.raises: [Defect], gcsafe.}
+    ## Every time a peer send us a subscription (even to an unknown topic),
+    ## we have to store it, which may be an attack vector.
+    ## This callback can be used to reject topic we're not interested in
 
-  PubSub* = ref object of LPProtocol
+  PubSub* {.public.} = ref object of LPProtocol
     switch*: Switch                    # the switch used to dial/connect to peers
     peerInfo*: PeerInfo                # this peer's info
     topics*: Table[string, seq[TopicHandler]]      # the topics that _we_ are interested in
-    peers*: Table[PeerId, PubSubPeer]  ##\
-      ## Peers that we are interested to gossip with (but not necessarily
-      ## yet connected to)
-    triggerSelf*: bool                 # trigger own local handler on publish
-    verifySignature*: bool             # enable signature verification
-    sign*: bool                        # enable message signing
+    peers*: Table[PeerId, PubSubPeer]  #\
+      # Peers that we are interested to gossip with (but not necessarily
+      # yet connected to)
+    triggerSelf*: bool                 ## trigger own local handler on publish
+    verifySignature*: bool             ## enable signature verification
+    sign*: bool                        ## enable message signing
     validators*: Table[string, HashSet[ValidatorHandler]]
     observers: ref seq[PubSubObserver] # ref as in smart_ptr
-    msgIdProvider*: MsgIdProvider      # Turn message into message id (not nil)
+    msgIdProvider*: MsgIdProvider      ## Turn message into message id (not nil)
     msgSeqno*: uint64
-    anonymize*: bool                   # if we omit fromPeer and seqno from RPC messages we send
+    anonymize*: bool                   ## if we omit fromPeer and seqno from RPC messages we send
     subscriptionValidator*: SubscriptionValidator # callback used to validate subscriptions
-    topicsHigh*: int                  # the maximum number of topics a peer is allowed to subscribe to
+    topicsHigh*: int                  ## the maximum number of topics a peer is allowed to subscribe to
     maxMessageSize*: int          ##\ 
       ## the maximum raw message size we'll globally allow
       ## for finer tuning, check message size on topic validator
@@ -114,7 +123,7 @@ type
       ## lead to issues, from descoring to connection drops
       ##
       ## defaults to 1mB
-    rng*: ref BrHmacDrbgContext
+    rng*: ref HmacDrbgContext
 
     knownTopics*: HashSet[string]
 
@@ -411,7 +420,7 @@ method onTopicSubscription*(p: PubSub, topic: string, subscribed: bool) {.base.}
 
 proc unsubscribe*(p: PubSub,
                   topic: string,
-                  handler: TopicHandler) =
+                  handler: TopicHandler) {.public.} =
   ## unsubscribe from a ``topic`` string
   ##
   p.topics.withValue(topic, handlers):
@@ -424,12 +433,13 @@ proc unsubscribe*(p: PubSub,
 
     p.updateTopicMetrics(topic)
 
-proc unsubscribe*(p: PubSub, topics: openArray[TopicPair]) =
+proc unsubscribe*(p: PubSub, topics: openArray[TopicPair]) {.public.} =
   ## unsubscribe from a list of ``topic`` handlers
   for t in topics:
     p.unsubscribe(t.topic, t.handler)
 
-proc unsubscribeAll*(p: PubSub, topic: string) =
+proc unsubscribeAll*(p: PubSub, topic: string) {.public.} =
+  ## unsubscribe every `handler` from `topic`
   if topic notin p.topics:
     debug "unsubscribeAll called for an unknown topic", topic
   else:
@@ -441,15 +451,14 @@ proc unsubscribeAll*(p: PubSub, topic: string) =
 
 proc subscribe*(p: PubSub,
                 topic: string,
-                handler: TopicHandler) =
+                handler: TopicHandler) {.public.} =
   ## subscribe to a topic
   ##
   ## ``topic``   - a string topic to subscribe to
   ##
-  ## ``handler`` - is a user provided proc
-  ##               that will be triggered
-  ##               on every received message
-  ##
+  ## ``handler`` - user provided proc that
+  ##               will be triggered on every
+  ##               received message
 
   # Check that this is an allowed topic
   if p.subscriptionValidator != nil and p.subscriptionValidator(topic) == false:
@@ -470,8 +479,9 @@ proc subscribe*(p: PubSub,
 
 method publish*(p: PubSub,
                 topic: string,
-                data: seq[byte]): Future[int] {.base, async.} =
+                data: seq[byte]): Future[int] {.base, async, public.} =
   ## publish to a ``topic``
+  ##
   ## The return value is the number of neighbours that we attempted to send the
   ## message to, excluding self. Note that this is an optimistic number of
   ## attempts - the number of peers that actually receive the message might
@@ -488,24 +498,19 @@ method initPubSub*(p: PubSub)
   if p.msgIdProvider == nil:
     p.msgIdProvider = defaultMsgIdProvider
 
-method start*(p: PubSub) {.async, base.} =
-  ## start pubsub
-  discard
-
-method stop*(p: PubSub) {.async, base.} =
-  ## stopt pubsub
-  discard
-
 method addValidator*(p: PubSub,
                      topic: varargs[string],
-                     hook: ValidatorHandler) {.base.} =
+                     hook: ValidatorHandler) {.base, public.} =
+  ## Add a validator to a `topic`. Each new message received in this
+  ## will be sent to `hook`. `hook` can return either `Accept`,
+  ## `Ignore` or `Reject` (which can descore the peer)
   for t in topic:
     trace "adding validator for topic", topicId = t
     p.validators.mgetOrPut(t, HashSet[ValidatorHandler]()).incl(hook)
 
 method removeValidator*(p: PubSub,
                         topic: varargs[string],
-                        hook: ValidatorHandler) {.base.} =
+                        hook: ValidatorHandler) {.base, public.} =
   for t in topic:
     p.validators.withValue(t, validators):
       validators[].excl(hook)
@@ -553,9 +558,9 @@ proc init*[PubParams: object | bool](
   msgIdProvider: MsgIdProvider = defaultMsgIdProvider,
   subscriptionValidator: SubscriptionValidator = nil,
   maxMessageSize: int = 1024 * 1024,
-  rng: ref BrHmacDrbgContext = newRng(),
+  rng: ref HmacDrbgContext = newRng(),
   parameters: PubParams = false): P
-  {.raises: [Defect, InitializationError].} =
+  {.raises: [Defect, InitializationError], public.} =
   let pubsub =
     when PubParams is bool:
       P(switch: switch,
@@ -598,9 +603,9 @@ proc init*[PubParams: object | bool](
 
   return pubsub
 
-proc addObserver*(p: PubSub; observer: PubSubObserver) = p.observers[] &= observer
+proc addObserver*(p: PubSub; observer: PubSubObserver) {.public.} = p.observers[] &= observer
 
-proc removeObserver*(p: PubSub; observer: PubSubObserver) =
+proc removeObserver*(p: PubSub; observer: PubSubObserver) {.public.} =
   let idx = p.observers[].find(observer)
   if idx != -1:
     p.observers[].del(idx)

libp2p/protocols/pubsub/pubsubpeer.nim

@@ -1,11 +1,11 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 {.push raises: [Defect].}
 

libp2p/protocols/pubsub/rpc/message.nim

@@ -1,11 +1,11 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 {.push raises: [Defect].}
 

libp2p/protocols/pubsub/rpc/messages.nim

@@ -1,11 +1,11 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 {.push raises: [Defect].}
 

libp2p/protocols/pubsub/rpc/protobuf.nim

@@ -1,11 +1,11 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 {.push raises: [Defect].}
 

libp2p/protocols/pubsub/timedcache.nim

@@ -1,11 +1,11 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 {.push raises: [Defect].}
 

libp2p/protocols/relay.nim

@@ -1,11 +1,11 @@
-## Nim-LibP2P
-## Copyright (c) 2022 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 {.push raises: [Defect].}
 
@@ -466,6 +466,7 @@ proc dial*(self: RelayTransport, ma: MultiAddress): Future[Connection] {.async,
   trace "Dial", relayPeerId, relayAddrs, dstPeerId
 
   let conn = await self.relay.switch.dial(relayPeerId, @[ relayAddrs ], RelayCodec)
+  conn.dir = Direction.Out
   result = await self.relay.dialPeer(conn, dstPeerId, @[])
 
 method dial*(

libp2p/protocols/secure/noise.nim

@@ -1,18 +1,18 @@
-## Nim-LibP2P
-## Copyright (c) 2020 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 {.push raises: [Defect].}
 
 import std/[oids, strformat]
 import chronos
 import chronicles
-import bearssl
+import bearssl/[rand, hash]
 import stew/[endians2, byteutils]
 import nimcrypto/[utils, sha2, hmac]
 import ../../stream/[connection, streamseq]
@@ -78,7 +78,7 @@ type
     rs: Curve25519Key
 
   Noise* = ref object of Secure
-    rng: ref BrHmacDrbgContext
+    rng: ref HmacDrbgContext
     localPrivateKey: PrivateKey
     localPublicKey: seq[byte]
     noiseKeys: KeyPair
@@ -106,7 +106,7 @@ func shortLog*(conn: NoiseConnection): auto =
 
 chronicles.formatIt(NoiseConnection): shortLog(it)
 
-proc genKeyPair(rng: var BrHmacDrbgContext): KeyPair =
+proc genKeyPair(rng: var HmacDrbgContext): KeyPair =
   result.privateKey = Curve25519Key.random(rng)
   result.publicKey = result.privateKey.public()
 
@@ -602,7 +602,7 @@ method init*(p: Noise) {.gcsafe.} =
 
 proc new*(
   T: typedesc[Noise],
-  rng: ref BrHmacDrbgContext,
+  rng: ref HmacDrbgContext,
   privateKey: PrivateKey,
   outgoing: bool = true,
   commonPrologue: seq[byte] = @[]): T =

libp2p/protocols/secure/plaintext.nim

@@ -1,11 +1,11 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 {.push raises: [Defect].}
 

libp2p/protocols/secure/secio.nim

@@ -1,16 +1,17 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 {.push raises: [Defect].}
 
 import std/[oids, strformat]
-import chronos, chronicles, stew/endians2, bearssl
+import bearssl/rand
+import chronos, chronicles, stew/endians2
 import nimcrypto/[hmac, sha2, sha, hash, rijndael, twofish, bcmode]
 import secure,
        ../../stream/connection,
@@ -37,7 +38,7 @@ const
 
 type
   Secio* = ref object of Secure
-    rng: ref BrHmacDrbgContext
+    rng: ref HmacDrbgContext
     localPrivateKey: PrivateKey
     localPublicKey: PublicKey
     remotePublicKey: PublicKey
@@ -304,7 +305,7 @@ method handshake*(s: Secio, conn: Connection, initiator: bool = false): Future[S
     localPeerId: PeerId
     localBytesPubkey = s.localPublicKey.getBytes().tryGet()
 
-  brHmacDrbgGenerate(s.rng[], localNonce)
+  hmacDrbgGenerate(s.rng[], localNonce)
 
   var request = createProposal(localNonce,
                                localBytesPubkey,
@@ -428,7 +429,7 @@ method init(s: Secio) {.gcsafe.} =
 
 proc new*(
   T: typedesc[Secio],
-  rng: ref BrHmacDrbgContext,
+  rng: ref HmacDrbgContext,
   localPrivateKey: PrivateKey): T =
   let pkRes = localPrivateKey.getPublicKey()
   if pkRes.isErr:

libp2p/protocols/secure/secure.nim

@@ -1,16 +1,16 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 {.push raises: [Defect].}
 
 import std/[strformat]
-import chronos, chronicles, bearssl
+import chronos, chronicles
 import ../protocol,
        ../../stream/streamseq,
        ../../stream/connection,
@@ -152,6 +152,8 @@ method readOnce*(s: SecureConn,
       s.isEof = true
       await s.close()
       raise err
+    except CancelledError as exc:
+      raise exc
     except CatchableError as err:
       debug "Error while reading message from secure connection, closing.",
         error = err.name,

libp2p/routing_record.nim

@@ -1,11 +1,11 @@
-## Nim-Libp2p
-## Copyright (c) 2021 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module implements Routing Records.
 

libp2p/signed_envelope.nim

@@ -1,11 +1,11 @@
-## Nim-Libp2p
-## Copyright (c) 2021 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module implements Signed Envelope.
 

libp2p/stream/bufferstream.nim

@@ -1,11 +1,11 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 {.push raises: [Defect].}
 
@@ -134,6 +134,10 @@ method readOnce*(s: BufferStream,
     let buf =
       try:
         await s.readQueue.popFirst()
+      except CancelledError as exc:
+        # Not very efficient, but shouldn't happen often
+        s.readBuf.assign(@(p.toOpenArray(0, rbytes - 1)) & @(s.readBuf.data))
+        raise exc
       except CatchableError as exc:
         # When an exception happens here, the Bufferstream is effectively
         # broken and no more reads will be valid - for now, return EOF if it's

libp2p/stream/chronosstream.nim

@@ -1,11 +1,11 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 {.push raises: [Defect].}
 

libp2p/stream/connection.nim

@@ -1,11 +1,11 @@
-## Nim-LibP2P
-## Copyright (c) 2020 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 {.push raises: [Defect].}
 

libp2p/stream/lpstream.nim

@@ -1,11 +1,13 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+## Length Prefixed stream implementation
 
 {.push raises: [Defect].}
 
@@ -15,6 +17,7 @@ import chronicles, chronos, metrics
 import ../varint,
        ../peerinfo,
        ../multiaddress,
+       ../utility,
        ../errors
 
 export errors
@@ -132,26 +135,32 @@ method initStream*(s: LPStream) {.base.} =
   inc getStreamTracker(s.objName).opened
   trace "Stream created", s, objName = s.objName, dir = $s.dir
 
-proc join*(s: LPStream): Future[void] =
+proc join*(s: LPStream): Future[void] {.public.} =
+  ## Wait for the stream to be closed
   s.closeEvent.wait()
 
-method closed*(s: LPStream): bool {.base.} =
+method closed*(s: LPStream): bool {.base, public.} =
   s.isClosed
 
-method atEof*(s: LPStream): bool {.base.} =
+method atEof*(s: LPStream): bool {.base, public.} =
   s.isEof
 
 method readOnce*(
   s: LPStream,
   pbytes: pointer,
   nbytes: int):
-  Future[int] {.base, async.} =
+  Future[int] {.base, async, public.} =
+  ## Reads whatever is available in the stream,
+  ## up to `nbytes`. Will block if nothing is
+  ## available
   doAssert(false, "not implemented!")
 
 proc readExactly*(s: LPStream,
                   pbytes: pointer,
                   nbytes: int):
-                  Future[void] {.async.} =
+                  Future[void] {.async, public.} =
+  ## Waits for `nbytes` to be available, then read
+  ## them and return them
   if s.atEof:
     raise newLPStreamEOFError()
 
@@ -180,7 +189,8 @@ proc readExactly*(s: LPStream,
 proc readLine*(s: LPStream,
                limit = 0,
                sep = "\r\n"): Future[string]
-               {.async.} =
+               {.async, public.} =
+  ## Reads up to `limit` bytes are read, or a `sep` is found
   # TODO replace with something that exploits buffering better
   var lim = if limit <= 0: -1 else: limit
   var state = 0
@@ -206,7 +216,7 @@ proc readLine*(s: LPStream,
       if len(result) == lim:
         break
 
-proc readVarint*(conn: LPStream): Future[uint64] {.async, gcsafe.} =
+proc readVarint*(conn: LPStream): Future[uint64] {.async, gcsafe, public.} =
   var
     buffer: array[10, byte]
 
@@ -225,7 +235,7 @@ proc readVarint*(conn: LPStream): Future[uint64] {.async, gcsafe.} =
   if true: # can't end with a raise apparently
     raise (ref InvalidVarintError)(msg: "Cannot parse varint")
 
-proc readLp*(s: LPStream, maxSize: int): Future[seq[byte]] {.async, gcsafe.} =
+proc readLp*(s: LPStream, maxSize: int): Future[seq[byte]] {.async, gcsafe, public.} =
   ## read length prefixed msg, with the length encoded as a varint
   let
     length = await s.readVarint()
@@ -241,10 +251,11 @@ proc readLp*(s: LPStream, maxSize: int): Future[seq[byte]] {.async, gcsafe.} =
   await s.readExactly(addr res[0], res.len)
   return res
 
-method write*(s: LPStream, msg: seq[byte]): Future[void] {.base.} =
+method write*(s: LPStream, msg: seq[byte]): Future[void] {.base, public.} =
+  # Write `msg` to stream, waiting for the write to be finished
   doAssert(false, "not implemented!")
 
-proc writeLp*(s: LPStream, msg: openArray[byte]): Future[void] =
+proc writeLp*(s: LPStream, msg: openArray[byte]): Future[void] {.public.} =
   ## Write `msg` with a varint-encoded length prefix
   let vbytes = PB.toBytes(msg.len().uint64)
   var buf = newSeqUninitialized[byte](msg.len() + vbytes.len)
@@ -252,10 +263,10 @@ proc writeLp*(s: LPStream, msg: openArray[byte]): Future[void] =
   buf[vbytes.len..<buf.len] = msg
   s.write(buf)
 
-proc writeLp*(s: LPStream, msg: string): Future[void] =
+proc writeLp*(s: LPStream, msg: string): Future[void] {.public.} =
   writeLp(s, msg.toOpenArrayByte(0, msg.high))
 
-proc write*(s: LPStream, msg: string): Future[void] =
+proc write*(s: LPStream, msg: string): Future[void] {.public.} =
   s.write(msg.toBytes())
 
 method closeImpl*(s: LPStream): Future[void] {.async, base.} =
@@ -266,7 +277,7 @@ method closeImpl*(s: LPStream): Future[void] {.async, base.} =
   s.closeEvent.fire()
   trace "Closed stream", s, objName = s.objName, dir = $s.dir
 
-method close*(s: LPStream): Future[void] {.base, async.} = # {.raises [Defect].}
+method close*(s: LPStream): Future[void] {.base, async, public.} = # {.raises [Defect].}
   ## close the stream - this may block, but will not raise exceptions
   ##
   if s.isClosed:
@@ -280,7 +291,7 @@ method close*(s: LPStream): Future[void] {.base, async.} = # {.raises [Defect].}
   # itself must implement this - once-only check as well, with their own field
   await closeImpl(s)
 
-proc closeWithEOF*(s: LPStream): Future[void] {.async.} =
+proc closeWithEOF*(s: LPStream): Future[void] {.async, public.} =
   ## Close the stream and wait for EOF - use this with half-closed streams where
   ## an EOF is expected to arrive from the other end.
   ##

libp2p/switch.nim

@@ -1,11 +1,15 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+## The switch is the core of libp2p, which brings together the
+## transports, the connection manager, the upgrader and other
+## parts to allow programs to use libp2p
 
 {.push raises: [Defect].}
 
@@ -37,6 +41,7 @@ import stream/connection,
        peerid,
        peerstore,
        errors,
+       utility,
        dialer
 
 export connmanager, upgrade, dialer, peerstore
@@ -56,7 +61,7 @@ const
   ConcurrentUpgrades* = 4
 
 type
-    Switch* = ref object of Dial
+    Switch* {.public.} = ref object of Dial
       peerInfo*: PeerInfo
       connManager*: ConnManager
       transports*: seq[Transport]
@@ -65,25 +70,36 @@ type
       dialer*: Dial
       peerStore*: PeerStore
       nameResolver*: NameResolver
+      started: bool
+      concurrentUpgrades: int
 
 proc addConnEventHandler*(s: Switch,
                           handler: ConnEventHandler,
-                          kind: ConnEventKind) =
+                          kind: ConnEventKind) {.public.} =
+  ## Adds a ConnEventHandler, which will be triggered when
+  ## a connection to a peer is created or dropped.
+  ## There may be multiple connections per peer.
+  ##
+  ## The handler should not raise.
   s.connManager.addConnEventHandler(handler, kind)
 
 proc removeConnEventHandler*(s: Switch,
                              handler: ConnEventHandler,
-                             kind: ConnEventKind) =
+                             kind: ConnEventKind) {.public.} =
   s.connManager.removeConnEventHandler(handler, kind)
 
 proc addPeerEventHandler*(s: Switch,
                           handler: PeerEventHandler,
-                          kind: PeerEventKind) =
+                          kind: PeerEventKind) {.public.} =
+  ## Adds a PeerEventHandler, which will be triggered when
+  ## a peer connects or disconnects from us.
+  ##
+  ## The handler should not raise.
   s.connManager.addPeerEventHandler(handler, kind)
 
 proc removePeerEventHandler*(s: Switch,
                              handler: PeerEventHandler,
-                             kind: PeerEventKind) =
+                             kind: PeerEventKind) {.public.} =
   s.connManager.removePeerEventHandler(handler, kind)
 
 method addTransport*(s: Switch,
@@ -91,32 +107,39 @@ method addTransport*(s: Switch,
   s.transports &= t
   s.dialer.addTransport(t)
 
-proc isConnected*(s: Switch, peerId: PeerId): bool =
+proc isConnected*(s: Switch, peerId: PeerId): bool {.public.} =
   ## returns true if the peer has one or more
-  ## associated connections (sockets)
+  ## associated connections
   ##
 
   peerId in s.connManager
 
-proc disconnect*(s: Switch, peerId: PeerId): Future[void] {.gcsafe.} =
+proc disconnect*(s: Switch, peerId: PeerId): Future[void] {.gcsafe, public.} =
+  ## Disconnect from a peer, waiting for the connection(s) to be dropped
   s.connManager.dropPeer(peerId)
 
 method connect*(
   s: Switch,
   peerId: PeerId,
   addrs: seq[MultiAddress],
-  forceDial = false): Future[void] =
+  forceDial = false): Future[void] {.public.} =
+  ## Connects to a peer without opening a stream to it
+
   s.dialer.connect(peerId, addrs, forceDial)
 
 method dial*(
   s: Switch,
   peerId: PeerId,
-  protos: seq[string]): Future[Connection] =
+  protos: seq[string]): Future[Connection] {.public.} =
+  ## Open a stream to a connected peer with the specified `protos`
+
   s.dialer.dial(peerId, protos)
 
 proc dial*(s: Switch,
            peerId: PeerId,
-           proto: string): Future[Connection] =
+           proto: string): Future[Connection] {.public.} =
+  ## Open a stream to a connected peer with the specified `proto`
+
   dial(s, peerId, @[proto])
 
 method dial*(
@@ -124,18 +147,26 @@ method dial*(
   peerId: PeerId,
   addrs: seq[MultiAddress],
   protos: seq[string],
-  forceDial = false): Future[Connection] =
+  forceDial = false): Future[Connection] {.public.} =
+  ## Connected to a peer and open a stream
+  ## with the specified `protos`
+
   s.dialer.dial(peerId, addrs, protos, forceDial)
 
 proc dial*(
   s: Switch,
   peerId: PeerId,
   addrs: seq[MultiAddress],
-  proto: string): Future[Connection] =
+  proto: string): Future[Connection] {.public.} =
+  ## Connected to a peer and open a stream
+  ## with the specified `proto`
+
   dial(s, peerId, addrs, @[proto])
 
 proc mount*[T: LPProtocol](s: Switch, proto: T, matcher: Matcher = nil)
-  {.gcsafe, raises: [Defect, LPError].} =
+  {.gcsafe, raises: [Defect, LPError], public.} =
+  ## mount a protocol to the switch
+
   if isNil(proto.handler):
     raise newException(LPError,
       "Protocol has to define a handle method or proc")
@@ -144,6 +175,9 @@ proc mount*[T: LPProtocol](s: Switch, proto: T, matcher: Matcher = nil)
     raise newException(LPError,
       "Protocol has to define a codec string")
 
+  if s.started and not proto.started:
+    raise newException(LPError, "Protocol not started")
+
   s.ms.addHandler(proto.codecs, proto, matcher)
   s.peerInfo.protocols.add(proto.codec)
 
@@ -213,9 +247,13 @@ proc accept(s: Switch, transport: Transport) {.async.} = # noraises
         await conn.close()
       return
 
-proc stop*(s: Switch) {.async.} =
+proc stop*(s: Switch) {.async, public.} =
+  ## Stop listening on every transport, and
+  ## close every active connections
+
   trace "Stopping switch"
 
+  s.started = false
   # close and cleanup all connections
   await s.connManager.close()
 
@@ -239,9 +277,13 @@ proc stop*(s: Switch) {.async.} =
     if not a.finished:
       a.cancel()
 
+  await s.ms.stop()
+
   trace "Switch stopped"
 
-proc start*(s: Switch) {.async, gcsafe.} =
+proc start*(s: Switch) {.async, gcsafe, public.} =
+  ## Start listening on every transport
+
   trace "starting switch for peer", peerInfo = s.peerInfo
   var startFuts: seq[Future[void]]
   for t in s.transports:
@@ -272,6 +314,10 @@ proc start*(s: Switch) {.async, gcsafe.} =
 
   s.peerInfo.update()
 
+  await s.ms.start()
+
+  s.started = true
+
   debug "Started libp2p node", peer = s.peerInfo
 
 proc newSwitch*(peerInfo: PeerInfo,
@@ -282,8 +328,9 @@ proc newSwitch*(peerInfo: PeerInfo,
                 connManager: ConnManager,
                 ms: MultistreamSelect,
                 nameResolver: NameResolver = nil,
-                peerStore = PeerStore.new()): Switch
-                {.raises: [Defect, LPError].} =
+                peerStore = PeerStore.new(),
+                concurrentUpgrades = ConcurrentUpgrades): Switch
+                {.raises: [Defect, LPError], public.} =
   if secureManagers.len == 0:
     raise newException(LPError, "Provide at least one secure manager")
 
@@ -294,7 +341,8 @@ proc newSwitch*(peerInfo: PeerInfo,
     connManager: connManager,
     peerStore: peerStore,
     dialer: Dialer.new(peerInfo.peerId, connManager, transports, ms, nameResolver),
-    nameResolver: nameResolver)
+    nameResolver: nameResolver,
+    concurrentUpgrades: ConcurrentUpgrades)
 
   switch.connManager.peerStore = peerStore
   switch.mount(identity)

libp2p/transcoder.nim

@@ -1,11 +1,11 @@
-## Nim-Libp2p
-## Copyright (c) 2018 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-Libp2p
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 ## This module implements transcoder interface.
 import vbuffer

libp2p/transports/tcptransport.nim

@@ -1,11 +1,13 @@
-## Nim-LibP2P
-## Copyright (c) 2019 Status Research & Development GmbH
-## Licensed under either of
-##  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
-##  * MIT license ([LICENSE-MIT](LICENSE-MIT))
-## at your option.
-## This file may not be copied, modified, or distributed except according to
-## those terms.
+# Nim-LibP2P
+# Copyright (c) 2022 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+## TCP transport implementation
 
 {.push raises: [Defect].}
 
@@ -20,7 +22,8 @@ import transport,
        ../multiaddress,
        ../stream/connection,
        ../stream/chronosstream,
-       ../upgrademngrs/upgrade
+       ../upgrademngrs/upgrade,
+       ../utility
 
 logScope:
   topics = "libp2p tcptransport"
@@ -118,7 +121,7 @@ proc connHandler*(self: TcpTransport,
 proc new*(
   T: typedesc[TcpTransport],
   flags: set[ServerFlags] = {},
-  upgrade: Upgrade): T =
+  upgrade: Upgrade): T {.public.} =
 
   let transport = T(
     flags: flags,