chore: add Nimble lock file

Co-authored-by: Ludovic Chenut <ludovic@status.im>

.github/workflows/doc.yml

@@ -19,7 +19,7 @@ jobs:
 
       - uses: jiro4989/setup-nim-action@v1
         with:
-          nim-version: 'stable'
+          nim-version: '1.6.x'
 
       - name: Generate doc
         run: |

.github/workflows/interop.yml

@@ -23,7 +23,7 @@ jobs:
 
       - name: Build image
         run: >
-          cd multidim-interop/impl/nim/v1.0 &&
+          cd transport-interop/impl/nim/v1.0 &&
           make commitSha=$GITHUB_SHA image_name=nim-libp2p-head
 
       - name: Create ping-version.json
@@ -45,10 +45,10 @@ jobs:
                 ]
             }
           EOF
-          
+
           ) > ${{ github.workspace }}/test_head.json
 
-      - uses: libp2p/test-plans/.github/actions/run-interop-ping-test@master
+      - uses: libp2p/test-plans/.github/actions/run-transport-interop-test@master
         with:
           test-filter: nim-libp2p-head
           extra-versions: ${{ github.workspace }}/test_head.json

.pinned

@@ -1,16 +1,17 @@
-bearssl;https://github.com/status-im/nim-bearssl@#9372f27a25d0718d3527afad6cc936f6a853f86e
+bearssl;https://github.com/status-im/nim-bearssl@#e4157639db180e52727712a47deaefcbbac6ec86
 chronicles;https://github.com/status-im/nim-chronicles@#32ac8679680ea699f7dbc046e8e0131cac97d41a
 chronos;https://github.com/status-im/nim-chronos@#ba143e029f35fd9b4cd3d89d007cc834d0d5ba3c
-dnsclient;https://github.com/ba0f3/dnsclient.nim@#2b3d4b4e35b5e698fbbeafe16a4fa757926a4673
-faststreams;https://github.com/status-im/nim-faststreams@#2a771bb91f8aae8520a5553955a2acce5fdd0c87
-httputils;https://github.com/status-im/nim-http-utils@#aad684d3758a74c1b327df93da2e956458410b48
-json_serialization;https://github.com/status-im/nim-json-serialization@#aa44ee61dd323022d4abe7cbf4e44668aad88454
-metrics;https://github.com/status-im/nim-metrics@#abf3acc7f06cee9ee2c287d2f31413dc3df4c04e
-nimcrypto;https://github.com/cheatfate/nimcrypto@#4014ef939b51e02053c2e16dd3481d47bc9267dd
-secp256k1;https://github.com/status-im/nim-secp256k1@#5fd81357839d57ef38fb17647bd5e31dfa9f55b8
-serialization;https://github.com/status-im/nim-serialization@#f0860e1c25acf26ef5e6ea231c7c0537c793b555
-stew;https://github.com/status-im/nim-stew@#000eeb14a34832e6c95303e6508e2925db56be7c
+dnsclient;https://github.com/ba0f3/dnsclient.nim@#23214235d4784d24aceed99bbfe153379ea557c8
+faststreams;https://github.com/status-im/nim-faststreams@#720fc5e5c8e428d9d0af618e1e27c44b42350309
+httputils;https://github.com/status-im/nim-http-utils@#3b491a40c60aad9e8d3407443f46f62511e63b18
+json_serialization;https://github.com/status-im/nim-json-serialization@#85b7ea093cb85ee4f433a617b97571bd709d30df
+metrics;https://github.com/status-im/nim-metrics@#6142e433fc8ea9b73379770a788017ac528d46ff
+nimcrypto;https://github.com/cheatfate/nimcrypto@#1c8d6e3caf3abc572136ae9a1da81730c4eb4288
+results;https://github.com/arnetheduck/nim-results@#f3c666a272c69d70cb41e7245e7f6844797303ad
+secp256k1;https://github.com/status-im/nim-secp256k1@#7246d91c667f4cc3759fdd50339caa45a2ecd8be
+serialization;https://github.com/status-im/nim-serialization@#4bdbc29e54fe54049950e352bb969aab97173b35
+stew;https://github.com/status-im/nim-stew@#3159137d9a3110edb4024145ce0ba778975de40e
 testutils;https://github.com/status-im/nim-testutils@#dfc4c1b39f9ded9baf6365014de2b4bfb4dafc34
-unittest2;https://github.com/status-im/nim-unittest2@#b178f47527074964f76c395ad0dfc81cf118f379
-websock;https://github.com/status-im/nim-websock@#3696e3f3a5b938e478e473a6089bf8de386d2f04
-zlib;https://github.com/status-im/nim-zlib@#d65ee2a7611eb9f0ef0e7350caed6e93ccfa9651
+unittest2;https://github.com/status-im/nim-unittest2@#2300fa9924a76e6c96bc4ea79d043e3a0f27120c
+websock;https://github.com/status-im/nim-websock@#f8ed9b40a5ff27ad02a3c237c4905b0924e3f982
+zlib;https://github.com/status-im/nim-zlib@#38b72eda9d70067df4a953f56b5ed59630f2a17b

libp2p.nimble

@@ -17,7 +17,7 @@ requires "nim >= 1.6.0",
          "secp256k1",
          "stew#head",
          "websock",
-         "unittest2 >= 0.0.5 & < 0.1.0"
+         "unittest2"
 
 let nimc = getEnv("NIMC", "nim") # Which nim compiler to use
 let lang = getEnv("NIMLANG", "c") # Which backend (c/cpp/js)

libp2p/builders.nim

@@ -25,7 +25,7 @@ import
   muxers/[muxer, mplex/mplex, yamux/yamux],
   protocols/[identify, secure/secure, secure/noise, rendezvous],
   protocols/connectivity/[autonat/server, relay/relay, relay/client, relay/rtransport],
-  connmanager, upgrademngrs/muxedupgrade,
+  connmanager, upgrademngrs/muxedupgrade, observedaddrmanager,
   nameresolving/nameresolver,
   errors, utility
 
@@ -59,6 +59,7 @@ type
     circuitRelay: Relay
     rdv: RendezVous
     services: seq[Service]
+    observedAddrManager: ObservedAddrManager
 
 proc new*(T: type[SwitchBuilder]): T {.public.} =
   ## Creates a SwitchBuilder
@@ -201,6 +202,10 @@ proc withServices*(b: SwitchBuilder, services: seq[Service]): SwitchBuilder =
   b.services = services
   b
 
+proc withObservedAddrManager*(b: SwitchBuilder, observedAddrManager: ObservedAddrManager): SwitchBuilder =
+  b.observedAddrManager = observedAddrManager
+  b
+
 proc build*(b: SwitchBuilder): Switch
   {.raises: [LPError], public.} =
 
@@ -223,11 +228,16 @@ proc build*(b: SwitchBuilder): Switch
       protoVersion = b.protoVersion,
       agentVersion = b.agentVersion)
 
+  let identify =
+    if b.observedAddrManager != nil:
+      Identify.new(peerInfo, b.sendSignedPeerRecord, b.observedAddrManager)
+    else:
+      Identify.new(peerInfo, b.sendSignedPeerRecord)
+
   let
-    identify = Identify.new(peerInfo, b.sendSignedPeerRecord)
     connManager = ConnManager.new(b.maxConnsPerPeer, b.maxConnections, b.maxIn, b.maxOut)
     ms = MultistreamSelect.new()
-    muxedUpgrade = MuxedUpgrade.new(b.muxers, secureManagerInstances, connManager, ms)
+    muxedUpgrade = MuxedUpgrade.new(b.muxers, secureManagerInstances, ms)
 
   let
     transports = block:

libp2p/crypto/crypto.nim

@@ -65,11 +65,13 @@ when supported(PKScheme.Ed25519):
   import ed25519/ed25519
 when supported(PKScheme.Secp256k1):
   import secp
+when supported(PKScheme.ECDSA):
+  import ecnist
 
-# We are still importing `ecnist` because, it is used for SECIO handshake,
-# but it will be impossible to create ECNIST keys or import ECNIST keys.
+  # These used to be declared in `crypto` itself
+  export ecnist.ephemeral, ecnist.ECDHEScheme
 
-import ecnist, bearssl/rand, bearssl/hash as bhash
+import bearssl/rand, bearssl/hash as bhash
 import ../protobuf/minprotobuf, ../vbuffer, ../multihash, ../multicodec
 import nimcrypto/[rijndael, twofish, sha2, hash, hmac]
 # We use `ncrutils` for constant-time hexadecimal encoding/decoding procedures.
@@ -86,8 +88,6 @@ type
     Sha256,
     Sha512
 
-  ECDHEScheme* = EcCurveKind
-
   PublicKey* = object
     case scheme*: PKScheme
     of PKScheme.RSA:
@@ -879,34 +879,6 @@ proc mac*(secret: Secret, id: int): seq[byte] {.inline.} =
   offset += secret.ivsize + secret.keysize
   copyMem(addr result[0], unsafeAddr secret.data[offset], secret.macsize)
 
-proc ephemeral*(
-    scheme: ECDHEScheme,
-    rng: var HmacDrbgContext): CryptoResult[EcKeyPair] =
-  ## Generate ephemeral keys used to perform ECDHE.
-  var keypair: EcKeyPair
-  if scheme == Secp256r1:
-    keypair = ? EcKeyPair.random(Secp256r1, rng).orError(KeyError)
-  elif scheme == Secp384r1:
-    keypair = ? EcKeyPair.random(Secp384r1, rng).orError(KeyError)
-  elif scheme == Secp521r1:
-    keypair = ? EcKeyPair.random(Secp521r1, rng).orError(KeyError)
-  ok(keypair)
-
-proc ephemeral*(
-    scheme: string, rng: var HmacDrbgContext): CryptoResult[EcKeyPair] =
-  ## Generate ephemeral keys used to perform ECDHE using string encoding.
-  ##
-  ## Currently supported encoding strings are P-256, P-384, P-521, if encoding
-  ## string is not supported P-521 key will be generated.
-  if scheme == "P-256":
-    ephemeral(Secp256r1, rng)
-  elif scheme == "P-384":
-    ephemeral(Secp384r1, rng)
-  elif scheme == "P-521":
-    ephemeral(Secp521r1, rng)
-  else:
-    ephemeral(Secp521r1, rng)
-
 proc getOrder*(remotePubkey, localNonce: openArray[byte],
                localPubkey, remoteNonce: openArray[byte]): CryptoResult[int] =
   ## Compare values and calculate `order` parameter.

libp2p/crypto/ecnist.nim

@@ -994,3 +994,33 @@ proc verify*[T: byte|char](sig: EcSignature, message: openArray[T],
     # Clear context with initial value
     kv.init(addr hc.vtable)
     result = (res == 1)
+
+type ECDHEScheme* = EcCurveKind
+
+proc ephemeral*(
+    scheme: ECDHEScheme,
+    rng: var HmacDrbgContext): EcResult[EcKeyPair] =
+  ## Generate ephemeral keys used to perform ECDHE.
+  var keypair: EcKeyPair
+  if scheme == Secp256r1:
+    keypair = ? EcKeyPair.random(Secp256r1, rng)
+  elif scheme == Secp384r1:
+    keypair = ? EcKeyPair.random(Secp384r1, rng)
+  elif scheme == Secp521r1:
+    keypair = ? EcKeyPair.random(Secp521r1, rng)
+  ok(keypair)
+
+proc ephemeral*(
+    scheme: string, rng: var HmacDrbgContext): EcResult[EcKeyPair] =
+  ## Generate ephemeral keys used to perform ECDHE using string encoding.
+  ##
+  ## Currently supported encoding strings are P-256, P-384, P-521, if encoding
+  ## string is not supported P-521 key will be generated.
+  if scheme == "P-256":
+    ephemeral(Secp256r1, rng)
+  elif scheme == "P-384":
+    ephemeral(Secp384r1, rng)
+  elif scheme == "P-521":
+    ephemeral(Secp521r1, rng)
+  else:
+    ephemeral(Secp521r1, rng)

libp2p/discovery/discoverymngr.nim

@@ -122,20 +122,15 @@ proc request*[T](dm: DiscoveryManager, value: T): DiscoveryQuery =
   pa.add(value)
   return dm.request(pa)
 
-proc advertise*(dm: DiscoveryManager, pa: PeerAttributes) =
+proc advertise*[T](dm: DiscoveryManager, value: T) =
   for i in dm.interfaces:
-    i.toAdvertise = pa
+    i.toAdvertise.add(value)
     if i.advertiseLoop.isNil:
       i.advertisementUpdated = newAsyncEvent()
       i.advertiseLoop = i.advertise()
     else:
       i.advertisementUpdated.fire()
 
-proc advertise*[T](dm: DiscoveryManager, value: T) =
-  var pa: PeerAttributes
-  pa.add(value)
-  dm.advertise(pa)
-
 template forEach*(query: DiscoveryQuery, code: untyped) =
   ## Will execute `code` for each discovered peer. The
   ## peer attritubtes are available through the variable

libp2p/discovery/rendezvousinterface.nim

@@ -19,6 +19,7 @@ type
     rdv*: RendezVous
     timeToRequest: Duration
     timeToAdvertise: Duration
+    ttl: Duration
 
   RdvNamespace* = distinct string
 
@@ -62,12 +63,16 @@ method advertise*(self: RendezVousInterface) {.async.} =
 
     self.advertisementUpdated.clear()
     for toAdv in toAdvertise:
-      await self.rdv.advertise(toAdv, self.timeToAdvertise)
+      try:
+        await self.rdv.advertise(toAdv, self.ttl)
+      except CatchableError as error:
+        debug "RendezVous advertise error: ", msg = error.msg
 
     await sleepAsync(self.timeToAdvertise) or self.advertisementUpdated.wait()
 
 proc new*(T: typedesc[RendezVousInterface],
           rdv: RendezVous,
           ttr: Duration = 1.minutes,
-          tta: Duration = MinimumDuration): RendezVousInterface =
-  T(rdv: rdv, timeToRequest: ttr, timeToAdvertise: tta)
+          tta: Duration = 1.minutes,
+          ttl: Duration = MinimumDuration): RendezVousInterface =
+  T(rdv: rdv, timeToRequest: ttr, timeToAdvertise: tta, ttl: ttl)

libp2p/errors.nim

@@ -19,7 +19,8 @@ func toException*(e: string): ref LPError =
 # sadly nim needs more love for hygienic templates
 # so here goes the macro, its based on the proc/template version
 # and uses quote do so it's quite readable
-macro checkFutures*[T](futs: seq[Future[T]], exclude: untyped = []): untyped =
+# TODO https://github.com/nim-lang/Nim/issues/22936
+macro checkFutures*[F](futs: seq[F], exclude: untyped = []): untyped =
   let nexclude = exclude.len
   case nexclude
   of 0:

libp2p/multiaddress.nim

@@ -398,6 +398,9 @@ const
     MAProtocol(
       mcodec: multiCodec("quic"), kind: Marker, size: 0
     ),
+    MAProtocol(
+      mcodec: multiCodec("quic-v1"), kind: Marker, size: 0
+    ),
     MAProtocol(
       mcodec: multiCodec("ip6zone"), kind: Length, size: 0,
       coder: TranscoderIP6Zone

libp2p/multicodec.nim

@@ -193,6 +193,7 @@ const MultiCodecList = [
   ("https", 0x01BB),
   ("tls", 0x01C0),
   ("quic", 0x01CC),
+  ("quic-v1", 0x01CD),
   ("ws", 0x01DD),
   ("wss", 0x01DE),
   ("p2p-websocket-star", 0x01DF), # not in multicodec list

libp2p/muxers/yamux/yamux.nim

@@ -186,6 +186,7 @@ proc remoteClosed(channel: YamuxChannel) {.async.} =
 method closeImpl*(channel: YamuxChannel) {.async, gcsafe.} =
   if not channel.closedLocally:
     channel.closedLocally = true
+    channel.isEof = true
 
     if channel.isReset == false and channel.sendQueue.len == 0:
       await channel.conn.write(YamuxHeader.data(channel.id, 0, {Fin}))
@@ -249,6 +250,7 @@ method readOnce*(
     await channel.closedRemotely or channel.receivedData.wait()
     if channel.closedRemotely.done() and channel.recvQueue.len == 0:
       channel.returnedEof = true
+      channel.isEof = true
       return 0
 
   let toRead = min(channel.recvQueue.len, nbytes)
@@ -454,6 +456,7 @@ method handle*(m: Yamux) {.async, gcsafe.} =
             if header.streamId in m.flushed:
               m.flushed.del(header.streamId)
             if header.streamId mod 2 == m.currentId mod 2:
+              debug "Peer used our reserved stream id, skipping", id=header.streamId, currentId=m.currentId, peerId=m.connection.peerId
               raise newException(YamuxError, "Peer used our reserved stream id")
             let newStream = m.createStream(header.streamId, false)
             if m.channels.len >= m.maxChannCount:

libp2p/protocols/connectivity/dcutr/client.nim

@@ -66,7 +66,7 @@ proc startSync*(self: DcutrClient, switch: Switch, remotePeerId: PeerId, addrs:
 
     if peerDialableAddrs.len > self.maxDialableAddrs:
         peerDialableAddrs = peerDialableAddrs[0..<self.maxDialableAddrs]
-    var futs = peerDialableAddrs.mapIt(switch.connect(stream.peerId, @[it], forceDial = true, reuseConnection = false))
+    var futs = peerDialableAddrs.mapIt(switch.connect(stream.peerId, @[it], forceDial = true, reuseConnection = false, upgradeDir = Direction.In))
     try:
       discard await anyCompleted(futs).wait(self.connectTimeout)
       debug "Dcutr initiator has directly connected to the remote peer."

libp2p/protocols/connectivity/dcutr/core.nim

@@ -56,5 +56,10 @@ proc send*(conn: Connection, msgType: MsgType, addrs: seq[MultiAddress]) {.async
   let pb = DcutrMsg(msgType: msgType, addrs: addrs).encode()
   await conn.writeLp(pb.buffer)
 
-proc getHolePunchableAddrs*(addrs: seq[MultiAddress]): seq[MultiAddress] =
-  addrs.filterIt(TCP.match(it))
+proc getHolePunchableAddrs*(addrs: seq[MultiAddress]): seq[MultiAddress] {.raises: [LPError]} =
+  var result = newSeq[MultiAddress]()
+  for a in addrs:
+      # This is necessary to also accept addrs like /ip4/198.51.100/tcp/1234/p2p/QmYyQSo1c1Ym7orWxLYvCrM2EmxFTANf8wXmmE7DWjhx5N
+      if [TCP, mapAnd(TCP_DNS, P2PPattern), mapAnd(TCP_IP, P2PPattern)].anyIt(it.match(a)):
+        result.add(a[0..1].tryGet())
+  return result

libp2p/protocols/connectivity/dcutr/server.nim

@@ -56,7 +56,7 @@ proc new*(T: typedesc[Dcutr], switch: Switch, connectTimeout = 15.seconds, maxDi
 
       if peerDialableAddrs.len > maxDialableAddrs:
         peerDialableAddrs = peerDialableAddrs[0..<maxDialableAddrs]
-      var futs = peerDialableAddrs.mapIt(switch.connect(stream.peerId, @[it], forceDial = true, reuseConnection = false, upgradeDir = Direction.In))
+      var futs = peerDialableAddrs.mapIt(switch.connect(stream.peerId, @[it], forceDial = true, reuseConnection = false, upgradeDir = Direction.Out))
       try:
         discard await anyCompleted(futs).wait(connectTimeout)
         debug "Dcutr receiver has directly connected to the remote peer."

libp2p/protocols/connectivity/relay/rconn.nim

@@ -47,6 +47,7 @@ proc new*(
   limitDuration: uint32,
   limitData: uint64): T =
   let rc = T(conn: conn, limitDuration: limitDuration, limitData: limitData)
+  rc.dir = conn.dir
   rc.initStream()
   if limitDuration > 0:
     proc checkDurationConnection() {.async.} =

libp2p/protocols/identify.nim

@@ -21,6 +21,7 @@ import ../protobuf/minprotobuf,
        ../peerid,
        ../crypto/crypto,
        ../multiaddress,
+       ../multicodec,
        ../protocols/protocol,
        ../utility,
        ../errors,
@@ -139,12 +140,13 @@ proc decodeMsg*(buf: seq[byte]): Opt[IdentifyInfo] =
 proc new*(
   T: typedesc[Identify],
   peerInfo: PeerInfo,
-  sendSignedPeerRecord = false
+  sendSignedPeerRecord = false,
+  observedAddrManager = ObservedAddrManager.new(),
   ): T =
   let identify = T(
     peerInfo: peerInfo,
     sendSignedPeerRecord: sendSignedPeerRecord,
-    observedAddrManager: ObservedAddrManager.new(),
+    observedAddrManager: observedAddrManager,
   )
   identify.init()
   identify
@@ -186,8 +188,12 @@ proc identify*(self: Identify,
   info.peerId = peer
 
   info.observedAddr.withValue(observed):
-    if not self.observedAddrManager.addObservation(observed):
-      debug "Observed address is not valid", observedAddr = observed
+    # Currently, we use the ObservedAddrManager only to find our dialable external NAT address. Therefore, addresses
+    # like "...\p2p-circuit\p2p\..." and "\p2p\..." are not useful to us.
+    if observed.contains(multiCodec("p2p-circuit")).get(false) or P2PPattern.matchPartial(observed):
+      trace "Not adding address to ObservedAddrManager.", observed
+    elif not self.observedAddrManager.addObservation(observed):
+      trace "Observed address is not valid.", observedAddr = observed
   return info
 
 proc new*(T: typedesc[IdentifyPush], handler: IdentifyPushHandler = nil): T {.public.} =

libp2p/protocols/perf/client.nim

@@ -0,0 +1,47 @@
+# Nim-LibP2P
+# Copyright (c) 2023 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+## `Perf <https://github.com/libp2p/specs/blob/master/perf/perf.md>`_ protocol specification
+
+import chronos, chronicles, sequtils
+import stew/endians2
+import ./core, ../../stream/connection
+
+logScope:
+  topics = "libp2p perf"
+
+type PerfClient* = ref object of RootObj
+
+proc perf*(_: typedesc[PerfClient], conn: Connection,
+           sizeToWrite: uint64 = 0, sizeToRead: uint64 = 0):
+           Future[Duration] {.async, public.} =
+  var
+    size = sizeToWrite
+    buf: array[PerfSize, byte]
+  let start = Moment.now()
+  trace "starting performance benchmark", conn, sizeToWrite, sizeToRead
+
+  await conn.write(toSeq(toBytesBE(sizeToRead)))
+  while size > 0:
+    let toWrite = min(size, PerfSize)
+    await conn.write(buf[0..<toWrite])
+    size -= toWrite
+
+  await conn.close()
+
+  size = sizeToRead
+
+  while size > 0:
+    let toRead = min(size, PerfSize)
+    await conn.readExactly(addr buf[0], toRead.int)
+    size = size - toRead
+
+  let duration = Moment.now() - start
+  trace "finishing performance benchmark", duration
+  return duration

libp2p/protocols/perf/core.nim

@@ -0,0 +1,14 @@
+# Nim-LibP2P
+# Copyright (c) 2023 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+## `Perf <https://github.com/libp2p/specs/blob/master/perf/perf.md>`_ protocol specification
+
+const
+  PerfCodec* = "/perf/1.0.0"
+  PerfSize* = 65536

libp2p/protocols/perf/server.nim

@@ -0,0 +1,60 @@
+# Nim-LibP2P
+# Copyright (c) 2023 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+## `Perf <https://github.com/libp2p/specs/blob/master/perf/perf.md>`_ protocol specification
+
+{.push raises: [].}
+
+import chronos, chronicles
+import stew/endians2
+import ./core,
+       ../protocol,
+       ../../stream/connection,
+       ../../utility
+
+export chronicles, connection
+
+logScope:
+  topics = "libp2p perf"
+
+type Perf* = ref object of LPProtocol
+
+proc new*(T: typedesc[Perf]): T {.public.} =
+  var p = T()
+  proc handle(conn: Connection, proto: string) {.async, gcsafe, closure.} =
+    var bytesRead = 0
+    try:
+      trace "Received benchmark performance check", conn
+      var
+        sizeBuffer: array[8, byte]
+        size: uint64
+      await conn.readExactly(addr sizeBuffer[0], 8)
+      size = uint64.fromBytesBE(sizeBuffer)
+
+      var toReadBuffer: array[PerfSize, byte]
+      try:
+        while true:
+          bytesRead += await conn.readOnce(addr toReadBuffer[0], PerfSize)
+      except CatchableError as exc:
+        discard
+
+      var buf: array[PerfSize, byte]
+      while size > 0:
+        let toWrite = min(size, PerfSize)
+        await conn.write(buf[0..<toWrite])
+        size -= toWrite
+    except CancelledError as exc:
+      raise exc
+    except CatchableError as exc:
+      trace "exception in perf handler", exc = exc.msg, conn
+    await conn.close()
+
+  p.handler = handle
+  p.codec = PerfCodec
+  return p

libp2p/protocols/pubsub/floodsub.nim

@@ -15,7 +15,7 @@ import ./pubsub,
        ./pubsubpeer,
        ./timedcache,
        ./peertable,
-       ./rpc/[message, messages],
+       ./rpc/[message, messages, protobuf],
        ../../crypto/crypto,
        ../../stream/connection,
        ../../peerid,
@@ -95,7 +95,16 @@ method unsubscribePeer*(f: FloodSub, peer: PeerId) =
 
 method rpcHandler*(f: FloodSub,
                    peer: PubSubPeer,
-                   rpcMsg: RPCMsg) {.async.} =
+                   data: seq[byte]) {.async.} =
+
+  var rpcMsg = decodeRpcMsg(data).valueOr:
+    debug "failed to decode msg from peer", peer, err = error
+    raise newException(CatchableError, "")
+
+  trace "decoded msg from peer", peer, msg = rpcMsg.shortLog
+  # trigger hooks
+  peer.recvObservers(rpcMsg)
+
   for i in 0..<min(f.topicsHigh, rpcMsg.subscriptions.len):
     template sub: untyped = rpcMsg.subscriptions[i]
     f.handleSubscribe(peer, sub.topic, sub.subscribe)

libp2p/protocols/pubsub/gossipsub.nim

@@ -13,13 +13,14 @@
 
 import std/[sets, sequtils]
 import chronos, chronicles, metrics
+import chronos/ratelimit
 import ./pubsub,
        ./floodsub,
        ./pubsubpeer,
        ./peertable,
        ./mcache,
        ./timedcache,
-       ./rpc/[messages, message],
+       ./rpc/[messages, message, protobuf],
        ../protocol,
        ../../stream/connection,
        ../../peerinfo,
@@ -40,6 +41,8 @@ logScope:
 declareCounter(libp2p_gossipsub_failed_publish, "number of failed publish")
 declareCounter(libp2p_gossipsub_invalid_topic_subscription, "number of invalid topic subscriptions that happened")
 declareCounter(libp2p_gossipsub_duplicate_during_validation, "number of duplicates received during message validation")
+declareCounter(libp2p_gossipsub_idontwant_saved_messages, "number of duplicates avoided by idontwant")
+declareCounter(libp2p_gossipsub_saved_bytes, "bytes saved by gossipsub optimizations", labels=["kind"])
 declareCounter(libp2p_gossipsub_duplicate, "number of duplicates received")
 declareCounter(libp2p_gossipsub_received, "number of messages received (deduplicated)")
 
@@ -74,7 +77,10 @@ proc init*(_: type[GossipSubParams]): GossipSubParams =
       behaviourPenaltyWeight: -1.0,
       behaviourPenaltyDecay: 0.999,
       disconnectBadPeers: false,
-      enablePX: false
+      enablePX: false,
+      bandwidthEstimatebps: 100_000_000, # 100 Mbps or 12.5 MBps
+      overheadRateLimit: Opt.none(tuple[bytes: int, interval: Duration]),
+      disconnectPeerAboveRateLimit: false
     )
 
 proc validateParameters*(parameters: GossipSubParams): Result[void, cstring] =
@@ -147,7 +153,7 @@ method init*(g: GossipSub) =
   g.codecs &= GossipSubCodec
   g.codecs &= GossipSubCodec_10
 
-method onNewPeer(g: GossipSub, peer: PubSubPeer) =
+method onNewPeer*(g: GossipSub, peer: PubSubPeer) =
   g.withPeerStats(peer.peerId) do (stats: var PeerStats):
     # Make sure stats and peer information match, even when reloading peer stats
     # from a previous connection
@@ -156,7 +162,7 @@ method onNewPeer(g: GossipSub, peer: PubSubPeer) =
     peer.behaviourPenalty = stats.behaviourPenalty
 
     # Check if the score is below the threshold and disconnect the peer if necessary
-    g.disconnectBadPeerCheck(peer, stats.score)
+    g.disconnectIfBadScorePeer(peer, stats.score)
 
   peer.iHaveBudget = IHavePeerBudget
   peer.pingBudget = PingsPeerBudget
@@ -201,8 +207,8 @@ method unsubscribePeer*(g: GossipSub, peer: PeerId) =
 
   for t in toSeq(g.gossipsub.keys):
     g.gossipsub.removePeer(t, pubSubPeer)
-    # also try to remove from explicit table here
-    g.explicit.removePeer(t, pubSubPeer)
+    # also try to remove from direct peers table here
+    g.subscribedDirectPeers.removePeer(t, pubSubPeer)
 
   for t in toSeq(g.fanout.keys):
     g.fanout.removePeer(t, pubSubPeer)
@@ -241,7 +247,7 @@ proc handleSubscribe*(g: GossipSub,
     # subscribe remote peer to the topic
     discard g.gossipsub.addPeer(topic, peer)
     if peer.peerId in g.parameters.directPeers:
-      discard g.explicit.addPeer(topic, peer)
+      discard g.subscribedDirectPeers.addPeer(topic, peer)
   else:
     trace "peer unsubscribed from topic"
 
@@ -255,7 +261,7 @@ proc handleSubscribe*(g: GossipSub,
 
     g.fanout.removePeer(topic, peer)
     if peer.peerId in g.parameters.directPeers:
-      g.explicit.removePeer(topic, peer)
+      g.subscribedDirectPeers.removePeer(topic, peer)
 
   trace "gossip peers", peers = g.gossipsub.peers(topic), topic
 
@@ -306,12 +312,13 @@ proc validateAndRelay(g: GossipSub,
     var seenPeers: HashSet[PubSubPeer]
     discard g.validationSeen.pop(msgIdSalted, seenPeers)
     libp2p_gossipsub_duplicate_during_validation.inc(seenPeers.len.int64)
+    libp2p_gossipsub_saved_bytes.inc((msg.data.len * seenPeers.len).int64, labelValues = ["validation_duplicate"])
 
     case validation
     of ValidationResult.Reject:
       debug "Dropping message after validation, reason: reject",
         msgId = shortLog(msgId), peer
-      g.punishInvalidMessage(peer, msg.topicIds)
+      await g.punishInvalidMessage(peer, msg)
       return
     of ValidationResult.Ignore:
       debug "Dropping message after validation, reason: ignore",
@@ -333,6 +340,9 @@ proc validateAndRelay(g: GossipSub,
       g.floodsub.withValue(t, peers): toSendPeers.incl(peers[])
       g.mesh.withValue(t, peers): toSendPeers.incl(peers[])
 
+      # add direct peers
+      toSendPeers.incl(g.subscribedDirectPeers.getOrDefault(t))
+
     # Don't send it to source peer, or peers that
     # sent it during validation
     toSendPeers.excl(peer)
@@ -349,6 +359,8 @@ proc validateAndRelay(g: GossipSub,
       for heDontWant in peer.heDontWants:
         if msgId in heDontWant:
           seenPeers.incl(peer)
+          libp2p_gossipsub_idontwant_saved_messages.inc
+          libp2p_gossipsub_saved_bytes.inc(msg.data.len.int64, labelValues = ["idontwant"])
           break
     toSendPeers.excl(seenPeers)
 
@@ -369,9 +381,57 @@ proc validateAndRelay(g: GossipSub,
   except CatchableError as exc:
     info "validateAndRelay failed", msg=exc.msg
 
+proc dataAndTopicsIdSize(msgs: seq[Message]): int =
+  msgs.mapIt(it.data.len + it.topicIds.mapIt(it.len).foldl(a + b, 0)).foldl(a + b, 0)
+
+proc rateLimit*(g: GossipSub, peer: PubSubPeer, rpcMsgOpt: Opt[RPCMsg], msgSize: int) {.raises:[PeerRateLimitError, CatchableError], async.} =
+  # In this way we count even ignored fields by protobuf
+
+  var rmsg = rpcMsgOpt.valueOr:
+    peer.overheadRateLimitOpt.withValue(overheadRateLimit):
+      if not overheadRateLimit.tryConsume(msgSize):
+        libp2p_gossipsub_peers_rate_limit_hits.inc(labelValues = [peer.getAgent()]) # let's just measure at the beginning for test purposes.
+        debug "Peer sent a msg that couldn't be decoded and it's above rate limit.", peer, uselessAppBytesNum = msgSize
+        if g.parameters.disconnectPeerAboveRateLimit:
+          await g.disconnectPeer(peer)
+          raise newException(PeerRateLimitError, "Peer disconnected because it's above rate limit.")
+
+    raise newException(CatchableError, "Peer msg couldn't be decoded")
+
+  let usefulMsgBytesNum =
+    if g.verifySignature:
+      byteSize(rmsg.messages)
+    else:
+      dataAndTopicsIdSize(rmsg.messages)
+
+  var uselessAppBytesNum = msgSize - usefulMsgBytesNum
+  rmsg.control.withValue(control):
+    uselessAppBytesNum -= (byteSize(control.ihave) + byteSize(control.iwant))
+
+  peer.overheadRateLimitOpt.withValue(overheadRateLimit):
+    if not overheadRateLimit.tryConsume(uselessAppBytesNum):
+      libp2p_gossipsub_peers_rate_limit_hits.inc(labelValues = [peer.getAgent()]) # let's just measure at the beginning for test purposes.
+      debug "Peer sent too much useless application data and it's above rate limit.", peer, msgSize, uselessAppBytesNum, rmsg
+      if g.parameters.disconnectPeerAboveRateLimit:
+        await g.disconnectPeer(peer)
+        raise newException(PeerRateLimitError, "Peer disconnected because it's above rate limit.")
+
 method rpcHandler*(g: GossipSub,
                   peer: PubSubPeer,
-                  rpcMsg: RPCMsg) {.async.} =
+                  data: seq[byte]) {.async.} =
+
+  let msgSize = data.len
+  var rpcMsg = decodeRpcMsg(data).valueOr:
+    debug "failed to decode msg from peer", peer, err = error
+    await rateLimit(g, peer, Opt.none(RPCMsg), msgSize)
+    return
+
+  trace "decoded msg from peer", peer, msg = rpcMsg.shortLog
+  await rateLimit(g, peer, Opt.some(rpcMsg), msgSize)
+
+  # trigger hooks
+  peer.recvObservers(rpcMsg)
+
   if rpcMsg.ping.len in 1..<64 and peer.pingBudget > 0:
     g.send(peer, RPCMsg(pong: rpcMsg.ping))
     peer.pingBudget.dec
@@ -432,14 +492,14 @@ method rpcHandler*(g: GossipSub,
       # always validate if signature is present or required
       debug "Dropping message due to failed signature verification",
         msgId = shortLog(msgId), peer
-      g.punishInvalidMessage(peer, msg.topicIds)
+      await g.punishInvalidMessage(peer, msg)
       continue
 
     if msg.seqno.len > 0 and msg.seqno.len != 8:
       # if we have seqno should be 8 bytes long
       debug "Dropping message due to invalid seqno length",
         msgId = shortLog(msgId), peer
-      g.punishInvalidMessage(peer, msg.topicIds)
+      await g.punishInvalidMessage(peer, msg)
       continue
 
     # g.anonymize needs no evaluation when receiving messages
@@ -511,32 +571,38 @@ method publish*(g: GossipSub,
 
   var peers: HashSet[PubSubPeer]
 
-  if g.parameters.floodPublish:
-    # With flood publishing enabled, the mesh is used when propagating messages from other peers,
-    # but a peer's own messages will always be published to all known peers in the topic.
-    for peer in g.gossipsub.getOrDefault(topic):
-      if peer.score >= g.parameters.publishThreshold:
-        trace "publish: including flood/high score peer", peer
-        peers.incl(peer)
-
   # add always direct peers
-  peers.incl(g.explicit.getOrDefault(topic))
+  peers.incl(g.subscribedDirectPeers.getOrDefault(topic))
 
   if topic in g.topics: # if we're subscribed use the mesh
     peers.incl(g.mesh.getOrDefault(topic))
 
-  if peers.len < g.parameters.dLow and g.parameters.floodPublish == false:
-    # not subscribed or bad mesh, send to fanout peers
-    # disable for floodPublish, since we already sent to every good peer
-    #
+  if g.parameters.floodPublish:
+    # With flood publishing enabled, the mesh is used when propagating messages from other peers,
+    # but a peer's own messages will always be published to all known peers in the topic, limited
+    # to the amount of peers we can send it to in one heartbeat
+    var maxPeersToFlodOpt: Opt[int64]
+    if g.parameters.bandwidthEstimatebps > 0:
+      let
+        bandwidth = (g.parameters.bandwidthEstimatebps) div 8 div 1000 # Divisions are to convert it to Bytes per ms TODO replace with bandwidth estimate
+        msToTransmit = max(data.len div bandwidth, 1)
+      maxPeersToFlodOpt = Opt.some(max(g.parameters.heartbeatInterval.milliseconds div msToTransmit, g.parameters.dLow))
+
+    for peer in g.gossipsub.getOrDefault(topic):
+      maxPeersToFlodOpt.withValue(maxPeersToFlod):
+        if peers.len >= maxPeersToFlod: break
+      if peer.score >= g.parameters.publishThreshold:
+        trace "publish: including flood/high score peer", peer
+        peers.incl(peer)
+
+  if peers.len < g.parameters.dLow:
+    # not subscribed, or bad mesh, send to fanout peers
     var fanoutPeers = g.fanout.getOrDefault(topic).toSeq()
-    if fanoutPeers.len == 0:
+    if fanoutPeers.len < g.parameters.dLow:
       g.replenishFanout(topic)
       fanoutPeers = g.fanout.getOrDefault(topic).toSeq()
 
     g.rng.shuffle(fanoutPeers)
-    if fanoutPeers.len + peers.len > g.parameters.d:
-      fanoutPeers.setLen(g.parameters.d - peers.len)
 
     for fanPeer in fanoutPeers:
       peers.incl(fanPeer)
@@ -554,7 +620,6 @@ method publish*(g: GossipSub,
     debug "No peers for topic, skipping publish",  peersOnTopic = topicPeers.len,
                                                    connectedPeers = topicPeers.filterIt(it.connected).len,
                                                    topic
-    # skipping topic as our metrics finds that heavy
     libp2p_gossipsub_failed_publish.inc()
     return 0
 
@@ -590,15 +655,16 @@ method publish*(g: GossipSub,
     libp2p_pubsub_messages_published.inc(peers.len.int64, labelValues = ["generic"])
 
   trace "Published message to peers", peers=peers.len
-
   return peers.len
 
 proc maintainDirectPeer(g: GossipSub, id: PeerId, addrs: seq[MultiAddress]) {.async.} =
-  let peer = g.peers.getOrDefault(id)
-  if isNil(peer):
+  if id notin g.peers:
     trace "Attempting to dial a direct peer", peer = id
+    if g.switch.isConnected(id):
+      warn "We are connected to a direct peer, but it isn't a GossipSub peer!", id
+      return
     try:
-      await g.switch.connect(id, addrs)
+      await g.switch.connect(id, addrs, forceDial = true)
       # populate the peer after it's connected
       discard g.getOrCreatePeer(id, g.codecs)
     except CancelledError as exc:
@@ -657,3 +723,13 @@ method initPubSub*(g: GossipSub)
 
   # init gossip stuff
   g.mcache = MCache.init(g.parameters.historyGossip, g.parameters.historyLength)
+
+method getOrCreatePeer*(
+    g: GossipSub,
+    peerId: PeerId,
+    protos: seq[string]): PubSubPeer =
+
+  let peer = procCall PubSub(g).getOrCreatePeer(peerId, protos)
+  g.parameters.overheadRateLimit.withValue(overheadRateLimit):
+    peer.overheadRateLimitOpt = Opt.some(TokenBucket.new(overheadRateLimit.bytes, overheadRateLimit.interval))
+  return peer

libp2p/protocols/pubsub/gossipsub/behavior.nim

@@ -106,10 +106,11 @@ proc handleGraft*(g: GossipSub,
     let topic = graft.topicId
     trace "peer grafted topic", peer, topic
 
-    # It is an error to GRAFT on a explicit peer
+    # It is an error to GRAFT on a direct peer
     if peer.peerId in g.parameters.directPeers:
       # receiving a graft from a direct peer should yield a more prominent warning (protocol violation)
-      warn "an explicit peer attempted to graft us, peering agreements should be reciprocal",
+      # we are trusting direct peer not to abuse this
+      warn "a direct peer attempted to graft us, peering agreements should be reciprocal",
         peer, topic
       # and such an attempt should be logged and rejected with a PRUNE
       prunes.add(ControlPrune(
@@ -340,7 +341,7 @@ proc rebalanceMesh*(g: GossipSub, topic: string, metrics: ptr MeshMetrics = nil)
             # avoid negative score peers
             it.score >= 0.0 and
             it notin currentMesh[] and
-            # don't pick explicit peers
+            # don't pick direct peers
             it.peerId notin g.parameters.directPeers and
             # and avoid peers we are backing off
             it.peerId notin backingOff:
@@ -380,7 +381,7 @@ proc rebalanceMesh*(g: GossipSub, topic: string, metrics: ptr MeshMetrics = nil)
             it notin currentMesh[] and
             # avoid negative score peers
             it.score >= 0.0 and
-            # don't pick explicit peers
+            # don't pick direct peers
             it.peerId notin g.parameters.directPeers and
             # and avoid peers we are backing off
             it.peerId notin backingOff:
@@ -482,7 +483,7 @@ proc rebalanceMesh*(g: GossipSub, topic: string, metrics: ptr MeshMetrics = nil)
               # avoid negative score peers
               it.score >= median.score and
               it notin currentMesh[] and
-              # don't pick explicit peers
+              # don't pick direct peers
               it.peerId notin g.parameters.directPeers and
               # and avoid peers we are backing off
               it.peerId notin backingOff:
@@ -556,8 +557,8 @@ proc replenishFanout*(g: GossipSub, topic: string) {.raises: [].} =
   logScope: topic
   trace "about to replenish fanout"
 
-  let currentMesh = g.mesh.getOrDefault(topic)
   if g.fanout.peers(topic) < g.parameters.dLow:
+    let currentMesh = g.mesh.getOrDefault(topic)
     trace "replenishing fanout", peers = g.fanout.peers(topic)
     for peer in g.gossipsub.getOrDefault(topic):
       if peer in currentMesh: continue

libp2p/protocols/pubsub/gossipsub/scoring.nim

@@ -11,9 +11,12 @@
 
 import std/[tables, sets]
 import chronos, chronicles, metrics
+import chronos/ratelimit
 import "."/[types]
 import ".."/[pubsubpeer]
+import ../rpc/messages
 import "../../.."/[peerid, multiaddress, switch, utils/heartbeat]
+import ../pubsub
 
 logScope:
   topics = "libp2p gossipsub"
@@ -27,6 +30,7 @@ declareGauge(libp2p_gossipsub_peers_score_invalidMessageDeliveries, "Detailed go
 declareGauge(libp2p_gossipsub_peers_score_appScore, "Detailed gossipsub scoring metric", labels = ["agent"])
 declareGauge(libp2p_gossipsub_peers_score_behaviourPenalty, "Detailed gossipsub scoring metric", labels = ["agent"])
 declareGauge(libp2p_gossipsub_peers_score_colocationFactor, "Detailed gossipsub scoring metric", labels = ["agent"])
+declarePublicCounter(libp2p_gossipsub_peers_rate_limit_hits, "The number of times peers were above their rate limit", labels = ["agent"])
 
 proc init*(_: type[TopicParams]): TopicParams =
   TopicParams(
@@ -85,27 +89,18 @@ proc colocationFactor(g: GossipSub, peer: PubSubPeer): float64 =
 
 {.pop.}
 
-proc disconnectPeer(g: GossipSub, peer: PubSubPeer) {.async.} =
-  let agent =
-    when defined(libp2p_agents_metrics):
-      if peer.shortAgent.len > 0:
-        peer.shortAgent
-      else:
-        "unknown"
-    else:
-      "unknown"
-  libp2p_gossipsub_bad_score_disconnection.inc(labelValues = [agent])
-
+proc disconnectPeer*(g: GossipSub, peer: PubSubPeer) {.async.} =
   try:
     await g.switch.disconnect(peer.peerId)
   except CatchableError as exc: # Never cancelled
     trace "Failed to close connection", peer, error = exc.name, msg = exc.msg
 
-proc disconnectBadPeerCheck*(g: GossipSub, peer: PubSubPeer, score: float64) =
+proc disconnectIfBadScorePeer*(g: GossipSub, peer: PubSubPeer, score: float64) =
   if g.parameters.disconnectBadPeers and score < g.parameters.graylistThreshold and
      peer.peerId notin g.parameters.directPeers:
     debug "disconnecting bad score peer", peer, score = peer.score
     asyncSpawn(g.disconnectPeer(peer))
+    libp2p_gossipsub_bad_score_disconnection.inc(labelValues = [peer.getAgent()])
 
 proc updateScores*(g: GossipSub) = # avoid async
   ## https://github.com/libp2p/specs/blob/master/pubsub/gossipsub/gossipsub-v1.1.md#the-score-function
@@ -175,14 +170,7 @@ proc updateScores*(g: GossipSub) = # avoid async
         score += topicScore * topicParams.topicWeight
 
       # Score metrics
-      let agent =
-        when defined(libp2p_agents_metrics):
-          if peer.shortAgent.len > 0:
-            peer.shortAgent
-          else:
-            "unknown"
-        else:
-          "unknown"
+      let agent = peer.getAgent()
       libp2p_gossipsub_peers_score_firstMessageDeliveries.inc(info.firstMessageDeliveries, labelValues = [agent])
       libp2p_gossipsub_peers_score_meshMessageDeliveries.inc(info.meshMessageDeliveries, labelValues = [agent])
       libp2p_gossipsub_peers_score_meshFailurePenalty.inc(info.meshFailurePenalty, labelValues = [agent])
@@ -219,14 +207,7 @@ proc updateScores*(g: GossipSub) = # avoid async
     score += colocationFactor * g.parameters.ipColocationFactorWeight
 
     # Score metrics
-    let agent =
-      when defined(libp2p_agents_metrics):
-        if peer.shortAgent.len > 0:
-          peer.shortAgent
-        else:
-          "unknown"
-      else:
-        "unknown"
+    let agent = peer.getAgent()
     libp2p_gossipsub_peers_score_appScore.inc(peer.appScore, labelValues = [agent])
     libp2p_gossipsub_peers_score_behaviourPenalty.inc(peer.behaviourPenalty, labelValues = [agent])
     libp2p_gossipsub_peers_score_colocationFactor.inc(colocationFactor, labelValues = [agent])
@@ -246,8 +227,7 @@ proc updateScores*(g: GossipSub) = # avoid async
 
     trace "updated peer's score", peer, score = peer.score, n_topics, is_grafted
 
-    g.disconnectBadPeerCheck(peer, stats.score)
-
+    g.disconnectIfBadScorePeer(peer, stats.score)
     libp2p_gossipsub_peers_scores.inc(peer.score, labelValues = [agent])
 
   for peer in evicting:
@@ -260,8 +240,18 @@ proc scoringHeartbeat*(g: GossipSub) {.async.} =
     trace "running scoring heartbeat", instance = cast[int](g)
     g.updateScores()
 
-proc punishInvalidMessage*(g: GossipSub, peer: PubSubPeer, topics: seq[string]) =
-  for tt in topics:
+proc punishInvalidMessage*(g: GossipSub, peer: PubSubPeer, msg: Message) {.async.}  =
+  let uselessAppBytesNum = msg.data.len
+  peer.overheadRateLimitOpt.withValue(overheadRateLimit):
+    if not overheadRateLimit.tryConsume(uselessAppBytesNum):
+      debug "Peer sent invalid message and it's above rate limit", peer, uselessAppBytesNum
+      libp2p_gossipsub_peers_rate_limit_hits.inc(labelValues = [peer.getAgent()]) # let's just measure at the beginning for test purposes.
+      if g.parameters.disconnectPeerAboveRateLimit:
+        await g.disconnectPeer(peer)
+        raise newException(PeerRateLimitError, "Peer disconnected because it's above rate limit.")
+
+
+  for tt in msg.topicIds:
     let t = tt
     if t notin g.topics:
       continue

libp2p/protocols/pubsub/gossipsub/types.nim

@@ -142,6 +142,11 @@ type
     disconnectBadPeers*: bool
     enablePX*: bool
 
+    bandwidthEstimatebps*: int # This is currently used only for limting flood publishing. 0 disables flood-limiting completely
+
+    overheadRateLimit*: Opt[tuple[bytes: int, interval: Duration]]
+    disconnectPeerAboveRateLimit*: bool
+
   BackoffTable* = Table[string, Table[PeerId, Moment]]
   ValidationSeenTable* = Table[MessageId, HashSet[PubSubPeer]]
 
@@ -156,7 +161,7 @@ type
     mesh*: PeerTable                           # peers that we send messages to when we are subscribed to the topic
     fanout*: PeerTable                         # peers that we send messages to when we're not subscribed to the topic
     gossipsub*: PeerTable                      # peers that are subscribed to a topic
-    explicit*: PeerTable                       # directpeers that we keep alive explicitly
+    subscribedDirectPeers*: PeerTable          # directpeers that we keep alive
     backingOff*: BackoffTable                  # peers to backoff from when replenishing the mesh
     lastFanoutPubSub*: Table[string, Moment]   # last publish time for fanout topics
     gossip*: Table[string, seq[ControlIHave]]  # pending gossip

libp2p/protocols/pubsub/pubsub.nim

@@ -17,6 +17,7 @@
 
 import std/[tables, sequtils, sets, strutils]
 import chronos, chronicles, metrics
+import chronos/ratelimit
 import ./errors as pubsub_errors,
        ./pubsubpeer,
        ./rpc/[message, messages, protobuf],
@@ -263,7 +264,7 @@ proc updateMetrics*(p: PubSub, rpcMsg: RPCMsg) =
 
 method rpcHandler*(p: PubSub,
                    peer: PubSubPeer,
-                   rpcMsg: RPCMsg): Future[void] {.base, async.} =
+                   data: seq[byte]): Future[void] {.base, async.} =
   ## Handler that must be overridden by concrete implementation
   raiseAssert "Unimplemented"
 
@@ -278,10 +279,11 @@ method onPubSubPeerEvent*(p: PubSub, peer: PubSubPeer, event: PubSubPeerEvent) {
   of PubSubPeerEventKind.Disconnected:
     discard
 
-proc getOrCreatePeer*(
+method getOrCreatePeer*(
     p: PubSub,
     peerId: PeerId,
-    protos: seq[string]): PubSubPeer =
+    protos: seq[string]): PubSubPeer {.base, gcsafe.} =
+
   p.peers.withValue(peerId, peer):
     return peer[]
 
@@ -354,9 +356,9 @@ method handleConn*(p: PubSub,
   ##    that we're interested in
   ##
 
-  proc handler(peer: PubSubPeer, msg: RPCMsg): Future[void] =
+  proc handler(peer: PubSubPeer, data: seq[byte]): Future[void] =
     # call pubsub rpc handler
-    p.rpcHandler(peer, msg)
+    p.rpcHandler(peer, data)
 
   let peer = p.getOrCreatePeer(conn.peerId, @[proto])
 

libp2p/protocols/pubsub/pubsubpeer.nim

@@ -12,6 +12,7 @@
 import std/[sequtils, strutils, tables, hashes, options, sets, deques]
 import stew/results
 import chronos, chronicles, nimcrypto/sha2, metrics
+import chronos/ratelimit
 import rpc/[messages, message, protobuf],
        ../../peerid,
        ../../peerinfo,
@@ -32,6 +33,8 @@ when defined(libp2p_expensive_metrics):
   declareCounter(libp2p_pubsub_skipped_sent_messages, "number of sent skipped messages", labels = ["id"])
 
 type
+  PeerRateLimitError* = object of CatchableError
+
   PubSubObserver* = ref object
     onRecv*: proc(peer: PubSubPeer; msgs: var RPCMsg) {.gcsafe, raises: [].}
     onSend*: proc(peer: PubSubPeer; msgs: var RPCMsg) {.gcsafe, raises: [].}
@@ -66,8 +69,9 @@ type
     maxMessageSize: int
     appScore*: float64 # application specific score
     behaviourPenalty*: float64 # the eventual penalty score
+    overheadRateLimitOpt*: Opt[TokenBucket]
 
-  RPCHandler* = proc(peer: PubSubPeer, msg: RPCMsg): Future[void]
+  RPCHandler* = proc(peer: PubSubPeer, data: seq[byte]): Future[void]
     {.gcsafe, raises: [].}
 
 when defined(libp2p_agents_metrics):
@@ -107,7 +111,7 @@ func outbound*(p: PubSubPeer): bool =
   else:
     false
 
-proc recvObservers(p: PubSubPeer, msg: var RPCMsg) =
+proc recvObservers*(p: PubSubPeer, msg: var RPCMsg) =
   # trigger hooks
   if not(isNil(p.observers)) and p.observers[].len > 0:
     for obs in p.observers[]:
@@ -134,26 +138,19 @@ proc handle*(p: PubSubPeer, conn: Connection) {.async.} =
           conn, peer = p, closed = conn.closed,
           data = data.shortLog
 
-        var rmsg = decodeRpcMsg(data).valueOr:
-          debug "failed to decode msg from peer",
-            conn, peer = p, closed = conn.closed,
-            err = error
-          break
-        data = newSeq[byte]() # Release memory
-
-        trace "decoded msg from peer",
-          conn, peer = p, closed = conn.closed,
-          msg = rmsg.shortLog
-        # trigger hooks
-        p.recvObservers(rmsg)
-
         when defined(libp2p_expensive_metrics):
           for m in rmsg.messages:
             for t in m.topicIDs:
               # metrics
               libp2p_pubsub_received_messages.inc(labelValues = [$p.peerId, t])
 
-        await p.handler(p, rmsg)
+        await p.handler(p, data)
+        data = newSeq[byte]() # Release memory
+    except PeerRateLimitError as exc:
+      debug "Peer rate limit exceeded, exiting read while", conn, peer = p, error = exc.msg
+    except CatchableError as exc:
+      debug "Exception occurred in PubSubPeer.handle",
+        conn, peer = p, closed = conn.closed, exc = exc.msg
     finally:
       await conn.close()
   except CancelledError:
@@ -245,7 +242,7 @@ proc sendEncoded*(p: PubSubPeer, msg: seq[byte]) {.raises: [], async.} =
     return
 
   if msg.len > p.maxMessageSize:
-    info "trying to send a too big for pubsub", maxSize=p.maxMessageSize, msgSize=msg.len
+    info "trying to send a msg too big for pubsub", maxSize=p.maxMessageSize, msgSize=msg.len
     return
 
   if p.sendConn == nil:
@@ -272,9 +269,42 @@ proc sendEncoded*(p: PubSubPeer, msg: seq[byte]) {.raises: [], async.} =
 
     await conn.close() # This will clean up the send connection
 
-proc send*(p: PubSubPeer, msg: RPCMsg, anonymize: bool) {.raises: [].} =
-  trace "sending msg to peer", peer = p, rpcMsg = shortLog(msg)
+iterator splitRPCMsg(peer: PubSubPeer, rpcMsg: RPCMsg, maxSize: int, anonymize: bool): seq[byte] =
+  ## This iterator takes an `RPCMsg` and sequentially repackages its Messages into new `RPCMsg` instances.
+  ## Each new `RPCMsg` accumulates Messages until reaching the specified `maxSize`. If a single Message
+  ## exceeds the `maxSize` when trying to fit into an empty `RPCMsg`, the latter is skipped as too large to send.
+  ## Every constructed `RPCMsg` is then encoded, optionally anonymized, and yielded as a sequence of bytes.
 
+  var currentRPCMsg = rpcMsg
+  currentRPCMsg.messages = newSeq[Message]()
+
+  var currentSize = byteSize(currentRPCMsg)
+
+  for msg in rpcMsg.messages:
+    let msgSize = byteSize(msg)
+
+    # Check if adding the next message will exceed maxSize
+    if float(currentSize + msgSize) * 1.1 > float(maxSize): # Guessing 10% protobuf overhead
+      if currentRPCMsg.messages.len == 0:
+        trace "message too big to sent", peer, rpcMsg = shortLog(currentRPCMsg)
+        continue # Skip this message
+
+      trace "sending msg to peer", peer, rpcMsg = shortLog(currentRPCMsg)
+      yield encodeRpcMsg(currentRPCMsg, anonymize)
+      currentRPCMsg = RPCMsg()
+      currentSize = 0
+
+    currentRPCMsg.messages.add(msg)
+    currentSize += msgSize
+
+  # Check if there is a non-empty currentRPCMsg left to be added
+  if currentSize > 0 and currentRPCMsg.messages.len > 0:
+    trace "sending msg to peer", peer, rpcMsg = shortLog(currentRPCMsg)
+    yield encodeRpcMsg(currentRPCMsg, anonymize)
+  else:
+    trace "message too big to sent", peer, rpcMsg = shortLog(currentRPCMsg)
+
+proc send*(p: PubSubPeer, msg: RPCMsg, anonymize: bool) {.raises: [].} =
   # When sending messages, we take care to re-encode them with the right
   # anonymization flag to ensure that we're not penalized for sending invalid
   # or malicious data on the wire - in particular, re-encoding protects against
@@ -292,7 +322,13 @@ proc send*(p: PubSubPeer, msg: RPCMsg, anonymize: bool) {.raises: [].} =
     sendMetrics(msg)
     encodeRpcMsg(msg, anonymize)
 
-  asyncSpawn p.sendEncoded(encoded)
+  if encoded.len > p.maxMessageSize and msg.messages.len > 1:
+    for encodedSplitMsg in splitRPCMsg(p, msg, p.maxMessageSize, anonymize):
+      asyncSpawn p.sendEncoded(encodedSplitMsg)
+  else:
+    # If the message size is within limits, send it as is
+    trace "sending msg to peer", peer = p, rpcMsg = shortLog(msg)
+    asyncSpawn p.sendEncoded(encoded)
 
 proc canAskIWant*(p: PubSubPeer, msgId: MessageId): bool =
   for sentIHave in p.sentIHaves.mitems():
@@ -307,7 +343,8 @@ proc new*(
   getConn: GetConn,
   onEvent: OnEvent,
   codec: string,
-  maxMessageSize: int): T =
+  maxMessageSize: int,
+  overheadRateLimitOpt: Opt[TokenBucket] = Opt.none(TokenBucket)): T =
 
   result = T(
     getConn: getConn,
@@ -315,7 +352,18 @@ proc new*(
     codec: codec,
     peerId: peerId,
     connectedFut: newFuture[void](),
-    maxMessageSize: maxMessageSize
+    maxMessageSize: maxMessageSize,
+    overheadRateLimitOpt: overheadRateLimitOpt
   )
   result.sentIHaves.addFirst(default(HashSet[MessageId]))
   result.heDontWants.addFirst(default(HashSet[MessageId]))
+
+proc getAgent*(peer: PubSubPeer): string =
+  return
+    when defined(libp2p_agents_metrics):
+      if peer.shortAgent.len > 0:
+        peer.shortAgent
+      else:
+        "unknown"
+    else:
+      "unknown"

libp2p/protocols/pubsub/rpc/messages.nim

@@ -9,7 +9,7 @@
 
 {.push raises: [].}
 
-import options, sequtils
+import options, sequtils, sugar
 import "../../.."/[
         peerid,
         routing_record,
@@ -18,6 +18,14 @@ import "../../.."/[
 
 export options
 
+proc expectedFields[T](t: typedesc[T], existingFieldNames: seq[string]) {.raises: [CatchableError].} =
+  var fieldNames: seq[string]
+  for name, _ in fieldPairs(T()):
+    fieldNames &= name
+  if fieldNames != existingFieldNames:
+    fieldNames.keepIf(proc(it: string): bool = it notin existingFieldNames)
+    raise newException(CatchableError, $T & " fields changed, please search for and revise all relevant procs. New fields: " & $fieldNames)
+
 type
     PeerInfoMsg* = object
       peerId*: PeerId
@@ -116,3 +124,54 @@ func shortLog*(m: RPCMsg): auto =
     messages: mapIt(m.messages, it.shortLog),
     control: m.control.get(ControlMessage()).shortLog
   )
+
+static: expectedFields(PeerInfoMsg, @["peerId", "signedPeerRecord"])
+proc byteSize(peerInfo: PeerInfoMsg): int =
+  peerInfo.peerId.len + peerInfo.signedPeerRecord.len
+
+static: expectedFields(SubOpts, @["subscribe", "topic"])
+proc byteSize(subOpts: SubOpts): int =
+  1 + subOpts.topic.len # 1 byte for the bool
+
+static: expectedFields(Message, @["fromPeer", "data", "seqno", "topicIds", "signature", "key"])
+proc byteSize*(msg: Message): int =
+  msg.fromPeer.len + msg.data.len + msg.seqno.len +
+         msg.signature.len + msg.key.len + msg.topicIds.foldl(a + b.len, 0)
+
+proc byteSize*(msgs: seq[Message]): int =
+  msgs.foldl(a + b.byteSize, 0)
+
+static: expectedFields(ControlIHave, @["topicId", "messageIds"])
+proc byteSize(controlIHave: ControlIHave): int =
+  controlIHave.topicId.len + controlIHave.messageIds.foldl(a + b.len, 0)
+
+proc byteSize*(ihaves: seq[ControlIHave]): int =
+  ihaves.foldl(a + b.byteSize, 0)
+
+static: expectedFields(ControlIWant, @["messageIds"])
+proc byteSize(controlIWant: ControlIWant): int =
+  controlIWant.messageIds.foldl(a + b.len, 0)
+
+proc byteSize*(iwants: seq[ControlIWant]): int =
+  iwants.foldl(a + b.byteSize, 0)
+
+static: expectedFields(ControlGraft, @["topicId"])
+proc byteSize(controlGraft: ControlGraft): int =
+  controlGraft.topicId.len
+
+static: expectedFields(ControlPrune, @["topicId", "peers", "backoff"])
+proc byteSize(controlPrune: ControlPrune): int =
+  controlPrune.topicId.len + controlPrune.peers.foldl(a + b.byteSize, 0) + 8 # 8 bytes for uint64
+
+static: expectedFields(ControlMessage, @["ihave", "iwant", "graft", "prune", "idontwant"])
+proc byteSize(control: ControlMessage): int =
+  control.ihave.foldl(a + b.byteSize, 0) + control.iwant.foldl(a + b.byteSize, 0) +
+  control.graft.foldl(a + b.byteSize, 0) + control.prune.foldl(a + b.byteSize, 0) +
+  control.idontwant.foldl(a + b.byteSize, 0)
+
+static: expectedFields(RPCMsg, @["subscriptions", "messages", "control", "ping", "pong"])
+proc byteSize*(rpc: RPCMsg): int =
+  result = rpc.subscriptions.foldl(a + b.byteSize, 0) + byteSize(rpc.messages) +
+           rpc.ping.len + rpc.pong.len
+  rpc.control.withValue(ctrl):
+    result += ctrl.byteSize

libp2p/protocols/rendezvous.nim

@@ -469,6 +469,8 @@ proc advertisePeer(rdv: RendezVous,
         trace "Unexpected register response", peer, msgType = msgRecv.msgType
       elif msgRecv.registerResponse.tryGet().status != ResponseStatus.Ok:
         trace "Refuse to register", peer, response = msgRecv.registerResponse
+      else:
+        trace "Successfully registered", peer, response = msgRecv.registerResponse
     except CatchableError as exc:
       trace "exception in the advertise", error = exc.msg
     finally:
@@ -476,9 +478,9 @@ proc advertisePeer(rdv: RendezVous,
   await rdv.sema.acquire()
   discard await advertiseWrap().withTimeout(5.seconds)
 
-proc advertise*(rdv: RendezVous,
+method advertise*(rdv: RendezVous,
                 ns: string,
-                ttl: Duration = MinimumDuration) {.async.} =
+                ttl: Duration = MinimumDuration) {.async, base.} =
   let sprBuff = rdv.switch.peerInfo.signedPeerRecord.encode().valueOr:
     raise newException(RendezVousError, "Wrong Signed Peer Record")
   if ns.len notin 1..255:

libp2p/transports/tcptransport.nim

@@ -219,7 +219,7 @@ method accept*(self: TcpTransport): Future[Connection] {.async, gcsafe.} =
 
   try:
     if self.acceptFuts.len <= 0:
-      self.acceptFuts = self.servers.mapIt(it.accept())
+      self.acceptFuts = self.servers.mapIt(Future[StreamTransport](it.accept()))
 
     if self.acceptFuts.len <= 0:
       return

libp2p/transports/wstransport.nim

@@ -108,7 +108,7 @@ type
     flags: set[ServerFlags]
     handshakeTimeout: Duration
     factories: seq[ExtFactory]
-    rng: Rng
+    rng: ref HmacDrbgContext
 
 proc secure*(self: WsTransport): bool =
   not (isNil(self.tlsPrivateKey) or isNil(self.tlsCertificate))
@@ -276,6 +276,8 @@ method accept*(self: WsTransport): Future[Connection] {.async, gcsafe.} =
     debug "AsyncStream Error", exc = exc.msg
   except TransportTooManyError as exc:
     debug "Too many files opened", exc = exc.msg
+  except TransportAbortedError as exc:
+    debug "Connection aborted", exc = exc.msg
   except AsyncTimeoutError as exc:
     debug "Timed out", exc = exc.msg
   except TransportUseClosedError as exc:
@@ -327,7 +329,7 @@ proc new*(
   tlsFlags: set[TLSFlags] = {},
   flags: set[ServerFlags] = {},
   factories: openArray[ExtFactory] = [],
-  rng: Rng = nil,
+  rng: ref HmacDrbgContext = nil,
   handshakeTimeout = DefaultHeadersTimeout): T {.public.} =
   ## Creates a secure WebSocket transport
 
@@ -346,7 +348,7 @@ proc new*(
   upgrade: Upgrade,
   flags: set[ServerFlags] = {},
   factories: openArray[ExtFactory] = [],
-  rng: Rng = nil,
+  rng: ref HmacDrbgContext = nil,
   handshakeTimeout = DefaultHeadersTimeout): T {.public.} =
   ## Creates a clear-text WebSocket transport
 

libp2p/upgrademngrs/muxedupgrade.nim

@@ -91,13 +91,11 @@ proc new*(
   T: type MuxedUpgrade,
   muxers: seq[MuxerProvider],
   secureManagers: openArray[Secure] = [],
-  connManager: ConnManager,
   ms: MultistreamSelect): T =
 
   let upgrader = T(
     muxers: muxers,
     secureManagers: @secureManagers,
-    connManager: connManager,
     ms: ms)
 
   upgrader.streamHandler = proc(conn: Connection)

libp2p/upgrademngrs/upgrade.nim

@@ -35,7 +35,6 @@ type
 
   Upgrade* = ref object of RootObj
     ms*: MultistreamSelect
-    connManager*: ConnManager
     secureManagers*: seq[Secure]
 
 method upgrade*(

libp2p/utility.nim

@@ -70,6 +70,10 @@ template safeConvert*[T: SomeInteger, S: Ordinal](value: S): T =
   else:
     {.error: "Source and target types have an incompatible range low..high".}
 
+proc capLen*[T](s: var seq[T], length: Natural) =
+  if s.len > length:
+    s.setLen(length)
+
 template exceptionToAssert*(body: untyped): untyped =
   block:
     var res: type(body)

nimble.lock

@@ -0,0 +1,219 @@
+{
+  "version": 2,
+  "packages": {
+    "unittest2": {
+      "version": "0.2.1",
+      "vcsRevision": "262b697f38d6b6f1e7462d3b3ab81d79b894e336",
+      "url": "https://github.com/status-im/nim-unittest2",
+      "downloadMethod": "git",
+      "dependencies": [],
+      "checksums": {
+        "sha1": "1bac3a8355441edeed1ef3134e7436d6fb5d4498"
+      }
+    },
+    "bearssl": {
+      "version": "0.2.1",
+      "vcsRevision": "e4157639db180e52727712a47deaefcbbac6ec86",
+      "url": "https://github.com/status-im/nim-bearssl",
+      "downloadMethod": "git",
+      "dependencies": [
+        "unittest2"
+      ],
+      "checksums": {
+        "sha1": "a5086fd5c0af2b852f34c0cc6e4cff93a98f97ec"
+      }
+    },
+    "results": {
+      "version": "0.4.0",
+      "vcsRevision": "f3c666a272c69d70cb41e7245e7f6844797303ad",
+      "url": "https://github.com/arnetheduck/nim-results",
+      "downloadMethod": "git",
+      "dependencies": [],
+      "checksums": {
+        "sha1": "51e08ca9524db98dc909fb39192272cc2b5451c7"
+      }
+    },
+    "stew": {
+      "version": "0.1.0",
+      "vcsRevision": "2c2544aec13536304438be045bfdd22452741466",
+      "url": "https://github.com/status-im/nim-stew",
+      "downloadMethod": "git",
+      "dependencies": [
+        "results",
+        "unittest2"
+      ],
+      "checksums": {
+        "sha1": "0d3c4f15a4cff934ec30e5b2d5fe590922839a4e"
+      }
+    },
+    "faststreams": {
+      "version": "0.3.0",
+      "vcsRevision": "720fc5e5c8e428d9d0af618e1e27c44b42350309",
+      "url": "https://github.com/status-im/nim-faststreams",
+      "downloadMethod": "git",
+      "dependencies": [
+        "stew",
+        "unittest2"
+      ],
+      "checksums": {
+        "sha1": "ab178ba25970b95d953434b5d86b4d60396ccb64"
+      }
+    },
+    "serialization": {
+      "version": "0.2.0",
+      "vcsRevision": "4bdbc29e54fe54049950e352bb969aab97173b35",
+      "url": "https://github.com/status-im/nim-serialization",
+      "downloadMethod": "git",
+      "dependencies": [
+        "faststreams",
+        "unittest2",
+        "stew"
+      ],
+      "checksums": {
+        "sha1": "c8c99a387aae488e7008aded909ebfe662e74450"
+      }
+    },
+    "json_serialization": {
+      "version": "0.1.5",
+      "vcsRevision": "85b7ea093cb85ee4f433a617b97571bd709d30df",
+      "url": "https://github.com/status-im/nim-json-serialization",
+      "downloadMethod": "git",
+      "dependencies": [
+        "serialization",
+        "stew"
+      ],
+      "checksums": {
+        "sha1": "c6b30565292acf199b8be1c62114726e354af59e"
+      }
+    },
+    "nimcrypto": {
+      "version": "0.6.0",
+      "vcsRevision": "1c8d6e3caf3abc572136ae9a1da81730c4eb4288",
+      "url": "https://github.com/cheatfate/nimcrypto",
+      "downloadMethod": "git",
+      "dependencies": [],
+      "checksums": {
+        "sha1": "da3b105ad6bd7beef25c69f03afccb5e5233d483"
+      }
+    },
+    "secp256k1": {
+      "version": "0.6.0.3.2",
+      "vcsRevision": "7246d91c667f4cc3759fdd50339caa45a2ecd8be",
+      "url": "https://github.com/status-im/nim-secp256k1",
+      "downloadMethod": "git",
+      "dependencies": [
+        "stew",
+        "nimcrypto"
+      ],
+      "checksums": {
+        "sha1": "aa0f88a68f67cef07f9f4a365a0121a2217dab81"
+      }
+    },
+    "httputils": {
+      "version": "0.3.0",
+      "vcsRevision": "3b491a40c60aad9e8d3407443f46f62511e63b18",
+      "url": "https://github.com/status-im/nim-http-utils",
+      "downloadMethod": "git",
+      "dependencies": [
+        "stew",
+        "unittest2"
+      ],
+      "checksums": {
+        "sha1": "1331f33585eda05d1e50385fa7871c3bf2a449d7"
+      }
+    },
+    "chronos": {
+      "version": "3.2.0",
+      "vcsRevision": "ba143e029f35fd9b4cd3d89d007cc834d0d5ba3c",
+      "url": "https://github.com/status-im/nim-chronos",
+      "downloadMethod": "git",
+      "dependencies": [
+        "stew",
+        "bearssl",
+        "httputils",
+        "unittest2"
+      ],
+      "checksums": {
+        "sha1": "5783067584ac6812eb64b8454ea6f9c97ff1262a"
+      }
+    },
+    "metrics": {
+      "version": "0.0.1",
+      "vcsRevision": "6142e433fc8ea9b73379770a788017ac528d46ff",
+      "url": "https://github.com/status-im/nim-metrics",
+      "downloadMethod": "git",
+      "dependencies": [
+        "chronos"
+      ],
+      "checksums": {
+        "sha1": "16ba266012d32d49631ca00add8e4698343758e0"
+      }
+    },
+    "testutils": {
+      "version": "0.5.0",
+      "vcsRevision": "dfc4c1b39f9ded9baf6365014de2b4bfb4dafc34",
+      "url": "https://github.com/status-im/nim-testutils",
+      "downloadMethod": "git",
+      "dependencies": [
+        "unittest2"
+      ],
+      "checksums": {
+        "sha1": "756d0757c4dd06a068f9d38c7f238576ba5ee897"
+      }
+    },
+    "chronicles": {
+      "version": "0.10.3",
+      "vcsRevision": "32ac8679680ea699f7dbc046e8e0131cac97d41a",
+      "url": "https://github.com/status-im/nim-chronicles",
+      "downloadMethod": "git",
+      "dependencies": [
+        "testutils",
+        "json_serialization"
+      ],
+      "checksums": {
+        "sha1": "79f09526d4d9b9196dd2f6a75310d71a890c4f88"
+      }
+    },
+    "zlib": {
+      "version": "0.1.0",
+      "vcsRevision": "a2f44bb7f65571a894227ff6fde9298a104e03a5",
+      "url": "https://github.com/status-im/nim-zlib",
+      "downloadMethod": "git",
+      "dependencies": [
+        "stew"
+      ],
+      "checksums": {
+        "sha1": "edbf76ebdecb63d302d1883fe4b23b2eb0608cb7"
+      }
+    },
+    "websock": {
+      "version": "0.1.0",
+      "vcsRevision": "f8ed9b40a5ff27ad02a3c237c4905b0924e3f982",
+      "url": "https://github.com/status-im/nim-websock",
+      "downloadMethod": "git",
+      "dependencies": [
+        "chronos",
+        "httputils",
+        "chronicles",
+        "stew",
+        "nimcrypto",
+        "bearssl",
+        "zlib"
+      ],
+      "checksums": {
+        "sha1": "94f836ae589056b2deb04bdfdcd614fff80adaf5"
+      }
+    },
+    "dnsclient": {
+      "version": "0.3.4",
+      "vcsRevision": "23214235d4784d24aceed99bbfe153379ea557c8",
+      "url": "https://github.com/ba0f3/dnsclient.nim",
+      "downloadMethod": "git",
+      "dependencies": [],
+      "checksums": {
+        "sha1": "65262c7e533ff49d6aca5539da4bc6c6ce132f40"
+      }
+    }
+  },
+  "tasks": {}
+}

tests/pubsub/testgossipinternal.nim

@@ -1,41 +1,31 @@
-include ../../libp2p/protocols/pubsub/gossipsub
+# Nim-LibP2P
+# Copyright (c) 2023 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
 
 {.used.}
 
-import std/[options, deques]
+import std/[options, deques, sequtils, enumerate, algorithm]
 import stew/byteutils
 import ../../libp2p/builders
 import ../../libp2p/errors
 import ../../libp2p/crypto/crypto
 import ../../libp2p/stream/bufferstream
+import ../../libp2p/protocols/pubsub/[pubsub, gossipsub, mcache, mcache, peertable]
+import ../../libp2p/protocols/pubsub/rpc/[message, messages]
 import ../../libp2p/switch
 import ../../libp2p/muxers/muxer
+import ../../libp2p/protocols/pubsub/rpc/protobuf
+import utils
 
 import ../helpers
 
-type
-  TestGossipSub = ref object of GossipSub
-
 proc noop(data: seq[byte]) {.async, gcsafe.} = discard
 
-proc getPubSubPeer(p: TestGossipSub, peerId: PeerId): PubSubPeer =
-  proc getConn(): Future[Connection] =
-    p.switch.dial(peerId, GossipSubCodec)
-
-  let pubSubPeer = PubSubPeer.new(peerId, getConn, nil, GossipSubCodec, 1024 * 1024)
-  debug "created new pubsub peer", peerId
-
-  p.peers[peerId] = pubSubPeer
-
-  onNewPeer(p, pubSubPeer)
-  pubSubPeer
-
-proc randomPeerId(): PeerId =
-  try:
-    PeerId.init(PrivateKey.random(ECDSA, rng[]).get()).tryGet()
-  except CatchableError as exc:
-    raise newException(Defect, exc.msg)
-
 const MsgIdSuccess = "msg id gen success"
 
 suite "GossipSub internal":
@@ -170,7 +160,7 @@ suite "GossipSub internal":
   asyncTest "`replenishFanout` Degree Lo":
     let gossipSub = TestGossipSub.init(newStandardSwitch())
 
-    proc handler(peer: PubSubPeer, msg: RPCMsg) {.async.} =
+    proc handler(peer: PubSubPeer, data: seq[byte]) {.async.} =
       discard
 
     let topic = "foobar"
@@ -197,7 +187,7 @@ suite "GossipSub internal":
   asyncTest "`dropFanoutPeers` drop expired fanout topics":
     let gossipSub = TestGossipSub.init(newStandardSwitch())
 
-    proc handler(peer: PubSubPeer, msg: RPCMsg) {.async.} =
+    proc handler(peer: PubSubPeer, data: seq[byte]) {.async.} =
       discard
 
     let topic = "foobar"
@@ -227,7 +217,7 @@ suite "GossipSub internal":
   asyncTest "`dropFanoutPeers` leave unexpired fanout topics":
     let gossipSub = TestGossipSub.init(newStandardSwitch())
 
-    proc handler(peer: PubSubPeer, msg: RPCMsg) {.async.} =
+    proc handler(peer: PubSubPeer, data: seq[byte]) {.async.} =
       discard
 
     let topic1 = "foobar1"
@@ -264,7 +254,7 @@ suite "GossipSub internal":
   asyncTest "`getGossipPeers` - should gather up to degree D non intersecting peers":
     let gossipSub = TestGossipSub.init(newStandardSwitch())
 
-    proc handler(peer: PubSubPeer, msg: RPCMsg) {.async.} =
+    proc handler(peer: PubSubPeer, data: seq[byte]) {.async.} =
       discard
 
     let topic = "foobar"
@@ -325,7 +315,7 @@ suite "GossipSub internal":
   asyncTest "`getGossipPeers` - should not crash on missing topics in mesh":
     let gossipSub = TestGossipSub.init(newStandardSwitch())
 
-    proc handler(peer: PubSubPeer, msg: RPCMsg) {.async.} =
+    proc handler(peer: PubSubPeer, data: seq[byte]) {.async.} =
       discard
 
     let topic = "foobar"
@@ -365,7 +355,7 @@ suite "GossipSub internal":
   asyncTest "`getGossipPeers` - should not crash on missing topics in fanout":
     let gossipSub = TestGossipSub.init(newStandardSwitch())
 
-    proc handler(peer: PubSubPeer, msg: RPCMsg) {.async.} =
+    proc handler(peer: PubSubPeer, data: seq[byte]) {.async.} =
       discard
 
     let topic = "foobar"
@@ -406,7 +396,7 @@ suite "GossipSub internal":
   asyncTest "`getGossipPeers` - should not crash on missing topics in gossip":
     let gossipSub = TestGossipSub.init(newStandardSwitch())
 
-    proc handler(peer: PubSubPeer, msg: RPCMsg) {.async.} =
+    proc handler(peer: PubSubPeer, data: seq[byte]) {.async.} =
       discard
 
     let topic = "foobar"
@@ -447,7 +437,7 @@ suite "GossipSub internal":
   asyncTest "Drop messages of topics without subscription":
     let gossipSub = TestGossipSub.init(newStandardSwitch())
 
-    proc handler(peer: PubSubPeer, msg: RPCMsg) {.async.} =
+    proc handler(peer: PubSubPeer, data: seq[byte]) {.async.} =
       check false
 
     let topic = "foobar"
@@ -470,7 +460,7 @@ suite "GossipSub internal":
       let peer = gossipSub.getPubSubPeer(peerId)
       inc seqno
       let msg = Message.init(peerId, ("bar" & $i).toBytes(), topic, some(seqno))
-      await gossipSub.rpcHandler(peer, RPCMsg(messages: @[msg]))
+      await gossipSub.rpcHandler(peer, encodeRpcMsg(RPCMsg(messages: @[msg]), false))
 
     check gossipSub.mcache.msgs.len == 0
 
@@ -481,7 +471,7 @@ suite "GossipSub internal":
     let gossipSub = TestGossipSub.init(newStandardSwitch())
     gossipSub.parameters.disconnectBadPeers = true
     gossipSub.parameters.appSpecificWeight = 1.0
-    proc handler(peer: PubSubPeer, msg: RPCMsg) {.async.} =
+    proc handler(peer: PubSubPeer, data: seq[byte]) {.async.} =
       check false
 
     let topic = "foobar"
@@ -525,7 +515,7 @@ suite "GossipSub internal":
     conn.peerId = peerId
     let peer = gossipSub.getPubSubPeer(peerId)
 
-    await gossipSub.rpcHandler(peer, lotOfSubs)
+    await gossipSub.rpcHandler(peer, encodeRpcMsg(lotOfSubs, false))
 
     check:
       gossipSub.gossipsub.len == gossipSub.topicsHigh
@@ -656,7 +646,7 @@ suite "GossipSub internal":
   asyncTest "handleIHave/Iwant tests":
     let gossipSub = TestGossipSub.init(newStandardSwitch())
 
-    proc handler(peer: PubSubPeer, msg: RPCMsg) {.async.} =
+    proc handler(peer: PubSubPeer, data: seq[byte]) {.async.} =
       check false
     proc handler2(topic: string, data: seq[byte]) {.async.} = discard
 
@@ -727,3 +717,130 @@ suite "GossipSub internal":
 
     await allFuturesThrowing(conns.mapIt(it.close()))
     await gossipSub.switch.stop()
+
+  proc setupTest(): Future[tuple[gossip0: GossipSub, gossip1: GossipSub, receivedMessages: ref HashSet[seq[byte]]]] {.async.} =
+    let
+      nodes = generateNodes(2, gossip = true, verifySignature = false)
+    discard await allFinished(
+        nodes[0].switch.start(),
+        nodes[1].switch.start()
+      )
+
+    await nodes[1].switch.connect(nodes[0].switch.peerInfo.peerId, nodes[0].switch.peerInfo.addrs)
+
+    var receivedMessages = new(HashSet[seq[byte]])
+
+    proc handlerA(topic: string, data: seq[byte]) {.async, gcsafe.} =
+      receivedMessages[].incl(data)
+
+    proc handlerB(topic: string, data: seq[byte]) {.async, gcsafe.} =
+      discard
+
+    nodes[0].subscribe("foobar", handlerA)
+    nodes[1].subscribe("foobar", handlerB)
+    await waitSubGraph(nodes, "foobar")
+
+    var gossip0: GossipSub = GossipSub(nodes[0])
+    var gossip1: GossipSub = GossipSub(nodes[1])
+
+    return (gossip0, gossip1, receivedMessages)
+
+  proc teardownTest(gossip0: GossipSub, gossip1: GossipSub) {.async.} =
+    await allFuturesThrowing(
+      gossip0.switch.stop(),
+      gossip1.switch.stop()
+    )
+
+  proc createMessages(gossip0: GossipSub, gossip1: GossipSub, size1: int, size2: int): tuple[iwantMessageIds: seq[MessageId], sentMessages: HashSet[seq[byte]]] =
+    var iwantMessageIds = newSeq[MessageId]()
+    var sentMessages = initHashSet[seq[byte]]()
+
+    for i, size in enumerate([size1, size2]):
+      let data = newSeqWith[byte](size, i.byte)
+      sentMessages.incl(data)
+
+      let msg = Message.init(gossip1.peerInfo.peerId, data, "foobar", some(uint64(i + 1)))
+      let iwantMessageId = gossip1.msgIdProvider(msg).expect(MsgIdSuccess)
+      iwantMessageIds.add(iwantMessageId)
+      gossip1.mcache.put(iwantMessageId, msg)
+
+      let peer = gossip1.peers[(gossip0.peerInfo.peerId)]
+      peer.sentIHaves[^1].incl(iwantMessageId)
+
+    return (iwantMessageIds, sentMessages)
+
+  asyncTest "e2e - Split IWANT replies when individual messages are below maxSize but combined exceed maxSize":
+    # This test checks if two messages, each below the maxSize, are correctly split when their combined size exceeds maxSize.
+    # Expected: Both messages should be received.
+    let (gossip0, gossip1, receivedMessages) = await setupTest()
+
+    let messageSize = gossip1.maxMessageSize div 2 + 1
+    let (iwantMessageIds, sentMessages) = createMessages(gossip0, gossip1, messageSize, messageSize)
+
+    gossip1.broadcast(gossip1.mesh["foobar"], RPCMsg(control: some(ControlMessage(
+      ihave: @[ControlIHave(topicId: "foobar", messageIds: iwantMessageIds)]
+    ))))
+
+    checkExpiring: receivedMessages[] == sentMessages
+    check receivedMessages[].len == 2
+
+    await teardownTest(gossip0, gossip1)
+
+  asyncTest "e2e - Discard IWANT replies when both messages individually exceed maxSize":
+    # This test checks if two messages, each exceeding the maxSize, are discarded and not sent.
+    # Expected: No messages should be received.
+    let (gossip0, gossip1, receivedMessages) = await setupTest()
+
+    let messageSize = gossip1.maxMessageSize + 10
+    let (bigIWantMessageIds, sentMessages) = createMessages(gossip0, gossip1, messageSize, messageSize)
+
+    gossip1.broadcast(gossip1.mesh["foobar"], RPCMsg(control: some(ControlMessage(
+      ihave: @[ControlIHave(topicId: "foobar", messageIds: bigIWantMessageIds)]
+    ))))
+
+    await sleepAsync(300.milliseconds)
+    checkExpiring: receivedMessages[].len == 0
+
+    await teardownTest(gossip0, gossip1)
+
+  asyncTest "e2e - Process IWANT replies when both messages are below maxSize":
+    # This test checks if two messages, both below the maxSize, are correctly processed and sent.
+    # Expected: Both messages should be received.
+    let (gossip0, gossip1, receivedMessages) = await setupTest()
+    let size1 = gossip1.maxMessageSize div 2
+    let size2 = gossip1.maxMessageSize div 3
+    let (bigIWantMessageIds, sentMessages) = createMessages(gossip0, gossip1, size1, size2)
+
+    gossip1.broadcast(gossip1.mesh["foobar"], RPCMsg(control: some(ControlMessage(
+      ihave: @[ControlIHave(topicId: "foobar", messageIds: bigIWantMessageIds)]
+    ))))
+
+    checkExpiring: receivedMessages[] == sentMessages
+    check receivedMessages[].len == 2
+
+    await teardownTest(gossip0, gossip1)
+
+  asyncTest "e2e - Split IWANT replies when one message is below maxSize and the other exceeds maxSize":
+    # This test checks if, when given two messages where one is below maxSize and the other exceeds it, only the smaller message is processed and sent.
+    # Expected: Only the smaller message should be received.
+    let (gossip0, gossip1, receivedMessages) = await setupTest()
+    let maxSize = gossip1.maxMessageSize
+    let size1 = maxSize div 2
+    let size2 = maxSize + 10
+    let (bigIWantMessageIds, sentMessages) = createMessages(gossip0, gossip1, size1, size2)
+
+    gossip1.broadcast(gossip1.mesh["foobar"], RPCMsg(control: some(ControlMessage(
+      ihave: @[ControlIHave(topicId: "foobar", messageIds: bigIWantMessageIds)]
+    ))))
+
+    var smallestSet: HashSet[seq[byte]]
+    let seqs = toSeq(sentMessages)
+    if seqs[0] < seqs[1]:
+      smallestSet.incl(seqs[0])
+    else:
+      smallestSet.incl(seqs[1])
+
+    checkExpiring: receivedMessages[] == smallestSet
+    check receivedMessages[].len == 1
+
+    await teardownTest(gossip0, gossip1)

tests/pubsub/testgossipsub.nim

@@ -10,8 +10,9 @@
 {.used.}
 
 import sequtils, options, tables, sets, sugar
-import chronos, stew/byteutils
+import chronos, stew/byteutils, chronos/ratelimit
 import chronicles
+import metrics
 import utils, ../../libp2p/[errors,
                             peerid,
                             peerinfo,
@@ -20,6 +21,7 @@ import utils, ../../libp2p/[errors,
                             crypto/crypto,
                             protocols/pubsub/pubsub,
                             protocols/pubsub/gossipsub,
+                            protocols/pubsub/gossipsub/scoring,
                             protocols/pubsub/pubsubpeer,
                             protocols/pubsub/peertable,
                             protocols/pubsub/timedcache,
@@ -628,7 +630,6 @@ suite "GossipSub":
       "foobar" in gossip1.gossipsub
       "foobar" notin gossip2.gossipsub
       not gossip1.mesh.hasPeerId("foobar", gossip2.peerInfo.peerId)
-      not gossip1.fanout.hasPeerId("foobar", gossip2.peerInfo.peerId)
 
     await allFuturesThrowing(
       nodes[0].switch.stop(),
@@ -637,6 +638,79 @@ suite "GossipSub":
 
     await allFuturesThrowing(nodesFut.concat())
 
+  # Helper procedures to avoid repetition
+  proc setupNodes(count: int): seq[PubSub] =
+    generateNodes(count, gossip = true)
+
+  proc startNodes(nodes: seq[PubSub]) {.async.} =
+    await allFuturesThrowing(
+      nodes.mapIt(it.switch.start())
+    )
+
+  proc stopNodes(nodes: seq[PubSub]) {.async.} =
+    await allFuturesThrowing(
+      nodes.mapIt(it.switch.stop())
+    )
+
+  proc connectNodes(nodes: seq[PubSub], target: PubSub) {.async.} =
+    proc handler(topic: string, data: seq[byte]) {.async, gcsafe.} =
+      check topic == "foobar"
+
+    for node in nodes:
+      node.subscribe("foobar", handler)
+      await node.switch.connect(target.peerInfo.peerId, target.peerInfo.addrs)
+
+  proc baseTestProcedure(nodes: seq[PubSub], gossip1: GossipSub, numPeersFirstMsg: int, numPeersSecondMsg: int) {.async.} =
+    proc handler(topic: string, data: seq[byte]) {.async, gcsafe.} =
+      check topic == "foobar"
+
+    block setup:
+      for i in 0..<50:
+        if (await nodes[0].publish("foobar", ("Hello!" & $i).toBytes())) == 19:
+          break setup
+        await sleepAsync(10.milliseconds)
+      check false
+
+    check (await nodes[0].publish("foobar", newSeq[byte](2_500_000))) == numPeersFirstMsg
+    check (await nodes[0].publish("foobar", newSeq[byte](500_001))) == numPeersSecondMsg
+
+    # Now try with a mesh
+    gossip1.subscribe("foobar", handler)
+    checkExpiring: gossip1.mesh.peers("foobar") > 5
+
+    # use a different length so that the message is not equal to the last
+    check (await nodes[0].publish("foobar", newSeq[byte](500_000))) == numPeersSecondMsg
+
+  # Actual tests
+  asyncTest "e2e - GossipSub floodPublish limit":
+
+    let
+      nodes = setupNodes(20)
+      gossip1 = GossipSub(nodes[0])
+
+    gossip1.parameters.floodPublish = true
+    gossip1.parameters.heartbeatInterval = milliseconds(700)
+
+    await startNodes(nodes)
+    await connectNodes(nodes[1..^1], nodes[0])
+    await baseTestProcedure(nodes, gossip1, gossip1.parameters.dLow, 17)
+    await stopNodes(nodes)
+
+  asyncTest "e2e - GossipSub floodPublish limit with bandwidthEstimatebps = 0":
+
+    let
+      nodes = setupNodes(20)
+      gossip1 = GossipSub(nodes[0])
+
+    gossip1.parameters.floodPublish = true
+    gossip1.parameters.heartbeatInterval = milliseconds(700)
+    gossip1.parameters.bandwidthEstimatebps = 0
+
+    await startNodes(nodes)
+    await connectNodes(nodes[1..^1], nodes[0])
+    await baseTestProcedure(nodes, gossip1, nodes.len - 1, nodes.len - 1)
+    await stopNodes(nodes)
+
   asyncTest "e2e - GossipSub with multiple peers":
     var runs = 10
 
@@ -856,3 +930,136 @@ suite "GossipSub":
 
     await allFuturesThrowing(nodesFut.concat())
 
+  proc initializeGossipTest(): Future[(seq[PubSub], GossipSub, GossipSub)] {.async.} =
+    let nodes = generateNodes(
+      2,
+      gossip = true,
+      overheadRateLimit = Opt.some((20, 1.millis)))
+
+    discard await allFinished(
+      nodes[0].switch.start(),
+      nodes[1].switch.start(),
+    )
+
+    await subscribeNodes(nodes)
+
+    proc handle(topic: string, data: seq[byte]) {.async, gcsafe.} = discard
+
+    let gossip0 = GossipSub(nodes[0])
+    let gossip1 = GossipSub(nodes[1])
+
+    gossip0.subscribe("foobar", handle)
+    gossip1.subscribe("foobar", handle)
+    await waitSubGraph(nodes, "foobar")
+
+    # Avoid being disconnected by failing signature verification
+    gossip0.verifySignature = false
+    gossip1.verifySignature = false
+
+    return (nodes, gossip0,  gossip1)
+
+  proc currentRateLimitHits(): float64 =
+    try:
+      libp2p_gossipsub_peers_rate_limit_hits.valueByName("libp2p_gossipsub_peers_rate_limit_hits_total", @["nim-libp2p"])
+    except KeyError:
+      0
+
+  asyncTest "e2e - GossipSub should not rate limit decodable messages below the size allowed":
+    let rateLimitHits = currentRateLimitHits()
+    let (nodes, gossip0, gossip1) = await initializeGossipTest()
+
+    gossip0.broadcast(gossip0.mesh["foobar"], RPCMsg(messages: @[Message(topicIDs: @["foobar"], data: newSeq[byte](10))]))
+    await sleepAsync(300.millis)
+
+    check currentRateLimitHits() == rateLimitHits
+    check gossip1.switch.isConnected(gossip0.switch.peerInfo.peerId) == true
+
+    # Disconnect peer when rate limiting is enabled
+    gossip1.parameters.disconnectPeerAboveRateLimit = true
+    gossip0.broadcast(gossip0.mesh["foobar"], RPCMsg(messages: @[Message(topicIDs: @["foobar"], data: newSeq[byte](12))]))
+    await sleepAsync(300.millis)
+
+    check gossip1.switch.isConnected(gossip0.switch.peerInfo.peerId) == true
+    check currentRateLimitHits() == rateLimitHits
+
+    await stopNodes(nodes)
+
+  asyncTest "e2e - GossipSub should rate limit undecodable messages above the size allowed":
+    let rateLimitHits = currentRateLimitHits()
+
+    let (nodes, gossip0, gossip1) = await initializeGossipTest()
+
+    # Simulate sending an undecodable message
+    await gossip1.peers[gossip0.switch.peerInfo.peerId].sendEncoded(newSeqWith[byte](33, 1.byte))
+    await sleepAsync(300.millis)
+
+    check currentRateLimitHits() == rateLimitHits + 1
+    check gossip1.switch.isConnected(gossip0.switch.peerInfo.peerId) == true
+
+    # Disconnect peer when rate limiting is enabled
+    gossip1.parameters.disconnectPeerAboveRateLimit = true
+    await gossip0.peers[gossip1.switch.peerInfo.peerId].sendEncoded(newSeqWith[byte](35, 1.byte))
+
+    checkExpiring gossip1.switch.isConnected(gossip0.switch.peerInfo.peerId) == false
+    check currentRateLimitHits() == rateLimitHits + 2
+
+    await stopNodes(nodes)
+
+  asyncTest "e2e - GossipSub should rate limit decodable messages above the size allowed":
+    let rateLimitHits = currentRateLimitHits()
+    let (nodes, gossip0, gossip1) = await initializeGossipTest()
+
+    let msg = RPCMsg(control: some(ControlMessage(prune: @[
+        ControlPrune(topicID: "foobar", peers: @[
+            PeerInfoMsg(peerId: PeerId(data: newSeq[byte](33)))
+        ], backoff: 123'u64)
+    ])))
+    gossip0.broadcast(gossip0.mesh["foobar"], msg)
+    await sleepAsync(300.millis)
+
+    check currentRateLimitHits() == rateLimitHits + 1
+    check gossip1.switch.isConnected(gossip0.switch.peerInfo.peerId) == true
+
+    # Disconnect peer when rate limiting is enabled
+    gossip1.parameters.disconnectPeerAboveRateLimit = true
+    let msg2 = RPCMsg(control: some(ControlMessage(prune: @[
+        ControlPrune(topicID: "foobar", peers: @[
+            PeerInfoMsg(peerId: PeerId(data: newSeq[byte](35)))
+        ], backoff: 123'u64)
+    ])))
+    gossip0.broadcast(gossip0.mesh["foobar"], msg2)
+
+    checkExpiring gossip1.switch.isConnected(gossip0.switch.peerInfo.peerId) == false
+    check currentRateLimitHits() == rateLimitHits + 2
+
+    await stopNodes(nodes)
+
+  asyncTest "e2e - GossipSub should rate limit invalid messages above the size allowed":
+    let rateLimitHits = currentRateLimitHits()
+    let (nodes, gossip0, gossip1) = await initializeGossipTest()
+
+    let topic = "foobar"
+    proc execValidator(topic: string, message: messages.Message): Future[ValidationResult] {.raises: [].} =
+      let res = newFuture[ValidationResult]()
+      res.complete(ValidationResult.Reject)
+      res
+
+    gossip0.addValidator(topic, execValidator)
+    gossip1.addValidator(topic, execValidator)
+
+    let msg = RPCMsg(messages: @[Message(topicIDs: @[topic], data: newSeq[byte](40))])
+
+    gossip0.broadcast(gossip0.mesh[topic], msg)
+    await sleepAsync(300.millis)
+
+    check currentRateLimitHits() == rateLimitHits + 1
+    check gossip1.switch.isConnected(gossip0.switch.peerInfo.peerId) == true
+
+    # Disconnect peer when rate limiting is enabled
+    gossip1.parameters.disconnectPeerAboveRateLimit = true
+    gossip0.broadcast(gossip0.mesh[topic], RPCMsg(messages: @[Message(topicIDs: @[topic], data: newSeq[byte](35))]))
+
+    checkExpiring gossip1.switch.isConnected(gossip0.switch.peerInfo.peerId) == false
+    check currentRateLimitHits() == rateLimitHits + 2
+
+    await stopNodes(nodes)

tests/pubsub/testgossipsub2.nim

@@ -167,36 +167,44 @@ suite "GossipSub":
 
   asyncTest "GossipSub directPeers: always forward messages":
     let
-      nodes = generateNodes(2, gossip = true)
+      nodes = generateNodes(3, gossip = true)
 
       # start switches
       nodesFut = await allFinished(
         nodes[0].switch.start(),
         nodes[1].switch.start(),
+        nodes[2].switch.start(),
       )
 
     await GossipSub(nodes[0]).addDirectPeer(nodes[1].switch.peerInfo.peerId, nodes[1].switch.peerInfo.addrs)
     await GossipSub(nodes[1]).addDirectPeer(nodes[0].switch.peerInfo.peerId, nodes[0].switch.peerInfo.addrs)
+    await GossipSub(nodes[1]).addDirectPeer(nodes[2].switch.peerInfo.peerId, nodes[2].switch.peerInfo.addrs)
+    await GossipSub(nodes[2]).addDirectPeer(nodes[1].switch.peerInfo.peerId, nodes[1].switch.peerInfo.addrs)
 
     var handlerFut = newFuture[void]()
     proc handler(topic: string, data: seq[byte]) {.async, gcsafe.} =
       check topic == "foobar"
       handlerFut.complete()
+    proc noop(topic: string, data: seq[byte]) {.async, gcsafe.} =
+      check topic == "foobar"
 
-    nodes[0].subscribe("foobar", handler)
-    nodes[1].subscribe("foobar", handler)
+    nodes[0].subscribe("foobar", noop)
+    nodes[1].subscribe("foobar", noop)
+    nodes[2].subscribe("foobar", handler)
 
     tryPublish await nodes[0].publish("foobar", toBytes("hellow")), 1
 
-    await handlerFut
+    await handlerFut.wait(2.seconds)
 
     # peer shouldn't be in our mesh
     check "foobar" notin GossipSub(nodes[0]).mesh
     check "foobar" notin GossipSub(nodes[1]).mesh
+    check "foobar" notin GossipSub(nodes[2]).mesh
 
     await allFuturesThrowing(
       nodes[0].switch.stop(),
-      nodes[1].switch.stop()
+      nodes[1].switch.stop(),
+      nodes[2].switch.stop()
     )
 
     await allFuturesThrowing(nodesFut.concat())

tests/pubsub/testmessage.nim

@@ -2,10 +2,10 @@ import unittest2
 
 {.used.}
 
-import options
+import options, strutils
 import stew/byteutils
 import ../../libp2p/[peerid, peerinfo,
-                     crypto/crypto,
+                     crypto/crypto as crypto,
                      protocols/pubsub/errors,
                      protocols/pubsub/rpc/message,
                      protocols/pubsub/rpc/messages]
@@ -28,7 +28,7 @@ suite "Message":
         """08011240B9EA7F0357B5C1247E4FCB5AD09C46818ECB07318CA84711875F4C6C
         E6B946186A4EB44E0D714B2A2D48263D75CF52D30BEF9D9AE2A9FEB7DAF1775F
         E731065A"""
-      seckey = PrivateKey.init(fromHex(stripSpaces(pkHex)))
+      seckey = PrivateKey.init(crypto.fromHex(stripSpaces(pkHex)))
                 .expect("valid private key bytes")
       peer = PeerInfo.new(seckey)
       msg = Message.init(some(peer), @[], "topic", some(seqno), sign = true)
@@ -46,7 +46,7 @@ suite "Message":
         """08011240B9EA7F0357B5C1247E4FCB5AD09C46818ECB07318CA84711875F4C6C
         E6B946186A4EB44E0D714B2A2D48263D75CF52D30BEF9D9AE2A9FEB7DAF1775F
         E731065A"""
-      seckey = PrivateKey.init(fromHex(stripSpaces(pkHex)))
+      seckey = PrivateKey.init(crypto.fromHex(stripSpaces(pkHex)))
                 .expect("valid private key bytes")
       peer = PeerInfo.new(seckey)
 
@@ -64,7 +64,7 @@ suite "Message":
         """08011240B9EA7F0357B5C1247E4FCB5AD09C46818ECB07318CA84711875F4C6C
         E6B946186A4EB44E0D714B2A2D48263D75CF52D30BEF9D9AE2A9FEB7DAF1775F
         E731065A"""
-      seckey = PrivateKey.init(fromHex(stripSpaces(pkHex)))
+      seckey = PrivateKey.init(crypto.fromHex(stripSpaces(pkHex)))
                 .expect("valid private key bytes")
       peer = PeerInfo.new(seckey)
       msg = Message.init(some(peer), @[], "topic", uint64.none, sign = true)
@@ -73,3 +73,55 @@ suite "Message":
     check:
       msgIdResult.isErr
       msgIdResult.error == ValidationResult.Reject
+
+  test "byteSize for RPCMsg":
+    var msg = Message(
+      fromPeer: PeerId(data: @['a'.byte, 'b'.byte]),  # 2 bytes
+      data: @[1'u8, 2, 3],  # 3 bytes
+      seqno: @[4'u8, 5],  # 2 bytes
+      signature: @['c'.byte, 'd'.byte],  # 2 bytes
+      key: @[6'u8, 7],  # 2 bytes
+      topicIds: @["abc", "defgh"]  # 3 + 5 = 8 bytes
+    )
+
+    var peerInfo = PeerInfoMsg(
+      peerId: PeerId(data: @['e'.byte]),  # 1 byte
+      signedPeerRecord: @['f'.byte, 'g'.byte]  # 2 bytes
+    )
+
+    var controlIHave = ControlIHave(
+      topicId: "ijk",  # 3 bytes
+      messageIds: @[ @['l'.byte], @['m'.byte, 'n'.byte] ]  # 1 + 2 = 3 bytes
+    )
+
+    var controlIWant = ControlIWant(
+      messageIds: @[ @['o'.byte, 'p'.byte], @['q'.byte] ]  # 2 + 1 = 3 bytes
+    )
+
+    var controlGraft = ControlGraft(
+      topicId: "rst"  # 3 bytes
+    )
+
+    var controlPrune = ControlPrune(
+      topicId: "uvw",  # 3 bytes
+      peers: @[peerInfo, peerInfo],  # (1 + 2) * 2 = 6 bytes
+      backoff: 12345678  # 8 bytes for uint64
+    )
+
+    var control = ControlMessage(
+      ihave: @[controlIHave, controlIHave],  # (3 + 3) * 2 = 12 bytes
+      iwant: @[controlIWant],  # 3 bytes
+      graft: @[controlGraft],  # 3 bytes
+      prune: @[controlPrune],  # 3 + 6 + 8 = 17 bytes
+      idontwant: @[controlIWant]  # 3 bytes
+    )
+
+    var rpcMsg = RPCMsg(
+      subscriptions: @[SubOpts(subscribe: true, topic: "a".repeat(12)), SubOpts(subscribe: false, topic: "b".repeat(14))],  # 1 + 12 + 1 + 14 = 28 bytes
+      messages: @[msg, msg],  # 19 * 2 = 38 bytes
+      ping: @[1'u8, 2],  # 2 bytes
+      pong: @[3'u8, 4],  # 2 bytes
+      control: some(control)  # 12 + 3 + 3 + 17 + 3 = 38 bytes
+    )
+
+    check byteSize(rpcMsg) == 28 + 38 + 2 + 2 + 38  # Total: 108 bytes

tests/pubsub/utils.nim

@@ -5,20 +5,43 @@ const
   libp2p_pubsub_anonymize {.booldefine.} = false
 
 import hashes, random, tables, sets, sequtils
-import chronos, stew/[byteutils, results]
+import chronos, stew/[byteutils, results], chronos/ratelimit
 import ../../libp2p/[builders,
                      protocols/pubsub/errors,
                      protocols/pubsub/pubsub,
+                     protocols/pubsub/pubsubpeer,
                      protocols/pubsub/gossipsub,
                      protocols/pubsub/floodsub,
                      protocols/pubsub/rpc/messages,
                      protocols/secure/secure]
+import ../helpers
 import chronicles
 
 export builders
 
 randomize()
 
+type
+  TestGossipSub* = ref object of GossipSub
+
+proc getPubSubPeer*(p: TestGossipSub, peerId: PeerId): PubSubPeer =
+  proc getConn(): Future[Connection] =
+    p.switch.dial(peerId, GossipSubCodec)
+
+  let pubSubPeer = PubSubPeer.new(peerId, getConn, nil, GossipSubCodec, 1024 * 1024)
+  debug "created new pubsub peer", peerId
+
+  p.peers[peerId] = pubSubPeer
+
+  onNewPeer(p, pubSubPeer)
+  pubSubPeer
+
+proc randomPeerId*(): PeerId =
+  try:
+    PeerId.init(PrivateKey.random(ECDSA, rng[]).get()).tryGet()
+  except CatchableError as exc:
+    raise newException(Defect, exc.msg)
+
 func defaultMsgIdProvider*(m: Message): Result[MessageId, ValidationResult] =
   let mid =
     if m.seqno.len > 0 and m.fromPeer.data.len > 0:
@@ -44,7 +67,8 @@ proc generateNodes*(
   sendSignedPeerRecord = false,
   unsubscribeBackoff = 1.seconds,
   maxMessageSize: int = 1024 * 1024,
-  enablePX: bool = false): seq[PubSub] =
+  enablePX: bool = false,
+  overheadRateLimit: Opt[tuple[bytes: int, interval: Duration]] = Opt.none(tuple[bytes: int, interval: Duration])): seq[PubSub] =
 
   for i in 0..<num:
     let switch = newStandardSwitch(secureManagers = secureManagers, sendSignedPeerRecord = sendSignedPeerRecord)
@@ -57,7 +81,7 @@ proc generateNodes*(
         msgIdProvider = msgIdProvider,
         anonymize = anonymize,
         maxMessageSize = maxMessageSize,
-        parameters = (var p = GossipSubParams.init(); p.floodPublish = false; p.historyLength = 20; p.historyGossip = 20; p.unsubscribeBackoff = unsubscribeBackoff; p.enablePX = enablePX; p))
+        parameters = (var p = GossipSubParams.init(); p.floodPublish = false; p.historyLength = 20; p.historyGossip = 20; p.unsubscribeBackoff = unsubscribeBackoff; p.enablePX = enablePX; p.overheadRateLimit = overheadRateLimit; p))
       # set some testing params, to enable scores
       g.topicParams.mgetOrPut("foobar", TopicParams.init()).topicWeight = 1.0
       g.topicParams.mgetOrPut("foo", TopicParams.init()).topicWeight = 1.0

tests/testcrypto.nim

@@ -11,6 +11,7 @@
 
 ## Test vectors was made using Go implementation
 ## https://github.com/libp2p/go-libp2p-crypto/blob/master/key.go
+from std/strutils import toUpper
 import unittest2
 import bearssl/hash
 import nimcrypto/utils
@@ -382,6 +383,31 @@ suite "Key interface test suite":
         toHex(checkseckey) == stripSpaces(PrivateKeys[i])
         toHex(checkpubkey) == stripSpaces(PublicKeys[i])
 
+  test "Spec test vectors":
+    # https://github.com/libp2p/specs/pull/537
+    const keys = [
+      (private: "08031279307702010104203E5B1FE9712E6C314942A750BD67485DE3C1EFE85B1BFB520AE8F9AE3DFA4A4CA00A06082A8648CE3D030107A14403420004DE3D300FA36AE0E8F5D530899D83ABAB44ABF3161F162A4BC901D8E6ECDA020E8B6D5F8DA30525E71D6851510C098E5C47C646A597FB4DCEC034E9F77C409E62",
+      public: "0803125b3059301306072a8648ce3d020106082a8648ce3d03010703420004de3d300fa36ae0e8f5d530899d83abab44abf3161f162a4bc901d8e6ecda020e8b6d5f8da30525e71d6851510c098e5c47c646a597fb4dcec034e9f77c409e62"),
+      (private: "080112407e0830617c4a7de83925dfb2694556b12936c477a0e1feb2e148ec9da60fee7d1ed1e8fae2c4a144b8be8fd4b47bf3d3b34b871c3cacf6010f0e42d474fce27e",
+      public: "080112201ed1e8fae2c4a144b8be8fd4b47bf3d3b34b871c3cacf6010f0e42d474fce27e"),
+      (private: "0802122053DADF1D5A164D6B4ACDB15E24AA4C5B1D3461BDBD42ABEDB0A4404D56CED8FB",
+      public: "08021221037777e994e452c21604f91de093ce415f5432f701dd8cd1a7a6fea0e630bfca99"),
+      (private: "080012ae123082092a0201000282020100e1beab071d08200bde24eef00d049449b07770ff9910257b2d7d5dda242ce8f0e2f12e1af4b32d9efd2c090f66b0f29986dbb645dae9880089704a94e5066d594162ae6ee8892e6ec70701db0a6c445c04778eb3de1293aa1a23c3825b85c6620a2bc3f82f9b0c309bc0ab3aeb1873282bebd3da03c33e76c21e9beb172fd44c9e43be32e2c99827033cf8d0f0c606f4579326c930eb4e854395ad941256542c793902185153c474bed109d6ff5141ebf9cd256cf58893a37f83729f97e7cb435ec679d2e33901d27bb35aa0d7e20561da08885ef0abbf8e2fb48d6a5487047a9ecb1ad41fa7ed84f6e3e8ecd5d98b3982d2a901b4454991766da295ab78822add5612a2df83bcee814cf50973e80d7ef38111b1bd87da2ae92438a2c8cbcc70b31ee319939a3b9c761dbc13b5c086d6b64bf7ae7dacc14622375d92a8ff9af7eb962162bbddebf90acb32adb5e4e4029f1c96019949ecfbfeffd7ac1e3fbcc6b6168c34be3d5a2e5999fcbb39bba7adbca78eab09b9bc39f7fa4b93411f4cc175e70c0a083e96bfaefb04a9580b4753c1738a6a760ae1afd851a1a4bdad231cf56e9284d832483df215a46c1c21bdf0c6cfe951c18f1ee4078c79c13d63edb6e14feaeffabc90ad317e4875fe648101b0864097e998f0ca3025ef9638cd2b0caecd3770ab54a1d9c6ca959b0f5dcbc90caeefc4135baca6fd475224269bbe1b02030100010282020100a472ffa858efd8588ce59ee264b957452f3673acdf5631d7bfd5ba0ef59779c231b0bc838a8b14cae367b6d9ef572c03c7883b0a3c652f5c24c316b1ccfd979f13d0cd7da20c7d34d9ec32dfdc81ee7292167e706d705efde5b8f3edfcba41409e642f8897357df5d320d21c43b33600a7ae4e505db957c1afbc189d73f0b5d972d9aaaeeb232ca20eebd5de6fe7f29d01470354413cc9a0af1154b7af7c1029adcd67c74b4798afeb69e09f2cb387305e73a1b5f450202d54f0ef096fe1bde340219a1194d1ac9026e90b366cce0c59b239d10e4888f52ca1780824d39ae01a6b9f4dd6059191a7f12b2a3d8db3c2868cd4e5a5862b8b625a4197d52c6ac77710116ebd3ced81c4d91ad5fdfbed68312ebce7eea45c1833ca3acf7da2052820eacf5c6b07d086dabeb893391c71417fd8a4b1829ae2cf60d1749d0e25da19530d889461c21da3492a8dc6ccac7de83ac1c2185262c7473c8cc42f547cc9864b02a8073b6aa54a037d8c0de3914784e6205e83d97918b944f11b877b12084c0dd1d36592f8a4f8b8da5bb404c3d2c079b22b6ceabfbcb637c0dbe0201f0909d533f8bf308ada47aee641a012a494d31b54c974e58b87f140258258bb82f31692659db7aa07e17a5b2a0832c24e122d3a8babcc9ee74cbb07d3058bb85b15f6f6b2674aba9fd34367be9782d444335fbed31e3c4086c652597c27104938b47fa10282010100e9fdf843c1550070ca711cb8ff28411466198f0e212511c3186623890c0071bf6561219682fe7dbdfd81176eba7c4faba21614a20721e0fcd63768e6d925688ecc90992059ac89256e0524de90bf3d8a052ce6a9f6adafa712f3107a016e20c80255c9e37d8206d1bc327e06e66eb24288da866b55904fd8b59e6b2ab31bc5eab47e597093c63fab7872102d57b4c589c66077f534a61f5f65127459a33c91f6db61fc431b1ae90be92b4149a3255291baf94304e3efb77b1107b5a3bda911359c40a53c347ff9100baf8f36dc5cd991066b5bdc28b39ed644f404afe9213f4d31c9d4e40f3a5f5e3c39bebeb244e84137544e1a1839c1c8aaebf0c78a7fad590282010100f6fa1f1e6b803742d5490b7441152f500970f46feb0b73a6e4baba2aaf3c0e245ed852fc31d86a8e46eb48e90fac409989dfee45238f97e8f1f8e83a136488c1b04b8a7fb695f37b8616307ff8a8d63e8cfa0b4fb9b9167ffaebabf111aa5a4344afbabd002ae8961c38c02da76a9149abdde93eb389eb32595c29ba30d8283a7885218a5a9d33f7f01dbdf85f3aad016c071395491338ec318d39220e1c7bd69d3d6b520a13a30d745c102b827ad9984b0dd6aed73916ffa82a06c1c111e7047dcd2668f988a0570a71474992eecf416e068f029ec323d5d635fd24694fc9bf96973c255d26c772a95bf8b7f876547a5beabf86f06cd21b67994f944e7a5493028201010095b02fd30069e547426a8bea58e8a2816f33688dac6c6f6974415af8402244a22133baedf34ce499d7036f3f19b38eb00897c18949b0c5a25953c71aeeccfc8f6594173157cc854bd98f16dffe8f28ca13b77eb43a2730585c49fc3f608cd811bb54b03b84bddaa8ef910988567f783012266199667a546a18fd88271fbf63a45ae4fd4884706da8befb9117c0a4d73de5172f8640b1091ed8a4aea3ed4641463f5ff6a5e3401ad7d0c92811f87956d1fd5f9a1d15c7f3839a08698d9f35f9d966e5000f7cb2655d7b6c4adcd8a9d950ea5f61bb7c9a33c17508f9baa313eecfee4ae493249ebe05a5d7770bbd3551b2eeb752e3649e0636de08e3d672e66cb90282010100ad93e4c31072b063fc5ab5fe22afacece775c795d0efdf7c704cfc027bde0d626a7646fc905bb5a80117e3ca49059af14e0160089f9190065be9bfecf12c3b2145b211c8e89e42dd91c38e9aa23ca73697063564f6f6aa6590088a738722df056004d18d7bccac62b3bafef6172fc2a4b071ea37f31eff7a076bcab7dd144e51a9da8754219352aef2c73478971539fa41de4759285ea626fa3c72e7085be47d554d915bbb5149cb6ef835351f231043049cd941506a034bf2f8767f3e1e42ead92f91cb3d75549b57ef7d56ac39c2d80d67f6a2b4ca192974bfc5060e2dd171217971002193dba12e7e4133ab201f07500a90495a38610279b13a48d54f0c99028201003e3a1ac0c2b67d54ed5c4bbe04a7db99103659d33a4f9d35809e1f60c282e5988dddc964527f3b05e6cc890eab3dcb571d66debf3a5527704c87264b3954d7265f4e8d2c637dd89b491b9cf23f264801f804b90454d65af0c4c830d1aef76f597ef61b26ca857ecce9cb78d4f6c2218c00d2975d46c2b013fbf59b750c3b92d8d3ed9e6d1fd0ef1ec091a5c286a3fe2dead292f40f380065731e2079ebb9f2a7ef2c415ecbb488da98f3a12609ca1b6ec8c734032c8bd513292ff842c375d4acd1b02dfb206b24cd815f8e2f9d4af8e7dea0370b19c1b23cc531d78b40e06e1119ee2e08f6f31c6e2e8444c568d13c5d451a291ae0c9f1d4f27d23b3a00d60ad",
+      public: "080012a60430820222300d06092a864886f70d01010105000382020f003082020a0282020100e1beab071d08200bde24eef00d049449b07770ff9910257b2d7d5dda242ce8f0e2f12e1af4b32d9efd2c090f66b0f29986dbb645dae9880089704a94e5066d594162ae6ee8892e6ec70701db0a6c445c04778eb3de1293aa1a23c3825b85c6620a2bc3f82f9b0c309bc0ab3aeb1873282bebd3da03c33e76c21e9beb172fd44c9e43be32e2c99827033cf8d0f0c606f4579326c930eb4e854395ad941256542c793902185153c474bed109d6ff5141ebf9cd256cf58893a37f83729f97e7cb435ec679d2e33901d27bb35aa0d7e20561da08885ef0abbf8e2fb48d6a5487047a9ecb1ad41fa7ed84f6e3e8ecd5d98b3982d2a901b4454991766da295ab78822add5612a2df83bcee814cf50973e80d7ef38111b1bd87da2ae92438a2c8cbcc70b31ee319939a3b9c761dbc13b5c086d6b64bf7ae7dacc14622375d92a8ff9af7eb962162bbddebf90acb32adb5e4e4029f1c96019949ecfbfeffd7ac1e3fbcc6b6168c34be3d5a2e5999fcbb39bba7adbca78eab09b9bc39f7fa4b93411f4cc175e70c0a083e96bfaefb04a9580b4753c1738a6a760ae1afd851a1a4bdad231cf56e9284d832483df215a46c1c21bdf0c6cfe951c18f1ee4078c79c13d63edb6e14feaeffabc90ad317e4875fe648101b0864097e998f0ca3025ef9638cd2b0caecd3770ab54a1d9c6ca959b0f5dcbc90caeefc4135baca6fd475224269bbe1b0203010001"),
+
+    ]
+    for (private, public) in keys:
+      var seckey = PrivateKey.init(fromHex(private)).expect("private key")
+      var pubkey = PublicKey.init(fromHex(public)).expect("public key")
+      var calckey = seckey.getPublicKey().expect("public key")
+      check:
+        pubkey == calckey
+      var checkseckey = seckey.getBytes().expect("private key")
+      var checkpubkey = pubkey.getBytes().expect("public key")
+      check:
+        toHex(checkseckey) == stripSpaces(private).toUpper()
+        toHex(checkpubkey) == stripSpaces(public).toUpper()
+
   test "Generate/Sign/Serialize/Deserialize/Verify test":
     var msg = "message to sign"
     var bmsg = cast[seq[byte]](msg)

tests/testdcutr.nim

@@ -57,14 +57,15 @@ suite "Dcutr":
     for t in behindNATSwitch.transports:
       t.networkReachability = NetworkReachability.NotReachable
 
-    await DcutrClient.new().startSync(behindNATSwitch, publicSwitch.peerInfo.peerId, behindNATSwitch.peerInfo.addrs)
-    .wait(300.millis)
+    expect CatchableError:
+      # we can't hole punch when both peers are in the same machine. This means that the simultaneous dialings will result
+      # in two connections attemps, instead of one. This dial is going to fail because the dcutr client is acting as the
+      # tcp simultaneous incoming upgrader in the dialer which works only in the simultaneous open case.
+      await DcutrClient.new().startSync(behindNATSwitch, publicSwitch.peerInfo.peerId, behindNATSwitch.peerInfo.addrs)
+      .wait(300.millis)
 
     checkExpiring:
-      # we can't hole punch when both peers are in the same machine. This means that the simultaneous dialings will result
-      # in two connections attemps, instead of one. The server dial is going to fail because it is acting as the
-      # tcp simultaneous incoming upgrader in the dialer which works only in the simultaneous open case, but the client
-      # dial will succeed.
+      # we still expect a new connection to be open by the receiver peer acting as the dcutr server
       behindNATSwitch.connManager.connCount(publicSwitch.peerInfo.peerId) == 2
 
     await allFutures(behindNATSwitch.stop(), publicSwitch.stop())
@@ -83,8 +84,8 @@ suite "Dcutr":
     body
 
     checkExpiring:
-      # no connection will be open by the receiver peer acting as the dcutr server
-      behindNATSwitch.connManager.connCount(publicSwitch.peerInfo.peerId) == 1
+      # we still expect a new connection to be open by the receiver peer acting as the dcutr server
+      behindNATSwitch.connManager.connCount(publicSwitch.peerInfo.peerId) == 2
 
     await allFutures(behindNATSwitch.stop(), publicSwitch.stop())
 
@@ -142,13 +143,16 @@ suite "Dcutr":
     for t in behindNATSwitch.transports:
       t.networkReachability = NetworkReachability.NotReachable
 
-    await DcutrClient.new().startSync(behindNATSwitch, publicSwitch.peerInfo.peerId, behindNATSwitch.peerInfo.addrs)
-    .wait(300.millis)
+    expect CatchableError:
+      # we can't hole punch when both peers are in the same machine. This means that the simultaneous dialings will result
+      # in two connections attemps, instead of one. This dial is going to fail because the dcutr client is acting as the
+      # tcp simultaneous incoming upgrader in the dialer which works only in the simultaneous open case.
+      await DcutrClient.new().startSync(behindNATSwitch, publicSwitch.peerInfo.peerId, behindNATSwitch.peerInfo.addrs)
+      .wait(300.millis)
 
     checkExpiring:
-      # we can't hole punch when both peers are in the same machine. This means that the simultaneous dialings will result
-      # in two connections attemps, instead of one. The server dial is going to fail, but the client dial will succeed.
-      behindNATSwitch.connManager.connCount(publicSwitch.peerInfo.peerId) == 2
+      # we still expect a new connection to be open by the receiver peer acting as the dcutr server
+      behindNATSwitch.connManager.connCount(publicSwitch.peerInfo.peerId) == 1
 
     await allFutures(behindNATSwitch.stop(), publicSwitch.stop())
 
@@ -175,3 +179,19 @@ suite "Dcutr":
         raise newException(CatchableError, "error")
 
     await ductrServerTest(connectProc)
+
+  test "should return valid TCP/IP and TCP/DNS addresses only":
+    let testAddrs = @[MultiAddress.init("/ip4/192.0.2.1/tcp/1234").tryGet(),
+                      MultiAddress.init("/ip4/203.0.113.5/tcp/5678/p2p/QmYyQSo1c1Ym7orWxLYvCrM2EmxFTANf8wXmmE7DWjhx5N").tryGet(),
+                      MultiAddress.init("/ip6/::1/tcp/9012").tryGet(),
+                      MultiAddress.init("/dns4/example.com/tcp/3456/p2p/QmYyQSo1c1Ym7orWxLYvCrM2EmxFTANf8wXmmE7DWjhx5N").tryGet(),
+                      MultiAddress.init("/ip4/198.51.100.42/udp/7890").tryGet()]
+
+    let expected = @[MultiAddress.init("/ip4/192.0.2.1/tcp/1234").tryGet(),
+                     MultiAddress.init("/ip4/203.0.113.5/tcp/5678").tryGet(),
+                     MultiAddress.init("/ip6/::1/tcp/9012").tryGet(),
+                     MultiAddress.init("/dns4/example.com/tcp/3456").tryGet()]
+
+    let result = getHolePunchableAddrs(testAddrs)
+
+    check result == expected

tests/testhpservice.nim

@@ -193,31 +193,17 @@ suite "Hole Punching":
     await privatePeerSwitch2.connect(privatePeerSwitch1.peerInfo.peerId, (await privatePeerRelayAddr1))
     privatePeerSwitch2.connectStub = rcvConnectStub
 
-    checkExpiring:
-      # we can't hole punch when both peers are in the same machine. This means that the simultaneous dialings will result
-      # in two connections attemps, instead of one. The server dial is going to fail because it is acting as the
-      # tcp simultaneous incoming upgrader in the dialer which works only in the simultaneous open case, but the client
-      # dial will succeed.
-      privatePeerSwitch1.connManager.connCount(privatePeerSwitch2.peerInfo.peerId) == 1 and
-        not isRelayed(privatePeerSwitch1.connManager.selectMuxer(privatePeerSwitch2.peerInfo.peerId).connection)
+    # wait for hole punching to finish in the background
+    await sleepAsync(600.millis)
 
     await allFuturesThrowing(
       privatePeerSwitch1.stop(), privatePeerSwitch2.stop(), switchRelay.stop(),
       switchAux.stop(), switchAux2.stop(), switchAux3.stop(), switchAux4.stop())
 
   asyncTest "Hole punching when peers addresses are private":
-    proc connectStub(self: SwitchStub,
-                    peerId: PeerId,
-                    addrs: seq[MultiAddress],
-                    forceDial = false,
-                    reuseConnection = true,
-                    upgradeDir = Direction.Out): Future[void] {.async.} =
-      self.connectStub = nil # this stub should be called only once
-      await sleepAsync(100.millis) # avoid simultaneous dialing that causes address in use error
-      await self.switch.connect(peerId, addrs, forceDial, reuseConnection, upgradeDir)
-    await holePunchingTest(nil, connectStub, NotReachable)
+    await holePunchingTest(nil, nil, NotReachable)
 
-  asyncTest "Hole punching when there is an error during unilateral direct connection":
+  asyncTest "Hole punching when peers addresses are private and there is an error in the initiator side":
 
     proc connectStub(self: SwitchStub,
                     peerId: PeerId,

tests/testmultiaddress.nim

@@ -60,6 +60,7 @@ const
     "/ip4/127.0.0.1/tcp/1234",
     "/ip4/127.0.0.1/tcp/1234/",
     "/ip4/127.0.0.1/udp/1234/quic",
+    "/ip4/192.168.80.3/udp/33422/quic-v1",
     "/ip4/127.0.0.1/ipfs/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC",
     "/ip4/127.0.0.1/ipfs/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC/tcp/1234",
     "/ip4/127.0.0.1/p2p/QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC",

tests/testnoise.nim

@@ -72,7 +72,7 @@ proc createSwitch(ma: MultiAddress; outgoing: bool, secio: bool = false): (Switc
       [Secure(Noise.new(rng, privateKey, outgoing = outgoing))]
     connManager = ConnManager.new()
     ms = MultistreamSelect.new()
-    muxedUpgrade = MuxedUpgrade.new(muxers, secureManagers, connManager, ms)
+    muxedUpgrade = MuxedUpgrade.new(muxers, secureManagers, ms)
     transports = @[Transport(TcpTransport.new(upgrade = muxedUpgrade))]
 
   let switch = newSwitch(

tests/testrelayv2.nim

@@ -19,14 +19,22 @@ import ./helpers
 import std/times
 import stew/byteutils
 
-proc createSwitch(r: Relay): Switch =
-  result = SwitchBuilder.new()
+proc createSwitch(r: Relay = nil, useYamux: bool = false): Switch =
+  var builder = SwitchBuilder.new()
     .withRng(newRng())
     .withAddresses(@[ MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet() ])
     .withTcpTransport()
-    .withMplex()
+
+  if useYamux:
+    builder = builder.withYamux()
+  else:
+    builder = builder.withMplex()
+
+  if r != nil:
+    builder = builder.withCircuitRelay(r)
+
+  return builder
     .withNoise()
-    .withCircuitRelay(r)
     .build()
 
 suite "Circuit Relay V2":
@@ -122,308 +130,310 @@ suite "Circuit Relay V2":
       expect(ReservationError):
         discard await cl1.reserve(src2.peerInfo.peerId, addrs)
 
-  suite "Connection":
-    asyncTeardown:
-      checkTrackers()
-    var
-      customProtoCodec {.threadvar.}: string
-      proto {.threadvar.}: LPProtocol
-      ttl {.threadvar.}: int
-      ldur {.threadvar.}: uint32
-      ldata {.threadvar.}: uint64
-      srcCl {.threadvar.}: RelayClient
-      dstCl {.threadvar.}: RelayClient
-      rv2 {.threadvar.}: Relay
-      src {.threadvar.}: Switch
-      dst {.threadvar.}: Switch
-      rel {.threadvar.}: Switch
-      rsvp {.threadvar.}: Rsvp
-      conn {.threadvar.}: Connection
-
-    asyncSetup:
-      customProtoCodec = "/test"
-      proto = new LPProtocol
-      proto.codec = customProtoCodec
-      ttl = 60
-      ldur = 120
-      ldata = 16384
-      srcCl = RelayClient.new()
-      dstCl = RelayClient.new()
-      src = createSwitch(srcCl)
-      dst = createSwitch(dstCl)
-      rel = newStandardSwitch()
-
-    asyncTest "Connection succeed":
-      proto.handler = proc(conn: Connection, proto: string) {.async.} =
-        check: "test1" == string.fromBytes(await conn.readLp(1024))
-        await conn.writeLp("test2")
-        check: "test3" == string.fromBytes(await conn.readLp(1024))
-        await conn.writeLp("test4")
-        await conn.close()
-      rv2 = Relay.new(reservationTTL=initDuration(seconds=ttl),
-                      limitDuration=ldur,
-                      limitData=ldata)
-      rv2.setup(rel)
-      rel.mount(rv2)
-      dst.mount(proto)
-
-      await rel.start()
-      await src.start()
-      await dst.start()
-
-      let addrs = MultiAddress.init($rel.peerInfo.addrs[0] & "/p2p/" &
-                                $rel.peerInfo.peerId & "/p2p-circuit").get()
-
-      await src.connect(rel.peerInfo.peerId, rel.peerInfo.addrs)
-      await dst.connect(rel.peerInfo.peerId, rel.peerInfo.addrs)
-
-      rsvp = await dstCl.reserve(rel.peerInfo.peerId, rel.peerInfo.addrs)
-
-      conn = await src.dial(dst.peerInfo.peerId, @[ addrs ], customProtoCodec)
-      await conn.writeLp("test1")
-      check: "test2" == string.fromBytes(await conn.readLp(1024))
-      await conn.writeLp("test3")
-      check: "test4" == string.fromBytes(await conn.readLp(1024))
-      await allFutures(conn.close())
-      await allFutures(src.stop(), dst.stop(), rel.stop())
-
-    asyncTest "Connection duration exceeded":
-      ldur = 3
-      proto.handler = proc(conn: Connection, proto: string) {.async.} =
-        check "wanna sleep?" == string.fromBytes(await conn.readLp(1024))
-        await conn.writeLp("yeah!")
-        check "go!" == string.fromBytes(await conn.readLp(1024))
-        await sleepAsync(chronos.timer.seconds(ldur + 1))
-        await conn.writeLp("that was a cool power nap")
-        await conn.close()
-      rv2 = Relay.new(reservationTTL=initDuration(seconds=ttl),
-                      limitDuration=ldur,
-                      limitData=ldata)
-      rv2.setup(rel)
-      rel.mount(rv2)
-      dst.mount(proto)
-
-      await rel.start()
-      await src.start()
-      await dst.start()
-
-      let addrs = MultiAddress.init($rel.peerInfo.addrs[0] & "/p2p/" &
-                                $rel.peerInfo.peerId & "/p2p-circuit").get()
-
-      await src.connect(rel.peerInfo.peerId, rel.peerInfo.addrs)
-      await dst.connect(rel.peerInfo.peerId, rel.peerInfo.addrs)
-
-      rsvp = await dstCl.reserve(rel.peerInfo.peerId, rel.peerInfo.addrs)
-      conn = await src.dial(dst.peerInfo.peerId, @[ addrs ], customProtoCodec)
-      await conn.writeLp("wanna sleep?")
-      check: "yeah!" == string.fromBytes(await conn.readLp(1024))
-      await conn.writeLp("go!")
-      expect(LPStreamEOFError):
-        discard await conn.readLp(1024)
-      await allFutures(conn.close())
-      await allFutures(src.stop(), dst.stop(), rel.stop())
-
-    asyncTest "Connection data exceeded":
-      ldata = 1000
-      proto.handler = proc(conn: Connection, proto: string) {.async.} =
-        check "count me the better story you know" == string.fromBytes(await conn.readLp(1024))
-        await conn.writeLp("do you expect a lorem ipsum or...?")
-        check "surprise me!" == string.fromBytes(await conn.readLp(1024))
-        await conn.writeLp("""Call me Ishmael. Some years ago--never mind how long
-precisely--having little or no money in my purse, and nothing
-particular to interest me on shore, I thought I would sail about a
-little and see the watery part of the world. It is a way I have of
-driving off the spleen and regulating the circulation. Whenever I
-find myself growing grim about the mouth; whenever it is a damp,
-drizzly November in my soul; whenever I find myself involuntarily
-pausing before coffin warehouses, and bringing up the rear of every
-funeral I meet; and especially whenever my hypos get such an upper
-hand of me, that it requires a strong moral principle to prevent me
-from deliberately stepping into the street, and methodically knocking
-people's hats off--then, I account it high time to get to sea as soon
-as I can. This is my substitute for pistol and ball. With a
-philosophical flourish Cato throws himself upon his sword; I quietly
-take to the ship.""")
-      rv2 = Relay.new(reservationTTL=initDuration(seconds=ttl),
-                      limitDuration=ldur,
-                      limitData=ldata)
-      rv2.setup(rel)
-      rel.mount(rv2)
-      dst.mount(proto)
-
-      await rel.start()
-      await src.start()
-      await dst.start()
-
-      let addrs = MultiAddress.init($rel.peerInfo.addrs[0] & "/p2p/" &
-                                $rel.peerInfo.peerId & "/p2p-circuit").get()
-
-      await src.connect(rel.peerInfo.peerId, rel.peerInfo.addrs)
-      await dst.connect(rel.peerInfo.peerId, rel.peerInfo.addrs)
-
-      rsvp = await dstCl.reserve(rel.peerInfo.peerId, rel.peerInfo.addrs)
-      conn = await src.dial(dst.peerInfo.peerId, @[ addrs ], customProtoCodec)
-      await conn.writeLp("count me the better story you know")
-      check: "do you expect a lorem ipsum or...?" == string.fromBytes(await conn.readLp(1024))
-      await conn.writeLp("surprise me!")
-      expect(LPStreamEOFError):
-        discard await conn.readLp(1024)
-      await allFutures(conn.close())
-      await allFutures(src.stop(), dst.stop(), rel.stop())
-
-    asyncTest "Reservation ttl expire during connection":
-      ttl = 3
-      proto.handler = proc(conn: Connection, proto: string) {.async.} =
-        check: "test1" == string.fromBytes(await conn.readLp(1024))
-        await conn.writeLp("test2")
-        check: "test3" == string.fromBytes(await conn.readLp(1024))
-        await conn.writeLp("test4")
-        await conn.close()
-      rv2 = Relay.new(reservationTTL=initDuration(seconds=ttl),
-                      limitDuration=ldur,
-                      limitData=ldata)
-      rv2.setup(rel)
-      rel.mount(rv2)
-      dst.mount(proto)
-
-      await rel.start()
-      await src.start()
-      await dst.start()
-
-      let addrs = MultiAddress.init($rel.peerInfo.addrs[0] & "/p2p/" &
-                                $rel.peerInfo.peerId & "/p2p-circuit").get()
-
-      await src.connect(rel.peerInfo.peerId, rel.peerInfo.addrs)
-      await dst.connect(rel.peerInfo.peerId, rel.peerInfo.addrs)
-
-      rsvp = await dstCl.reserve(rel.peerInfo.peerId, rel.peerInfo.addrs)
-      conn = await src.dial(dst.peerInfo.peerId, @[ addrs ], customProtoCodec)
-      await conn.writeLp("test1")
-      check: "test2" == string.fromBytes(await conn.readLp(1024))
-      await conn.writeLp("test3")
-      check: "test4" == string.fromBytes(await conn.readLp(1024))
-      await src.disconnect(rel.peerInfo.peerId)
-      await sleepAsync(chronos.timer.seconds(ttl + 1))
-
-      expect(DialFailedError):
-        check: conn.atEof()
-        await conn.close()
-        await src.connect(rel.peerInfo.peerId, rel.peerInfo.addrs)
-        conn = await src.dial(dst.peerInfo.peerId, @[ addrs ], customProtoCodec)
-      await allFutures(conn.close())
-      await allFutures(src.stop(), dst.stop(), rel.stop())
-
-    asyncTest "Connection over relay":
-      # src => rel => rel2 => dst
-      # rel2 reserve rel
-      # dst reserve rel2
-      # src try to connect with dst
-      proto.handler = proc(conn: Connection, proto: string) {.async.} =
-        raise newException(CatchableError, "Should not be here")
-      let
-        rel2Cl = RelayClient.new(canHop = true)
-        rel2 = createSwitch(rel2Cl)
-        rv2 = Relay.new()
-      rv2.setup(rel)
-      rel.mount(rv2)
-      dst.mount(proto)
-      await rel.start()
-      await rel2.start()
-      await src.start()
-      await dst.start()
-
-      let
-        addrs = @[ MultiAddress.init($rel.peerInfo.addrs[0] & "/p2p/" &
-                                     $rel.peerInfo.peerId & "/p2p-circuit/p2p/" &
-                                     $rel2.peerInfo.peerId & "/p2p/" &
-                                     $rel2.peerInfo.peerId & "/p2p-circuit").get() ]
-
-      await src.connect(rel.peerInfo.peerId, rel.peerInfo.addrs)
-      await rel2.connect(rel.peerInfo.peerId, rel.peerInfo.addrs)
-      await dst.connect(rel2.peerInfo.peerId, rel2.peerInfo.addrs)
-
-      rsvp = await rel2Cl.reserve(rel.peerInfo.peerId, rel.peerInfo.addrs)
-      let rsvp2 = await dstCl.reserve(rel2.peerInfo.peerId, rel2.peerInfo.addrs)
-
-      expect(DialFailedError):
-        conn = await src.dial(dst.peerInfo.peerId, addrs, customProtoCodec)
-      await allFutures(conn.close())
-      await allFutures(src.stop(), dst.stop(), rel.stop(), rel2.stop())
-
-    asyncTest "Connection using ClientRelay":
+  for (useYamux, muxName) in [(false, "Mplex"), (true, "Yamux")]:
+    suite "Circuit Relay V2 Connection using " & muxName:
+      asyncTeardown:
+        checkTrackers()
       var
-        protoABC = new LPProtocol
-        protoBCA = new LPProtocol
-        protoCAB = new LPProtocol
-      protoABC.codec = "/abctest"
-      protoABC.handler = proc(conn: Connection, proto: string) {.async.} =
-        check: "testABC1" == string.fromBytes(await conn.readLp(1024))
-        await conn.writeLp("testABC2")
-        check: "testABC3" == string.fromBytes(await conn.readLp(1024))
-        await conn.writeLp("testABC4")
-        await conn.close()
-      protoBCA.codec = "/bcatest"
-      protoBCA.handler = proc(conn: Connection, proto: string) {.async.} =
-        check: "testBCA1" == string.fromBytes(await conn.readLp(1024))
-        await conn.writeLp("testBCA2")
-        check: "testBCA3" == string.fromBytes(await conn.readLp(1024))
-        await conn.writeLp("testBCA4")
-        await conn.close()
-      protoCAB.codec = "/cabtest"
-      protoCAB.handler = proc(conn: Connection, proto: string) {.async.} =
-        check: "testCAB1" == string.fromBytes(await conn.readLp(1024))
-        await conn.writeLp("testCAB2")
-        check: "testCAB3" == string.fromBytes(await conn.readLp(1024))
-        await conn.writeLp("testCAB4")
-        await conn.close()
+        customProtoCodec {.threadvar.}: string
+        proto {.threadvar.}: LPProtocol
+        ttl {.threadvar.}: int
+        ldur {.threadvar.}: uint32
+        ldata {.threadvar.}: uint64
+        srcCl {.threadvar.}: RelayClient
+        dstCl {.threadvar.}: RelayClient
+        rv2 {.threadvar.}: Relay
+        src {.threadvar.}: Switch
+        dst {.threadvar.}: Switch
+        rel {.threadvar.}: Switch
+        rsvp {.threadvar.}: Rsvp
+        conn {.threadvar.}: Connection
 
-      let
-        clientA = RelayClient.new(canHop = true)
-        clientB = RelayClient.new(canHop = true)
-        clientC = RelayClient.new(canHop = true)
-        switchA = createSwitch(clientA)
-        switchB = createSwitch(clientB)
-        switchC = createSwitch(clientC)
+      asyncSetup:
+        customProtoCodec = "/test"
+        proto = new LPProtocol
+        proto.codec = customProtoCodec
+        ttl = 60
+        ldur = 120
+        ldata = 16384
+        srcCl = RelayClient.new()
+        dstCl = RelayClient.new()
+        src = createSwitch(srcCl, useYamux)
+        dst = createSwitch(dstCl, useYamux)
+        rel = createSwitch(nil, useYamux)
 
-      switchA.mount(protoBCA)
-      switchB.mount(protoCAB)
-      switchC.mount(protoABC)
+      asyncTest "Connection succeed":
+        proto.handler = proc(conn: Connection, proto: string) {.async.} =
+          check: "test1" == string.fromBytes(await conn.readLp(1024))
+          await conn.writeLp("test2")
+          check: "test3" == string.fromBytes(await conn.readLp(1024))
+          await conn.writeLp("test4")
+          await conn.close()
+        rv2 = Relay.new(reservationTTL=initDuration(seconds=ttl),
+                        limitDuration=ldur,
+                        limitData=ldata)
+        rv2.setup(rel)
+        rel.mount(rv2)
+        dst.mount(proto)
 
-      await switchA.start()
-      await switchB.start()
-      await switchC.start()
+        await rel.start()
+        await src.start()
+        await dst.start()
 
-      let
-        addrsABC = MultiAddress.init($switchB.peerInfo.addrs[0] & "/p2p/" &
-                                     $switchB.peerInfo.peerId & "/p2p-circuit").get()
-        addrsBCA = MultiAddress.init($switchC.peerInfo.addrs[0] & "/p2p/" &
-                                     $switchC.peerInfo.peerId & "/p2p-circuit").get()
-        addrsCAB = MultiAddress.init($switchA.peerInfo.addrs[0] & "/p2p/" &
-                                     $switchA.peerInfo.peerId & "/p2p-circuit").get()
+        let addrs = MultiAddress.init($rel.peerInfo.addrs[0] & "/p2p/" &
+                                  $rel.peerInfo.peerId & "/p2p-circuit").get()
 
-      await switchA.connect(switchB.peerInfo.peerId, switchB.peerInfo.addrs)
-      await switchB.connect(switchC.peerInfo.peerId, switchC.peerInfo.addrs)
-      await switchC.connect(switchA.peerInfo.peerId, switchA.peerInfo.addrs)
-      let rsvpABC = await clientA.reserve(switchC.peerInfo.peerId, switchC.peerInfo.addrs)
-      let rsvpBCA = await clientB.reserve(switchA.peerInfo.peerId, switchA.peerInfo.addrs)
-      let rsvpCAB = await clientC.reserve(switchB.peerInfo.peerId, switchB.peerInfo.addrs)
-      let connABC = await switchA.dial(switchC.peerInfo.peerId, @[ addrsABC ], "/abctest")
-      let connBCA = await switchB.dial(switchA.peerInfo.peerId, @[ addrsBCA ], "/bcatest")
-      let connCAB = await switchC.dial(switchB.peerInfo.peerId, @[ addrsCAB ], "/cabtest")
+        await src.connect(rel.peerInfo.peerId, rel.peerInfo.addrs)
+        await dst.connect(rel.peerInfo.peerId, rel.peerInfo.addrs)
 
-      await connABC.writeLp("testABC1")
-      await connBCA.writeLp("testBCA1")
-      await connCAB.writeLp("testCAB1")
-      check:
-        "testABC2" == string.fromBytes(await connABC.readLp(1024))
-        "testBCA2" == string.fromBytes(await connBCA.readLp(1024))
-        "testCAB2" == string.fromBytes(await connCAB.readLp(1024))
-      await connABC.writeLp("testABC3")
-      await connBCA.writeLp("testBCA3")
-      await connCAB.writeLp("testCAB3")
-      check:
-        "testABC4" == string.fromBytes(await connABC.readLp(1024))
-        "testBCA4" == string.fromBytes(await connBCA.readLp(1024))
-        "testCAB4" == string.fromBytes(await connCAB.readLp(1024))
-      await allFutures(connABC.close(), connBCA.close(), connCAB.close())
-      await allFutures(switchA.stop(), switchB.stop(), switchC.stop())
+        rsvp = await dstCl.reserve(rel.peerInfo.peerId, rel.peerInfo.addrs)
+
+        conn = await src.dial(dst.peerInfo.peerId, @[ addrs ], customProtoCodec)
+        await conn.writeLp("test1")
+        check: "test2" == string.fromBytes(await conn.readLp(1024))
+        await conn.writeLp("test3")
+        check: "test4" == string.fromBytes(await conn.readLp(1024))
+        await allFutures(conn.close())
+        await allFutures(src.stop(), dst.stop(), rel.stop())
+
+      asyncTest "Connection duration exceeded":
+        ldur = 3
+        proto.handler = proc(conn: Connection, proto: string) {.async.} =
+          check "wanna sleep?" == string.fromBytes(await conn.readLp(1024))
+          await conn.writeLp("yeah!")
+          check "go!" == string.fromBytes(await conn.readLp(1024))
+          await sleepAsync(chronos.timer.seconds(ldur + 1))
+          await conn.writeLp("that was a cool power nap")
+          await conn.close()
+        rv2 = Relay.new(reservationTTL=initDuration(seconds=ttl),
+                        limitDuration=ldur,
+                        limitData=ldata)
+        rv2.setup(rel)
+        rel.mount(rv2)
+        dst.mount(proto)
+
+        await rel.start()
+        await src.start()
+        await dst.start()
+
+        let addrs = MultiAddress.init($rel.peerInfo.addrs[0] & "/p2p/" &
+                                  $rel.peerInfo.peerId & "/p2p-circuit").get()
+
+        await src.connect(rel.peerInfo.peerId, rel.peerInfo.addrs)
+        await dst.connect(rel.peerInfo.peerId, rel.peerInfo.addrs)
+
+        rsvp = await dstCl.reserve(rel.peerInfo.peerId, rel.peerInfo.addrs)
+        conn = await src.dial(dst.peerInfo.peerId, @[ addrs ], customProtoCodec)
+        await conn.writeLp("wanna sleep?")
+        check: "yeah!" == string.fromBytes(await conn.readLp(1024))
+        await conn.writeLp("go!")
+        expect(LPStreamEOFError):
+          discard await conn.readLp(1024)
+        await allFutures(conn.close())
+        await allFutures(src.stop(), dst.stop(), rel.stop())
+
+      asyncTest "Connection data exceeded":
+        ldata = 1000
+        proto.handler = proc(conn: Connection, proto: string) {.async.} =
+          check "count me the better story you know" == string.fromBytes(await conn.readLp(1024))
+          await conn.writeLp("do you expect a lorem ipsum or...?")
+          check "surprise me!" == string.fromBytes(await conn.readLp(1024))
+          await conn.writeLp("""Call me Ishmael. Some years ago--never mind how long
+  precisely--having little or no money in my purse, and nothing
+  particular to interest me on shore, I thought I would sail about a
+  little and see the watery part of the world. It is a way I have of
+  driving off the spleen and regulating the circulation. Whenever I
+  find myself growing grim about the mouth; whenever it is a damp,
+  drizzly November in my soul; whenever I find myself involuntarily
+  pausing before coffin warehouses, and bringing up the rear of every
+  funeral I meet; and especially whenever my hypos get such an upper
+  hand of me, that it requires a strong moral principle to prevent me
+  from deliberately stepping into the street, and methodically knocking
+  people's hats off--then, I account it high time to get to sea as soon
+  as I can. This is my substitute for pistol and ball. With a
+  philosophical flourish Cato throws himself upon his sword; I quietly
+  take to the ship.""")
+        rv2 = Relay.new(reservationTTL=initDuration(seconds=ttl),
+                        limitDuration=ldur,
+                        limitData=ldata)
+        rv2.setup(rel)
+        rel.mount(rv2)
+        dst.mount(proto)
+
+        await rel.start()
+        await src.start()
+        await dst.start()
+
+        let addrs = MultiAddress.init($rel.peerInfo.addrs[0] & "/p2p/" &
+                                  $rel.peerInfo.peerId & "/p2p-circuit").get()
+
+        await src.connect(rel.peerInfo.peerId, rel.peerInfo.addrs)
+        await dst.connect(rel.peerInfo.peerId, rel.peerInfo.addrs)
+
+        rsvp = await dstCl.reserve(rel.peerInfo.peerId, rel.peerInfo.addrs)
+        conn = await src.dial(dst.peerInfo.peerId, @[ addrs ], customProtoCodec)
+        await conn.writeLp("count me the better story you know")
+        check: "do you expect a lorem ipsum or...?" == string.fromBytes(await conn.readLp(1024))
+        await conn.writeLp("surprise me!")
+        expect(LPStreamEOFError):
+          discard await conn.readLp(1024)
+        await allFutures(conn.close())
+        await allFutures(src.stop(), dst.stop(), rel.stop())
+
+      asyncTest "Reservation ttl expire during connection":
+        ttl = 3
+        proto.handler = proc(conn: Connection, proto: string) {.async.} =
+          check: "test1" == string.fromBytes(await conn.readLp(1024))
+          await conn.writeLp("test2")
+          check: "test3" == string.fromBytes(await conn.readLp(1024))
+          await conn.writeLp("test4")
+          await conn.close()
+        rv2 = Relay.new(reservationTTL=initDuration(seconds=ttl),
+                        limitDuration=ldur,
+                        limitData=ldata)
+        rv2.setup(rel)
+        rel.mount(rv2)
+        dst.mount(proto)
+
+        await rel.start()
+        await src.start()
+        await dst.start()
+
+        let addrs = MultiAddress.init($rel.peerInfo.addrs[0] & "/p2p/" &
+                                  $rel.peerInfo.peerId & "/p2p-circuit").get()
+
+        await src.connect(rel.peerInfo.peerId, rel.peerInfo.addrs)
+        await dst.connect(rel.peerInfo.peerId, rel.peerInfo.addrs)
+
+        rsvp = await dstCl.reserve(rel.peerInfo.peerId, rel.peerInfo.addrs)
+        conn = await src.dial(dst.peerInfo.peerId, @[ addrs ], customProtoCodec)
+        await conn.writeLp("test1")
+        check: "test2" == string.fromBytes(await conn.readLp(1024))
+        await conn.writeLp("test3")
+        check: "test4" == string.fromBytes(await conn.readLp(1024))
+        await src.disconnect(rel.peerInfo.peerId)
+        await sleepAsync(chronos.timer.seconds(ttl + 1))
+
+        expect(DialFailedError):
+          check: conn.atEof()
+          await conn.close()
+          await src.connect(rel.peerInfo.peerId, rel.peerInfo.addrs)
+          conn = await src.dial(dst.peerInfo.peerId, @[ addrs ], customProtoCodec)
+        await allFutures(conn.close())
+        await allFutures(src.stop(), dst.stop(), rel.stop())
+
+      asyncTest "Connection over relay":
+        # src => rel => rel2 => dst
+        # rel2 reserve rel
+        # dst reserve rel2
+        # src try to connect with dst
+        proto.handler = proc(conn: Connection, proto: string) {.async.} =
+          raise newException(CatchableError, "Should not be here")
+        let
+          rel2Cl = RelayClient.new(canHop = true)
+          rel2 = createSwitch(rel2Cl, useYamux)
+          rv2 = Relay.new()
+        rv2.setup(rel)
+        rel.mount(rv2)
+        dst.mount(proto)
+        await rel.start()
+        await rel2.start()
+        await src.start()
+        await dst.start()
+
+        let
+          addrs = @[ MultiAddress.init($rel.peerInfo.addrs[0] & "/p2p/" &
+                                       $rel.peerInfo.peerId & "/p2p-circuit/p2p/" &
+                                       $rel2.peerInfo.peerId & "/p2p/" &
+                                       $rel2.peerInfo.peerId & "/p2p-circuit").get() ]
+
+        await src.connect(rel.peerInfo.peerId, rel.peerInfo.addrs)
+        await rel2.connect(rel.peerInfo.peerId, rel.peerInfo.addrs)
+        await dst.connect(rel2.peerInfo.peerId, rel2.peerInfo.addrs)
+
+        rsvp = await rel2Cl.reserve(rel.peerInfo.peerId, rel.peerInfo.addrs)
+        let rsvp2 = await dstCl.reserve(rel2.peerInfo.peerId, rel2.peerInfo.addrs)
+
+        expect(DialFailedError):
+          conn = await src.dial(dst.peerInfo.peerId, addrs, customProtoCodec)
+        if not conn.isNil():
+          await allFutures(conn.close())
+        await allFutures(src.stop(), dst.stop(), rel.stop(), rel2.stop())
+
+      asyncTest "Connection using ClientRelay":
+        var
+          protoABC = new LPProtocol
+          protoBCA = new LPProtocol
+          protoCAB = new LPProtocol
+        protoABC.codec = "/abctest"
+        protoABC.handler = proc(conn: Connection, proto: string) {.async.} =
+          check: "testABC1" == string.fromBytes(await conn.readLp(1024))
+          await conn.writeLp("testABC2")
+          check: "testABC3" == string.fromBytes(await conn.readLp(1024))
+          await conn.writeLp("testABC4")
+          await conn.close()
+        protoBCA.codec = "/bcatest"
+        protoBCA.handler = proc(conn: Connection, proto: string) {.async.} =
+          check: "testBCA1" == string.fromBytes(await conn.readLp(1024))
+          await conn.writeLp("testBCA2")
+          check: "testBCA3" == string.fromBytes(await conn.readLp(1024))
+          await conn.writeLp("testBCA4")
+          await conn.close()
+        protoCAB.codec = "/cabtest"
+        protoCAB.handler = proc(conn: Connection, proto: string) {.async.} =
+          check: "testCAB1" == string.fromBytes(await conn.readLp(1024))
+          await conn.writeLp("testCAB2")
+          check: "testCAB3" == string.fromBytes(await conn.readLp(1024))
+          await conn.writeLp("testCAB4")
+          await conn.close()
+
+        let
+          clientA = RelayClient.new(canHop = true)
+          clientB = RelayClient.new(canHop = true)
+          clientC = RelayClient.new(canHop = true)
+          switchA = createSwitch(clientA, useYamux)
+          switchB = createSwitch(clientB, useYamux)
+          switchC = createSwitch(clientC, useYamux)
+
+        switchA.mount(protoBCA)
+        switchB.mount(protoCAB)
+        switchC.mount(protoABC)
+
+        await switchA.start()
+        await switchB.start()
+        await switchC.start()
+
+        let
+          addrsABC = MultiAddress.init($switchB.peerInfo.addrs[0] & "/p2p/" &
+                                       $switchB.peerInfo.peerId & "/p2p-circuit").get()
+          addrsBCA = MultiAddress.init($switchC.peerInfo.addrs[0] & "/p2p/" &
+                                       $switchC.peerInfo.peerId & "/p2p-circuit").get()
+          addrsCAB = MultiAddress.init($switchA.peerInfo.addrs[0] & "/p2p/" &
+                                       $switchA.peerInfo.peerId & "/p2p-circuit").get()
+
+        await switchA.connect(switchB.peerInfo.peerId, switchB.peerInfo.addrs)
+        await switchB.connect(switchC.peerInfo.peerId, switchC.peerInfo.addrs)
+        await switchC.connect(switchA.peerInfo.peerId, switchA.peerInfo.addrs)
+        let rsvpABC = await clientA.reserve(switchC.peerInfo.peerId, switchC.peerInfo.addrs)
+        let rsvpBCA = await clientB.reserve(switchA.peerInfo.peerId, switchA.peerInfo.addrs)
+        let rsvpCAB = await clientC.reserve(switchB.peerInfo.peerId, switchB.peerInfo.addrs)
+        let connABC = await switchA.dial(switchC.peerInfo.peerId, @[ addrsABC ], "/abctest")
+        let connBCA = await switchB.dial(switchA.peerInfo.peerId, @[ addrsBCA ], "/bcatest")
+        let connCAB = await switchC.dial(switchB.peerInfo.peerId, @[ addrsCAB ], "/cabtest")
+
+        await connABC.writeLp("testABC1")
+        await connBCA.writeLp("testBCA1")
+        await connCAB.writeLp("testCAB1")
+        check:
+          "testABC2" == string.fromBytes(await connABC.readLp(1024))
+          "testBCA2" == string.fromBytes(await connBCA.readLp(1024))
+          "testCAB2" == string.fromBytes(await connCAB.readLp(1024))
+        await connABC.writeLp("testABC3")
+        await connBCA.writeLp("testBCA3")
+        await connCAB.writeLp("testCAB3")
+        check:
+          "testABC4" == string.fromBytes(await connABC.readLp(1024))
+          "testBCA4" == string.fromBytes(await connBCA.readLp(1024))
+          "testCAB4" == string.fromBytes(await connCAB.readLp(1024))
+        await allFutures(connABC.close(), connBCA.close(), connCAB.close())
+        await allFutures(switchA.stop(), switchB.stop(), switchC.stop())

tests/testrendezvous.nim

@@ -14,6 +14,7 @@ import chronos
 import ../libp2p/[protocols/rendezvous,
                   switch,
                   builders,]
+import ../libp2p/discovery/[rendezvousinterface, discoverymngr]
 import ./helpers
 
 proc createSwitch(rdv: RendezVous = RendezVous.new()): Switch =

tests/testrendezvousinterface.nim

@@ -0,0 +1,73 @@
+{.used.}
+
+# Nim-Libp2p
+# Copyright (c) 2023 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+import sequtils, strutils
+import chronos
+import ../libp2p/[protocols/rendezvous,
+                  switch,
+                  builders,]
+import ../libp2p/discovery/[rendezvousinterface, discoverymngr]
+import ./helpers
+
+proc createSwitch(rdv: RendezVous = RendezVous.new()): Switch =
+  SwitchBuilder.new()
+    .withRng(newRng())
+    .withAddresses(@[ MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet() ])
+    .withTcpTransport()
+    .withMplex()
+    .withNoise()
+    .withRendezVous(rdv)
+    .build()
+
+type
+  MockRendezVous = ref object of RendezVous
+    numAdvertiseNs1: int
+    numAdvertiseNs2: int
+
+  MockErrorRendezVous = ref object of MockRendezVous
+
+method advertise*(self: MockRendezVous, namespace: string, ttl: Duration) {.async.} =
+  if namespace == "ns1":
+    self.numAdvertiseNs1 += 1
+  elif namespace == "ns2":
+    self.numAdvertiseNs2 += 1
+  # Forward the call to the actual implementation
+  await procCall RendezVous(self).advertise(namespace, ttl)
+
+method advertise*(self: MockErrorRendezVous, namespace: string, ttl: Duration) {.async.} =
+  await procCall MockRendezVous(self).advertise(namespace, ttl)
+  raise newException(CatchableError, "MockErrorRendezVous.advertise")
+
+suite "RendezVous Interface":
+  teardown:
+    checkTrackers()
+
+  proc baseTimeToAdvertiseTest(rdv: MockRendezVous) {.async.} =
+    let
+      tta = 100.milliseconds
+      ttl = 2.hours
+      client = createSwitch(rdv)
+      dm = DiscoveryManager()
+
+    await client.start()
+    dm.add(RendezVousInterface.new(rdv = rdv, tta = tta, ttl = ttl))
+    dm.advertise(RdvNamespace("ns1"))
+    dm.advertise(RdvNamespace("ns2"))
+
+    checkExpiring: rdv.numAdvertiseNs1 >= 5
+    checkExpiring: rdv.numAdvertiseNs2 >= 5
+    await client.stop()
+
+  asyncTest "Check timeToAdvertise interval":
+    await baseTimeToAdvertiseTest(MockRendezVous.new(newRng()))
+
+  asyncTest "Check timeToAdvertise interval when there is an error":
+    await baseTimeToAdvertiseTest(MockErrorRendezVous.new(newRng()))