refactor(noise): reduce memory usage (#1497)

libp2p/protocols/secure/noise.nim

@@ -15,11 +15,12 @@ import chronicles
 import bearssl/[rand, hash]
 import stew/[endians2, byteutils]
 import nimcrypto/[utils, sha2, hmac]
-import ../../stream/[connection, streamseq]
+import ../../stream/[connection]
 import ../../peerid
 import ../../peerinfo
 import ../../protobuf/minprotobuf
 import ../../utility
+import ../../utils/bytesview
 
 import secure, ../../crypto/[crypto, chacha20poly1305, curve25519, hkdf]
 
@@ -237,13 +238,14 @@ template write_e(): untyped =
   # Sets e (which must be empty) to GENERATE_KEYPAIR().
   # Appends e.public_key to the buffer. Calls MixHash(e.public_key).
   hs.e = genKeyPair(p.rng[])
-  msg.add hs.e.publicKey
   hs.ss.mixHash(hs.e.publicKey)
 
+  hs.e.publicKey.getBytes
+
 template write_s(): untyped =
   trace "noise write s"
   # Appends EncryptAndHash(s.public_key) to the buffer.
-  msg.add hs.ss.encryptAndHash(hs.s.publicKey)
+  hs.ss.encryptAndHash(hs.s.publicKey)
 
 template dh_ee(): untyped =
   trace "noise dh ee"
@@ -281,9 +283,10 @@ template read_e(): untyped =
   # Sets re (which must be empty) to the next DHLEN bytes from the message.
   # Calls MixHash(re.public_key).
   hs.re[0 .. Curve25519Key.high] = msg.toOpenArray(0, Curve25519Key.high)
-  msg.consume(Curve25519Key.len)
   hs.ss.mixHash(hs.re)
 
+  Curve25519Key.len
+
 template read_s(): untyped =
   trace "noise read s", size = msg.len
   # Sets temp to the next DHLEN + 16 bytes of the message if HasKey() == True,
@@ -300,7 +303,7 @@ template read_s(): untyped =
       Curve25519Key.len
   hs.rs[0 .. Curve25519Key.high] = hs.ss.decryptAndHash(msg.toOpenArray(0, rsLen - 1))
 
-  msg.consume(rsLen)
+  rsLen
 
 proc readFrame(
     sconn: Connection
@@ -316,28 +319,25 @@ proc readFrame(
   await sconn.readExactly(addr buffer[0], buffer.len)
   return buffer
 
-proc writeFrame(
-    sconn: Connection, buf: openArray[byte]
-): Future[void] {.async: (raises: [CancelledError, LPStreamError], raw: true).} =
-  doAssert buf.len <= uint16.high.int
-  var
-    lesize = buf.len.uint16
-    besize = lesize.toBytesBE
-    outbuf = newSeqOfCap[byte](besize.len + buf.len)
-  trace "writeFrame", sconn, size = lesize, data = shortLog(buf)
-  outbuf &= besize
-  outbuf &= buf
-  sconn.write(outbuf)
-
 proc receiveHSMessage(
     sconn: Connection
 ): Future[seq[byte]] {.async: (raises: [CancelledError, LPStreamError], raw: true).} =
   readFrame(sconn)
 
-proc sendHSMessage(
-    sconn: Connection, buf: openArray[byte]
-): Future[void] {.async: (raises: [CancelledError, LPStreamError], raw: true).} =
-  writeFrame(sconn, buf)
+template sendHSMessage(sconn: Connection, parts: varargs[seq[byte]]): untyped =
+  # sends message (message frame) using multiple seq[byte] that 
+  # concatenated represent entire mesage.
+
+  var msgSize: int
+  for p in parts:
+    msgSize += p.len
+
+  trace "sendHSMessage", sconn, size = msgSize
+  doAssert msgSize <= uint16.high.int
+
+  await sconn.write(@(msgSize.uint16.toBytesBE))
+  for p in parts:
+    await sconn.write(p)
 
 proc handshakeXXOutbound(
     p: Noise, conn: Connection, p2pSecret: seq[byte]
@@ -348,38 +348,30 @@ proc handshakeXXOutbound(
   try:
     hs.ss.mixHash(p.commonPrologue)
     hs.s = p.noiseKeys
+    var remoteP2psecret: seq[byte]
 
-    # -> e
-    var msg: StreamSeq
+    block: # -> e
+      let ebytes = write_e()
+      # IK might use this btw!
+      let hbytes = hs.ss.encryptAndHash([])
 
-    write_e()
+      conn.sendHSMessage(ebytes, hbytes)
 
-    # IK might use this btw!
-    msg.add hs.ss.encryptAndHash([])
+    block: # <- e, ee, s, es
+      var msg = BytesView.init(await conn.receiveHSMessage())
+      msg.consume(read_e())
+      dh_ee()
+      msg.consume(read_s())
+      dh_es()
+      remoteP2psecret = hs.ss.decryptAndHash(msg.data())
 
-    await conn.sendHSMessage(msg.data)
+    block: # -> s, se
+      let sbytes = write_s()
+      dh_se()
+      # last payload must follow the encrypted way of sending
+      let hbytes = hs.ss.encryptAndHash(p2pSecret)
 
-    # <- e, ee, s, es
-
-    msg.assign(await conn.receiveHSMessage())
-
-    read_e()
-    dh_ee()
-    read_s()
-    dh_es()
-
-    let remoteP2psecret = hs.ss.decryptAndHash(msg.data)
-    msg.clear()
-
-    # -> s, se
-
-    write_s()
-    dh_se()
-
-    # last payload must follow the encrypted way of sending
-    msg.add hs.ss.encryptAndHash(p2pSecret)
-
-    await conn.sendHSMessage(msg.data)
+      conn.sendHSMessage(sbytes, hbytes)
 
     let (cs1, cs2) = hs.ss.split()
     return
@@ -397,41 +389,30 @@ proc handshakeXXInbound(
   try:
     hs.ss.mixHash(p.commonPrologue)
     hs.s = p.noiseKeys
+    var remoteP2psecret: seq[byte]
 
-    # -> e
+    block: # <- e
+      var msg = BytesView.init(await conn.receiveHSMessage())
+      msg.consume(read_e())
+      # we might use this early data one day, keeping it here for clarity
+      let earlyData {.used.} = hs.ss.decryptAndHash(msg.data())
 
-    var msg: StreamSeq
-    msg.add(await conn.receiveHSMessage())
+    block: # -> e, ee, s, es
+      let ebytes = write_e()
+      dh_ee()
+      let sbytes = write_s()
+      dh_es()
+      let hbytes = hs.ss.encryptAndHash(p2pSecret)
 
-    read_e()
+      conn.sendHSMessage(ebytes, sbytes, hbytes)
 
-    # we might use this early data one day, keeping it here for clarity
-    let earlyData {.used.} = hs.ss.decryptAndHash(msg.data)
+    block: # <- s, se
+      var msg = BytesView.init(await conn.receiveHSMessage())
+      msg.consume(read_s())
+      dh_se()
+      remoteP2psecret = hs.ss.decryptAndHash(msg.data())
 
-    # <- e, ee, s, es
-
-    msg.consume(msg.len)
-
-    write_e()
-    dh_ee()
-    write_s()
-    dh_es()
-
-    msg.add hs.ss.encryptAndHash(p2pSecret)
-
-    await conn.sendHSMessage(msg.data)
-    msg.clear()
-
-    # -> s, se
-
-    msg.add(await conn.receiveHSMessage())
-
-    read_s()
-    dh_se()
-
-    let
-      remoteP2psecret = hs.ss.decryptAndHash(msg.data)
-      (cs1, cs2) = hs.ss.split()
+    let (cs1, cs2) = hs.ss.split()
     return
       HandshakeResult(cs1: cs1, cs2: cs2, remoteP2psecret: remoteP2psecret, rs: hs.rs)
   finally:

libp2p/utils/bytesview.nim

@@ -0,0 +1,28 @@
+# Nim-Libp2p
+# Copyright (c) 2025 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+type BytesView* = object
+  data: seq[byte]
+  rpos: int
+
+proc init*(t: typedesc[BytesView], data: sink seq[byte]): BytesView =
+  BytesView(data: data, rpos: 0)
+
+func len*(v: BytesView): int {.inline.} =
+  v.data.len - v.rpos
+
+func consume*(v: var BytesView, n: int) {.inline.} =
+  doAssert v.data.len >= v.rpos + n
+  v.rpos += n
+
+template toOpenArray*(v: BytesView, b, e: int): openArray[byte] =
+  v.data.toOpenArray(v.rpos + b, v.rpos + e - b)
+
+template data*(v: BytesView): openArray[byte] =
+  v.data.toOpenArray(v.rpos, v.data.len - 1)

tests/testbytesview.nim

@@ -0,0 +1,33 @@
+{.used.}
+
+# Nim-Libp2p
+# Copyright (c) 2025 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+import unittest2
+import ../libp2p/utils/bytesview
+
+suite "BytesView":
+  test "basics":
+    var b = BytesView.init(@[byte 1, 2, 3, 4, 5, 6])
+    check b.len() == 6
+    check @(b.data()) == @([byte 1, 2, 3, 4, 5, 6])
+    check @(b.toOpenArray(1, 3)) == @([byte 2, 3])
+
+    b.consume(2)
+    check b.len() == 4
+    check @(b.data()) == @([byte 3, 4, 5, 6])
+    check @(b.toOpenArray(1, 3)) == @([byte 4, 5])
+
+    b.consume(2)
+    check b.len() == 2
+    check @(b.data()) == @([byte 5, 6])
+
+    b.consume(2)
+    check b.len() == 0
+    check b.data().len == 0

tests/testnative.nim

@@ -11,7 +11,7 @@
 
 import
   testvarint, testconnection, testbridgestream, testminprotobuf, teststreamseq,
-  testsemaphore, testheartbeat, testfuture, testzeroqueue
+  testsemaphore, testheartbeat, testfuture, testzeroqueue, testbytesview
 
 import testminasn1, testrsa, testecnist, tested25519, testsecp256k1, testcrypto