Compare commits

...

585 Commits

Author SHA1 Message Date
Gabriel Cruz
fd4100e19c test: dummy pr 2025-05-28 15:31:41 -03:00
richΛrd
cd60b254a0 chore(version): update libp2p.nimble to 1.10.1 (#1390) 2025-05-21 07:40:11 -04:00
richΛrd
b88cdcdd4b chore: make quic optional (#1389) 2025-05-20 21:04:30 -04:00
vladopajic
4a5e06cb45 revert: disable transport interop with zig-v0.0.1 (#1372) (#1383) 2025-05-20 14:20:42 +02:00
vladopajic
fff3a7ad1f chore(hp): add timeout on dial (#1378) 2025-05-20 11:10:01 +00:00
Miran
05c894d487 fix(ci): test Nim 2.2 (#1385) 2025-05-19 15:51:56 -03:00
vladopajic
8850e9ccd9 ci(test): reduce timeout (#1376) 2025-05-19 15:34:16 +00:00
Ivan FB
2746531851 chore(dialer): capture possible exception (#1381) 2025-05-19 10:57:04 -04:00
vladopajic
2856db5490 ci(interop): disable transport interop with zig-v0.0.1 (#1372) 2025-05-15 20:04:41 +00:00
AYAHASSAN287
b29e78ccae test(gossipsub): block5 protobuf test cases (#1204)
Co-authored-by: Radoslaw Kaminski <radoslaw@status.im>
2025-05-15 16:32:03 +01:00
Gabriel Cruz
c9761c3588 chore: improve README.md text (#1373) 2025-05-15 12:35:01 +00:00
richΛrd
e4ef21e07c chore: bump quic (#1371)
Co-authored-by: Gabriel Cruz <8129788+gmelodie@users.noreply.github.com>
2025-05-14 21:06:38 +00:00
Miran
61429aa0d6 chore: fix import warnings (#1370) 2025-05-14 19:08:46 +00:00
Radosław Kamiński
c1ef011556 test(gossipsub): refactor testgossipinternal (#1366) 2025-05-14 17:15:31 +01:00
vladopajic
cd1424c09f chore(interop): use the same redis dependency (#1364) 2025-05-14 15:49:51 +00:00
Miran
878d627f93 chore: update dependencies (#1368) 2025-05-14 10:51:08 -03:00
richΛrd
1d6385ddc5 chore: bump quic (#1361)
Co-authored-by: Gabriel Cruz <8129788+gmelodie@users.noreply.github.com>
2025-05-14 11:40:13 +00:00
Gabriel Cruz
873f730b4e chore: change nim-stew dep tagging (#1362) 2025-05-13 21:46:07 -04:00
Radosław Kamiński
1c1547b137 test(gossipsub): Topic Membership Tests - updated (#1363) 2025-05-13 16:22:49 +01:00
Álex
9997f3e3d3 test(gossipsub): control message (#1191)
Co-authored-by: Radoslaw Kaminski <radoslaw@status.im>
2025-05-13 10:54:07 -04:00
richΛrd
4d0b4ecc22 feat: interop (#1303) 2025-05-06 19:27:33 -04:00
Gabriel Cruz
ccb24b5f1f feat(cert): add certificate signing request (CSR) generation (#1355) 2025-05-06 18:56:51 +00:00
Marko Burčul
5cb493439d fix(ci): secrets token typo (#1357) 2025-05-05 09:49:42 -03:00
Ivan FB
24b284240a chore: add gcsafe pragma to removeValidator (#1356) 2025-05-02 18:39:00 +02:00
richΛrd
b0f77d24f9 chore(version): update libp2p.nimble to 1.10.0 (#1351) 2025-05-01 05:39:58 -04:00
richΛrd
e32ac492d3 chore: set @vacp2p/p2p team as codeowners of repo (#1352) 2025-05-01 05:03:54 -03:00
Gabriel Cruz
470a7f8cc5 chore: add libp2p CID codec (#1348) 2025-04-27 09:45:40 +00:00
Radosław Kamiński
b269fce289 test(gossipsub): reorganize tests by feature category (#1350) 2025-04-25 16:48:50 +01:00
vladopajic
bc4febe92c fix: git ignore for tests (#1349) 2025-04-24 15:36:46 +02:00
Radosław Kamiński
b5f9bfe0f4 test(gossipsub): optimise heartbeat interval and sleepAsync (#1342) 2025-04-23 18:10:16 +01:00
Gabriel Cruz
4ce1e8119b chore(readme): add gabe as a maintainer (#1346) 2025-04-23 15:57:32 +02:00
Miran
65136b38e2 chore: fix warnings (#1341)
Co-authored-by: vladopajic <vladopajic@users.noreply.github.com>
2025-04-22 19:45:53 +00:00
Gabriel Cruz
ffc114e8d9 chore: fix broken old status-im links (#1332) 2025-04-22 09:14:26 +00:00
Radosław Kamiński
f2be2d6ed5 test: include missing tests in testall (#1338) 2025-04-22 09:45:38 +01:00
Radosław Kamiński
ab690a06a6 test: combine tests (#1335) 2025-04-21 17:39:42 +01:00
Radosław Kamiński
10cdaf14c5 chore(ci): decouple examples from unit tests (#1334) 2025-04-21 16:31:50 +01:00
Radosław Kamiński
ebbfb63c17 chore(test): remove unused flags and simplify testpubsub (#1328) 2025-04-17 13:38:27 +02:00
Álex
ac25da6cea test(gossipsub): message propagation (#1184)
Co-authored-by: Radoslaw Kaminski <radoslaw@status.im>
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
2025-04-14 15:49:13 +01:00
Gabriel Cruz
fb41972ba3 chore: rendezvous improvements (#1319) 2025-04-11 13:31:24 +00:00
Richard Ramos
504d1618af fix: bump nim-quic 2025-04-10 17:54:20 -04:00
richΛrd
0f91b23f12 fix: do not use while loop for quic transport errors (#1317) 2025-04-10 21:47:42 +00:00
vladopajic
5ddd62a8b9 chore(git): ignore auto generated test binaries (#1320) 2025-04-10 13:39:04 +00:00
vladopajic
e7f13a7e73 refactor: utilize singe bridgedConnections (#1309) 2025-04-10 08:37:26 -04:00
vladopajic
89e825fb0d fix(quic): continue accept when client certificate is incorrect (#1312) 2025-04-08 21:03:47 +02:00
richΛrd
1b706e84fa chore: bump nim-quic (#1314) 2025-04-08 10:04:31 -04:00
richΛrd
5cafcb70dc chore: remove range checks from rendezvous (#1306) 2025-04-07 12:16:18 +00:00
vladopajic
8c71266058 chore(readme): add quic and memory transports (#1311) 2025-04-04 15:07:31 +00:00
vladopajic
9c986c5c13 feat(transport): add memory transport (#1304) 2025-04-04 15:43:34 +02:00
vladopajic
3d0451d7f2 chore(protocols): remove deprecated utilities (#1305) 2025-04-04 08:44:36 +00:00
richΛrd
b1f65c97ae fix: unsafe string usage (#1308) 2025-04-03 15:33:08 -04:00
vladopajic
5584809fca chore(certificate): update test vectors (#1294) 2025-04-01 17:15:26 +02:00
richΛrd
7586f17b15 fix: set peerId on incoming Quic connection (#1302) 2025-03-31 09:38:30 -04:00
richΛrd
0e16d873c8 feat: withQuicTransport (#1301) 2025-03-30 04:44:49 +00:00
richΛrd
b11acd2118 chore: update quic and expect exception in test (#1300)
Co-authored-by: vladopajic <vladopajic@users.noreply.github.com>
2025-03-27 12:19:49 -04:00
vladopajic
1376f5b077 chore(quic): add tests with invalid certs (#1297) 2025-03-27 15:19:14 +01:00
richΛrd
340ea05ae5 feat: quic (#1265)
Co-authored-by: vladopajic <vladopajic@users.noreply.github.com>
2025-03-26 10:17:15 -04:00
vladopajic
024ec51f66 feat(certificate): add date verification (#1299) 2025-03-25 11:50:25 +01:00
richΛrd
efe453df87 refactor: use openssl instead of mbedtls (#1298) 2025-03-24 10:22:52 -04:00
vladopajic
c0f4d903ba feat(certificate): set distinguishable issuer name with peer id (#1296) 2025-03-21 12:38:02 +00:00
vladopajic
28f2b268ae chore(certificate): cosmetics (#1293) 2025-03-19 17:02:14 +00:00
vladopajic
5abb6916b6 feat: X.509 certificate validation (#1292) 2025-03-19 15:40:14 +00:00
richΛrd
e6aec94c0c chore: use token per repo in autobump task (#1288) 2025-03-18 17:12:52 +00:00
vladopajic
9eddc7c662 chore: specify exceptions (#1284) 2025-03-17 13:09:18 +00:00
richΛrd
028c730a4f chore: remove python dependency (#1287) 2025-03-17 08:04:30 -04:00
richΛrd
3c93bdaf80 chore(version): update libp2p.nimble to 1.9.0 (#1282)
Co-authored-by: kaiserd <1684595+kaiserd@users.noreply.github.com>
2025-03-14 15:56:51 +00:00
vladopajic
037b99997e chore(ci): remove AppVeyor config (#1281) 2025-03-10 21:09:40 +00:00
vladopajic
e67744bf2a chore(connmanager): propagate CancelledError (#1276) 2025-03-05 17:31:46 +00:00
vladopajic
5843e6fb4f chore(protocol): handler to propagate CancelledError (#1275) 2025-03-05 15:04:29 +00:00
vladopajic
f0ff7e4c69 fix(ci): transport interoperability action (#1277) 2025-03-04 19:00:08 +01:00
diegomrsantos
24808ad534 feat(tls-certificate): generate and parse libp2p tls certificate (#1209)
Co-authored-by: Richard Ramos <info@richardramos.me>
2025-03-04 08:05:40 -04:00
vladopajic
c4bccef138 chore(relay): specify raised exceptions (#1274) 2025-03-03 15:18:27 +01:00
vladopajic
adf2345adb chore: specify raised exceptions in miscellaneous places (#1269) 2025-02-28 09:19:53 -04:00
vladopajic
f7daad91e6 chore(protocols): specify raised exceptions (part 2) (#1268) 2025-02-28 08:06:51 -04:00
vladopajic
65052d7b59 chore(transports): specify raised exceptions (#1266) 2025-02-27 10:42:05 +01:00
vladopajic
b07ec5c0c6 chore(protocol): list raised exceptions (#1260) 2025-02-25 17:33:24 -04:00
vladopajic
f4c94ddba1 chore(dialer): list raised exceptions (#1264) 2025-02-24 20:10:20 +01:00
vladopajic
a7ec485ca9 chore(transports): list raised exceptions (#1255) 2025-02-24 16:42:27 +01:00
vladopajic
86b6469e35 chore: list raised exceptions in switch services (#1251)
Co-authored-by: richΛrd <info@richardramos.me>
2025-02-24 15:33:06 +01:00
Ivan FB
3e16ca724d chore: add trace log to analyse remote stream reset (#1253) 2025-02-24 08:24:27 -04:00
vladopajic
93dd5a6768 chore(connmanager): specify raised exceptions (#1263) 2025-02-19 15:55:36 +00:00
richΛrd
ec43d0cb9f chore: refactors to remove .closure., .gcsafe for .async. procs, and added callback compatibility to daemonapi (#1240) 2025-02-19 15:00:44 +00:00
vladopajic
8469a750e7 chore: add description of public pragma (#1262) 2025-02-19 14:14:50 +01:00
vladopajic
fc6ac07ce8 ci: utilize github action for nph check (#1257) 2025-02-17 15:42:58 +00:00
vladopajic
79cdc31b37 ci: remove commit message check (#1256) 2025-02-17 12:35:42 +00:00
vladopajic
be33ad6ac7 chore: specify raising exceptions in daemon module (#1249) 2025-02-14 15:36:29 +01:00
vladopajic
a6e45d6157 chore: list raised exceptions in utils module (#1252)
part of https://github.com/vacp2p/nim-libp2p/issues/962 effort.
2025-02-13 11:27:29 -04:00
richΛrd
37e0f61679 chore: update maintainers (#1248) 2025-02-11 14:32:12 -04:00
vladopajic
5d382b6423 style: fix code style with nph (#1246) 2025-02-11 15:39:08 +00:00
richΛrd
78a4344054 refactor(pubsub): do not raise exceptions on publish (#1244) 2025-02-07 01:30:21 +00:00
Ivan FB
a4f0a638e7 chore: add isClosedRemotely public bool attribute in lpstream (#1242)
Also adds the gcsafe to getStreams method so that we can invoke it from async procs
2025-02-06 09:54:00 -04:00
richΛrd
c5aa3736f9 chore(version): update libp2p.nimble to 1.8.0 (#1236)
This PR's commit is planned to be tagged as v1.8.0
2025-01-22 08:45:56 -04:00
richΛrd
b0f83fd48c fix: use target repository branch as suffix for nim-libp2p-auto-bump- (#1238)
The branch `nim-libp2p-auto-bump-unstable` was not getting autobumped
because at one point of time in nim-libp2p `unstable` branch was renamed
to `master` branch. Since the workflow file depended on `GITHUB_REF`, it
was instead updating `nim-libp2p-auto-bump-master` instead of
`nim-libp2p-auto-bump-unstable`
2025-01-15 14:25:21 -04:00
richΛrd
d6e5094095 fix(pubsub): revert async: raises: [] annotation for TopicHandler and ValidatorHandler (#1237) 2025-01-15 03:35:24 +00:00
richΛrd
483e1d91ba feat: idontwant on publish (#1230)
Closes #1224 

The following changes were done:
1. Once a message is broadcasted, if it exceeds the threeshold to
consider it a large message it will send an IDONTWANT message before
publishing the message

4f9c21f699/libp2p/protocols/pubsub/gossipsub.nim (L800-L801)
2. Extracts the logic to broadcast an IDONTWANT message and to verify
the size of a message into separate functions
3. Adds a template to filter values of a hashset without modifying the
original
2025-01-14 16:59:38 -04:00
richΛrd
d215bb21e0 fix(multiaddress): don't re-export minprotobuf (#1235) 2025-01-14 15:47:06 -04:00
richΛrd
61ac0c5b95 feat(pubsub): add {.async: (raises).} annotations (#1233)
This PR adds `{.async: (raises).}` annotations to the pubsub package.
The cases in which a `raises:[CatchableError]` was added were due to not
being part of the package and should probably be changed in a separate
PR
2025-01-14 17:01:02 +01:00
diegomrsantos
1fa30f07e8 chore(ci): add arm64 for macOS (#1212)
This PR adds the macOS 14 GitHub runner that uses the arm64 cpu.
2024-12-20 21:18:56 -04:00
richΛrd
39d0451a10 chore: validate PR titles and commits, and autoassign PRs (#1227) 2024-12-20 15:42:54 +01:00
richΛrd
4dc7a89f45 chore: use latest nimpng and nico (#1229) 2024-12-17 14:37:06 -04:00
richΛrd
fd26f93b80 fix(ci): use nim 2.0 branch (#1228) 2024-12-09 19:41:08 +00:00
Etan Kissling
dd2c74d413 feat(nameresolving): Add {.async: (raises).} annotations (#1214)
Modernize `nameresolving` modules to track `{.async: (raises).}`.

---------

Co-authored-by: kaiserd <1684595+kaiserd@users.noreply.github.com>
Co-authored-by: richΛrd <info@richardramos.me>
2024-12-09 12:43:21 +01:00
Eugene Kabanov
b7e0df127f Fix PeerStore missing remote endpoints of established connection. (#1226) 2024-11-27 14:49:41 +02:00
kaiserd
f591e692fc chore(version): update libp2p.nimble to 1.7.1 (#1223)
minor version for rendezvous fix
2024-11-09 10:51:33 +07:00
NagyZoltanPeter
8855bce085 fix:missing raises pragma for one of RendezVous.new (#1222) 2024-11-08 11:11:51 +01:00
kaiserd
ed5670408b chore(version): update libp2p.nimble to 1.7.0 (#1213)
This commit is planned to be tagged with v1.7.0.
The only feature added in this version are configurable min and max TTL
for rendezvous.

Co-authored-by: ksr <kaiserd@users.noreply.github.com>
2024-11-03 18:15:33 +00:00
Álex
97192a3c80 fix(ci): nim 2.0 dependency failure (#1216)
Broken due to an update to Nimble 0.16.2 for the version-2-0 branch.
This PR sets the ref to the previous commit. Also, updates the key
naming so it fits better.

Nim's commit list: https://github.com/nim-lang/Nim/commits/version-2-0/

# Important Note
In this PR some required job's names were changed. They need to be
updated at repo-level so this PR can be merged.
2024-10-31 17:56:13 +00:00
Álex
294d06323c docs(test): handle IHAVE / IWANT tests (#1202)
Add documentation as requested.
2024-10-31 17:23:24 +00:00
Álex
a3b8729cbe fix(ci): Daily workflows report (#1200)
Fix daily workflows' green tick when jobs failed.

Based of off: https://stackoverflow.com/a/58859404

Closes:  https://github.com/vacp2p/nim-libp2p/issues/1197

---------

Co-authored-by: Diego <diego@status.im>
2024-10-10 12:10:49 +00:00
Álex
6c970911f2 fix(CI): free disk space on interop transport job (#1206)
Readd the free disk space job, as space issues still happen when
building images.

---------

Co-authored-by: Diego <diego@status.im>
2024-10-10 09:42:25 +00:00
Álex
5d48776b02 chore(ci): Enable S3 caching for interop (#1193)
- Adds our S3 bucket for caching docker images as Protocol Labs shut
down their shared one.
- Remove the free disk space workaround that prevented the jobs from
failing for using too much space for the images.

---------

Co-authored-by: diegomrsantos <diego@status.im>
2024-09-26 09:56:09 +00:00
Simon-Pierre Vivier
d389d96789 feat: rendezvous refactor (#1183)
Hello!

This PR aim to refactor rendezvous code so that it is easier to impl.
Waku rdv strategy. The hardcoded min and max TTL were out of range with
what we needed and specifying which peers to interact with is also
needed since Waku deals with peers on multiple separate shards.

I tried to keep the changes to a minimum, specifically I did not change
the name of any public procs which result in less than descriptive names
in some cases. I also wanted to return results instead of raising
exceptions but didn't. Would it be acceptable to do so?

Please advise on best practices, thank you.

---------

Co-authored-by: Ludovic Chenut <ludovic@status.im>
2024-09-25 09:11:57 +00:00
kaiserd
09fe199b6b chore(version): update libp2p.nimble to 1.6.0 (#1196)
Updating libp2p.nimble to 1.6.0.
This commit is planned to be tagged with 1.6.0.

The only new feature in this version is the highly experimental Quick
transport. Still, it is a feature and justifies a minor version upgrade.
It also contains a few fixes and improvements (see commit history).

Co-authored-by: ksr <kaiserd@users.noreply.github.com>
2024-09-19 20:06:45 +00:00
diegomrsantos
68306cf1f1 chore: fix devel compilation issues (#1195)
- fixes https://github.com/vacp2p/nim-libp2p/issues/1194.
- fixes ambiguous `KeyError`
- removes an unnecessary type param for `newSeqWith`
- fixes tests for `safeConvert`

The main fixes relate to Nim 2.2 being more strict and not accepting
calls with a wrong number of type parameters.
2024-09-19 11:35:50 +00:00
Tanguy
b37133ca43 feat(transport): add experimental QUIC Transport (not production ready) (#725)
Our quic effort is blocked by bearssl not supporting TLS1.3, but since
Mark did most of the work to implement Quic here:
https://github.com/status-im/nim-libp2p/pull/563 and on nim-quic, this
PR is going to bring encryption-less Quic into nim-libp2p
This allows us to test it, and make sure it doesn't bitrot.

Heavily WiP:
- [X] Extract code from #563
- [X] Create custom muxer & upgrader
- [X] Basic E2E switch test working
- [x] Update nim-quic to get address informations in libp2p (for
`observed address` and port 0 resolving)
- [ ] More tests
- [ ] Cleanup

Since encryption is not yet supported, we're not compatible with any
other libp2ps, and have to rely on home made protocols to retrieve the
peer's id

---------

Co-authored-by: markspanbroek <mark@spanbroek.net>
Co-authored-by: Diego <diego@status.im>
2024-09-12 09:32:14 +00:00
diegomrsantos
3e3df07269 chore: add support to merge queues (#1192)
This change is necessary before enabling merge queues. The `merge_group`
event is needed to trigger the GitHub Actions workflow when a pull
request is added to a merge queue.

The merge queue provides the same benefits as the Require branches to be
up to date before merging branch protection but does not require a pull
request author to update their pull request branch and wait for status
checks to finish before trying to merge. More info on
https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/configuring-pull-request-merges/managing-a-merge-queue.
2024-09-10 16:10:24 +00:00
Etan Kissling
1771534030 fix(multiaddress): Raise MaError instead of LPError on & (#1145)
`LPError` is the top level error type of libp2p, it makes more sense to
raise a multi address specific subtype to avoid requiring callers to
catch errors too broadly. As `MaError` inherits from `LPError`, existing
error handlers will still work fine.

---------

Co-authored-by: diegomrsantos <diego@status.im>
2024-09-10 13:24:58 +00:00
diegomrsantos
21a444197c chore: move mm:refc config and remove skipParentCfg (#1190)
- Moving the `mm:refc` configuration to the main `config.nims` file
makes it be used in tests and tasks that build examples.
- It's not clear why `skipParentCfg`was being used and it doesn't seem
to be necessary.
2024-09-10 13:51:54 +02:00
Ivan FB
966996542e chore(connmanager): also show peerID and dir when too many conns (#1185)
Bring a bit more detail when the "Too many connections for peer" message
is being logged. Particularly, we are adding the offending `peerId` and
the stream direction

Co-authored-by: diegomrsantos <diego@status.im>
2024-09-09 13:52:01 +00:00
diegomrsantos
8070b21825 fix(transport): tcp accept fixes (#1170)
Address the comments in https://github.com/vacp2p/nim-libp2p/pull/1164
2024-09-09 11:49:33 +00:00
Cofson
d98152f266 Create funding.json (#1188) 2024-09-05 18:25:50 +02:00
diegomrsantos
47a51983b5 chore(CI): rollback nph change (#1187)
Rollbacks to the previous `nph` command that shows what files aren't
formatted.
2024-09-03 09:01:35 +00:00
Álex
70754cd575 ci: Enable conditional SAT solving (#1177)
* Add conditional SAT dependency solving to a new daily job.

closes: https://github.com/vacp2p/nim-libp2p/issues/1174
2024-09-02 15:33:16 +02:00
tersec
a1811e7395 fix(transport): libp2p compilation in Nim version-2-0 and devel (#1186)
For example:
```
/Users/runner/work/nim-libp2p/nim-libp2p/nimbledeps/pkgs2/websock-0.1.0-94f836ae589056b2deb04bdfdcd614fff80adaf5/websock/http/client.nim(173, 5) template/generic instantiation of `async` from here
/Users/runner/work/nim-libp2p/nim-libp2p/nimbledeps/pkgs2/websock-0.1.0-94f836ae589056b2deb04bdfdcd614fff80adaf5/websock/http/client.nim(165, 1) Warning: The raises pragma doesn't work on async procedures - use `async: (raises: [...]) instead. [User]
/Users/runner/work/nim-libp2p/nim-libp2p/nimbledeps/pkgs2/websock-0.1.0-94f836ae589056b2deb04bdfdcd614fff80adaf5/websock/websock.nim(257, 5) template/generic instantiation of `async` from here
/Users/runner/work/nim-libp2p/nim-libp2p/nimbledeps/pkgs2/websock-0.1.0-94f836ae589056b2deb04bdfdcd614fff80adaf5/websock/websock.nim(251, 1) Warning: The raises pragma doesn't work on async procedures - use `async: (raises: [...]) instead. [User]
/Users/runner/work/nim-libp2p/nim-libp2p/libp2p/transports/wstransport.nim(77, 18) template/generic instantiation of `async` from here
/Users/runner/work/nim-libp2p/nim-libp2p/libp2p/transports/wstransport.nim(83, 10) template/generic instantiation of `setResult` from here
/Users/runner/work/nim-libp2p/nim-libp2p/libp2p/transports/wstransport.nim(78, 26) template/generic instantiation of `mapExceptions` from here
/Users/runner/work/nim-libp2p/nim-libp2p/nimbledeps/pkgs2/chronos-4.0.2-c5e9517b9189713210e2abab8b77a68da71ded12/chronos/internal/asyncmacro.nim(542, 60) Error: expression 'value(cast[type(recv(s.session, pbytes, nbytes))](chronosInternalRetFuture.internalChild))' is of type 'int' and has to be used (or discarded); start of expression here: /Users/runner/work/nim-libp2p/nim-libp2p/libp2p/transports/wstransport.nim(78, 26)
stack trace: (most recent call last)
```
from
https://github.com/vacp2p/nim-libp2p/actions/runs/10655841970/job/29533846606?pr=1145

For minimal example of this:
```nim
template g(body: untyped) =
  try:
    body
  except CatchableError:
    raise newException(CatchableError, "")

discard g(0)
```

Also, even in 2.0.8, a variation doesn't work:
```
template g(body: untyped) = body
discard g(0)
```
2024-09-01 21:49:41 +00:00
Álex
c6e8fadbda fix(ci): Daily workflow parent's name (#1182)
* Fix daily workflows' parent's name.
2024-08-23 15:57:08 +02:00
Álex
48846d69cb chore(logs): remove duplicate msg key (#1180)
* Remove `msg` parameter key in logs.

closes: https://github.com/vacp2p/nim-libp2p/issues/1176
2024-08-14 17:19:54 +02:00
kaiserd
18a2e79ce2 chore(version): update libp2p.nimble to 1.5.0 (#1166) 2024-08-13 19:00:45 +02:00
Ludovic Chenut
55cc5434fe fix(yamux): future leak (#1171) 2024-08-12 19:21:16 +02:00
diegomrsantos
cde5ed7e8c fix: infinite loop when connection is aborted before being accepted (#1164) 2024-08-07 20:54:24 +02:00
Álex
6ec038d29a chore: Cleanup CI (#1117)
* Standardise names
* Update actions versions
* Update parameter handling
* Split jobs into steps

---------

Co-authored-by: kaiserd <1684595+kaiserd@users.noreply.github.com>
2024-08-06 16:24:13 +02:00
Álex
fdae9e4b42 fix(test): interop transport (#1159)
Free disk space before running the steps of interop's transport test.
The original job has enough space but in our repo it crashes midway due
to missing disk space.

---------

Co-authored-by: kaiserd <1684595+kaiserd@users.noreply.github.com>
Co-authored-by: Diego <diego@status.im>
2024-08-02 15:53:37 +02:00
gabrielmer
a60f0c5532 feat: adding onValidated observer (#1128)
### Description

Adding an `onValidated` observer which will run every time a message is
received and validated. This comes from the necessity of precisely track
message deliveries and network activity.

`onRecv` observers run before any check is performed on the received
message, which means that it runs every time a duplicate or invalid
message arrives, which is inefficient and inaccurate for our purpose of
tracking only received, unique and valid messages. Therefore, adding
this extra option of running an observer for every message after all
validation checks pass.
2024-08-01 18:50:13 +03:00
diegomrsantos
62f2d85f11 fix: add gcc 14 support (#1151)
- Add ubuntu-24-gcc-14 target on CI.
https://github.com/vacp2p/nim-libp2p/issues/1156 prevents us from using
only Ubuntu 24.
- Made the changes necessary to support gcc 14. More info on
https://github.com/status-im/nim-bearssl/pull/62

Fixes https://github.com/vacp2p/nim-libp2p/issues/1150
2024-08-01 14:50:44 +02:00
Álex
e5e319c1a9 fix(ci): windows-amd64 (Nim version-1-6) (#1160)
The failure is due to incompatibility (in caching) after Nimble's
v.0.14.0 update, where they changed the dependencies directory name from
`pkgs` to `pkgs2`.
This PR includes the nim branch in the cache key to avoid the directory
name issue.

In the future, if we deprecate support for Nim 1.6 we may remove this.

fixes https://github.com/vacp2p/nim-libp2p/issues/1157

---------

Co-authored-by: Diego <diego@status.im>
2024-08-01 12:54:18 +02:00
gabrielmer
f8d4da6421 chore: setting dialing canceled log to trace (#1153)
We are trying to reduce the logs load in our fleets, and one of the most
recurrent one is

```
Dialing canceled                           topics="libp2p dialer" tid=1 file=dialer.nim:67 err="Future operation cancelled!" peerId=16U*XAFJX3
```

which is quite spammy and doesn't give much info. 

In addition to that, its corresponding `Dialing address` log is in
trace.

So adjusting the log level of `Dialing canceled` to trace :)
2024-07-23 14:14:18 +02:00
diegomrsantos
b5fb7b3a97 chore: update os images on ci (#1149)
The main motivation was to update the Ubuntu version on the daily job as
it seemed it wasn't supported anymore.

macOS 14 fails immediately with no error msg, so we are using 13 for
now.
2024-07-11 10:22:16 +02:00
diegomrsantos
fa19bbbbb7 fix: support ipv6 dual stack (#1148)
Fixes https://github.com/vacp2p/nim-libp2p/issues/1147
2024-07-10 18:08:52 +02:00
diegomrsantos
86563cbddd chore: enable Nim 2.0.x and fix compilation issues (#1146)
This PR enables Nim 2.0.x with `refc` garbage collector on CI.

The following compilation error had to be fixed: Error: undeclared
identifier: 'acceptHandler`gensym435'; if declared in a template, this
identifier may be inconsistently marked inject or gensym
2024-07-10 14:24:12 +02:00
diegomrsantos
be801602f6 fix: run workflows on master (#1144)
The `unstable` branch wasn't removed from some workflows and `interop`
wasn't running on `master`.
2024-07-05 17:33:34 +02:00
kaiserd
94d93cbf25 chore(version): update libp2p.nimble to 1.4.0 (#1143) 2024-07-03 15:10:11 +02:00
diegomrsantos
78f0855419 feat: add maxSize to TimedCache (#1132) 2024-07-01 22:00:51 +02:00
diegomrsantos
2195313dba feat: iDontWant is sent only for gossipsub 1.2 or higher (#1135) 2024-06-25 19:32:08 +02:00
Ludovic Chenut
100f3188ed feat(peerEvents): add a peerEvent Identified (#843)
Co-authored-by: diegomrsantos <diego@status.im>
2024-06-21 13:06:59 +02:00
Ludovic Chenut
d1d53ff369 chore(yamux): change closedRemotely from Future into AsyncEvent (#1133) 2024-06-21 12:11:18 +02:00
gabrielmer
0f27f896ab chore: improve max outgoing connections log (#1129) 2024-06-20 11:51:13 +02:00
diegomrsantos
0be7144e34 fix(CI): rebuild website job (#1125) 2024-06-19 14:40:01 +00:00
diegomrsantos
fba6dc31b0 chore: add .git-blame-ignore-revs (#1130) 2024-06-19 15:36:56 +02:00
diegomrsantos
02f6e6127c fix(readme): update links (#1126) 2024-06-14 15:27:55 +02:00
diegomrsantos
1d826ee26f fix(CI): generate website job (#1124) 2024-06-13 15:28:47 +02:00
diegomrsantos
7498258f7c fix(gossipsub): pubsubpeer is created with wrong gossipsub version (#1116) 2024-06-13 12:25:48 +02:00
diegomrsantos
4618f4c68f fix(tests): flaky testdaemon (#1123) 2024-06-13 09:07:36 +00:00
diegomrsantos
3bf8a2907f fix(tests): testautorelay (#1121) 2024-06-12 14:31:09 +00:00
diegomrsantos
96bfefc928 feat(gossipsub): support version 1.2.0 (#1106) 2024-06-12 15:46:47 +02:00
diegomrsantos
dc83a1e9b6 chore(formatting): format the whole codebase using nph 0.5.1 (#1118) 2024-06-11 17:18:06 +02:00
kaiserd
d0af3fbe85 chore(version): update libp2p.nimble to 1.3.0 (#1119) 2024-06-07 13:25:47 +00:00
diegomrsantos
120549e313 fix(services): setup services before peerinfo is updated (#1120) 2024-06-07 11:48:44 +02:00
diegomrsantos
bccb305cf5 feat(service): add wildcard address resolver (#1099)
Co-authored-by: Ludovic Chenut <ludovic@status.im>
2024-06-06 11:05:45 +00:00
Álex
f9a6ef06cf Update go-libp2p-daemon build version. (#1111) 2024-06-04 21:42:24 +02:00
diegomrsantos
8cb7dbb425 fix(multicodec): remove unnecessary "!=" operator (#1112) 2024-06-04 16:42:30 +02:00
omahs
368c9765f7 chore: fix typos (#1110) 2024-06-03 20:05:40 +02:00
kaiserd
d6feb1bbc2 chore(peer-score): enhance score trace logs cont' (#1108) 2024-06-03 14:57:20 +02:00
Ivan FB
3f5b5cee75 chore(peer-scoring): enhance score trace logs (#1107) 2024-06-03 12:25:47 +02:00
Jacek Sieka
8a4e8a00a2 Send IDONTWANT before validating message (#1103) 2024-06-03 10:34:05 +02:00
kaiserd
77d40c34f4 chore(README): small PRs (#1098) 2024-05-29 11:40:45 +02:00
diegomrsantos
2fa2c4425f fix(yamux): set EoF when remote peer half closes the stream in yamux (#1086) 2024-05-24 14:11:27 +02:00
kaiserd
0911cb20f4 chore(gossipsub): cleanups (#1096) 2024-05-15 18:57:15 +02:00
Jacek Sieka
3ca49a2f40 fix(transport): various tcp transport races (#1095)
Co-authored-by: diegomrsantos <diego@status.im>
2024-05-14 07:10:34 +02:00
diegomrsantos
1b91b97499 fix(CI): rename branch from unstable to master in bumper workflow (#1097) 2024-05-10 15:42:43 +02:00
Jacek Sieka
21cbe3a91a chore: cleanups (#1092)
* remove cruft
* remove redundant error handling (reduces warnings)
* remove redundant copying
2024-05-08 14:33:26 +02:00
diegomrsantos
88e233db81 fix: Asynchronous task [sendMsgSlow()] was cancelled [FutureDefect] (#1094) 2024-05-07 15:44:14 +02:00
Jacek Sieka
84659af45b avoid latency/copy when sending low-priority messages to fast peers (#1060) 2024-05-02 12:26:16 +02:00
Jacek Sieka
aef44ed1ce salt idontwant (#1090) 2024-05-02 12:18:55 +02:00
Jacek Sieka
02c96fc003 Improve memory efficiency of seen cache (#1073) 2024-05-01 18:38:24 +02:00
kaiserd
c4da9be32c chore: empty commit to trigger new commit hooks after renaming branch (#1089) 2024-05-01 17:00:53 +02:00
Diego
2b5319622c Revert "always allow new data be received if the recvWindow is > 0"
This reverts commit 5cbb473d1b.
2024-04-25 15:01:29 +02:00
Diego
5cbb473d1b always allow new data be received if the recvWindow is > 0 2024-04-25 14:55:19 +02:00
Ivan FB
b30b2656d5 fix: reset accept fut in stop (#1082)
Co-authored-by: diegomrsantos <diego@status.im>
2024-04-17 22:44:41 +02:00
diegomrsantos
89cad5a3ba fix: remove explicit param from GossipSubParams constructor (#1080) 2024-04-09 20:14:59 +02:00
Ludovic Chenut
09b3e11956 fix: valueOr and withValue utilities (#1079) 2024-04-04 17:15:50 +02:00
Etan Kissling
03f67d3db5 add support for setting protocol handlers with {.raises.} annotation (#1064) 2024-03-28 09:42:31 +01:00
diegomrsantos
bb97a9de79 improvement: create a new gossipsub constructor (#1078) 2024-03-27 11:54:15 +01:00
diegomrsantos
1a707e1264 feat: add max number of elements to non-prio queue (#1077) 2024-03-25 22:00:11 +01:00
Álex Cabeza Romero
458b0885dd fix(issue-1052): Single topic for RPC Message (#1061) 2024-03-25 12:06:34 +01:00
Jacek Sieka
a2027003cd Avoid unnecessary rate limit message copy (#1067) 2024-03-21 13:11:40 +00:00
Etan Kissling
c5db35d9b0 annotate upgrademngrs with {.async: (raises).} (#1068) 2024-03-21 08:19:57 +01:00
Jacek Sieka
d1e51beb7f Remove secio (#1072) 2024-03-20 14:53:56 +01:00
Etan Kissling
275d649287 move header.length check in yamux to original location (#1069) 2024-03-20 13:35:44 +01:00
Jacek Sieka
467b5b4f0c avoid cancelling send future (#1075) 2024-03-20 10:54:32 +00:00
Ivan FB
fdf53d18cd libp2p/dialer.nim: tiny log change to make it clearer a connection upgrade (#1071) 2024-03-18 11:38:23 +01:00
Etan Kissling
48a3ac06ff {.async: (raises).} for MultistreamSelect (#1066) 2024-03-12 21:05:53 +01:00
Etan Kissling
49a92e5641 avoid pointless exception raising in dcutr/server (#1063) 2024-03-12 18:29:01 +01:00
Etan Kissling
08a48faf41 {.async: (raises).} annotations for protocols/secure (#1059) 2024-03-07 11:22:22 +00:00
Etan Kissling
61b299e411 {.async: (raises).} for relay/utils.nim (#1058) 2024-03-07 10:45:25 +01:00
Etan Kissling
ca01ee06a8 clean up triple lookup and avoid KeyError when adding muxer (#1057) 2024-03-06 06:49:45 +01:00
Etan Kissling
6c43ab3fce default MultiAddress param for newStandardSwitch does not raise (#1056) 2024-03-06 06:48:13 +01:00
Jacek Sieka
ae13a0d583 Send priority with queue fix (#1051)
Co-authored-by: Diego <diego@status.im>
2024-03-05 15:05:21 +00:00
Etan Kissling
28609597d1 add {.async: (raises).} to libp2p/stream modules (#1050)
Co-authored-by: Dmitriy Ryajov <dryajov@gmail.com>
Co-authored-by: Jacek Sieka <jacek@status.im>
2024-03-05 07:06:27 +00:00
Etan Kissling
8294d5b9df document known --mm:orc crash (#1039) 2024-03-04 19:34:09 +01:00
Etan Kissling
78e83889ee define proper parent error type for YamuxError (#1040) 2024-03-04 19:26:27 +01:00
Etan Kissling
7603b8de5e catch WebSocketError in wstransport (#1049) 2024-03-04 00:27:35 +01:00
Etan Kissling
8cccd54125 avoid triple lookup in m.flushed yamux table (#1045) 2024-03-04 00:27:13 +01:00
Etan Kissling
18e00a741b avoid KeyError in edge case of yamux handler (#1044) 2024-03-04 00:24:18 +01:00
Etan Kissling
ee264fdf11 in yamux, do not write {Rst} packet to stream that's in use (#1041) 2024-03-04 00:23:42 +01:00
Etan Kissling
9059a8aced use race instead of or to avoid lockup (#1042) 2024-03-04 00:06:32 +01:00
Etan Kissling
0b753e7cf2 don't forget closing the stream when final {Fin} fails in yamux (#1043) 2024-03-04 00:05:59 +01:00
Etan Kissling
d43c5feab0 do not log yamux buffers without sanitization (trace log level) (#1046) 2024-03-04 00:04:37 +01:00
Etan Kissling
1609fd7197 change SecioError and NoiseError to descendants of LPStreamError (#1047) 2024-03-04 00:04:25 +01:00
Etan Kissling
42cd78e95b remove unused LPStreamError types (#1048) 2024-03-04 00:03:44 +01:00
Etan Kissling
44cada9c55 use new Chronos trackCounter APIs for leaks checks in tests (#1038) 2024-03-03 18:13:37 +01:00
Etan Kissling
6c873481ac move allFutureThrowing helper to tests (#1037)
Co-authored-by: Jacek Sieka <jacek@status.im>
2024-03-01 18:06:26 +01:00
Etan Kissling
d08ce17144 remove unused MultiBase.encode(..., Cid) function (#1036) 2024-03-01 14:07:52 +01:00
Etan Kissling
bd6ead95ef increase tolerance of simple heartbeat test (#1034) 2024-03-01 14:06:42 +01:00
Etan Kissling
53e3825e07 fix typo in ProtoMessage.toString() (#1033) 2024-02-29 15:56:47 +01:00
diegomrsantos
e9b456162a use chronos 4.0.0 (#1030) 2024-02-27 13:26:50 +01:00
diegomrsantos
250024f6cc fix: move transport interop tests to nim-libp2p repo (#1031) 2024-02-27 10:55:43 +01:00
Eugene Kabanov
fec632d28d Fix empty path crash issue for MultiAddresses unix, ip6zone, dns***. (#1025)
This issue was discovered by @0xTylerHolmes . Thank you!
2024-02-22 15:57:11 +01:00
Ludovic Chenut
349496e40f feat: Yamux timeout (#1029) 2024-02-22 10:21:34 +01:00
diegomrsantos
7faa0fac23 fix: allFuturesThrowing compilation issue on daily (#1026) 2024-02-19 19:46:34 +01:00
Diego
c5e4f8e12d Revert "feat: message prioritization with immediate peer-published dispatch and queuing for other msgs (#1015)"
This reverts commit fe4ff79885.
2024-02-19 13:47:37 +01:00
diegomrsantos
fe4ff79885 feat: message prioritization with immediate peer-published dispatch and queuing for other msgs (#1015) 2024-02-16 10:54:16 +01:00
Álex Cabeza Romero
aa4ebb0b3c docs(general): Improve docs (#1021) 2024-02-15 16:14:26 +01:00
diegomrsantos
e0f70b7177 improvement: enhanced checkExpiring macro with custom timeout (#1023) 2024-02-09 11:51:27 +01:00
Ludovic Chenut
c1dfd58772 fix: yamux metrics (#1022) 2024-02-08 12:36:58 +01:00
Álex Cabeza Romero
04af0c4323 test(flaky): Log checkExpiring failure (#1018)
Add simple logging mechanism on checkExpiring failure.
2024-02-06 17:47:13 +01:00
Ludovic Chenut
eb0890cd6f docs: add comments and improve yamux readability (#1006) 2024-02-02 15:14:02 +01:00
Álex Cabeza Romero
9bc5ec1566 tests(flaky): Increase check timeouts (#995)
Increase checkExpiring timeouts to verify impact on flaky tests.
2024-01-31 23:46:12 +00:00
diegomrsantos
5594bcb33e fix: more metrics issues when libp2p_expensive_metrics is enabled (#1016) 2024-01-30 16:55:55 +01:00
diegomrsantos
d46bcdb6ac fix: compilation issue when libp2p_expensive_metrics is enabled. (#1014) 2024-01-29 11:31:11 +01:00
diegomrsantos
9468bb6b4d fix(hole-punching-interop): update nim to 1.6.16 (#1012) 2024-01-26 11:15:40 +01:00
diegomrsantos
2725be64ba fix: use a temp var in withValue (#1010) 2024-01-18 16:25:56 +01:00
diegomrsantos
e3c967ad19 improvement(ci): improve ci daily workflows (#1002) 2023-12-18 20:14:33 +01:00
Ludovic Chenut
d2c98bd87d improvement(yamux): make the window size configurable (#987)
Co-authored-by: Diego <diego@status.im>
2023-12-15 16:30:50 +01:00
Ivan FB
3011ba4326 libp2p/multiaddress.nim: use of IpAddress instead of ValidIpAddress (#1001) 2023-12-12 12:53:36 +01:00
Etan Kissling
c6566707fa include connection info when logging identify message (#991) 2023-12-05 18:44:16 +01:00
diegomrsantos
3be681ec4d feat: add hole-punching interop tests (#998) 2023-12-05 18:37:33 +01:00
Jacek Sieka
2ede0fa40c remove redundant gcsafe annotations (#999) 2023-12-05 08:05:32 +01:00
Roman Zajic
7c195ab927 fix: remove forgotten "matrix-prep" job (#997) 2023-12-02 09:56:50 +08:00
Roman Zajic
3230407ffe fix: move workflows for Nim Devel and legacy i386 from "Daily" (#968) 2023-12-01 17:47:47 +08:00
diegomrsantos
deb72c8580 fix(dcutr): update the DCUtR initiator transport direction to Inbound (#994) 2023-11-29 17:38:47 +01:00
diegomrsantos
ce0685c272 fix(identify): do not add p2p and relayed addrs to observed addr manager (#990) 2023-11-21 18:24:35 +01:00
diegomrsantos
1f4b090227 fix(yamux): doesn't work in a Relayv2 connection (#979)
Co-authored-by: Ludovic Chenut <ludovic@status.im>
2023-11-21 16:03:29 +01:00
diegomrsantos
fb05f5ae22 fix(dcutr): handle tcp/p2p addresses (#989) 2023-11-20 17:06:17 +01:00
diegomrsantos
e12f65f193 fix(multiaddress): add quic-v1 multiaddress support (#988) 2023-11-20 11:09:56 +01:00
diegomrsantos
4b3bc4f819 Make ObservedAddrManager injectable (#970) 2023-11-20 11:06:02 +01:00
diegomrsantos
6791f5e7bb fix(dcutr): make the dcutr client inbound and the server outbound (#983) 2023-11-17 10:46:35 +01:00
diegomrsantos
08d9c84aca Remove unittest2 range (#986) 2023-11-17 08:20:02 +01:00
Jacek Sieka
4e7eaba67a fix chronos v4 compat (#982) 2023-11-16 16:54:34 +01:00
diegomrsantos
5f7a3ab829 fix: doc workflow (#985) 2023-11-16 15:58:05 +01:00
diegomrsantos
ebef85c9d7 Rate limit fixes (#965) 2023-11-09 14:20:28 +01:00
diegomrsantos
3fc1236659 Revert "Prevent concurrent IWANT of the same message (#943)" (#977) 2023-11-03 15:24:27 +01:00
Ludovic Chenut
fc4e9a8bb8 Fix WS transport when the connection aborts (#967) 2023-10-23 17:12:20 +02:00
Tanguy
60f953629d Remove ConnManager from Upgrade (#959) 2023-10-13 12:08:17 +00:00
diegomrsantos
18b0f726df Rate Limit tests (#953) 2023-10-05 15:12:07 +00:00
diegomrsantos
459f6851e7 Add a flag if a peer should be disconnected when above rate limit (#954) 2023-10-05 14:51:27 +02:00
Tanguy
575344e2e9 Update interop CI name (#956) 2023-10-05 10:54:24 +02:00
diegomrsantos
75871817ee Split msgs in iwant response if bigger than limit (#944) 2023-10-02 11:39:28 +02:00
diegomrsantos
61929aed6c Improve rdv advertise (#951)
Co-authored-by: Ludovic Chenut <ludovic@status.im>
2023-09-27 15:52:22 +02:00
diegomrsantos
56599f5b9d GossipSub Traffic scoring (#920) 2023-09-22 16:45:08 +02:00
Tanguy
b2eac7ecbd GS: Relay messages to direct peers (#949) 2023-09-15 17:22:02 +02:00
Tanguy
20b0e40f7d Fix doc generation CI (#948) 2023-09-08 12:21:04 +02:00
Tanguy
ff77d52851 IDontWant metrics (#946) 2023-09-06 16:05:59 +00:00
Tanguy
545a31d4f0 Bump dependencies (#947) 2023-09-06 17:52:43 +02:00
Jacek Sieka
b76bac752f avoid importing ecnist when not needed (#942) 2023-08-30 11:39:48 +02:00
diegomrsantos
c6aa085e98 Prevent concurrent IWANT of the same message (#943) 2023-08-21 16:34:24 +02:00
Ludovic Chenut
e03547ea3e Perf protocol (#925) 2023-08-14 17:25:55 +02:00
diegomrsantos
f80ce3133c Bandwidth estimate as a parameter (#941) 2023-08-14 17:03:46 +02:00
Tanguy
d6263bf751 nim-websock new version compatibility (#939) 2023-08-02 17:10:31 +02:00
Tanguy
56c23a286a Add specs crypto tests (#938) 2023-08-01 15:28:38 +02:00
Tanguy
7a369dd1bf GossipSub: Limit flood publishing (#911)
Co-authored-by: Diego <diego@status.im>
2023-07-31 11:13:51 +02:00
Tanguy
b784167805 GossipSub: IDontWant (#934) 2023-07-28 10:58:05 +02:00
Tanguy
440461b24b GS: improve handleIHave (#922) 2023-07-11 12:17:50 +02:00
Jacek Sieka
fab1340020 avoid a few zeroMem (#932) 2023-07-11 12:17:28 +02:00
Tanguy
1721f078c7 Fix crash on empty write (#930) 2023-07-10 13:52:08 +00:00
Ivan Folgueira Bande
74c402ed9d wstransport.nim: avoid re-raising 'TransportOsError' to avoid stopping switch.accept (#929) 2023-07-07 11:32:20 +02:00
diegomrsantos
c45f9705ab Gossipsub scoring improvements (#909) 2023-07-04 00:27:45 +02:00
Etan Kissling
81b861b34e avoid ProveField warning in crypto.init (#915) 2023-06-29 15:28:25 +02:00
Jacek Sieka
43359dd9d1 standard nimble env vars (#921) 2023-06-28 17:51:51 +02:00
diegomrsantos
f85d0f75ea Handling Opt[PeerId] in logging (#923) 2023-06-28 17:00:33 +02:00
Tanguy
66f9dc9167 Remove all Result.get()s & Option -> Opt (#902)
Co-authored-by: Ludovic Chenut <ludovic@status.im>
Co-authored-by: Diego <diego@status.im>
2023-06-28 16:44:58 +02:00
Tanguy
1c4d0832ce Add GossipSub ping (#912) 2023-06-21 10:40:10 +02:00
Tanguy
224f92e172 Fix #916 regression causing accept loop lockup (#919) 2023-06-20 14:18:49 +00:00
Tanguy
5efa089196 TCP transport: handle getObservedAddr errors (#918) 2023-06-20 10:25:29 +02:00
Tanguy
9d4c4307de Bumper: fix case where target is up to date (#917) 2023-06-15 18:37:01 +02:00
Tanguy
49dfa84c6f Transports: handle TransportAbortedError properly (#916) 2023-06-14 15:55:56 +00:00
Tanguy
a65b7b028f GossipSub: remove peer if we can't establish sendConn (#886) 2023-06-14 17:23:39 +02:00
diegomrsantos
67711478ce Consider dns as public address (#913) 2023-06-13 17:58:41 +02:00
Tanguy
c28d8bb353 WS Transport: handle 'tls/ws' (#914) 2023-06-12 15:45:53 +00:00
Tanguy
eb78292d9c Bump deps (#896) 2023-06-07 17:42:42 +02:00
Vaclav Pavlin
3725f6a95b chore: add basic metrics for rendezvous (#905) 2023-06-07 15:45:06 +02:00
diegomrsantos
3640b4dd89 Autonat and HP changes (#899) 2023-06-07 15:26:58 +02:00
Tanguy
32085ca88a Allow to override TCP connection timeouts (#903) 2023-06-07 14:27:32 +02:00
Tanguy
c76d1e18ef Remove nim 1.2 support (#907) 2023-06-07 11:12:49 +00:00
Tanguy
41649f0999 Version 1.1.0 (#904) 2023-06-06 11:05:49 +00:00
diegomrsantos
67102873ba Fail only if all addresses in PeerRecord are invalid (#898)
Fixes https://github.com/waku-org/nwaku/issues/1768
2023-05-31 08:59:50 +02:00
diegomrsantos
d40d324160 fix missing import (#897) 2023-05-26 10:36:39 +02:00
diegomrsantos
a677b06273 Try a direct connection only if there isn't one already (#891) 2023-05-25 15:48:22 +02:00
diegomrsantos
6050cdef7e Refinement of Hole Punching Service (#892) 2023-05-25 15:47:00 +02:00
Tanguy
fedfa8e817 Fix bumper CI (#894) 2023-05-22 17:37:42 +02:00
diegomrsantos
6887b43777 Improve utility tests (#893) 2023-05-22 11:07:22 +02:00
Tanguy
225accd11b Less warnings (#813)
Co-authored-by: Diego <diego@status.im>
2023-05-18 10:24:17 +02:00
diegomrsantos
7d6bc545e0 Handle dns addrs in HP service (#890) 2023-05-16 14:59:02 +02:00
Tanguy
a1eb53b181 Fix gossipsub dOut handling (#883) 2023-04-26 13:44:45 +02:00
Tanguy
db629dca25 Fix network protocol metrics typo (#874) 2023-04-26 09:52:06 +02:00
diegomrsantos
a5666789b0 Hole Punching (#806)
Co-authored-by: Tanguy <tanguy@status.im>
2023-04-18 12:50:21 +02:00
diegomrsantos
b7726bf68f Dcutr (#824)
Co-authored-by: Tanguy <tanguy@status.im>
2023-04-14 16:23:19 +02:00
diegomrsantos
0221affe98 Invalid MA is ignored (#881) 2023-04-14 12:05:32 +00:00
diegomrsantos
edbd35b16c Fix interop tests (#882) 2023-04-13 19:38:34 +02:00
diegomrsantos
80cca0ecac Does not allow an empty MA (#877) 2023-04-06 14:19:01 +00:00
diegomrsantos
0041ed4cf8 Transport hole punching (#873)
Co-authored-by: Tanguy <tanguy@status.im>
2023-04-06 15:23:35 +02:00
Tanguy
95e98e8c51 Fix traffic metrics (#879) 2023-04-03 12:37:23 +00:00
Tanguy
4aa615c44c GossipSub: TimedEntry & shortAgent fixes (#858) 2023-04-03 11:05:01 +02:00
Tanguy
6b61ce8c91 GossipSub: Better IWANT handling (#875) 2023-04-03 10:56:20 +02:00
Alvaro Revuelta
53b060f8f0 Add getters for conns and streams (#878) 2023-03-31 00:16:39 +02:00
diegomrsantos
af5299f26c Create an ObservedAddrManager and add an AddressMapper in AutonatService and AutoRelayService (#871)
Co-authored-by: Tanguy <tanguy@status.im>
2023-03-24 15:42:49 +00:00
Tanguy
bac754e2ad Various gossipsub fixes (#827) 2023-03-21 17:13:25 +01:00
Tanguy
8d5ea43e2b Upgrade flow refactoring (#807) 2023-03-08 12:30:19 +01:00
Jacek Sieka
e573238705 reexport public types (#872) 2023-03-06 15:36:10 +00:00
Tanguy
c1a3bd8fee Fix pubsub CI logs (#861) 2023-03-01 16:59:44 +01:00
diegomrsantos
ddeb7b3bd4 Handle when peers ask each other at the same time (#865) 2023-02-21 17:49:41 +01:00
Tanguy
382b992e00 Interop tests (#864) 2023-02-20 14:26:53 +01:00
Tanguy
408dcf12bd Fix backward compatibility of #822 (#862) 2023-02-15 17:18:29 +01:00
Ludovic Chenut
0012b639c8 Fix testrelay (#860) 2023-02-15 11:18:42 +01:00
Tanguy
f7f1e89669 TCP Transport: enable NO_DELAY for clients (#822) 2023-02-14 10:35:44 +01:00
Tanguy
f14ada3dcf Move tests flags from 'nimble test' to 'config.nims' (#852) 2023-02-10 11:32:21 +01:00
diegomrsantos
444b837923 Autonat doesn't ask an incoming peer (#857) 2023-02-09 17:40:04 +01:00
diegomrsantos
f89bd0c77c Autonat dials dns addrs (#856) 2023-02-09 16:53:46 +01:00
diegomrsantos
e68186373b Dialing addrs concurrently in autonat (#855) 2023-02-07 18:51:17 +01:00
diegomrsantos
266c7b117a Add anyCompleted proc (#853) 2023-02-07 18:50:42 +01:00
Ludovic Chenut
0e28d3b828 Add the peerId to the transport.dial (#842) 2023-01-31 12:46:10 +01:00
diegomrsantos
4ace70d53b Connect is able to force a new connection (#849) 2023-01-25 11:19:03 +01:00
Tanguy
ca19f8fdbf Autonat service: handle connections limits (#846)
Co-authored-by: diegomrsantos <diego@status.im>
2023-01-24 17:04:42 +01:00
Tanguy
351bda2b56 Add expected connections to connmngr (#845)
Co-authored-by: diegomrsantos <diego@status.im>
2023-01-23 22:28:39 +00:00
Tanguy
7d9c43a5ce Fix CI for nim devel 1.6 (#848) 2023-01-23 15:53:15 +01:00
Tanguy
c11772c94e Happy new year! (#847) 2023-01-20 15:47:40 +01:00
Ludovic Chenut
489c115132 Autorelay service (#819) 2023-01-17 16:18:38 +01:00
Tanguy
166c0d1c87 Fix PubSub subscribe on connection race condition (#809) 2023-01-10 13:33:14 +01:00
Etan Kissling
ba451196e8 Avoid closing connection on channel EOF (#816) 2023-01-06 15:18:16 +01:00
diegomrsantos
9f658c151e Autonat refactoring (#834) 2023-01-06 11:14:38 +01:00
diegomrsantos
e304ad0f7e Remove unnecessary async (#836) 2023-01-05 15:02:52 +01:00
diegomrsantos
5e3323d43f More autonat tests (#833) 2022-12-23 19:10:15 +00:00
diegomrsantos
9532bff983 Ignore unknown answers (#831)
Co-authored-by: Tanguy <tanguy@status.im>
2022-12-23 16:49:25 +01:00
diegomrsantos
676786b00e Add Autonat timeout (#829)
Co-authored-by: Ludovic Chenut <ludovic@status.im>
2022-12-22 19:29:31 +00:00
diegomrsantos
d521c57b82 Handle dial error correctly (#830) 2022-12-22 17:33:59 +01:00
Tanguy
63e1872516 Fix yamux simultaneous closing (#823) 2022-12-16 18:14:40 +01:00
diegomrsantos
67ef25fae0 Autonat service (#814)
Co-authored-by: Tanguy <tanguy@status.im>
2022-12-16 11:32:00 +00:00
Tanguy
fe7a69e389 Fix pinning system on devel (#820) 2022-12-14 12:10:00 +01:00
Ludovic Chenut
a17cad710c Fix devel 32bits (#818) 2022-12-09 17:23:33 +01:00
Tanguy
3863a4cd21 Allow connect with known PeerId in MA (#817) 2022-12-08 16:11:55 +00:00
Tanguy
64cbbe1e0a Limit number of streams per protocol per peer (#811) 2022-12-01 12:20:40 +01:00
Tanguy
31ad4ae205 Fix website (#812) 2022-12-01 12:20:17 +01:00
diegomrsantos
b3d9360dfc Starting switch two times does not crash (#810) 2022-11-29 15:21:51 +00:00
Tanguy
1711c204ea Fix: big messages in pubsub (#808) 2022-11-24 16:43:45 +00:00
Tanguy
c43aacdc81 WS: add timeout to handleRequest (#796) 2022-11-22 10:44:08 +01:00
Tanguy
711609057c Reuseable install_nim action (#803) 2022-11-21 16:56:04 +01:00
Alvaro Revuelta
192cac6254 Store missing pubkey in PeerStore [KeyBook] (#804) 2022-11-21 10:55:25 +01:00
Tanguy
cc3c637c22 Daily CI: run devel with refc (#799) 2022-11-16 16:56:02 +01:00
Tanguy
afbb1b4d3c Fix Codecov (#802)
Also switch from 14 jobs to 1
2022-11-16 14:33:42 +01:00
Dmitriy Ryajov
8c2eca18dc Allow passing max message size (#800)
Co-authored-by: Tanguy <tanguy@status.im>
2022-11-15 14:01:14 +01:00
Tanguy
ce371f3bb4 Update README (#798) 2022-11-10 13:47:41 +01:00
diegomrsantos
23338fceaa Explicit dependency to unittest2 and update other packages (#797) 2022-11-07 22:55:22 +01:00
diegomrsantos
6ab6ab48ef Fix after gcsafe requirements change and other issues on devel (#795) 2022-11-05 02:04:05 +01:00
diegomrsantos
d9305bda84 Add Tor Transport support (#765) 2022-11-04 19:20:23 +01:00
Lorenzo Delgado
f95eda8bf6 Add libp2p label to chronicles log topics (#794) 2022-11-04 11:19:27 +01:00
Tanguy
a3e9d1ed80 Version 1.0.0 (#785) 2022-10-30 13:18:04 +01:00
Tanguy
c9c2f6acdb Tron example (#775)
Co-authored-by: lchenut <ludovic@status.im>
2022-10-30 08:49:02 +01:00
Tanguy
c39c1cbf68 Fix daily CI (#790) 2022-10-29 23:26:44 +02:00
lchenut
b4f96721af Add a tutorial for the discovery manager (#787)
Co-authored-by: Tanguy <tanguy@status.im>
2022-10-29 10:12:12 +00:00
Tanguy
4bce8f38c9 Remove sleepAsync from tests/testswitch (#792) 2022-10-27 23:10:24 +00:00
Tanguy
dc13ff81d3 Revert "Temporarily remove failing test (#788)" & fix tests (#789)
Co-authored-by: Ludovic Chenut <ludovic@status.im>
2022-10-26 11:07:31 +02:00
Tanguy
a69301f392 Merge remote-tracking branch 'origin/unstable' 2022-10-21 19:44:32 +02:00
Tanguy
4b105c6abd GossipSub tutorial (#784) 2022-10-21 17:00:36 +02:00
Tanguy
2e12c7ab73 Temporarily remove failing test (#788) 2022-10-21 16:59:53 +02:00
Tanguy
a086fcba72 Remove shallow copies (#782) 2022-10-20 14:52:02 +02:00
Tanguy
7b103e02f2 Allow public address mapping (#767) 2022-10-20 12:22:28 +02:00
lchenut
32233d36c8 Discovery interface (#783)
Co-authored-by: Tanguy <tanguy@status.im>
2022-10-03 22:00:00 +00:00
lchenut
1c99aca054 RendezVous Protocol (#751) 2022-09-30 10:41:04 +02:00
diegomrsantos
4f18dd30e9 Handle trying to write empty byte seq (#780) 2022-09-29 20:02:10 +02:00
Tanguy
0cd3554ce4 Bump deps (#779) 2022-09-29 10:29:51 +02:00
Tanguy
bcb8f5e3b6 Protobuf tutorial (#778) 2022-09-29 10:28:58 +02:00
Tanguy
eb78660702 Docs rework (#776) 2022-09-28 10:40:53 +02:00
Etan Kissling
103e199bc0 add lodestar to known lib p2p agents documentation (#766) 2022-09-26 11:48:03 +02:00
Tanguy
fa5d102370 Better dnsaddr resolving (#753) 2022-09-26 11:03:24 +02:00
diegomrsantos
a56c3bc296 Make observedAddr optional (#772)
Co-authored-by: Tanguy <tanguy@status.im>
2022-09-22 21:55:59 +02:00
Tanguy
5e7e009445 Move relay & autonat to connectivity folder (#769) 2022-09-15 09:43:40 +02:00
Tanguy
72abe822c0 Fix switch failed start (#770) 2022-09-15 07:06:32 +00:00
Tanguy
a001508490 Add codex & waku to autobump (#768) 2022-09-14 12:05:43 +00:00
lchenut
4d8b50d24c Specify EOF error (#759) 2022-09-14 10:58:41 +02:00
Tanguy
ef594e1e02 Only log multiple missed heartbeats as info (#763) 2022-09-12 17:09:10 +02:00
diegomrsantos
d8a9e93ff7 Add onion3 multiaddr support (#764) 2022-09-08 17:10:11 +02:00
Tanguy
abbeaab684 Keep connection alive when peer doesn't support pubsub (#754) 2022-09-06 13:20:42 +02:00
Jacek Sieka
dfbfbe6eb6 allow connection to a peer with unknown PeerId (#756)
Co-authored-by: Tanguy <tanguy@status.im>
2022-09-05 14:31:14 +02:00
lchenut
1de7508b64 Fix typos (#761) 2022-09-02 18:40:14 +00:00
Tanguy
3ffc03ed16 Gossipsub: Rebalance mesh immediately when peer sub/unsub (#719) 2022-09-02 10:24:54 +02:00
Tanguy
543358b262 Remove unused code 2022-09-02 10:09:12 +02:00
Miran
14d2c3f51e Configuration refactoring (#732)
move -d:nimRawSetjmp to nim.cfg
switch .cfg to .nims
add --skipParentCfg

Co-authored-by: Tanguy <tanguy@status.im>
2022-09-01 14:00:22 +02:00
lchenut
2332813873 Autonat protocol (#739) 2022-08-23 17:49:07 +02:00
Tanguy
124a7a5ffe Add missing stub logger (#752) 2022-08-03 19:04:17 +02:00
Tanguy
2d864633ea ConnManager connection tracking refacto (#749) 2022-08-03 16:48:19 +02:00
Miran
2fbe82bf9d make it more Nim 1.4+ compatible (#742) 2022-08-03 13:33:19 +02:00
Miran
20c02a5f23 ci: test Nim 1.6 branch (#747)
* ci: test Nim 1.6 branch
* drop `devel` from testing

Co-authored-by: Tanguy <tanguy@status.im>
2022-08-02 19:07:08 +02:00
lchenut
a9a7e7eb15 Yamux metrics and limits (#740)
* Add yamux channel gauge
* Add limit to channel
* Add recv/send queue length metrics
* Add yamux stream tracking
* Add timeout to YamuxChannel

Co-authored-by: Tanguy <tanguy@status.im>
2022-08-01 12:52:42 +00:00
lchenut
34c2fb8787 Circuit relay v2 (#717) 2022-08-01 14:31:22 +02:00
tersec
1e598a0239 topicIDs -> topicIds; PubsubPeerEvent -> PubSubPeerEvent; messageIDs -> messageIds (#748) 2022-07-27 17:14:05 +00:00
tersec
4ca1c2d7ed --styleCheck:error (#743)
* --styleCheck:error

* PeerID -> PeerId

* other libp2p styleCheck changes

* MessageID -> messageId; topicID -> topicId
2022-07-27 11:47:50 +00:00
Tanguy
83ad890535 Documentation site (#745) 2022-07-26 12:44:26 +02:00
Tanguy
0b0686ee94 Move auto bump to different CI job (#731) 2022-07-26 12:35:02 +02:00
Tanguy
93ac795aef Bump dependencies (#744) 2022-07-22 21:39:06 +02:00
lchenut
912873f8b3 Replace table by seq for storing muxers (#741)
Replace table by seq for stocking muxers
2022-07-22 12:54:09 +02:00
Ivan Yonchovski
78a65eebcc Add setup files (#738) 2022-07-12 21:03:23 +03:00
Tanguy
533e39ef94 Yamux (#704)
Co-authored-by: Ludovic Chenut <ludovic@status.im>
2022-07-04 15:19:21 +02:00
Tanguy
150fafbee8 Faster CI (#724)
* Only compile once in CI
* Fix codecov fetch
2022-07-01 20:20:42 +02:00
Tanguy
d0523fdc9d Documentation (#716) 2022-07-01 20:19:57 +02:00
lchenut
0ece5eaf12 Start/stop protocols (#730)
Starting/stopping a switch now starts/stops all protocols mounted on that switch
2022-06-30 11:21:33 +02:00
Jacek Sieka
e6440c43c2 lower log level of no-peers message (#735)
the number of peers sent to is returned from the function - the caller
can log this, if need be
2022-06-30 09:57:07 +02:00
Jacek Sieka
597abddba7 don't increase seqno when anonymizing (#734) 2022-06-30 09:56:49 +02:00
Tanguy
5d7024f2e0 Handle more cancellations (#733)
* Mplex: handle write cancellation
* LpChannel: Don't close connection on EOF
2022-06-24 11:11:23 +02:00
Tanguy
a7e335e1bb Remove Br prefix from BearSSL (#728)
ref status-im/nim-bearssl#27
2022-06-16 10:08:52 +02:00
Tanguy
718374d890 Merge pull request #718 from status-im/unstable
Unstable
2022-06-07 10:06:31 +02:00
lchenut
36f3132d9a Fix compilation error on nim 1.4 (#723) 2022-05-27 12:51:56 +02:00
Tanguy
ca3f4e8701 builder: Expose MaxChanCount (#722) 2022-05-25 15:59:42 +00:00
Tanguy
7323ecc9c4 Optimize rebalanceMesh (#708) 2022-05-25 12:59:33 +02:00
Tanguy
60becadcf9 Peer store refacto (#700)
There is now a global PeerStore structure (instead of having one for libp2p, one for waku, etc)

The user can create custom books for new types easily

Also add a pruning system to remove dead peers
2022-05-25 12:12:57 +02:00
Tanguy
1696d0c707 Set activity flag in WS (#721) 2022-05-24 19:41:05 +00:00
Tanguy
d4ff1c88e9 Less warnings (#710) 2022-05-24 15:10:57 +02:00
Tanguy
e536d7cb1b Bump deps (#720) 2022-05-19 17:29:09 +02:00
lchenut
13503f3799 Circuit relay v1 (#670)
Circuit relay v1
2022-05-18 10:19:37 +02:00
Tanguy
991549f391 Gossipsub scoring fixes (#709)
* Use decayInterval as a scoring heartbeat period
* Take mesh delivery window into account
2022-05-11 10:38:43 +02:00
lchenut
32ca1898d9 Gossipsub: Put Peer Exchange behind a flag (#715)
Add a flag to enable Peer Exchange in Gossipsub (disabled by default)
2022-05-10 10:39:43 +02:00
Tanguy
9ba5c069c8 Update SPR when the PeerInfo changes (#711) 2022-04-13 09:35:28 +02:00
Tanguy
c97befb387 Add tests for gossipsub direct peers (#707) 2022-04-12 14:03:31 +00:00
tersec
fc6b8f46f1 encrypt and decrypt empty sequences (#713)
* encrypt and decrypt empty sequences

* use assign in curve25519
2022-04-12 10:41:48 +00:00
Csaba Kiraly
9973b9466d expose more libp2p performance and queuing metrics (#678)
* gossipsub: adding duplicate arrival metrics

Adding counters for received deduplicated messages and for
duplicates recognized by the seen cache. Note that duplicates that
are not recognized (arrive after seenTTL) are not counted as
duplicates here either.

* gossipsub: adding mcache (message cache for responding IWANT) stats

It is generally assumed that IWANT messages arrive when mcache still
has the message. These stats are to verify this assumption.

* libp2p: adding internal TX queuing stats

Messages are queued in TX before getting written on the stream,
but we have no statistics about these queues. This patch adds
some queue length and queuing time related statistics.

* adding Grafana libp2p dashboard

Adding Grafana dashboard with newly exposed metrics.

* enable libp2p_mplex_metrics in nimble test

Signed-off-by: Csaba Kiraly <csaba.kiraly@gmail.com>
2022-04-06 16:00:24 +02:00
Tanguy
868ecab54f Pin go version in CI (#706) 2022-04-05 13:12:55 +02:00
Tanguy
84cbcd8f22 Bump dependencies (#703) 2022-03-29 09:23:24 +02:00
Tanguy
eaa72dcdbe WS Accept timeout (#699)
* Add timeout to WS accept
* Handle more WS errors
2022-03-17 10:16:48 +01:00
Tanguy
c7504d2446 Gossipsub peer exchange (#647)
* Signed envelopes and routing records
* Send signed peer record as part of identify (#649)
* Add SPR from identify to new peer book (#657)
* Send & receive gossipsub PX
* Add Signed Payload

Co-authored-by: Hanno Cornelius <68783915+jm-clius@users.noreply.github.com>
2022-03-14 09:39:30 +01:00
Tanguy
cba3ca3c3e Unstable (#695) 2022-02-25 09:41:32 +01:00
Eric Mastro
44a7260f07 fixes from #688 (#697)
* tests: invert message logic on expect from #688
* fix: export pubsub_errors for backward compatibility
2022-02-24 17:32:20 +01:00
Tanguy
c09d032133 Allow force dial (#696) 2022-02-24 17:31:47 +01:00
Tanguy
f98bf612bd Fix tests of #638 2022-02-21 18:14:43 +01:00
Tanguy
fd59cbc7a9 Fix shuffle of #638 2022-02-21 17:00:18 +01:00
Tanguy
bc318084f4 GS: Publish to fanout when mesh unhealthy (#638)
* Send to fanout when mesh unhealthy

* don't use fanout when floodPublish
2022-02-21 16:22:08 +01:00
Eric Mastro
3b718baa97 feat: allow msgIdProvider to fail (#688)
* feat: allow msgIdProvider to fail

Closes: #642.

Changes the return type of the msgIdProvider to `Result[MessageID, string]` so that message id generation can fail.

String error type was chosen as this `msgIdProvider` mainly because the failed message id generation drops the message and logs the error provided. Because `msgIdProvider` can be externally provided by library consumers, an enum didn’t make sense and a object seemed to be overkill. Exceptions could have been used as well, however, in this case, Result ergonomics were warranted and prevented wrapping quite a large block of code in try/except.

The `defaultMsgIdProvider` function previously allowed message id generation to fail silently for use in the tests: when seqno or source peerid were not valid, the message id generated was based on a hash of the message data and topic ids. The silent failing was moved to the `defaultMsgIdProvider` used only in the tests so that it could not fail silently in applications.

Unit tests were added for the `defaultMsgIdProvider`.

* Change MsgIdProvider error type to ValidationResult
2022-02-21 16:04:17 +01:00
Tanguy
9a7e3bda3c Bump dependencies (#694) 2022-02-10 14:21:12 +01:00
Tanguy
00e1f9342f Fix identify log for json_sink (#690) 2022-02-01 18:35:48 +01:00
Tanguy
07da14a7a7 Fix websocket EOF reading exception (#689) 2022-01-28 18:05:07 +00:00
Tanguy
c18830ad33 Score correctly on mesh peer unsub (#644)
* Score correctly on mesh peer unsub
* remove from mesh before removing from gossipsub
2022-01-15 12:47:41 +01:00
Tanguy
1a97d0a2f5 Validate pubsub subscriptions (#627)
* Check topic before subscribing
* Block subscribe to invalid topics
2022-01-14 12:40:30 -06:00
Ștefan Talpalaru
e72d03bc78 CI: test with multiple Nim versions by default (#684)
* CI: test with 1.2 & devel by default
* Skip buggy windows websocket test
2022-01-10 12:29:52 +01:00
Tanguy
388b92d58f Bump dependencies (#683) 2022-01-07 08:19:22 +01:00
Tanguy
f3dee6865c Chronos strict exception tracking (#652)
* Enable chronos strict exception tracking ( -d:chronosStrictException )
2022-01-05 16:27:33 +01:00
Dmitriy Ryajov
dffe4bed45 mitigate high traffic - drop faulty peers (#460)
Only allow 1 connection per peer

Co-authored-by: Tanguy <tanguy@status.im>
2022-01-04 11:21:24 +00:00
Tanguy
fb0d10b6fd Gossipsub: process messages concurrently (#680)
* Gossip sub: process messages concurrently

* Retries for flaky test
2021-12-27 11:17:00 +01:00
Tanguy
58f383e661 Bump unittest2
Now that we have pinned nimble deps, we get
surprises from the nim version updates
2021-12-17 09:13:17 +01:00
Tanguy
123bf290c3 Merge branch 'unstable' (#674) 2021-12-17 08:31:47 +01:00
Tanguy
1e7e95f5fd Pin dependencies for CI (#677)
Adds a pin-file for CI. When a dependency break something in the libp2p, we don't want for the CI to be
broken on every branch, just on a "bump" branch that will update the pin-file & fix the eventual interop issue.
2021-12-16 18:00:10 +00:00
Tanguy
df566e69db Fixes for style check (#676) 2021-12-16 11:05:20 +01:00
Tanguy
1152012bb0 Update README
Core developers, badges, module list were not up to date
2021-12-15 15:01:28 +01:00
Jacek Sieka
c49932b55a fast path for writes (#659)
avoids several copies of the various message buffers being kept alive
for the lifetime of the future
2021-12-14 10:55:17 +01:00
Tanguy
47a35e26d7 Typo: s/unsubcribeBackoff/unsubscribeBackoff (#675) 2021-12-14 10:50:57 +01:00
Etan Kissling
2373ee0061 harden ecnist byte export against uninitialized key (#671)
Currently, `ecnist`'s `toBytes` and `getBytes` methods operate only on
properly initialized keys. If an un-initialized key is given, an
`IndexError` may be raised if the key's `xlen` / `qlen` property is
larger than the maximum buffer size. This patch hardens those functions
to report a proper error in that case.
Note that the library functions called by `init` and `initRaw` already
reject data that does not have the expected length, so these new checks
should not be reachable in practice.
2021-12-13 18:46:25 +00:00
Etan Kissling
0be9180977 harden ProtoBuffer.finish() assert (#672)
This hardens the length check in `ProtoBuffer.finish` to account for
passed `options`.
2021-12-13 18:31:05 +00:00
Tanguy
8fe44e35c2 Fix flaky tests (#669)
* Fix push identify flaky test
* Fix mplex flaky tests
* Fix switch flaky test
2021-12-13 11:25:33 +01:00
Jacek Sieka
18dc233c9b link to style guide 2021-12-10 10:11:52 +01:00
Eric Mastro
fffa7e8cc2 fix: remove returned Futures from switch.start (#662)
* fix: remove returned Futures from switch.start

The proc `start` returned a seq of futures that was mean to be awaited by the caller. However, the start proc itself awaited each Future before returning it, so the ceremony requiring the caller to await the Future, and returning the Futures themselves was just used to handle errors. But we'll give a better way to handle errors in a future revision

Remove `switch.start` return type (implicit `Future[void]`)

Update tutorials and examples to reflect the change.

* Raise error during failed transport

Replaces logging of error, and adds comment that it should be replaced with a callback in a future PR.
2021-12-03 19:23:12 +01:00
Tanguy
6893bd9dbb Customizable gossipsub backoff on unsubscribe (#665)
* Customizable gossipsub backoff on unsubscribe
* change default to 5s
2021-12-02 14:47:40 +00:00
Eric Mastro
c19b966d23 fix (directchat): auto disconnect self once conn closed (#663)
When closing a connection of clientA(using the command “/disconnect”), the connection is then closed on clientA’s side. However, the connection remained open on clientB and clientB could continue sending message to clientA and clientA would receive those messages.

This PR listens for a closing connection on clientB, then also closes clientB’s connection.
2021-12-02 13:45:55 +01:00
Tanguy
b8c54068a3 Handle websock exceptions in handle (#658) 2021-11-26 13:55:35 +01:00
Tanguy
0dfac6fce7 Signed envelopes and routing records (#656) 2021-11-24 14:03:40 -06:00
Dmitriy Ryajov
73168b6eae Add support for multiple addresses to transports (#598)
* add test for multiple local addresses

* allow transports to listen on multiple addrs

* fix tcp transport accept

* check switch addrs are correct

* switch test to port 0

* close accepted peers on close

* ignore CancelledError in transport accept

* test ci

* only accept in accept loop

* avoid accept greedyness

* close acceptedPeers

* accept doesn't crash on cancelled fut

* add common transport test

* close conn on handling failure

* close accepted peers in two steps

* test for macos

* revert accept greedyness

* fix dialing cancel

* test chronos fix

* add ws

* ws cancellation

* small fix

* remove chronos blocked test

* fix testping

* Fix transport's switch start (like #609)

* bump chronos

* Websocket: handle both ws & wss

Co-authored-by: Tanguy Cizain <tanguycizain@gmail.com>
Co-authored-by: Tanguy <tanguy@status.im>
2021-11-24 14:01:12 -06:00
Tanguy
98184d9dfd Merge pull request #654 from status-im/ci_testslim
CI Multi nim: only test slim
2021-11-19 14:54:51 +01:00
Tanguy
5f18968c3f CI Multi nim: only test slim 2021-11-19 11:52:29 +01:00
Tanguy
32ede6da3c Merge pull request #651 from status-im/unstable
Unstable
2021-11-16 17:36:00 +01:00
Tanguy
6f779c47c8 Gossipsub: don't send to peers seen during validation (#648)
* Gossipsub: don't send to peers seen during validation

* Less error prone code

* add metric

* Fix metric

* remove dangling code test

* address comments

* don't allocate memory
2021-11-14 09:08:05 +01:00
Tanguy
5d1b10f3e7 Short agent fix (#645)
* Copy client type to muxed channel

* Fix incoming shortAgent
2021-11-08 18:33:37 +01:00
Tanguy
7d677f848f Short agent fix (#645)
* Copy client type to muxed channel

* Fix incoming shortAgent
2021-11-08 17:42:56 +01:00
Tanguy
c92125a1a4 Integrate dns resolving (#615)
* integrate dns

* give hostname to transport dial

* add hostname test

* switched to websock master

* Add dnsaddr dial test w multiple transports
2021-11-08 13:02:03 +01:00
Tanguy
e787fc35a6 Add examples to CI (#599)
* add examples to CI

* add markdown runner

* two tutorials
2021-11-08 13:00:44 +01:00
Zahary Karadjov
cf8a1a60a4 Work-around a Nim compilation error 2021-11-04 12:19:17 +02:00
Tanguy
e1d96a0f4d Remove isWire (#640) 2021-10-28 19:11:31 +02:00
Ștefan Talpalaru
b308affb98 CI: test multiple Nim compiler branches (#624)
* CI: test multiple Nim compiler branches

* make it a separate workflow and limit target branches

* make it a daily job
2021-10-26 14:47:30 -06:00
Tanguy
5885e03680 Add maxMessageSize option to pubsub (#634)
* Add maxMessageSize option to pubsub
* Switched default to 1mb
2021-10-25 12:58:38 +02:00
Tanguy
846baf3853 Various cleanups part 1 (#632)
* raise -> raise exc
* replace stdlib random with bearssl
* object init -> new
* Remove deprecated procs
* getMandatoryField
2021-10-25 10:26:32 +02:00
Tanguy
3669b90ceb Fix WS observed address (#631)
* Fix WS observed address

* Unify tcptransport & wstransport
2021-10-14 13:16:34 +02:00
Tanguy
75bfc1b5f7 Fix muxer bandwidth typo (#628) 2021-09-27 14:30:22 +02:00
Tanguy
1b2cdd6aec Merge branch 'master' into unstable 2021-09-09 13:22:45 +02:00
Tanguy
68af8122e9 Fix macos go-daemon CI (#622)
bump libp2p-go-daemon
2021-09-08 16:58:44 +02:00
Menduist
d02735dc46 Remove peer info (#610)
Peer Info is now for local peer data only.
For other peers info, use the peer store.

Previous reference to peer info are replaced with the peerid
2021-09-08 11:07:46 +02:00
Tanguy Cizain
8cddfde837 Rename getKey -> getPublicKey (#621)
* rename getKey to getPublicKey

* use publicKey directly in gossipsub

* update error messages
2021-09-02 12:03:40 +02:00
Tanguy Cizain
8ea90037e5 Fix gossipsub incoming graft backoff (#616)
* Fix gossipsub incoming graft backoff

* Improve debug messages

* clamp to 24h
2021-09-01 08:41:11 +02:00
Tanguy Cizain
62ca6dd9a0 Add missing test to testnative (#618)
* add missing tests

* fix testping
2021-09-01 08:38:24 +02:00
Tanguy Cizain
f274bfe19d DNS Addresses handling (#580)
* add 'dns' multiaddr protocol

* multiaddr: isWire is true for DNS protocols

* resolve dns on connect

* fix typo

* add dns test

* update resolveDns error handling

* handle multiple dns entries

* start of new resolver

* working dns resolver

* use the DnsResolver

* fix json logs

* small overhaul

* fix dns implem in lp2p

* update dnsclient repo

* add dns test to testnative

* dummy dns server for ut

* better mocked

* moved resolving to transport

* moved mockresolver to libp2p

* test resolve in switch test

* try multiple txt & track leaks

* raise e

* catchable error instead of exception

* save failed dns server

* moved resolve back to dialer

* remove nameresolver from dialer
2021-08-18 09:40:12 +02:00
Tanguy Cizain
e58658d822 Fix unstable autobump (#613) 2021-08-03 11:43:00 -06:00
Tanguy Cizain
af3be7966b Websocket Transport (#593)
* start of websocket transport

* more ws tests

* switch to common test

* add close to wsstream

* update ws & chronicles version

* cleanup

* removed multicodec

* clean ws outgoing connections

* renamed to websock

* removed stream from logs

* renamed ws to websock

* add connection closing test to common transport

* close incoming connection on ws stop

* renamed testwebsocket.nim -> testwstransport.nim

* removed raise todo

* split out/in connections

* add wss to tests

* Fix tls (#608)

* change log level

* fixed issue related to stopping

some cosmetic cleanup

* use `allFutures` to stop/close things

Prevent potential race conditions when stopping two or more transports

* misc

* point websock to server-case-object branch

* interop test with go

* removed websock version specification

* add daemon -> native ws test

* fix & test closed read/write

* update readOnce, thanks jangko

Co-authored-by: Dmitriy Ryajov <dryajov@gmail.com>
2021-08-03 15:48:03 +02:00
markspanbroek
8b438142ce Fixes failing peer store test in Nim 1.4.8 (#612)
* Fixes failing test in Nim 1.4.8

This tests failed because the order in which the elements
of a hashset are added to a seq is non-deterministic.

Co-authored-by: Tanguy Cizain <tanguycizain@gmail.com>
2021-07-28 12:06:18 +02:00
Tanguy Cizain
7fc0dfbd55 Muxer bandwidth metrics (#607)
* add stream bytes metrics

* renamed bw metric

* renamed

* new global metric
2021-07-26 16:12:36 +02:00
Tanguy Cizain
cb94baf9c4 Fix transport's switch start (#609)
* fix switch start

* don't fail eagerly on transport start

* handlecancel

* raise exc
2021-07-19 12:51:27 +02:00
Tanguy Cizain
c1b2d45d1b Track ChronosStream in test (#605)
* track chronos stream in tests

* use chronosstreamtrackername
2021-07-13 13:53:08 +02:00
Tanguy Cizain
008dcf7043 Faster CI (#603)
* ci on PR only

* add option to run ci manually

* thanks Stefan
2021-07-13 10:26:49 +02:00
Jacek Sieka
1d4f7c7a43 avoid borrow, breaks logging (#604) 2021-07-11 09:59:42 +02:00
Miran
c12f00c8b6 make isLiteral compatible with ARC and ORC (#602)
(cherry picked from commit 2d4dad0716)
2021-07-01 12:05:50 +02:00
Tanguy Cizain
26e47d7da5 Various transports improvement (#594)
* little transport cleanup

* rename TcpTransport.init -> TcpTransport.new

* moved transport e2e to common file

* remove localAddress

* rename testtransport -> testtcptransport

* add checktrackers to commontransports

* removed multicodec from transports
2021-06-30 10:59:30 +02:00
Dmitriy Ryajov
49f137049f Merge remote-tracking branch 'origin' into unstable 2021-06-29 15:43:11 -06:00
Dmitriy Ryajov
1681197d67 Merge branch 'master' into unstable 2021-06-29 15:42:22 -06:00
Dmitriy Ryajov
398b3a4ce6 Merge branch 'master' into unstable 2021-06-29 11:41:34 -06:00
Ben Tilford
72ccd1d1f5 Correcting paths and file/directory names (#596) 2021-06-27 22:36:57 -06:00
Giovanni Petrantoni
4be0cdcdb4 Docs cleanup (#557)
* remove pointless go_daemon page (made no sense)

* remove confusing go requirement from main readme

* wip directchat fixes

* use matchPartial for connect(wire)

* fix directchat

* revert wire changes

* rewrite directchat partially

* more readme updates

* fix things to follow the update on blog posts
2021-06-27 20:37:00 -06:00
Tanguy Cizain
7e3974f9c8 Fix flaky macos switch test (#592)
* sleep to avoid flaky switch test on macos

* Update tests/testswitch.nim

Co-authored-by: markspanbroek <mark@spanbroek.net>

Co-authored-by: markspanbroek <mark@spanbroek.net>
2021-06-23 09:27:52 -06:00
Tanguy Cizain
bee91538ef builder: add multiple local address support (#591) 2021-06-21 17:14:24 +02:00
Dmitriy Ryajov
b63e064b4a Remove asynccheck (#590)
* Merge master (#555)

* Revisit Floodsub (#543)

Fixes #525

add coverage to unsubscribeAll and testing

* add mounted protos to identify message (#546)

* add stable/unstable auto bumps

* fix auto-bump CI

* merge nbc auto bump with CI in order to bump only on CI success

* put conditional locks on nbc bump (#549)

* Fix minor exception issues (#550)

Makes code compatible with
https://github.com/status-im/nim-chronos/pull/166 without requiring it.

* fix nimbus ref for auto-bump stable's PR

* Split dialer (#542)

* extracting dialing logic to dialer

* exposing upgrade methods on transport

* cleanup

* fixing tests to use new interfaces

* add comments

* add base exception class and fix hierarchy

* fix imports

* `doAssert` is `ValueError` not `AssertionError`?

* revert back to `AssertionError`

Co-authored-by: Giovanni Petrantoni <7008900+sinkingsugar@users.noreply.github.com>
Co-authored-by: Jacek Sieka <jacek@status.im>

* Merge master (#555)

* Revisit Floodsub (#543)

Fixes #525

add coverage to unsubscribeAll and testing

* add mounted protos to identify message (#546)

* add stable/unstable auto bumps

* fix auto-bump CI

* merge nbc auto bump with CI in order to bump only on CI success

* put conditional locks on nbc bump (#549)

* Fix minor exception issues (#550)

Makes code compatible with
https://github.com/status-im/nim-chronos/pull/166 without requiring it.

* fix nimbus ref for auto-bump stable's PR

* Split dialer (#542)

* extracting dialing logic to dialer

* exposing upgrade methods on transport

* cleanup

* fixing tests to use new interfaces

* add comments

* add base exception class and fix hierarchy

* fix imports

* `doAssert` is `ValueError` not `AssertionError`?

* revert back to `AssertionError`

Co-authored-by: Giovanni Petrantoni <7008900+sinkingsugar@users.noreply.github.com>
Co-authored-by: Jacek Sieka <jacek@status.im>

* cleanup

Co-authored-by: Giovanni Petrantoni <7008900+sinkingsugar@users.noreply.github.com>
Co-authored-by: Jacek Sieka <jacek@status.im>
2021-06-14 17:21:44 -06:00
Tanguy Cizain
bed00ec43c Identify Push (#587)
* start of identifypush

* better pushidentify

* push identify test

* fix: make peerid optional
2021-06-14 11:08:47 -06:00
Tanguy Cizain
bd2e9a0462 Fix trackers tests (#588)
* Fix checkTracker typo

* fix testswich tests

* use tracker consts
2021-06-14 10:26:11 +02:00
Dmitriy Ryajov
c7d9770637 Merge master (#555)
* Revisit Floodsub (#543)

Fixes #525

add coverage to unsubscribeAll and testing

* add mounted protos to identify message (#546)

* add stable/unstable auto bumps

* fix auto-bump CI

* merge nbc auto bump with CI in order to bump only on CI success

* put conditional locks on nbc bump (#549)

* Fix minor exception issues (#550)

Makes code compatible with
https://github.com/status-im/nim-chronos/pull/166 without requiring it.

* fix nimbus ref for auto-bump stable's PR

* Split dialer (#542)

* extracting dialing logic to dialer

* exposing upgrade methods on transport

* cleanup

* fixing tests to use new interfaces

* add comments

* add base exception class and fix hierarchy

* fix imports

* `doAssert` is `ValueError` not `AssertionError`?

* revert back to `AssertionError`

Co-authored-by: Giovanni Petrantoni <7008900+sinkingsugar@users.noreply.github.com>
Co-authored-by: Jacek Sieka <jacek@status.im>
2021-06-11 16:34:41 -06:00
Dmitriy Ryajov
9e2b464e14 Cleanup testinterop imports (#589)
* cleanup exports/imports, specially for testinterop

* export pubsub
2021-06-11 16:34:40 -06:00
Dmitriy Ryajov
40e21e3375 Cleanup testinterop imports (#589)
* cleanup exports/imports, specially for testinterop

* export pubsub
2021-06-11 16:33:47 -06:00
Tanguy Cizain
93156447ba Peer Store implement part II (#586)
* Connect & Peer event handlers now receive a peerinfo

* small peerstore refacto

* implement peerstore in switch

* changed PeerStore to final ref object

* revert libp2p/builders.nim
2021-06-08 18:55:24 +02:00
Tanguy Cizain
55a3606ecb add ping protocol (#584)
* add ping protocol

* add ping protocol handler

* ping styling

* more ping tests

* switch ping to bearssl rng

* update ping style

* new cancellation test
2021-06-08 18:53:45 +02:00
Tanguy Cizain
caac8191d2 Change newXXXX procs to XXXX.new (#585)
* newBufferStream -> BufferStream.new

* newMultistream -> MultistreamSelect.new

* newSecio -> Secio.new

* newNoise -> Noise.new

* newPlainText -> PlainText.new

* newPubSubPeer -> PubSubPeer.new

* newIdentify -> Identify.new

* newMuxerProvider -> MuxerProvider.new
2021-06-07 09:32:08 +02:00
Ștefan Talpalaru
fbe888a3aa CI: refactor Nim compiler building (#579) 2021-06-02 12:26:03 -06:00
Dmitriy Ryajov
ac47964377 use pkg namespace 2021-06-02 12:26:02 -06:00
Dmitriy Ryajov
b56ca11b74 use raises defect 2021-06-02 12:26:02 -06:00
Dmitriy Ryajov
ce42674d80 avoid memory safety errors with nim 1.4.x 2021-06-02 12:26:01 -06:00
Dmitriy Ryajov
a6eea0c275 import chronicles otherwise compile breaks 2021-06-02 12:25:37 -06:00
Dmitriy Ryajov
1c3616e3a5 merge latest master 2021-06-02 12:25:36 -06:00
Dmitriy Ryajov
c949f14a99 Merge master to unstable (#570)
* Revisit Floodsub (#543)

Fixes #525

add coverage to unsubscribeAll and testing

* add mounted protos to identify message (#546)

* add stable/unstable auto bumps

* fix auto-bump CI

* merge nbc auto bump with CI in order to bump only on CI success

* put conditional locks on nbc bump (#549)

* Fix minor exception issues (#550)

Makes code compatible with
https://github.com/status-im/nim-chronos/pull/166 without requiring it.

* fix nimbus ref for auto-bump stable's PR

* use a builder pattern to build the switch (#551)

* use a builder pattern to build the switch

* with with

* more refs

* builders (#559)

* More builders (#560)

* address some issues pointed out in review

* re-add to prevent breaking other projects

* mem usage cleanups for pubsub (#564)

In `async` functions, a closure environment is created for variables
that cross an await boundary - this closure environment is kept in
memory for the lifetime of the associated future - this means that
although _some_ variables are no longer used, they still take up memory
for a long time.

In Nimbus, message validation is processed in batches meaning the future
of an incoming gossip message stays around for quite a while - this
leads to memory consumption peaks of 100-200 mb when there are many
attestations in the pipeline.

To avoid excessive memory usage, it's generally better to move non-async
code into proc's such that the variables therein can be released earlier
- this includes the many hidden variables introduced by macro and
template expansion (ie chronicles that does expensive exception
handling)

* move seen table salt to floodsub, use there as well
* shorten seen table salt to size of hash
* avoid unnecessary memory allocations and copies in a few places
* factor out message scoring
* avoid reencoding outgoing message for every peer
* keep checking validators until reject (in case there's both reject and
ignore)
* `readOnce` avoids `readExactly` overhead for single-byte read
* genericAssign -> assign2

* More gossip coverage (#553)

* add floodPublish test

* test delivery via control Iwant/have mechanics

* fix issues in control, and add testing

* fix possible backoff issue with pruned routine overriding it

* fix control messages (#566)

* remove unused control graft check in handleControl

* avoid sending empty Iwant messages

* Split dialer (#542)

* extracting dialing logic to dialer

* exposing upgrade methods on transport

* cleanup

* fixing tests to use new interfaces

* add comments

* add base exception class and fix hierarchy

* fix imports

* Merge master (#555)

* Revisit Floodsub (#543)

Fixes #525

add coverage to unsubscribeAll and testing

* add mounted protos to identify message (#546)

* add stable/unstable auto bumps

* fix auto-bump CI

* merge nbc auto bump with CI in order to bump only on CI success

* put conditional locks on nbc bump (#549)

* Fix minor exception issues (#550)

Makes code compatible with
https://github.com/status-im/nim-chronos/pull/166 without requiring it.

* fix nimbus ref for auto-bump stable's PR

* Split dialer (#542)

* extracting dialing logic to dialer

* exposing upgrade methods on transport

* cleanup

* fixing tests to use new interfaces

* add comments

* add base exception class and fix hierarchy

* fix imports

* `doAssert` is `ValueError` not `AssertionError`?

* revert back to `AssertionError`

Co-authored-by: Giovanni Petrantoni <7008900+sinkingsugar@users.noreply.github.com>
Co-authored-by: Jacek Sieka <jacek@status.im>

* Builders (#558)

* use a builder pattern to build the switch (#551)

* use a builder pattern to build the switch

* with with

* more refs

* Merge master (#555)

* Revisit Floodsub (#543)

Fixes #525

add coverage to unsubscribeAll and testing

* add mounted protos to identify message (#546)

* add stable/unstable auto bumps

* fix auto-bump CI

* merge nbc auto bump with CI in order to bump only on CI success

* put conditional locks on nbc bump (#549)

* Fix minor exception issues (#550)

Makes code compatible with
https://github.com/status-im/nim-chronos/pull/166 without requiring it.

* fix nimbus ref for auto-bump stable's PR

* Split dialer (#542)

* extracting dialing logic to dialer

* exposing upgrade methods on transport

* cleanup

* fixing tests to use new interfaces

* add comments

* add base exception class and fix hierarchy

* fix imports

* `doAssert` is `ValueError` not `AssertionError`?

* revert back to `AssertionError`

Co-authored-by: Giovanni Petrantoni <7008900+sinkingsugar@users.noreply.github.com>
Co-authored-by: Jacek Sieka <jacek@status.im>

* `doAssert` is `ValueError` not `AssertionError`?

* revert back to `AssertionError`

* fix builders

* more builder stuff

* more builders

Co-authored-by: Giovanni Petrantoni <7008900+sinkingsugar@users.noreply.github.com>
Co-authored-by: Jacek Sieka <jacek@status.im>

Co-authored-by: Giovanni Petrantoni <7008900+sinkingsugar@users.noreply.github.com>
Co-authored-by: Jacek Sieka <jacek@status.im>
2021-06-02 12:24:46 -06:00
Dmitriy Ryajov
5ee67f31bf Merge master (#562)
* builders (#559)

* More builders (#560)

* address some issues pointed out in review

* re-add to prevent breaking other projects

* Merge master (#555)

* Revisit Floodsub (#543)

Fixes #525

add coverage to unsubscribeAll and testing

* add mounted protos to identify message (#546)

* add stable/unstable auto bumps

* fix auto-bump CI

* merge nbc auto bump with CI in order to bump only on CI success

* put conditional locks on nbc bump (#549)

* Fix minor exception issues (#550)

Makes code compatible with
https://github.com/status-im/nim-chronos/pull/166 without requiring it.

* fix nimbus ref for auto-bump stable's PR

* Split dialer (#542)

* extracting dialing logic to dialer

* exposing upgrade methods on transport

* cleanup

* fixing tests to use new interfaces

* add comments

* add base exception class and fix hierarchy

* fix imports

* `doAssert` is `ValueError` not `AssertionError`?

* revert back to `AssertionError`

Co-authored-by: Giovanni Petrantoni <7008900+sinkingsugar@users.noreply.github.com>
Co-authored-by: Jacek Sieka <jacek@status.im>

Co-authored-by: Giovanni Petrantoni <7008900+sinkingsugar@users.noreply.github.com>
Co-authored-by: Jacek Sieka <jacek@status.im>
2021-06-02 12:24:46 -06:00
Dmitriy Ryajov
530e589e14 Builders (#558)
* use a builder pattern to build the switch (#551)

* use a builder pattern to build the switch

* with with

* more refs

* Merge master (#555)

* Revisit Floodsub (#543)

Fixes #525

add coverage to unsubscribeAll and testing

* add mounted protos to identify message (#546)

* add stable/unstable auto bumps

* fix auto-bump CI

* merge nbc auto bump with CI in order to bump only on CI success

* put conditional locks on nbc bump (#549)

* Fix minor exception issues (#550)

Makes code compatible with
https://github.com/status-im/nim-chronos/pull/166 without requiring it.

* fix nimbus ref for auto-bump stable's PR

* Split dialer (#542)

* extracting dialing logic to dialer

* exposing upgrade methods on transport

* cleanup

* fixing tests to use new interfaces

* add comments

* add base exception class and fix hierarchy

* fix imports

* `doAssert` is `ValueError` not `AssertionError`?

* revert back to `AssertionError`

Co-authored-by: Giovanni Petrantoni <7008900+sinkingsugar@users.noreply.github.com>
Co-authored-by: Jacek Sieka <jacek@status.im>

* `doAssert` is `ValueError` not `AssertionError`?

* revert back to `AssertionError`

* fix builders

* more builder stuff

* more builders

Co-authored-by: Giovanni Petrantoni <7008900+sinkingsugar@users.noreply.github.com>
Co-authored-by: Jacek Sieka <jacek@status.im>
2021-06-02 12:24:44 -06:00
Dmitriy Ryajov
ee49b76478 Merge master (#555)
* Revisit Floodsub (#543)

Fixes #525

add coverage to unsubscribeAll and testing

* add mounted protos to identify message (#546)

* add stable/unstable auto bumps

* fix auto-bump CI

* merge nbc auto bump with CI in order to bump only on CI success

* put conditional locks on nbc bump (#549)

* Fix minor exception issues (#550)

Makes code compatible with
https://github.com/status-im/nim-chronos/pull/166 without requiring it.

* fix nimbus ref for auto-bump stable's PR

* Split dialer (#542)

* extracting dialing logic to dialer

* exposing upgrade methods on transport

* cleanup

* fixing tests to use new interfaces

* add comments

* add base exception class and fix hierarchy

* fix imports

* `doAssert` is `ValueError` not `AssertionError`?

* revert back to `AssertionError`

Co-authored-by: Giovanni Petrantoni <7008900+sinkingsugar@users.noreply.github.com>
Co-authored-by: Jacek Sieka <jacek@status.im>
2021-06-02 12:24:07 -06:00
Dmitriy Ryajov
eef5dd0042 fix imports 2021-06-02 12:24:06 -06:00
Dmitriy Ryajov
8e3ef540ea add base exception class and fix hierarchy 2021-06-02 12:24:04 -06:00
Dmitriy Ryajov
a3c00af945 Split dialer (#542)
* extracting dialing logic to dialer

* exposing upgrade methods on transport

* cleanup

* fixing tests to use new interfaces

* add comments
2021-06-02 12:23:44 -06:00
Dmitriy Ryajov
3da656687b use LPError more consistently (#582)
* use LPError more consistently

* don't use Exceptino

* annotate with raises

* don't panic on concatenation

* further rework error handling
2021-06-02 15:39:10 +02:00
Dmitriy Ryajov
1b94c3feda fix #581 (#583) 2021-06-01 18:06:08 -06:00
Dmitriy Ryajov
34787728a3 don't use MaResult as default in newStandardSwitch (#578) 2021-05-24 16:48:18 -06:00
Dmitriy Ryajov
24132d7129 More raises cleanup (#575)
* use toException to map errors

* don't initialize address twice

* better error messages

* allow LPError to escape

* allow LPError to escape
2021-05-22 12:27:30 -06:00
Dmitriy Ryajov
ac4e060e1a adding raises defect across the codebase (#572)
* adding raises defect across the codebase

* use unittest2

* add windows deps caching

* update mingw link

* die on failed peerinfo initialization

* use result.expect instead of get

* use expect more consistently and rework inits

* use expect more consistently

* throw on missing public key

* remove unused closure annotation

* merge master
2021-05-21 10:27:01 -06:00
Jacek Sieka
9674a6a6f6 simplify connmanager (#573)
* no need to init orderedset
* array more simple than table
2021-05-19 08:49:55 +02:00
Jacek Sieka
83a20a992a gossipsub: unsubscribe fixes (#569)
* gossipsub: unsubscribe fixes

* fix KeyError when updating metric of unsubscribed topic
* fix unsubscribe message not being sent to all peers causing them to
keep thinking we're still subscribed
* release memory earlier in a few places

* floodsub fix
2021-05-07 00:43:45 +02:00
Giovanni Petrantoni
9f301964ed fix control messages (#566)
* remove unused control graft check in handleControl

* avoid sending empty Iwant messages
2021-04-28 10:03:03 +09:00
Giovanni Petrantoni
f81a085d0b More gossip coverage (#553)
* add floodPublish test

* test delivery via control Iwant/have mechanics

* fix issues in control, and add testing

* fix possible backoff issue with pruned routine overriding it
2021-04-22 18:51:22 +09:00
Jacek Sieka
e285d8bbf4 mem usage cleanups for pubsub (#564)
In `async` functions, a closure environment is created for variables
that cross an await boundary - this closure environment is kept in
memory for the lifetime of the associated future - this means that
although _some_ variables are no longer used, they still take up memory
for a long time.

In Nimbus, message validation is processed in batches meaning the future
of an incoming gossip message stays around for quite a while - this
leads to memory consumption peaks of 100-200 mb when there are many
attestations in the pipeline.

To avoid excessive memory usage, it's generally better to move non-async
code into proc's such that the variables therein can be released earlier
- this includes the many hidden variables introduced by macro and
template expansion (ie chronicles that does expensive exception
handling)

* move seen table salt to floodsub, use there as well
* shorten seen table salt to size of hash
* avoid unnecessary memory allocations and copies in a few places
* factor out message scoring
* avoid reencoding outgoing message for every peer
* keep checking validators until reject (in case there's both reject and
ignore)
* `readOnce` avoids `readExactly` overhead for single-byte read
* genericAssign -> assign2
2021-04-18 10:08:33 +02:00
Dmitriy Ryajov
6b930ae7e6 More builders (#560)
* address some issues pointed out in review

* re-add to prevent breaking other projects
2021-04-06 14:16:23 -06:00
Dmitriy Ryajov
290866dd62 builders (#559) 2021-04-05 16:06:45 -06:00
Giovanni Petrantoni
795a651839 use a builder pattern to build the switch (#551)
* use a builder pattern to build the switch

* with with

* more refs
2021-04-02 10:20:51 +09:00
Giovanni Petrantoni
df497660bc fix nimbus ref for auto-bump stable's PR 2021-03-23 16:46:24 +09:00
Jacek Sieka
54031c9e9b Fix minor exception issues (#550)
Makes code compatible with
https://github.com/status-im/nim-chronos/pull/166 without requiring it.
2021-03-23 07:45:25 +01:00
Giovanni Petrantoni
38333e45ae put conditional locks on nbc bump (#549) 2021-03-22 22:13:00 +09:00
Giovanni Petrantoni
3bf6acef23 merge nbc auto bump with CI in order to bump only on CI success 2021-03-16 09:31:06 +09:00
Giovanni Petrantoni
2d3595708c fix auto-bump CI 2021-03-16 08:45:37 +09:00
Giovanni Petrantoni
9175a9476e add stable/unstable auto bumps 2021-03-16 08:31:39 +09:00
Dmitriy Ryajov
f7a9d83545 add mounted protos to identify message (#546) 2021-03-15 15:29:05 -06:00
Giovanni Petrantoni
03bbdd2261 Revisit Floodsub (#543)
Fixes #525

add coverage to unsubscribeAll and testing
2021-03-15 13:16:03 -06:00
Giovanni Petrantoni
a54e1cc699 add coverage to disabled metrics (#545) 2021-03-15 21:36:57 +09:00
Giovanni Petrantoni
aeb18c4e41 now wild except: 2021-03-15 16:48:11 +09:00
Giovanni Petrantoni
c5b9fd3353 Try fix ci (#544)
* use apt-get

* try bump ubuntu ver
2021-03-15 14:50:36 +09:00
Giovanni Petrantoni
ba0a57ee34 fix reports number 2021-03-15 12:16:01 +09:00
Giovanni Petrantoni
996dc5be9c add testfloodsub coverage for different build options 2021-03-15 12:15:29 +09:00
Giovanni Petrantoni
4760df1e31 fix build with libp2p_agents_metrics switch 2021-03-15 01:42:47 +00:00
Jacek Sieka
70deac9e0d fix peer score accumulation (#541)
* fix accumulating peer score
* fix missing exception handling
* remove unnecessary initHashSet/initTable calls
* simplify peer stats management
* clean up tests a little
* fix some missing raises annotations
2021-03-09 13:22:52 +01:00
Giovanni Petrantoni
269d3df351 Consolidate metrics collection for mesh (#540)
* Consolidate metrics collection for mesh

* more fixes

* wrapping up

* Update libp2p/protocols/pubsub/gossipsub/behavior.nim

Co-authored-by: Jacek Sieka <jacek@status.im>

* Update libp2p/protocols/pubsub/gossipsub/behavior.nim

Co-authored-by: Jacek Sieka <jacek@status.im>

* Update libp2p/protocols/pubsub/gossipsub/behavior.nim

Co-authored-by: Jacek Sieka <jacek@status.im>

Co-authored-by: Jacek Sieka <jacek@status.im>
2021-03-03 22:11:21 +01:00
Giovanni Petrantoni
34c2fbeb16 small gossipsub metrics change 2021-03-03 08:41:21 +00:00
Giovanni Petrantoni
02ad017107 Gossipsub fixes and Initiator flagging fixes (#539)
* properly propagate initiator information for gossipsub

* Fix pubsubpeer lifetime management

* restore old behavior

* tests fixing

* clamp backoff time value received

* fix member name collisions

* internal test fixes

* better names and explaining of the importance of transport direction

* fixes
2021-03-03 08:23:40 +09:00
Giovanni Petrantoni
c1334c6d89 pubsubpeer better address management 2021-02-28 04:53:17 +00:00
Giovanni Petrantoni
7b2727d930 avoid leaking in peersInIP, don't depend on sendConn 2021-02-27 23:49:56 +09:00
Giovanni Petrantoni
67d0926e89 use in any case PeerID for peersInIP to avoid keeping references 2021-02-27 21:31:59 +09:00
Giovanni Petrantoni
fae38e0146 fix PubSubPeer hashing issues 2021-02-26 19:19:15 +09:00
Giovanni Petrantoni
45300c28a9 [SEC] gossipsub - handleIHAVE/handleIWANT recommendations & notes (#535)
Fixes #400
2021-02-26 14:27:42 +09:00
Giovanni Petrantoni
d7469b2286 [SEC] gossipsub - a peer score is not retained up until expiry if abusive peer unsubscribes (#534)
* [SEC] gossipsub - a peer score is not retained up until expiry if abusive peer unsubscribes
Fixes #402

* remove debug logging
2021-02-26 14:15:58 +09:00
Giovanni Petrantoni
c1d8317e3c fix badly merged code in gossipsub.colocationFactor 2021-02-26 12:39:57 +09:00
Giovanni Petrantoni
eac6cd3dbf Debt: cleanup warnings #426 (#536)
* testswitch cleanups

* Debt: cleanup warnings
Fixes #426
2021-02-25 09:24:49 -06:00
Giovanni Petrantoni
2015a878ad fix missing coverage (#533)
* investigate missing coverage

* test different pattern

* wip pattern testing

* try globastar
2021-02-23 12:50:43 +09:00
Giovanni Petrantoni
8236319a91 [SEC] gossipsub - handleGraft/handlePrune recommendations & notes (#530)
Fixes #401
2021-02-22 12:04:20 +09:00
Giovanni Petrantoni
1368bf7ecb test rebalanceMesh with low score peers (#529) 2021-02-22 10:05:25 +09:00
Giovanni Petrantoni
922cd92f94 don't check if peers have sendConn when disconnecting for bad scoring 2021-02-22 10:04:02 +09:00
Giovanni Petrantoni
51d8cd4ade [SEC] gossipsub - rebalanceMesh may prune up to D_lo on oversubscription (#531)
Fixes #403
2021-02-13 13:39:32 +09:00
Giovanni Petrantoni
e124e342b0 n subscription limits (#528)
* subscription high water, cleanups

* subscription limits test

* newline
2021-02-12 12:27:26 +09:00
Dmitriy Ryajov
12adefb4de add multi types to exports (#527)
* add multitypes to exports

* export standard setup
2021-02-10 11:42:46 -06:00
Dmitriy Ryajov
f4145ebbfa More exports cleanup (#522)
* annotate `SecureProtocol.Secio` as deprecated

* dont export varint

* add `errors` to exports - convenient error utils
2021-02-09 15:41:49 -06:00
Giovanni Petrantoni
fff54fa23c add more diagnostics when gossip publish fails 2021-02-09 18:42:59 +09:00
Ștefan Talpalaru
d9563d65ae support compilation with Nim-1.4 HEAD (#521) 2021-02-08 15:21:43 -06:00
Dmitriy Ryajov
2658181df9 Merge unstable (#518)
* Address Book POC implementation (#499)

* Address Book POC implementation

* Feat/peerstore impl (#505)

Co-authored-by: Hanno Cornelius <68783915+jm-clius@users.noreply.github.com>
2021-02-08 15:16:23 -06:00
Dmitriy Ryajov
4dea23c394 Remove secio usage and cleanup exports (#519)
* cleaned up exports

* remove secio use

* added more useful exports

* proper import
2021-02-08 14:33:34 -06:00
Giovanni Petrantoni
646557597d lower some gossipsub logging to debug level 2021-02-08 10:11:41 +09:00
Giovanni Petrantoni
fd73cf9f9d Refactor gossipsub into multiple modules (#515)
* Refactor gossipsub into multiple modules

* splitup further gossipsub

* move more mesh related stuff to behavior

* fix internal tests

* fix PubSubPeer.outbound flag, make it more reliable

* use discard rather then _
2021-02-06 09:13:04 +09:00
Dmitriy Ryajov
3213e5d377 add sane exports to libp2p (#517) 2021-02-05 11:57:32 -06:00
Dmitriy Ryajov
5c234ddd37 add hash proc to support using with containers (#516) 2021-02-05 10:12:44 -06:00
282 changed files with 43223 additions and 17360 deletions

View File

@@ -1,52 +0,0 @@
version: '{build}'
image: Visual Studio 2015
cache:
- NimBinaries
- p2pdCache
matrix:
# We always want 32 and 64-bit compilation
fast_finish: false
platform:
- x86
- x64
# when multiple CI builds are queued, the tested commit needs to be in the last X commits cloned with "--depth X"
clone_depth: 10
install:
- git submodule update --init --recursive
# use the newest versions documented here: https://www.appveyor.com/docs/windows-images-software/#mingw-msys-cygwin
- IF "%PLATFORM%" == "x86" SET PATH=C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin;%PATH%
- IF "%PLATFORM%" == "x64" SET PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH%
# build nim from our own branch - this to avoid the day-to-day churn and
# regressions of the fast-paced Nim development while maintaining the
# flexibility to apply patches
- curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh
- env MAKE="mingw32-make -j2" ARCH_OVERRIDE=%PLATFORM% bash build_nim.sh Nim csources dist/nimble NimBinaries
- SET PATH=%CD%\Nim\bin;%PATH%
# set path for produced Go binaries
- MKDIR goblin
- CD goblin
- SET GOPATH=%CD%
- SET PATH=%GOPATH%\bin;%PATH%
- CD ..
# install and build go-libp2p-daemon
- bash scripts/build_p2pd.sh p2pdCache v0.3.0
build_script:
- nimble install -y --depsOnly
test_script:
- nimble test
- nimble examples_build
deploy: off

1141
.assets/full-logo.svg Normal file

File diff suppressed because it is too large Load Diff

After

Width:  |  Height:  |  Size: 61 KiB

96
.assets/small-logo.svg Normal file
View File

@@ -0,0 +1,96 @@
<?xml version="1.0" encoding="UTF-8"?>
<svg xmlns="http://www.w3.org/2000/svg" xmlns:v="https://vecta.io/nano" xmlns:xlink="http://www.w3.org/1999/xlink" width="172.071" height="196.414" viewBox="0 0 45.527 51.968">
<g transform="matrix(.2822 0 0 .2822 -212.833275 -150.656248)">
<path d="M835.432 533.821l-12.483 9.783c-6.482-.207-19.197 1.251-26.086 3.769-6.346-4.04-11.923-8.5-11.923-8.5l-7.762 13.071c-4.444 2.375-8.906 5.046-12.883 8.58l-10.162-4.17c6.125 12.414 10.243 24.844 21.445 32.316 17.834-28.299 100.705-25.691 118.907-.16 11.764-6.165 16.339-19.429 20.965-31.674-.507.168-6.802 2.285-10.882 3.849-2.436-2.665-8.179-6.763-11.443-8.741-3.096-5.696-7.602-13.391-7.602-13.391s-5.337 3.988-11.523 8.34c-8.357-1.55-18.465-3.433-26.966-2.967-5.787-4.779-11.603-10.104-11.603-10.104z" fill="#f3d400" />
<g opacity=".9" transform="matrix(.9375 0 0 .9375 765.1166 550.13225)">
<path d="M99.952 106.898l.215-.107 24.755-14.248-24.97-14.535-24.97 14.374z" fill="#cc2a65" />
<use xlink:href="#B" fill="#a21d4c" />
<path d="M124.922 92.542l-24.755 14.248-.215.107v28.89l24.97-14.356z" fill="#b62454" />
<path d="M50.012 106.737l.215-.125 24.755-14.248-24.97-14.517-24.97 14.356z" fill="#c8d92b" />
<path d="M50.012 135.609v-28.872l-24.97-14.535v28.89h.018z" fill="#c2d02f" />
<path d="M74.982 92.381l-24.755 14.23-.215.125v28.872.018h.018l24.952-14.356v-.018z" fill="#b9be33" />
<path d="M74.982 121.253l.215-.107 24.755-14.248-24.97-14.535-24.97 14.374z" fill="#cc2a65" />
<use xlink:href="#B" x="-24.97" y="14.356" fill="#a21d4c" />
<path d="M99.952 106.898l-24.755 14.248-.215.107v28.89H75l24.952-14.356z" fill="#b62454" />
<path d="M124.905 121.415l.215-.125 24.737-14.23-24.952-14.535-24.97 14.356z" fill="#a159a2" />
<path d="M124.905 150.305v-28.89l-24.97-14.535v28.89h.018z" fill="#772a86" />
<path d="M149.875 107.059l-24.755 14.23-.215.125v28.89l24.97-14.356z" fill="#8e3b95" />
<path d="M74.982 92.345l.215-.125 24.737-14.248-24.952-14.517-24.97 14.356z" fill="#bec831" />
<path d="M74.982 121.217V92.345L50.012 77.81v28.89h.018z" fill="#a1a938" />
<path d="M99.952 77.989l-24.755 14.23-.215.125v28.872.018l24.97-14.356v-.018z" fill="#999b37" />
<path d="M75 60.645l.197-.125 24.755-14.23L75 31.755 50.029 46.11l24.952 14.535z" fill="#bec831" />
<path d="M74.982 89.535L75 60.645 50.029 46.11 50.012 75h.018z" fill="#a1a938" />
<path d="M99.97 46.307L75.197 60.52l-.197.125h-.018v28.89l24.97-14.338v-.018z" fill="#999b37" />
<path d="M99.952 75.179l.215-.107 24.755-14.23L99.97 46.306 75 60.644z" fill="#ee539a" />
<path d="M99.952 104.069v-28.89L75 60.644l-.018 28.89H75z" fill="#d01b68" />
<path d="M124.922 60.841l-24.755 14.23-.215.107v28.89l24.97-14.338.018-28.89z" fill="#ec0f68" />
<path d="M124.923 89.731l.215-.125 24.755-14.23-24.952-14.535h-.018l-24.97 14.338z" fill="#a159a2" />
<path d="M124.905 118.622l.018-28.89-24.97-14.535v28.872.018z" fill="#772a86" />
<path d="M149.893 75.376l-24.755 14.23-.215.125-.018 28.89h.018l24.97-14.356z" fill="#8e3b95" />
<path d="M50.03 75l.197-.125 24.755-14.23L50.03 46.109 25.06 60.447l24.952 14.535z" fill="#c8d92b" />
<path d="M50.012 103.872L50.03 75 25.06 60.447l-.018 28.89h.018z" fill="#c2d02f" />
<path d="M75 60.644l-24.773 14.23-.197.125-.018 28.872 24.97-14.338z" fill="#b9be33" />
<path d="M74.982 89.534l.215-.125 24.755-14.23L75 60.644l-24.97 14.338z" fill="#f7af19" />
<path d="M74.982 118.425v-.018.018-28.89L50.029 75l-.018 28.872.018.018z" fill="#f2901f" />
<path d="M99.952 75.179l-24.755 14.23-.215.125v28.89l24.97-14.356.018-28.89z" fill="#f9a120" />
<path d="M99.934 135.769l.215-.125 24.684-14.356-25.042-14.409L74.91 121.36z" fill="#833593" />
<path d="M100.077 164.66l-.143-28.89-25.042-14.409.143 28.89h.018z" fill="#652977" />
<path d="M124.833 121.288l-24.684 14.356-.215.125.143 28.89 24.899-14.481z" fill="#4d1f5b" />
<path d="M99.952 104.069l.215-.107 24.755-14.23L99.97 75.179h-.018l-24.97 14.356z" fill="#a159a2" />
<path d="M99.934 132.959l.018-28.89-24.97-14.535v28.89z" fill="#772a86" />
<path d="M124.922 89.732l-24.755 14.23-.215.107-.018 28.89h.018l24.97-14.338z" fill="#8e3b95" />
<path d="M25.042 121.074l.197-.125 24.755-14.248-24.952-14.517h-.018L.071 106.54l24.952 14.535z" fill="#f6dd03" />
<path d="M25.024 149.947h.018v-28.872L.071 106.54v28.89z" fill="#f9bb1d" />
<path d="M49.994 106.719l-24.755 14.23-.197.125v28.872.018l24.952-14.356h.018v-.018-28.872z" fill="#e9ae20" />
<path d="M25.06 89.338l.197-.125 24.755-14.23L25.06 60.447.089 74.803l24.952 14.535z" fill="#f6dd03" />
<path d="M25.042 118.228l.018-28.89L.089 74.803.072 103.675l.018.018z" fill="#f9bb1d" />
<path d="M50.03 75L25.257 89.212l-.197.125-.018 28.89 24.97-14.356v.018-.018z" fill="#e9ae20" />
<path d="M50.012 135.59l.215-.107 24.737-14.248L50.012 106.7l-24.97 14.374z" fill="#f7af19" />
<path d="M50.012 164.481v-28.89l-24.97-14.517v28.872.018z" fill="#f2901f" />
<path d="M74.964 121.235l-24.755 14.248-.197.107v28.89l24.97-14.356v-28.89z" fill="#f9a120" />
<path d="M50.012 103.872l.215-.107 24.755-14.23L50.03 74.982 25.06 89.338z" fill="#f7af19" />
<path d="M50.012 132.763v-28.89L25.06 89.338l-.018 28.89h.018z" fill="#f2901f" />
<path d="M74.982 89.535l-24.755 14.23-.215.107v28.89l24.97-14.338.018-28.89z" fill="#f9a120" />
<path d="M74.982 150.125l.197-.125 24.755-14.23-24.952-14.535h-.018l-24.952 14.356 24.952 14.535z" fill="#f7af19" />
<path d="M74.964 179.015h.018v-28.89l-24.97-14.517v28.872.018z" fill="#f2901f" />
<path d="M99.934 135.77L75.179 150l-.197.125v28.89l24.97-14.356v-28.89z" fill="#f9a120" />
<path d="M74.982 118.425l.215-.125 24.755-14.23L75 89.535h-.018l-24.97 14.338z" fill="#31838b" />
<path d="M74.964 147.297l.018-28.872-24.97-14.535v-.018 28.89.018z" fill="#22626c" />
<path d="M99.952 104.069L75.197 118.3l-.215.125-.018 28.872v.018h.018l24.97-14.356z" fill="#1b4b56" />
<path d="M74.982 28.962l.215-.125 24.737-14.248L74.982.072l-24.97 14.356 24.97 14.517z" fill="#bec831" />
<path d="M74.982 57.834V28.962l-24.97-14.535v28.89h.018z" fill="#a1a938" />
<path d="M99.952 14.606l-24.755 14.23-.215.125v28.872l24.97-14.356z" fill="#999b37" />
<path d="M74.964 28.944l.215-.125 24.755-14.23L74.982.054h-.018l-24.97 14.338z" fill="#a159a2" />
<path d="M74.946 57.835l.018-28.89-24.97-14.535v28.872.018z" fill="#772a86" />
<path d="M99.934 14.589l-24.755 14.23-.215.125-.018 28.89h.018l24.97-14.356z" fill="#8e3b95" />
<path d="M99.952 43.479l.215-.107 24.755-14.248-24.97-14.535-24.97 14.356z" fill="#ee539a" />
<use xlink:href="#B" y="-63.419" fill="#d01b68" />
<path d="M124.922 29.123l-24.755 14.248-.215.107v28.89l24.97-14.356z" fill="#ec0f68" />
<path d="M50.03 43.317l.215-.125L75 28.961 50.048 14.427h-.018L25.06 28.765z" fill="#31838b" />
<path d="M50.012 72.189l.018-28.872-24.97-14.535v-.018 28.89.018z" fill="#22626c" />
<path d="M75 28.961l-24.755 14.23-.215.125-.018 28.872v.018h.018L75 57.852z" fill="#1b4b56" />
<path d="M124.923 58.013l.215-.125 24.737-14.23-24.952-14.535-24.97 14.356z" fill="#cc2a65" />
<use xlink:href="#B" x="24.971" y="-48.884" fill="#a21d4c" />
<path d="M149.893 43.658l-24.755 14.23-.215.125v28.89l24.97-14.356z" fill="#b62454" />
<path d="M74.982 57.835l.215-.107 24.755-14.248-24.97-14.535L50.012 43.3z" fill="#c8d92b" />
<path d="M74.982 86.725v-28.89l-24.97-14.517V72.19l.018.018z" fill="#c2d02f" />
<path d="M99.952 43.479L75.197 57.727l-.215.107v28.89H75l24.952-14.356z" fill="#b9be33" />
<path d="M99.952 72.369l.215-.125 24.755-14.23-24.97-14.535-24.97 14.356z" fill="#33b4d7" />
<use xlink:href="#B" y="-34.529" fill="#209ac5" />
<path d="M124.922 58.014l-24.755 14.23-.215.125v28.89h.018l24.952-14.356z" fill="#0f8cae" />
<path d="M25.06 57.673l.197-.125L50.012 43.3 25.06 28.783h-.018L.089 43.139l24.952 14.535z" fill="#94d6e3" />
<path d="M25.042 86.546h.018V57.673L.089 43.139v28.89z" fill="#73ccdd" />
<path d="M50.012 43.318l-24.755 14.23-.197.125v28.872.018l24.952-14.356h.018v-.018-28.872z" fill="#3bafbb" />
<path d="M50.03 72.19l.215-.107 24.737-14.248L50.03 43.318 25.06 57.674z" fill="#94d6e3" />
<path d="M50.03 101.08V72.208v-.018L25.06 57.674v28.872.018z" fill="#73ccdd" />
<path d="M74.982 57.835L50.227 72.083l-.197.107v28.89L75 86.725v-28.89z" fill="#3bafbb" />
<path d="M75 86.724l.197-.107 24.755-14.248L75 57.834h-.018L50.029 72.189l24.952 14.535z" fill="#33b4d7" />
<path d="M74.982 115.614H75v-28.89l-24.97-14.517v28.872.018z" fill="#209ac5" />
<path d="M99.952 72.368L75.197 86.617l-.197.107v28.89l24.97-14.356v-28.89z" fill="#0f8cae" />
</g>
<path d="M759.126 567.007s10.273 21.02 16.364 35.698c25.549 33.869 90.792 36.224 119.235.656 9.484-17.619 16.733-36.357 16.733-36.357-7.297 10.862-20.094 18.056-27.408 22.095-5.197 2.861-17.189 4.59-17.189 4.59l-31.482-16.393-31.663 16.065s-11.832-1.91-17.189-4.426c-10.811-5.799-19.735-12.549-27.401-21.928z" fill="#ffe953" />
</g>
<defs>
<path id="B" d="M99.952 135.788v-28.89l-24.97-14.517v28.872l.018.018z" />
</defs>
</svg>

After

Width:  |  Height:  |  Size: 9.4 KiB

2
.git-blame-ignore-revs Normal file
View File

@@ -0,0 +1,2 @@
# Formatted with nph 0.5.1
dc83a1e9b68f00b3be7e09febdb1a3f877321b9a

1
.github/CODEOWNERS vendored Normal file
View File

@@ -0,0 +1 @@
* @vacp2p/p2p

133
.github/actions/install_nim/action.yml vendored Normal file
View File

@@ -0,0 +1,133 @@
name: Install Nim
inputs:
os:
description: "Operating system to build for"
required: true
cpu:
description: "CPU to build for"
default: "amd64"
nim_ref:
description: "Nim version"
default: "version-1-6"
shell:
description: "Shell to run commands in"
default: "bash --noprofile --norc -e -o pipefail"
runs:
using: "composite"
steps:
- name: Install build dependencies (Linux i386)
shell: ${{ inputs.shell }}
if: inputs.os == 'Linux' && inputs.cpu == 'i386'
run: |
sudo dpkg --add-architecture i386
sudo apt-get update -qq
sudo DEBIAN_FRONTEND='noninteractive' apt-get install \
--no-install-recommends -yq gcc-multilib g++-multilib \
libssl-dev:i386
mkdir -p external/bin
cat << EOF > external/bin/gcc
#!/bin/bash
exec $(which gcc) -m32 "\$@"
EOF
cat << EOF > external/bin/g++
#!/bin/bash
exec $(which g++) -m32 "\$@"
EOF
chmod 755 external/bin/gcc external/bin/g++
echo '${{ github.workspace }}/external/bin' >> $GITHUB_PATH
- name: MSYS2 (Windows i386)
if: inputs.os == 'Windows' && inputs.cpu == 'i386'
uses: msys2/setup-msys2@v2
with:
path-type: inherit
msystem: MINGW32
install: >-
base-devel
git
mingw-w64-i686-toolchain
- name: MSYS2 (Windows amd64)
if: inputs.os == 'Windows' && inputs.cpu == 'amd64'
uses: msys2/setup-msys2@v2
with:
path-type: inherit
install: >-
base-devel
git
mingw-w64-x86_64-toolchain
- name: Restore Nim DLLs dependencies (Windows) from cache
if: inputs.os == 'Windows'
id: windows-dlls-cache
uses: actions/cache@v4
with:
path: external/dlls
key: 'dlls'
- name: Install DLL dependencies (Windows)
shell: ${{ inputs.shell }}
if: >
steps.windows-dlls-cache.outputs.cache-hit != 'true' &&
inputs.os == 'Windows'
run: |
mkdir external
curl -L "https://nim-lang.org/download/windeps.zip" -o external/windeps.zip
7z x external/windeps.zip -oexternal/dlls
- name: Path to cached dependencies (Windows)
shell: ${{ inputs.shell }}
if: >
inputs.os == 'Windows'
run: |
echo '${{ github.workspace }}'"/external/dlls" >> $GITHUB_PATH
- name: Derive environment variables
shell: ${{ inputs.shell }}
run: |
if [[ '${{ inputs.cpu }}' == 'amd64' ]]; then
PLATFORM=x64
elif [[ '${{ inputs.cpu }}' == 'arm64' ]]; then
PLATFORM=arm64
else
PLATFORM=x86
fi
echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV
ncpu=
MAKE_CMD="make"
case '${{ inputs.os }}' in
'Linux')
ncpu=$(nproc)
;;
'macOS')
ncpu=$(sysctl -n hw.ncpu)
;;
'Windows')
ncpu=$NUMBER_OF_PROCESSORS
MAKE_CMD="mingw32-make"
;;
esac
[[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
echo "ncpu=$ncpu" >> $GITHUB_ENV
echo "MAKE_CMD=${MAKE_CMD}" >> $GITHUB_ENV
echo '${{ github.workspace }}/nim/bin' >> $GITHUB_PATH
- name: Restore Nim from cache
id: nim-cache
uses: actions/cache@v4
with:
path: '${{ github.workspace }}/nim'
key: ${{ inputs.os }}-${{ inputs.cpu }}-nim-${{ inputs.nim_ref }}-cache-${{ env.cache_nonce }}
- name: Build Nim and Nimble
shell: ${{ inputs.shell }}
if: ${{ steps.nim-cache.outputs.cache-hit != 'true' }}
run: |
# We don't want partial matches of the cache restored
rm -rf nim
curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh
env MAKE="${MAKE_CMD} -j${ncpu}" ARCH_OVERRIDE=${PLATFORM} NIM_COMMIT=${{ inputs.nim_ref }} \
QUICK_AND_DIRTY_COMPILER=1 QUICK_AND_DIRTY_NIMBLE=1 CC=gcc \
bash build_nim.sh nim csources dist/nimble NimBinaries

12
.github/workflows/auto_assign_pr.yml vendored Normal file
View File

@@ -0,0 +1,12 @@
name: Auto Assign PR to Creator
on:
pull_request:
types:
- opened
jobs:
assign_creator:
runs-on: ubuntu-latest
steps:
- uses: toshimaru/auto-author-assign@v1.6.2

View File

@@ -1,207 +1,122 @@
name: nim-libp2p CI
on: [push, pull_request]
name: Continuous Integration
on:
push:
branches:
- master
pull_request:
merge_group:
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
build:
test:
timeout-minutes: 40
strategy:
fail-fast: false
max-parallel: 20
matrix:
branch: [v1.2.6]
target:
# Unit tests
platform:
- os: linux
cpu: amd64
TEST_KIND: unit-tests
- os: linux
cpu: i386
TEST_KIND: unit-tests
- os: linux-gcc-14
cpu: amd64
- os: macos
cpu: amd64
TEST_KIND: unit-tests
- os: windows
cpu: i386
TEST_KIND: unit-tests
- os: macos-14
cpu: arm64
- os: windows
cpu: amd64
TEST_KIND: unit-tests
nim:
- ref: version-1-6
memory_management: refc
- ref: version-2-0
memory_management: refc
- ref: version-2-2
memory_management: refc
include:
- target:
- platform:
os: linux
builder: ubuntu-18.04
- target:
builder: ubuntu-22.04
shell: bash
- platform:
os: linux-gcc-14
builder: ubuntu-24.04
shell: bash
- platform:
os: macos
builder: macos-10.15
- target:
builder: macos-13
shell: bash
- platform:
os: macos-14
builder: macos-14
shell: bash
- platform:
os: windows
builder: windows-2019
name: '${{ matrix.target.os }}-${{ matrix.target.cpu }} (${{ matrix.branch }})'
builder: windows-2022
shell: msys2 {0}
defaults:
run:
shell: ${{ matrix.shell }}
name: '${{ matrix.platform.os }}-${{ matrix.platform.cpu }} (Nim ${{ matrix.nim.ref }})'
runs-on: ${{ matrix.builder }}
steps:
- name: Checkout nim-libp2p
uses: actions/checkout@v2
- name: Checkout
uses: actions/checkout@v4
with:
path: nim-libp2p
submodules: true
- name: Install build dependencies (Linux i386)
if: runner.os == 'Linux' && matrix.target.cpu == 'i386'
run: |
sudo dpkg --add-architecture i386
sudo apt-fast update -qq
sudo DEBIAN_FRONTEND='noninteractive' apt-fast install \
--no-install-recommends -yq gcc-multilib g++-multilib \
libssl-dev:i386
mkdir -p external/bin
cat << EOF > external/bin/gcc
#!/bin/bash
exec $(which gcc) -m32 "\$@"
EOF
cat << EOF > external/bin/g++
#!/bin/bash
exec $(which g++) -m32 "\$@"
EOF
chmod 755 external/bin/gcc external/bin/g++
echo '${{ github.workspace }}/external/bin' >> $GITHUB_PATH
- name: Install build dependencies (Windows)
if: runner.os == 'Windows'
shell: bash
run: |
mkdir external
if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then
arch=64
else
arch=32
fi
curl -L "https://nim-lang.org/download/mingw$arch-6.3.0.7z" -o "external/mingw$arch.7z"
curl -L "https://nim-lang.org/download/windeps.zip" -o external/windeps.zip
7z x "external/mingw$arch.7z" -oexternal/
7z x external/windeps.zip -oexternal/dlls
echo '${{ github.workspace }}'"/external/mingw$arch/bin" >> $GITHUB_PATH
echo '${{ github.workspace }}'"/external/dlls" >> $GITHUB_PATH
- name: Setup environment
shell: bash
run: echo '${{ github.workspace }}/nim/bin' >> $GITHUB_PATH
- name: Get latest Nim commit hash
id: versions
shell: bash
run: |
getHash() {
git ls-remote "https://github.com/$1" "${2:-HEAD}" | cut -f 1
}
nimHash=$(getHash nim-lang/Nim '${{ matrix.branch }}')
csourcesHash=$(getHash nim-lang/csources)
echo "::set-output name=nim::$nimHash"
echo "::set-output name=csources::$csourcesHash"
- name: Restore prebuilt Nim from cache
id: nim-cache
uses: actions/cache@v1
- name: Setup Nim
uses: "./.github/actions/install_nim"
with:
path: nim
key: "nim-${{ matrix.target.os }}-${{ matrix.target.cpu }}-${{ steps.versions.outputs.nim }}"
- name: Restore prebuilt csources from cache
if: steps.nim-cache.outputs.cache-hit != 'true'
id: csources-cache
uses: actions/cache@v1
with:
path: csources/bin
key: "csources-${{ matrix.target.os }}-${{ matrix.target.cpu }}-${{ steps.versions.outputs.csources }}"
- name: Checkout Nim csources
if: >
steps.csources-cache.outputs.cache-hit != 'true' &&
steps.nim-cache.outputs.cache-hit != 'true'
uses: actions/checkout@v2
with:
repository: nim-lang/csources
path: csources
ref: ${{ steps.versions.outputs.csources }}
- name: Checkout Nim
if: steps.nim-cache.outputs.cache-hit != 'true'
uses: actions/checkout@v2
with:
repository: nim-lang/Nim
path: nim
ref: ${{ steps.versions.outputs.nim }}
- name: Build Nim and associated tools
if: steps.nim-cache.outputs.cache-hit != 'true'
shell: bash
run: |
ncpu=
ext=
case '${{ runner.os }}' in
'Linux')
ncpu=$(nproc)
;;
'macOS')
ncpu=$(sysctl -n hw.ncpu)
;;
'Windows')
ncpu=$NUMBER_OF_PROCESSORS
ext=.exe
;;
esac
[[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
if [[ ! -e csources/bin/nim$ext ]]; then
make -C csources -j $ncpu CC=gcc ucpu='${{ matrix.target.cpu }}'
else
echo 'Using prebuilt csources'
fi
cp -v csources/bin/nim$ext nim/bin
cd nim
nim c koch
./koch boot -d:release
./koch tools -d:release
# clean up to save cache space
rm koch
rm -rf nimcache
rm -rf dist
rm -rf .git
# - name: Install library dependencies (Linux amd64)
# if: runner.os == 'Linux' && matrix.target.cpu == 'amd64'
# run: |
# sudo DEBIAN_FRONTEND='noninteractive' apt-fast install \
# --no-install-recommends -yq libgmp-dev
#
# - name: Install library dependencies (Linux i386)
# if: runner.os == 'Linux' && matrix.target.cpu == 'i386'
# run: |
# sudo DEBIAN_FRONTEND='noninteractive' apt-fast install \
# libgmp-dev:i386
#
# - name: Install library dependencies (macOS)
# if: runner.os == 'macOS'
# run: brew install gmp
#
# - name: Install library dependencies (Windows)
# if: runner.os == 'Windows'
# shell: bash
# run: |
# choco install msys2
# pacman -S mingw-w64-x86_64-gmp
os: ${{ matrix.platform.os }}
cpu: ${{ matrix.platform.cpu }}
shell: ${{ matrix.shell }}
nim_ref: ${{ matrix.nim.ref }}
- name: Setup Go
uses: actions/setup-go@v2
uses: actions/setup-go@v5
with:
go-version: '^1.15.5'
go-version: '~1.16.0' # That's the minimum Go version that works with arm.
- name: Install p2pd
shell: bash
run: |
cd nim-libp2p
V=1 bash scripts/build_p2pd.sh p2pdCache v0.3.0
V=1 bash scripts/build_p2pd.sh p2pdCache 124530a3
- name: Run nim-libp2p tests
shell: bash
- name: Restore deps from cache
id: deps-cache
uses: actions/cache@v3
with:
path: nimbledeps
# Using nim.ref as a simple way to differentiate between nimble using the "pkgs" or "pkgs2" directories.
# The change happened on Nimble v0.14.0. Also forcing the deps to be reinstalled on each os and cpu.
key: nimbledeps-${{ matrix.nim.ref }}-${{ matrix.builder }}-${{ matrix.platform.cpu }}-${{ hashFiles('.pinned') }} # hashFiles returns a different value on windows
- name: Install deps
if: ${{ steps.deps-cache.outputs.cache-hit != 'true' }}
run: |
export UCPU="$cpu"
cd nim-libp2p
nimble install -y --depsOnly
nimble install_pinned
- name: Use gcc 14
if : ${{ matrix.platform.os == 'linux-gcc-14'}}
run: |
# Add GCC-14 to alternatives
sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 14
# Set GCC-14 as the default
sudo update-alternatives --set gcc /usr/bin/gcc-14
- name: Run tests
run: |
nim --version
nimble --version
gcc --version
export NIMFLAGS="${NIMFLAGS} -d:libp2p_quic_support --mm:${{ matrix.nim.memory_management }}"
nimble test

View File

@@ -1,121 +0,0 @@
name: nim-libp2p codecov builds
on: [push, pull_request]
jobs:
GossipSub:
runs-on: ubuntu-20.04
strategy:
matrix:
nim-options: [
"",
"-d:libp2p_pubsub_anonymize=true -d:libp2p_pubsub_sign=false -d:libp2p_pubsub_verify=false",
"-d:libp2p_pubsub_sign=true -d:libp2p_pubsub_verify=true"
]
test-program: [
"tests/pubsub/testpubsub",
"tests/pubsub/testgossipinternal"
]
steps:
- uses: actions/checkout@v2
- name: Run
run: |
sudo apt-get update
sudo apt-get install -y lcov build-essential git curl
mkdir coverage
curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh
env MAKE="make -j${NPROC}" bash build_nim.sh Nim csources dist/nimble NimBinaries
export PATH="$PATH:$PWD/Nim/bin"
nimble install -y --depsOnly
export NIM_OPTIONS="--opt:speed -d:debug --verbosity:0 --hints:off --lineDir:on -d:chronicles_log_level=INFO --warning[CaseTransition]:off --warning[ObservableStores]:off --warning[LockLevel]:off --nimcache:nimcache --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage ${{ matrix.nim-options }}"
nim c $NIM_OPTIONS -r ${{ matrix.test-program }}
cd nimcache; rm *.c; cd ..
lcov --capture --directory nimcache --output-file coverage/coverage.info
lcov --extract coverage/coverage.info `pwd`/libp2p/* `pwd`/libp2p/**/* `pwd`/libp2p/**/**/* --output-file coverage/coverage.f.info
export COV_UUID=`cksum <<< "${{ matrix.test-program }} $NIM_OPTIONS" | cut -f 1 -d ' '`
genhtml coverage/coverage.f.info --output-directory coverage/$COV_UUID-output
echo ${{ matrix.test-program }} > coverage/$COV_UUID-nim_options.txt
echo $NIM_OPTIONS >> coverage/$COV_UUID-nim_options.txt
bash <(curl -s https://codecov.io/bash) -f coverage/coverage.f.info || echo "Codecov did not collect coverage reports"
- uses: actions/upload-artifact@master
with:
name: coverage
path: coverage
Tests:
runs-on: ubuntu-20.04
strategy:
matrix:
nim-options: [
""
]
test-program: [
"tests/testnative",
]
steps:
- uses: actions/checkout@v2
- name: Run
run: |
sudo apt-get update
sudo apt-get install -y lcov build-essential git curl
mkdir coverage
curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh
env MAKE="make -j${NPROC}" bash build_nim.sh Nim csources dist/nimble NimBinaries
export PATH="$PATH:$PWD/Nim/bin"
nimble install -y --depsOnly
export NIM_OPTIONS="--opt:speed -d:debug --verbosity:0 --hints:off --lineDir:on -d:chronicles_log_level=INFO --warning[CaseTransition]:off --warning[ObservableStores]:off --warning[LockLevel]:off --nimcache:nimcache --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage ${{ matrix.nim-options }}"
nim c $NIM_OPTIONS -r ${{ matrix.test-program }}
cd nimcache; rm *.c; cd ..
lcov --capture --directory nimcache --output-file coverage/coverage.info
lcov --extract coverage/coverage.info `pwd`/libp2p/* `pwd`/libp2p/**/* `pwd`/libp2p/**/**/* --output-file coverage/coverage.f.info
export COV_UUID=`cksum <<< "${{ matrix.test-program }} $NIM_OPTIONS" | cut -f 1 -d ' '`
genhtml coverage/coverage.f.info --output-directory coverage/$COV_UUID-output
echo ${{ matrix.test-program }} > coverage/$COV_UUID-nim_options.txt
echo $NIM_OPTIONS >> coverage/$COV_UUID-nim_options.txt
bash <(curl -s https://codecov.io/bash) -f coverage/coverage.f.info || echo "Codecov did not collect coverage reports"
- uses: actions/upload-artifact@master
with:
name: coverage
path: coverage
Filter:
runs-on: ubuntu-20.04
strategy:
matrix:
nim-options: [
"",
"-d:libp2p_pki_schemes=secp256k1",
"-d:libp2p_pki_schemes=secp256k1;ed25519",
"-d:libp2p_pki_schemes=secp256k1;ed25519;ecnist",
]
test-program: [
"tests/testpkifilter",
]
steps:
- uses: actions/checkout@v2
- name: Run
run: |
sudo apt-get update
sudo apt-get install -y lcov build-essential git curl
mkdir coverage
curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh
env MAKE="make -j${NPROC}" bash build_nim.sh Nim csources dist/nimble NimBinaries
export PATH="$PATH:$PWD/Nim/bin"
nimble install -y --depsOnly
export NIM_OPTIONS="--opt:speed -d:debug --verbosity:0 --hints:off --lineDir:on -d:chronicles_log_level=INFO --warning[CaseTransition]:off --warning[ObservableStores]:off --warning[LockLevel]:off --nimcache:nimcache --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage ${{ matrix.nim-options }}"
nim c $NIM_OPTIONS -r ${{ matrix.test-program }}
cd nimcache; rm *.c; cd ..
lcov --capture --directory nimcache --output-file coverage/coverage.info
lcov --extract coverage/coverage.info `pwd`/libp2p/* `pwd`/libp2p/**/* `pwd`/libp2p/**/**/* --output-file coverage/coverage.f.info
export COV_UUID=`cksum <<< "${{ matrix.test-program }} $NIM_OPTIONS" | cut -f 1 -d ' '`
genhtml coverage/coverage.f.info --output-directory coverage/$COV_UUID-output
echo ${{ matrix.test-program }} > coverage/$COV_UUID-nim_options.txt
echo $NIM_OPTIONS >> coverage/$COV_UUID-nim_options.txt
bash <(curl -s https://codecov.io/bash) -f coverage/coverage.f.info || echo "Codecov did not collect coverage reports"
- uses: actions/upload-artifact@master
with:
name: coverage
path: coverage

70
.github/workflows/coverage.yml vendored Normal file
View File

@@ -0,0 +1,70 @@
name: Coverage
on:
# On push to common branches, this computes the coverage that PRs will use for diff
push:
branches:
- master
pull_request:
merge_group:
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
codecov:
name: Run coverage and upload to codecov
runs-on: ubuntu-22.04
env:
CICOV: YES
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Setup Nim
uses: "./.github/actions/install_nim"
with:
os: linux
cpu: amd64
shell: bash
- name: Restore deps from cache
id: deps-cache
uses: actions/cache@v4
with:
path: nimbledeps
key: nimbledeps-${{ hashFiles('.pinned') }}
- name: Install deps
if: ${{ steps.deps-cache.outputs.cache-hit != 'true' }}
run: |
nimble install_pinned
- name: Setup coverage
run: |
sudo apt-get update
sudo apt-get install -y lcov build-essential git curl
mkdir coverage
- name: Run test suite with coverage flags
run: |
export NIMFLAGS="-d:libp2p_quic_support --lineDir:on --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage"
nimble testnative
nimble testpubsub
nimble testfilter
- name: Run coverage
run: |
find nimcache -name *.c -delete
lcov --capture --directory nimcache --output-file coverage/coverage.info
shopt -s globstar
ls `pwd`/libp2p/{*,**/*}.nim
lcov --extract coverage/coverage.info `pwd`/libp2p/{*,**/*}.nim --output-file coverage/coverage.f.info
genhtml coverage/coverage.f.info --output-directory coverage/output
- name: Upload coverage to codecov
run: |
bash <(curl -s https://codecov.io/bash) -f coverage/coverage.f.info || echo "Codecov did not collect coverage reports"

14
.github/workflows/daily_amd64.yml vendored Normal file
View File

@@ -0,0 +1,14 @@
name: Daily amd64
on:
schedule:
- cron: "30 6 * * *"
workflow_dispatch:
jobs:
test_amd64:
name: Daily amd64
uses: ./.github/workflows/daily_common.yml
with:
nim: "[{'ref': 'version-1-6', 'memory_management': 'refc'}, {'ref': 'version-2-0', 'memory_management': 'refc'}]"
cpu: "['amd64']"

101
.github/workflows/daily_common.yml vendored Normal file
View File

@@ -0,0 +1,101 @@
name: Daily Common
# Serves as base workflow for daily tasks, it's not run by itself.
on:
workflow_call:
inputs:
nim:
description: 'Nim Configuration'
required: true
type: string # Following this format: [{"ref": ..., "memory_management": ...}, ...]
cpu:
description: 'CPU'
required: true
type: string
exclude:
description: 'Exclude matrix configurations'
required: false
type: string
default: "[]"
use_sat_solver:
description: 'Install dependencies with SAT Solver'
required: false
type: boolean
default: false
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
delete_cache:
name: Delete github action's branch cache
runs-on: ubuntu-latest
steps:
- uses: snnaplab/delete-branch-cache-action@v1
test:
needs: delete_cache
timeout-minutes: 40
strategy:
fail-fast: false
matrix:
platform:
- os: linux
builder: ubuntu-22.04
shell: bash
- os: macos
builder: macos-13
shell: bash
- os: windows
builder: windows-2022
shell: msys2 {0}
nim: ${{ fromJSON(inputs.nim) }}
cpu: ${{ fromJSON(inputs.cpu) }}
exclude: ${{ fromJSON(inputs.exclude) }}
defaults:
run:
shell: ${{ matrix.platform.shell }}
name: '${{ matrix.platform.os }}-${{ matrix.cpu }} (Nim ${{ matrix.nim.ref }})'
runs-on: ${{ matrix.platform.builder }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup Nim
uses: "./.github/actions/install_nim"
with:
os: ${{ matrix.platform.os }}
shell: ${{ matrix.platform.shell }}
nim_ref: ${{ matrix.nim.ref }}
cpu: ${{ matrix.cpu }}
- name: Setup Go
uses: actions/setup-go@v5
with:
go-version: '~1.16.0'
cache: false
- name: Install p2pd
run: |
V=1 bash scripts/build_p2pd.sh p2pdCache 124530a3
- name: Install dependencies
run: |
nimble install -y --depsOnly
- name: Run tests
run: |
nim --version
nimble --version
if [[ "${{ inputs.use_sat_solver }}" == "true" ]]; then
dependency_solver="sat"
else
dependency_solver="legacy"
fi
export NIMFLAGS="${NIMFLAGS} -d:libp2p_quic_support --mm:${{ matrix.nim.memory_management }} --solver:${dependency_solver}"
nimble test

14
.github/workflows/daily_devel.yml vendored Normal file
View File

@@ -0,0 +1,14 @@
name: Daily Nim Devel
on:
schedule:
- cron: "30 6 * * *"
workflow_dispatch:
jobs:
test_nim_devel:
name: Daily Nim Devel
uses: ./.github/workflows/daily_common.yml
with:
nim: "[{'ref': 'devel', 'memory_management': 'orc'}]"
cpu: "['amd64']"

15
.github/workflows/daily_i386.yml vendored Normal file
View File

@@ -0,0 +1,15 @@
name: Daily i386
on:
schedule:
- cron: "30 6 * * *"
workflow_dispatch:
jobs:
test_i386:
name: Daily i386 (Linux)
uses: ./.github/workflows/daily_common.yml
with:
nim: "[{'ref': 'version-1-6', 'memory_management': 'refc'}, {'ref': 'version-2-0', 'memory_management': 'refc'}, {'ref': 'devel', 'memory_management': 'orc'}]"
cpu: "['i386']"
exclude: "[{'platform': {'os':'macos'}}, {'platform': {'os':'windows'}}]"

15
.github/workflows/daily_sat.yml vendored Normal file
View File

@@ -0,0 +1,15 @@
name: Daily SAT
on:
schedule:
- cron: "30 6 * * *"
workflow_dispatch:
jobs:
test_amd64:
name: Daily SAT
uses: ./.github/workflows/daily_common.yml
with:
nim: "[{'ref': 'version-2-0', 'memory_management': 'refc'}]"
cpu: "['amd64']"
use_sat_solver: true

53
.github/workflows/dependencies.yml vendored Normal file
View File

@@ -0,0 +1,53 @@
name: Dependencies
on:
push:
branches:
- master
workflow_dispatch:
jobs:
bumper:
# Pushes new refs to interested external repositories, so they can do early testing against libp2p's newer versions
runs-on: ubuntu-latest
name: Bump libp2p's version for ${{ matrix.target.repository }}:${{ matrix.target.ref }}
strategy:
fail-fast: false
matrix:
target:
- repository: status-im/nimbus-eth2
ref: unstable
token: ${{ secrets.ACTIONS_GITHUB_TOKEN_NIMBUS_ETH2 }}
- repository: waku-org/nwaku
ref: master
token: ${{ secrets.ACTIONS_GITHUB_TOKEN_NWAKU }}
- repository: codex-storage/nim-codex
ref: master
token: ${{ secrets.ACTIONS_GITHUB_TOKEN_NIM_CODEX }}
steps:
- name: Clone target repository
uses: actions/checkout@v4
with:
repository: ${{ matrix.target.repository }}
ref: ${{ matrix.target.ref}}
path: nbc
fetch-depth: 0
token: ${{ matrix.target.token }}
- name: Checkout this ref in target repository
run: |
cd nbc
git submodule update --init vendor/nim-libp2p
cd vendor/nim-libp2p
git checkout $GITHUB_SHA
- name: Push this ref to target repository
run: |
cd nbc
git config --global user.email "${{ github.actor }}@users.noreply.github.com"
git config --global user.name = "${{ github.actor }}"
git commit --allow-empty -a -m "auto-bump nim-libp2p"
git branch -D nim-libp2p-auto-bump-${{ matrix.target.ref }} || true
git switch -c nim-libp2p-auto-bump-${{ matrix.target.ref }}
git push -f origin nim-libp2p-auto-bump-${{ matrix.target.ref }}

111
.github/workflows/documentation.yml vendored Normal file
View File

@@ -0,0 +1,111 @@
name: Documentation Generation And Publishing
on:
push:
branches:
- master
workflow_dispatch:
jobs:
build:
timeout-minutes: 20
name: 'Generate & upload documentation'
runs-on: ubuntu-latest
continue-on-error: true
steps:
- name: Checkout
uses: actions/checkout@v4
with:
submodules: true
- uses: jiro4989/setup-nim-action@v1
with:
nim-version: '1.6.x'
- name: Generate doc
run: |
nim --version
nimble --version
nimble install_pinned
# nim doc can "fail", but the doc is still generated
nim doc --git.url:https://github.com/vacp2p/nim-libp2p --git.commit:${GITHUB_REF##*/} --outdir:${GITHUB_REF##*/} --project libp2p || true
# check that the folder exists
ls ${GITHUB_REF##*/}
- name: Clone the gh-pages branch
uses: actions/checkout@v4
with:
repository: vacp2p/nim-libp2p
ref: gh-pages
path: subdoc
submodules: true
fetch-depth: 0
- name: Commit & push
run: |
cd subdoc
# Update / create this branch doc
rm -rf ${GITHUB_REF##*/}
mv ../${GITHUB_REF##*/} .
# Remove .idx files
# NOTE: git also uses idx files in his
# internal folder, hence the `*` instead of `.`
find * -name "*.idx" -delete
git add .
git config --global user.email "${{ github.actor }}@users.noreply.github.com"
git config --global user.name = "${{ github.actor }}"
git commit -a -m "update docs for ${GITHUB_REF##*/}"
git push origin gh-pages
update_site:
name: 'Rebuild website'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- uses: actions/setup-python@v2
with:
python-version: 3.x
- uses: jiro4989/setup-nim-action@v1
with:
nim-version: 'stable'
- name: Generate website
run: pip install mkdocs-material && nimble -y website
- name: Clone the gh-pages branch
uses: actions/checkout@v4
with:
repository: vacp2p/nim-libp2p
ref: gh-pages
path: subdoc
fetch-depth: 0
- name: Commit & push
run: |
cd subdoc
# Ensure the latest changes are fetched and reset to the remote branch
git fetch origin gh-pages
git reset --hard origin/gh-pages
rm -rf docs
mv ../site docs
git add .
if git diff-index --quiet HEAD --; then
echo "No changes to commit"
else
git config --global user.email "${{ github.actor }}@users.noreply.github.com"
git config --global user.name "${{ github.actor }}"
git commit -m "update website"
git push origin gh-pages
fi

60
.github/workflows/examples.yml vendored Normal file
View File

@@ -0,0 +1,60 @@
name: Examples
on:
push:
branches:
- master
pull_request:
merge_group:
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
examples:
timeout-minutes: 30
strategy:
fail-fast: false
defaults:
run:
shell: bash
name: "Build Examples"
runs-on: ubuntu-22.04
steps:
- name: Checkout
uses: actions/checkout@v4
with:
submodules: true
- name: Setup Nim
uses: "./.github/actions/install_nim"
with:
shell: bash
os: linux
cpu: amd64
nim_ref: version-1-6
- name: Restore deps from cache
id: deps-cache
uses: actions/cache@v3
with:
path: nimbledeps
key: nimbledeps-${{ hashFiles('.pinned') }}
- name: Install deps
if: ${{ steps.deps-cache.outputs.cache-hit != 'true' }}
run: |
nimble install_pinned
- name: Build and run examples
run: |
nim --version
nimble --version
gcc --version
NIMFLAGS="${NIMFLAGS} --mm:${{ matrix.nim.memory_management }}"
nimble examples

60
.github/workflows/interop.yml vendored Normal file
View File

@@ -0,0 +1,60 @@
name: Interoperability Tests
on:
pull_request:
merge_group:
push:
branches:
- master
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
run-transport-interop:
name: Run transport interoperability tests
runs-on: ubuntu-22.04
steps:
- name: Free Disk Space
# For some reason we have space issues while running this action. Likely while building the image.
# This action will free up some space to avoid the issue.
uses: jlumbroso/free-disk-space@v1.3.1
with:
tool-cache: true
- uses: actions/checkout@v4
- uses: docker/setup-buildx-action@v3
- name: Build image
run: docker buildx build --load -t nim-libp2p-head -f tests/transport-interop/Dockerfile .
- name: Run tests
uses: libp2p/test-plans/.github/actions/run-transport-interop-test@master
with:
test-filter: nim-libp2p-head
# without suffix action fails because "hole-punching-interop" artifacts have
# the same name as "transport-interop" artifacts
test-results-suffix: transport-interop
extra-versions: ${{ github.workspace }}/tests/transport-interop/version.json
s3-cache-bucket: ${{ vars.S3_LIBP2P_BUILD_CACHE_BUCKET_NAME }}
s3-access-key-id: ${{ vars.S3_LIBP2P_BUILD_CACHE_AWS_ACCESS_KEY_ID }}
s3-secret-access-key: ${{ secrets.S3_LIBP2P_BUILD_CACHE_AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ vars.S3_LIBP2P_BUILD_CACHE_AWS_REGION }}
run-hole-punching-interop:
name: Run hole-punching interoperability tests
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v4
- uses: docker/setup-buildx-action@v3
- name: Build image
run: docker buildx build --load -t nim-libp2p-head -f tests/hole-punching-interop/Dockerfile .
- name: Run tests
uses: libp2p/test-plans/.github/actions/run-interop-hole-punch-test@master
with:
test-filter: nim-libp2p-head
extra-versions: ${{ github.workspace }}/tests/hole-punching-interop/version.json
s3-cache-bucket: ${{ vars.S3_LIBP2P_BUILD_CACHE_BUCKET_NAME }}
s3-access-key-id: ${{ vars.S3_LIBP2P_BUILD_CACHE_AWS_ACCESS_KEY_ID }}
s3-secret-access-key: ${{ secrets.S3_LIBP2P_BUILD_CACHE_AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ vars.S3_LIBP2P_BUILD_CACHE_AWS_REGION }}

27
.github/workflows/linters.yml vendored Normal file
View File

@@ -0,0 +1,27 @@
name: Linters
on:
pull_request:
merge_group:
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
nph:
name: NPH
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
fetch-depth: 2 # In PR, has extra merge commit: ^1 = PR, ^2 = base
- name: Check `nph` formatting
uses: arnetheduck/nph-action@v1
with:
version: 0.6.1
options: "examples libp2p tests tools *.nim*"
fail: true
suggest: true

View File

@@ -1,46 +0,0 @@
name: NBC Bump PR
on:
push:
branches:
- master
jobs:
bumpNBC:
runs-on: ubuntu-latest
steps:
- uses: status-im/github-app-token@v1
name: Generate token
id: generate-token
with:
app_id: ${{ secrets.BUMP_BOT_APP_ID }}
private_key: ${{ secrets.BUMP_BOT_APP_PRIVATE_KEY }}
- name: Clone NBC
uses: actions/checkout@v2
with:
repository: status-im/nimbus-eth2
ref: unstable
path: nbc
submodules: true
fetch-depth: 0
- name: Checkout this ref
run: |
cd nbc/vendor/nim-libp2p
git checkout $GITHUB_SHA
- name: Commit this bump
run: |
cd nbc
git config --global user.email "${{ github.actor }}@users.noreply.github.com"
git config --global user.name = "${{ github.actor }}"
git commit -a -m "auto-bump nim-libp2p"
- name: Make PR
uses: peter-evans/create-pull-request@v3.5.0
with:
branch: nim-libp2p-auto-bump
path: nbc
token: ${{ steps.generate-token.outputs.token }}
title: nim-libp2p auto bump

35
.github/workflows/pr_lint.yml vendored Normal file
View File

@@ -0,0 +1,35 @@
name: "Conventional Commits"
on:
pull_request:
types:
- opened
- edited
- reopened
- synchronize
jobs:
main:
name: Validate PR title
runs-on: ubuntu-latest
permissions:
pull-requests: write
steps:
- uses: amannn/action-semantic-pull-request@v5
id: lint_pr_title
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- uses: marocchino/sticky-pull-request-comment@v2
# When the previous steps fails, the workflow would stop. By adding this
# condition you can continue the execution with the populated error message.
if: always() && (steps.lint_pr_title.outputs.error_message != null)
with:
header: pr-title-lint-error
message: |
Pull requests titles must follow the [Conventional Commits specification](https://www.conventionalcommits.org/en/v1.0.0/)
# Delete a previous comment when the issue has been resolved
- if: ${{ steps.lint_pr_title.outputs.error_message == null }}
uses: marocchino/sticky-pull-request-comment@v2
with:
header: pr-title-lint-error
delete: true

13
.gitignore vendored
View File

@@ -1,4 +1,5 @@
nimcache/
nimbledeps/
# Executables shall be put in an ignored build/ directory
# Ignore dynamic, static libs and libtool archive files
@@ -12,3 +13,15 @@ build/
.vscode/
.DS_Store
tests/pubsub/testgossipsub
examples/*.md
nimble.develop
nimble.paths
go-libp2p-daemon/
# Ignore all test build files in tests folder (auto generated when running tests).
# First rule (`tests/**/test*[^.]*`) will ignore all binaries: has prefix test + does not have dot in name.
# Second and third rules are here to un-ignores all files with extension and Docker file,
# because it appears that vs code is skipping text search is some tests files without these rules.
tests/**/test*[^.]*
!tests/**/*.*
!tests/**/Dockerfile

19
.pinned Normal file
View File

@@ -0,0 +1,19 @@
bearssl;https://github.com/status-im/nim-bearssl@#34d712933a4e0f91f5e66bc848594a581504a215
chronicles;https://github.com/status-im/nim-chronicles@#32ac8679680ea699f7dbc046e8e0131cac97d41a
chronos;https://github.com/status-im/nim-chronos@#c04576d829b8a0a1b12baaa8bc92037501b3a4a0
dnsclient;https://github.com/ba0f3/dnsclient.nim@#23214235d4784d24aceed99bbfe153379ea557c8
faststreams;https://github.com/status-im/nim-faststreams@#c51315d0ae5eb2594d0bf41181d0e1aca1b3c01d
httputils;https://github.com/status-im/nim-http-utils@#79cbab1460f4c0cdde2084589d017c43a3d7b4f1
json_serialization;https://github.com/status-im/nim-json-serialization@#2b1c5eb11df3647a2cee107cd4cce3593cbb8bcf
metrics;https://github.com/status-im/nim-metrics@#6142e433fc8ea9b73379770a788017ac528d46ff
ngtcp2;https://github.com/status-im/nim-ngtcp2@#9456daa178c655bccd4a3c78ad3b8cce1f0add73
nimcrypto;https://github.com/cheatfate/nimcrypto@#19c41d6be4c00b4a2c8000583bd30cf8ceb5f4b1
quic;https://github.com/status-im/nim-quic.git@#a6c30263c95fc5ddb2ef4d197c09b282555c06b0
results;https://github.com/arnetheduck/nim-results@#df8113dda4c2d74d460a8fa98252b0b771bf1f27
secp256k1;https://github.com/status-im/nim-secp256k1@#f808ed5e7a7bfc42204ec7830f14b7a42b63c284
serialization;https://github.com/status-im/nim-serialization@#548d0adc9797a10b2db7f788b804330306293088
stew;https://github.com/status-im/nim-stew@#0db179256cf98eb9ce9ee7b9bc939f219e621f77
testutils;https://github.com/status-im/nim-testutils@#9e842bd58420d23044bc55e16088e8abbe93ce51
unittest2;https://github.com/status-im/nim-unittest2@#8b51e99b4a57fcfb31689230e75595f024543024
websock;https://github.com/status-im/nim-websock@#f8ed9b40a5ff27ad02a3c237c4905b0924e3f982
zlib;https://github.com/status-im/nim-zlib@#daa8723fd32299d4ca621c837430c29a5a11e19a

233
README.md
View File

@@ -1,13 +1,13 @@
<h1 align="center">
<a href="https://libp2p.io"><img width="250" src="https://github.com/libp2p/libp2p/blob/master/logo/black-bg-2.png?raw=true" alt="libp2p hex logo" /></a>
<a href="https://libp2p.io"><img width="250" src="./.assets/full-logo.svg?raw=true" alt="nim-libp2p logo" /></a>
</h1>
<h3 align="center">The Nim implementation of the libp2p Networking Stack.</h3>
<h3 align="center">The <a href="https://nim-lang.org/">Nim</a> implementation of the <a href="https://libp2p.io/">libp2p</a> Networking Stack.</h3>
<p align="center">
<a href="https://travis-ci.org/status-im/nim-libp2p"><img src="https://travis-ci.org/status-im/nim-libp2p.svg?branch=master" /></a>
<a href="https://ci.appveyor.com/project/nimbus/nim-libp2p/branch/master"><img src="https://ci.appveyor.com/api/projects/status/pqgif5bcie6cp3wi/branch/master?svg=true" /></a>
<a href="https://dev.azure.com/nimbus-dev/nim-libp2p/_build?definitionId=5&branchName=master"><img src="https://img.shields.io/azure-devops/build/nimbus-dev/dc5eed24-3f6c-4c06-8466-3d060abd6c8b/5/master?label=Azure%20%28Linux%2064-bit%2C%20Windows%2032-bit%2F64-bit%2C%20MacOS%2064-bit%29" /></a>
<a href="https://github.com/vacp2p/nim-libp2p/actions"><img src="https://github.com/vacp2p/nim-libp2p/actions/workflows/ci.yml/badge.svg" /></a>
<a href="https://codecov.io/gh/vacp2p/nim-libp2p"><img src="https://codecov.io/gh/vacp2p/nim-libp2p/branch/master/graph/badge.svg?token=UR5JRQ249W"/></a>
</p>
<p align="center">
@@ -16,168 +16,166 @@
<img src="https://img.shields.io/badge/nim-%3E%3D1.2.0-orange.svg?style=flat-square" />
</p>
## Introduction
An implementation of [libp2p](https://libp2p.io/) in Nim. Also provides a Nim wrapper of the [Libp2p Go daemon](https://github.com/libp2p/go-libp2p).
## Project Status
The current native Nim libp2p implementation support is experimental and shouldn't be relied on for production use. It is under active development and contributions are highly welcomed. :)
Check our [examples folder](/examples) to get started!
# Table of Contents
- [Background](#background)
- [Install](#install)
- [Prerequisite](#prerequisite)
- [Usage](#usage)
- [API](#api)
- [Getting Started](#getting-started)
- [Tutorials and Examples](#tutorials-and-examples)
- [Using the Go Daemon](#using-the-go-daemon)
- [Getting Started](#getting-started)
- [Testing](#testing)
- [Modules](#modules)
- [Users](#users)
- [Stability](#stability)
- [Development](#development)
- [Tests](#tests)
- [Packages](#packages)
- [Contribute](#contribute)
- [Core Developers](#core-developers)
- [Contribute](#contribute)
- [Contributors](#contributors)
- [Core Maintainers](#core-maintainers)
- [License](#license)
## Background
libp2p is a networking stack and library modularized out of [The IPFS Project](https://github.com/ipfs/ipfs), and bundled separately for other tools to use.
libp2p is a [Peer-to-Peer](https://en.wikipedia.org/wiki/Peer-to-peer) networking stack, with [implementations](https://github.com/libp2p/libp2p#implementations) in multiple languages derived from the same [specifications.](https://github.com/libp2p/specs)
libp2p is the product of a long and arduous quest of understanding; a deep dive into the internet's network stack and the peer-to-peer protocols from the past. Building large scale peer-to-peer systems has been complex and difficult in the last 15 years and libp2p is a way to fix that. It is a "network stack", a suite of networking protocols that cleanly separates concerns and enables sophisticated applications to only use the protocols they absolutely need, without giving up interoperability and upgradeability.
Building large scale peer-to-peer systems has been complex and difficult in the last 15 years and libp2p is a way to fix that. It strives to be a modular stack with secure defaults and useful protocols, while remaining open and extensible.
This is a native Nim implementation, using [chronos](https://github.com/status-im/nim-chronos) for asynchronous execution. It's used in production by a few [projects](#users)
libp2p grew out of IPFS, but it is built so that lots of people can use it, for lots of different projects.
- Learn more about libp2p at [**libp2p.io**](https://libp2p.io) and follow our evolving documentation efforts at [**docs.libp2p.io**](https://docs.libp2p.io).
- [Here](https://github.com/libp2p/libp2p#description) is an overview of libp2p and its implementations in other programming languages.
Learn more about libp2p at [**libp2p.io**](https://libp2p.io) and follow libp2p's documentation [**docs.libp2p.io**](https://docs.libp2p.io).
## Install
**Prerequisite**
- [Nim](https://nim-lang.org/install.html)
> The currently supported Nim versions are 1.6, 2.0 and 2.2.
```
nimble install libp2p
```
### Prerequisite
- [Nim](https://nim-lang.org/install.html)
## Usage
## Getting Started
You'll find the nim-libp2p documentation [here](https://vacp2p.github.io/nim-libp2p/docs/). See [examples](./examples) for simple usage patterns.
### API
The specification is available in the [docs/api](docs/api) folder.
### Testing
Remember you'll need to build the `go-libp2p-daemon` binary to run the `nim-libp2p` tests.
To do so, please follow the installation instructions in [daemonapi.md](examples/go-daemon/daemonapi.md).
### Getting Started
Please read the [GETTING_STARTED.md](docs/GETTING_STARTED.md) guide.
## Modules
List of packages modules implemented in nim-libp2p:
### Tutorials and Examples
Example code can be found in the [examples folder](/examples).
| Name | Description |
| ---------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------- |
| **Libp2p** | |
| [libp2p](libp2p/switch.nim) | The core of the project |
| [connmanager](libp2p/connmanager.nim) | Connection manager |
| [identify / push identify](libp2p/protocols/identify.nim) | [Identify](https://docs.libp2p.io/concepts/fundamentals/protocols/#identify) protocol |
| [ping](libp2p/protocols/ping.nim) | [Ping](https://docs.libp2p.io/concepts/fundamentals/protocols/#ping) protocol |
| [libp2p-daemon-client](libp2p/daemon/daemonapi.nim) | [go-daemon](https://github.com/libp2p/go-libp2p-daemon) nim wrapper |
| [interop-libp2p](tests/testinterop.nim) | Interop tests |
| **Transports** | |
| [libp2p-tcp](libp2p/transports/tcptransport.nim) | TCP transport |
| [libp2p-ws](libp2p/transports/wstransport.nim) | WebSocket & WebSocket Secure transport |
| [libp2p-tor](libp2p/transports/tortransport.nim) | Tor Transport |
| [libp2p-quic](libp2p/transports/quictransport.nim) | Quic Transport |
| [libp2p-memory](libp2p/transports/memorytransport.nim) | Memory Transport |
| **Secure Channels** | |
| [libp2p-noise](libp2p/protocols/secure/noise.nim) | [Noise](https://docs.libp2p.io/concepts/secure-comm/noise/) secure channel |
| [libp2p-plaintext](libp2p/protocols/secure/plaintext.nim) | Plain Text for development purposes |
| **Stream Multiplexers** | |
| [libp2p-mplex](libp2p/muxers/mplex/mplex.nim) | [MPlex](https://github.com/libp2p/specs/tree/master/mplex) multiplexer |
| [libp2p-yamux](libp2p/muxers/yamux/yamux.nim) | [Yamux](https://docs.libp2p.io/concepts/multiplex/yamux/) multiplexer |
| **Data Types** | |
| [peer-id](libp2p/peerid.nim) | [Cryptographic identifiers](https://docs.libp2p.io/concepts/fundamentals/peers/#peer-id) |
| [peer-store](libp2p/peerstore.nim) | [Address book of known peers](https://docs.libp2p.io/concepts/fundamentals/peers/#peer-store) |
| [multiaddress](libp2p/multiaddress.nim) | [Composable network addresses](https://github.com/multiformats/multiaddr) |
| [signed-envelope](libp2p/signed_envelope.nim) | [Signed generic data container](https://github.com/libp2p/specs/blob/master/RFC/0002-signed-envelopes.md) |
| [routing-record](libp2p/routing_record.nim) | [Signed peer dialing informations](https://github.com/libp2p/specs/blob/master/RFC/0003-routing-records.md) |
| [discovery manager](libp2p/discovery/discoverymngr.nim) | Discovery Manager |
| **Utilities** | |
| [libp2p-crypto](libp2p/crypto) | Cryptographic backend |
| [libp2p-crypto-secp256k1](libp2p/crypto/secp.nim) | |
| **Pubsub** | |
| [libp2p-pubsub](libp2p/protocols/pubsub/pubsub.nim) | Pub-Sub generic interface |
| [libp2p-floodsub](libp2p/protocols/pubsub/floodsub.nim) | FloodSub implementation |
| [libp2p-gossipsub](libp2p/protocols/pubsub/gossipsub.nim) | [GossipSub](https://docs.libp2p.io/concepts/publish-subscribe/) implementation |
#### Direct Chat Tutorial
- [Part I](https://our.status.im/nim-libp2p-tutorial-a-peer-to-peer-chat-example-1/): Set up the main function and use multi-thread for processing IO.
- [Part II](https://our.status.im/nim-libp2p-tutorial-a-peer-to-peer-chat-example-2/): Dial remote peer and allow customized user input commands.
- [Part III](https://our.status.im/nim-libp2p-tutorial-a-peer-to-peer-chat-example-3/): Configure and establish a libp2p node.
## Users
nim-libp2p is used by:
- [Nimbus](https://github.com/status-im/nimbus-eth2), an Ethereum client
- [nwaku](https://github.com/waku-org/nwaku), a decentralized messaging application
- [nim-codex](https://github.com/codex-storage/nim-codex), a decentralized storage application
- (open a pull request if you want to be included here)
### Using the Go Daemon
Please find the installation and usage intructions in [daemonapi.md](docs/api/libp2p/daemonapi.md).
## Stability
nim-libp2p has been used in production for over a year in high-stake scenarios, so its core is considered stable.
Some modules are more recent and less stable.
Examples can be found in the [examples/go-daemon folder](https://github.com/status-im/nim-libp2p/tree/readme/examples/go-daemon);
The versioning follows [semver](https://semver.org/), with some additions:
- Some of libp2p procedures are marked as `.public.`, they will remain compatible during each `MAJOR` version
- The rest of the procedures are considered internal, and can change at any `MINOR` version (but remain compatible for each new `PATCH`)
We aim to be compatible at all time with at least 2 Nim `MINOR` versions, currently `1.6 & 2.0`
## Development
**Clone and Install dependencies:**
Clone and Install dependencies:
```sh
git clone https://github.com/status-im/nim-libp2p
git clone https://github.com/vacp2p/nim-libp2p
cd nim-libp2p
nimble install
# to use dependencies computed by nimble
nimble install -dy
# OR to install the dependencies versions used in CI
nimble install_pinned
```
### Tests
#### Prerequisite
- [Go 1.12+](https://golang.org/dl/)
#### Run unit tests
Run unit tests:
```sh
# run all the unit tests
nimble test
```
This requires the go daemon to be available. To only run native tests, use `nimble testnative`.
Or use `nimble tasks` to show all available tasks.
### Packages
### Contribute
List of packages currently in existence for nim-libp2p:
The libp2p implementation in Nim is a work in progress. We welcome contributors to help out! Specifically, you can:
- Go through the modules and **check out existing issues**. This would be especially useful for modules in active development. Some knowledge of IPFS/libp2p may be required, as well as the infrastructure behind it.
- **Perform code reviews**. Feel free to let us know if you found anything that can a) speed up the project development b) ensure better quality and c) reduce possible future bugs.
- **Add tests**. Help nim-libp2p to be more robust by adding more tests to the [tests folder](tests/).
- **Small PRs**. Try to keep PRs atomic and digestible. This makes the review process and pinpointing bugs easier.
- **Code format**. Please format code using [nph](https://github.com/arnetheduck/nph) v0.5.1. This will ensure a consistent codebase and make PRs easier to review. A CI rule has been added to ensure that future commits are all formatted using the same nph version.
The code follows the [Status Nim Style Guide](https://status-im.github.io/nim-style-guide/).
#### Libp2p
- [libp2p](https://github.com/status-im/nim-libp2p)
- [libp2p-daemon-client](https://github.com/status-im/nim-libp2p/blob/master/libp2p/daemon/daemonapi.nim)
- [interop-libp2p](https://github.com/status-im/nim-libp2p/blob/master/tests/testinterop.nim)
### Contributors
<a href="https://github.com/vacp2p/nim-libp2p/graphs/contributors"><img src="https://contrib.rocks/image?repo=vacp2p/nim-libp2p" alt="nim-libp2p contributors"></a>
#### Transports
- [libp2p-tcp](https://github.com/status-im/nim-libp2p/blob/master/libp2p/transports/tcptransport.nim)
### Core Maintainers
<table>
<tbody>
<tr>
<td align="center"><a href="https://github.com/richard-ramos"><img src="https://avatars.githubusercontent.com/u/1106587?v=4?s=100" width="100px;" alt="Richard"/><br /><sub><b>Richard</b></sub></a></td>
<td align="center"><a href="https://github.com/vladopajic"><img src="https://avatars.githubusercontent.com/u/4353513?v=4?s=100" width="100px;" alt="Vlado"/><br /><sub><b>Vlado</b></sub></a></td>
<td align="center"><a href="https://github.com/gmelodie"><img src="https://avatars.githubusercontent.com/u/8129788?v=4?s=100" width="100px;" alt="Gabe"/><br /><sub><b>Gabe</b></sub></a></td>
</tr>
</tbody>
</table>
#### Secure Channels
- [libp2p-secio](https://github.com/status-im/nim-libp2p/blob/master/libp2p/protocols/secure/secio.nim)
### Compile time flags
#### Stream Multiplexers
- [libp2p-mplex](https://github.com/status-im/nim-libp2p/blob/master/libp2p/muxers/mplex/mplex.nim)
#### Utilities
- [libp2p-crypto](https://github.com/status-im/nim-libp2p/tree/master/libp2p/crypto)
- [libp2p-crypto-secp256k1](https://github.com/status-im/nim-libp2p/blob/master/libp2p/crypto/secp.nim)
#### Data Types
- [peer-id](https://github.com/status-im/nim-libp2p/blob/master/libp2p/peer.nim)
- [peer-info](https://github.com/status-im/nim-libp2p/blob/master/libp2p/peerinfo.nim)
#### Pubsub
- [libp2p-pubsub](https://github.com/status-im/nim-libp2p/blob/master/libp2p/protocols/pubsub/pubsub.nim)
- [libp2p-floodsub](https://github.com/status-im/nim-libp2p/blob/master/libp2p/protocols/pubsub/floodsub.nim)
- [libp2p-gossipsub](https://github.com/status-im/nim-libp2p/blob/master/libp2p/protocols/pubsub/gossipsub.nim)
Packages that exist in the original libp2p specs and are under active development:
- libp2p-daemon
- libp2p-webrtc-direct
- libp2p-webrtc-star
- libp2p-websockets
- libp2p-spdy
- libp2p-bootstrap
- libp2p-kad-dht
- libp2p-mdns
- libp2p-webrtc-star
- libp2p-delegated-content-routing
- libp2p-delegated-peer-routing
- libp2p-nat-mgnr
- libp2p-utils
** Note that the current stack reflects the minimal requirements for the upcoming Eth2 implementation.
### Tips and tricks
#### enable expensive metrics:
Enable quic transport support
```bash
nim c -d:libp2p_quic_support some_file.nim
```
Enable expensive metrics (ie, metrics with per-peer cardinality):
```bash
nim c -d:libp2p_expensive_metrics some_file.nim
```
#### use identify metrics
Set list of known libp2p agents for metrics:
```bash
nim c -d:libp2p_agents_metrics -d:KnownLibP2PAgents=nimbus,lighthouse,prysm,teku some_file.nim
nim c -d:libp2p_agents_metrics -d:KnownLibP2PAgents=nimbus,lighthouse,lodestar,prysm,teku some_file.nim
```
### specify gossipsub specific topics to measure
Specify gossipsub specific topics to measure in the metrics:
```bash
nim c -d:KnownLibP2PTopics=topic1,topic2,topic3 some_file.nim
```
## Contribute
The libp2p implementation in Nim is a work in progress. We welcome contributors to help out! Specifically, you can:
- Go through the modules and **check out existing issues**. This would be especially useful for modules in active development. Some knowledge of IPFS/libp2p may be required, as well as the infrastructure behind it.
- **Perform code reviews**. Feel free to let us know if you found anything that can a) speed up the project development b) ensure better quality and c) reduce possible future bugs.
- **Add tests**. Help nim-libp2p to be more robust by adding more tests to the [tests folder](https://github.com/status-im/nim-libp2p/tree/master/tests).
### Core Developers
[@cheatfate](https://github.com/cheatfate), [Dmitriy Ryajov](https://github.com/dryajov), [Giovanni Petrantoni](https://github.com/sinkingsugar), [Zahary Karadjov](https://github.com/zah)
## License
Licensed and distributed under either of
@@ -189,4 +187,3 @@ or
* Apache License, Version 2.0, ([LICENSE-APACHEv2](LICENSE-APACHEv2) or http://www.apache.org/licenses/LICENSE-2.0)
at your option. These files may not be copied, modified, or distributed except according to those terms.

View File

@@ -1,14 +1,8 @@
codecov:
notify:
require_ci_to_pass: true
# must be the number of coverage report builds
# notice that this number is for PRs;
# like this we disabled notify on pure branches report
# which is fine I guess
after_n_builds: 25
comment:
layout: "reach, diff, flags, files"
after_n_builds: 25 # must be the number of coverage report builds
coverage:
status:
project:
@@ -16,4 +10,4 @@ coverage:
# basic settings
target: auto
threshold: 5%
base: auto
base: auto

28
config.nims Normal file
View File

@@ -0,0 +1,28 @@
# to allow locking
if dirExists("nimbledeps/pkgs"):
switch("NimblePath", "nimbledeps/pkgs")
if dirExists("nimbledeps/pkgs2"):
switch("NimblePath", "nimbledeps/pkgs2")
switch("warning", "CaseTransition:off")
switch("warning", "ObservableStores:off")
switch("warning", "LockLevel:off")
--styleCheck:
usages
switch("warningAsError", "UseBase:on")
--styleCheck:
error
--mm:
refc
# reconsider when there's a version-2-2 branch worth testing with as we might switch to orc
# Avoid some rare stack corruption while using exceptions with a SEH-enabled
# toolchain: https://github.com/status-im/nimbus-eth2/issues/3121
if defined(windows) and not defined(vcc):
--define:
nimRawSetjmp
# begin Nimble config (version 1)
when fileExists("nimble.paths"):
include "nimble.paths"
# end Nimble config

View File

@@ -1,3 +0,0 @@
# API
Coming Soon...

View File

@@ -1,98 +0,0 @@
# Getting Started
Welcome to nim-libp2p! This guide will walk you through a peer to peer chat example. <br>
The full code can be found in [directchat.nim](examples/directchat.nim) under the examples folder.
### Direct Chat Example
To run nim-libp2p, add it to your project's nimble file and spawn a node as follows:
```nim
import tables
import chronos
import ../libp2p/[switch,
multistream,
protocols/identify,
connection,
transports/transport,
transports/tcptransport,
multiaddress,
peerinfo,
crypto/crypto,
peerid,
protocols/protocol,
muxers/muxer,
muxers/mplex/mplex,
protocols/secure/secio,
protocols/secure/secure]
const TestCodec = "/test/proto/1.0.0" # custom protocol string
type
TestProto = ref object of LPProtocol # declare a custom protocol
method init(p: TestProto) {.gcsafe.} =
# handle incoming connections in closure
proc handle(conn: Connection, proto: string) {.async, gcsafe.} =
echo "Got from remote - ", cast[string](await conn.readLp(1024))
await conn.writeLp("Hello!")
await conn.close()
p.codec = TestCodec # init proto with the correct string id
p.handler = handle # set proto handler
proc createSwitch(ma: MultiAddress): (Switch, PeerInfo) =
## Helper to create a swith
let seckey = PrivateKey.random(RSA) # use a random key for peer id
var peerInfo = PeerInfo.init(seckey) # create a peer id and assign
peerInfo.addrs.add(ma) # set this peer's multiaddresses (can be any number)
let identify = newIdentify(peerInfo) # create the identify proto
proc createMplex(conn: Connection): Muxer =
# helper proc to create multiplexers,
# use this to perform any custom setup up,
# such as adjusting timeout or anything else
# that the muxer requires
result = newMplex(conn)
let mplexProvider = newMuxerProvider(createMplex, MplexCodec) # create multiplexer
let transports = @[Transport(newTransport(TcpTransport))] # add all transports (tcp only for now, but can be anything in the future)
let muxers = {MplexCodec: mplexProvider}.toTable() # add all muxers
let secureManagers = {SecioCodec: Secure(newSecio(seckey))}.toTable() # setup the secio and any other secure provider
# create the switch
let switch = newSwitch(peerInfo,
transports,
identify,
muxers,
secureManagers)
result = (switch, peerInfo)
proc main() {.async, gcsafe.} =
let ma1: MultiAddress = Multiaddress.init("/ip4/0.0.0.0/tcp/0")
let ma2: MultiAddress = Multiaddress.init("/ip4/0.0.0.0/tcp/0")
var peerInfo1, peerInfo2: PeerInfo
var switch1, switch2: Switch
(switch1, peerInfo1) = createSwitch(ma1) # create node 1
# setup the custom proto
let testProto = new TestProto
testProto.init() # run it's init method to perform any required initialization
switch1.mount(testProto) # mount the proto
var switch1Fut = await switch1.start() # start the node
(switch2, peerInfo2) = createSwitch(ma2) # create node 2
var switch2Fut = await switch2.start() # start second node
let conn = await switch2.dial(switch1.peerInfo, TestCodec) # dial the first node
await conn.writeLp("Hello!") # writeLp send a length prefixed buffer over the wire
# readLp reads length prefixed bytes and returns a buffer without the prefix
echo "Remote responded with - ", cast[string](await conn.readLp(1024))
await allFutures(switch1.stop(), switch2.stop()) # close connections and shutdown all transports
await allFutures(switch1Fut & switch2Fut) # wait for all transports to shutdown
waitFor(main())
```

View File

@@ -1,56 +0,0 @@
# Table of Contents
- [Introduction](#introduction)
- [Installation](#installation)
- [Usage](#usage)
- [Example](#example)
- [Getting Started](#getting-started)
# Introduction
This is a libp2p-backed daemon wrapping the functionalities of go-libp2p for use in Nim. <br>
For more information about the go daemon, check out [this repository](https://github.com/libp2p/go-libp2p-daemon).
# Installation
```sh
# clone and install dependencies
git clone https://github.com/status-im/nim-libp2p
cd nim-libp2p
nimble install
# perform unit tests
nimble test
# update the git submodule to install the go daemon
git submodule update --init --recursive
go version
git clone https://github.com/libp2p/go-libp2p-daemon
cd go-libp2p-daemon
git checkout v0.0.1
go install ./...
cd ..
```
# Usage
## Example
Examples can be found in the [examples folder](https://github.com/status-im/nim-libp2p/tree/readme/examples/go-daemon)
## Getting Started
Try out the chat example. Full code can be found [here](https://github.com/status-im/nim-libp2p/blob/master/examples/chat.nim):
```bash
nim c -r --threads:on examples\chat.nim
```
This will output a peer ID such as `QmbmHfVvouKammmQDJck4hz33WvVktNEe7pasxz2HgseRu` which you can use in another instance to connect to it.
```bash
./example/chat
/connect QmbmHfVvouKammmQDJck4hz33WvVktNEe7pasxz2HgseRu
```
You can now chat between the instances!
![Chat example](https://imgur.com/caYRu8K.gif)

View File

@@ -1,29 +0,0 @@
# Introduction
This folder contains the documentation for each nim-libp2p module and the sample code for the tutorials.
# Table of Contents
### [Getting Started](GETTING_STARTED.md)
### Tutorials
- P2P Chat Example
- [part I](tutorial/directchat/start.nim)
- [part II](tutorial/directchat/second.nim)
### API Specifications
- libp2p
- [libp2p-daemon-client](api/libp2p/daemonapi.md)
- [interop-libp2p](api/libp2p/interop.md)
- transports
- [libp2p-tcp](api/transports/tcptransport.md)
- secure channels
- [libp2p-secio](api/secure_channels/secio.md)
- stream multiplexers
- [libp2p-mplex](api/stream_multiplexers/mplex.md)
- utilities
- [libp2p-crypto](api/utilities/crypto.md)
- [libp2p-crypto-secp256k1](api/utilities/secp256k1.md)
- data types
- [peer-id](api/data_types/peer.md)
- [peer-info](api/data_types/peerinfo.md)
- pubsub
- [libp2p-pubsub](api/pubsub/pubsub.md)
- [libp2p-floodsub](api/pubsub/floodsub.md)
- [libp2p-gossipsub](api/pubsub/gossipsub.md)

View File

@@ -1,149 +0,0 @@
when not(compileOption("threads")):
{.fatal: "Please, compile this program with the --threads:on option!".}
import tables, strformat, strutils
import chronos
import ../libp2p/[switch,
multistream,
crypto/crypto,
protocols/identify,
connection,
transports/transport,
transports/tcptransport,
multiaddress,
peerinfo,
peerid,
protocols/protocol,
protocols/secure/secure,
protocols/secure/secio,
muxers/muxer,
muxers/mplex/mplex]
const ChatCodec = "/nim-libp2p/chat/1.0.0"
const DefaultAddr = "/ip4/127.0.0.1/tcp/55505"
const Help = """
Commands: /[?|hep|connect|disconnect|exit]
help: Prints this help
connect: dials a remote peer
disconnect: ends current session
exit: closes the chat
"""
type ChatProto = ref object of LPProtocol
switch: Switch # a single entry point for dialing and listening to peer
transp: StreamTransport # transport streams between read & write file descriptor
conn: Connection # create and close read & write stream
connected: bool # if the node is connected to another peer
started: bool # if the node has started
# copied from https://github.com/status-im/nimbus-eth2/blob/0ed657e953740a92458f23033d47483ffa17ccb0/beacon_chain/eth2_network.nim#L109-L115
proc initAddress(T: type MultiAddress, str: string): T =
let address = MultiAddress.init(str)
if IPFS.match(address) and matchPartial(multiaddress.TCP, address):
result = address
else:
raise newException(MultiAddressError,
"Invalid bootstrap node multi-address")
proc dialPeer(p: ChatProto, address: string) {.async.} =
let multiAddr = MultiAddress.initAddress(address);
let parts = address.split("/")
let remotePeer = PeerInfo.init(parts[^1],
[multiAddr])
echo &"dialing peer: {multiAddr}"
p.conn = await p.switch.dial(remotePeer, ChatCodec)
p.connected = true
proc readAndPrint(p: ChatProto) {.async.} =
while true:
while p.connected:
echo cast[string](await p.conn.readLp(1024))
await sleepAsync(100.millis)
proc writeAndPrint(p: ChatProto) {.async.} =
while true:
if not p.connected:
echo "type an address or wait for a connection:"
echo "type /[help|?] for help"
let line = await p.transp.readLine()
if line.startsWith("/help") or line.startsWith("/?") or not p.started:
echo Help
continue
if line.startsWith("/disconnect"):
echo "Ending current session"
if p.connected and p.conn.closed.not:
await p.conn.close()
p.connected = false
elif line.startsWith("/connect"):
if p.connected:
var yesno = "N"
echo "a session is already in progress, do you want end it [y/N]?"
yesno = await p.transp.readLine()
if yesno.cmpIgnoreCase("y") == 0:
await p.conn.close()
p.connected = false
elif yesno.cmpIgnoreCase("n") == 0:
continue
else:
echo "unrecognized response"
continue
echo "enter address of remote peer"
let address = await p.transp.readLine()
if address.len > 0:
await p.dialPeer(address)
elif line.startsWith("/exit"):
if p.connected and p.conn.closed.not:
await p.conn.close()
p.connected = false
await p.switch.stop()
echo "quitting..."
quit(0)
else:
if p.connected:
await p.conn.writeLp(line)
else:
try:
if line.startsWith("/") and "ipfs" in line:
await p.dialPeer(line)
except:
echo &"unable to dial remote peer {line}"
echo getCurrentExceptionMsg()
proc readWriteLoop(p: ChatProto) {.async.} =
asyncCheck p.writeAndPrint() # execute the async function but does not block
asyncCheck p.readAndPrint()
proc processInput(rfd: AsyncFD) {.async.} =
let transp = fromPipe(rfd)
while true:
let a = await transp.readLine()
echo "You just entered: " & a
proc readInput(wfd: AsyncFD) {.thread.} =
## This procedure performs reading from `stdin` and sends data over
## pipe to main thread.
let transp = fromPipe(wfd)
while true:
let line = stdin.readLine()
discard waitFor transp.write(line & "\r\n")
proc main() {.async.} =
let (rfd, wfd) = createAsyncPipe()
if rfd == asyncInvalidPipe or wfd == asyncInvalidPipe:
raise newException(ValueError, "Could not initialize pipe!")
var thread: Thread[AsyncFD]
thread.createThread(readInput, wfd)
await processInput(rfd)
when isMainModule: # isMainModule = true when the module is compiled as the main file
waitFor(main())

View File

@@ -1,39 +0,0 @@
when not(compileOption("threads")):
{.fatal: "Please, compile this program with the --threads:on option!".}
import chronos # an efficient library for async
proc processInput(rfd: AsyncFD) {.async.} =
echo "Type something below to see if the multithread IO works:\nType 'exit' to exit."
let transp = fromPipe(rfd)
while true:
let a = await transp.readLine()
if a == "exit":
quit(0);
echo "You just entered: " & a
proc readInput(wfd: AsyncFD) {.thread.} =
## This procedure performs reading from `stdin` and sends data over
## pipe to main thread.
let transp = fromPipe(wfd)
while true:
let line = stdin.readLine()
discard waitFor transp.write(line & "\r\n")
proc main() {.async.} =
let (rfd, wfd) = createAsyncPipe()
if rfd == asyncInvalidPipe or wfd == asyncInvalidPipe:
raise newException(ValueError, "Could not initialize pipe!")
var thread: Thread[AsyncFD]
thread.createThread(readInput, wfd)
await processInput(rfd)
when isMainModule: # isMainModule = true when the module is compiled as the main file
waitFor(main())

View File

@@ -1,149 +0,0 @@
when not(compileOption("threads")):
{.fatal: "Please, compile this program with the --threads:on option!".}
import tables, strformat, strutils
import chronos
import ../libp2p/[switch,
multistream,
crypto/crypto,
protocols/identify,
connection,
transports/transport,
transports/tcptransport,
multiaddress,
peerinfo,
peerid,
protocols/protocol,
protocols/secure/secure,
protocols/secure/secio,
muxers/muxer,
muxers/mplex/mplex]
const ChatCodec = "/nim-libp2p/chat/1.0.0"
const DefaultAddr = "/ip4/127.0.0.1/tcp/55505"
const Help = """
Commands: /[?|hep|connect|disconnect|exit]
help: Prints this help
connect: dials a remote peer
disconnect: ends current session
exit: closes the chat
"""
type ChatProto = ref object of LPProtocol
switch: Switch # a single entry point for dialing and listening to peer
transp: StreamTransport # transport streams between read & write file descriptor
conn: Connection # create and close read & write stream
connected: bool # if the node is connected to another peer
started: bool # if the node has started
# copied from https://github.com/status-im/nimbus-eth2/blob/0ed657e953740a92458f23033d47483ffa17ccb0/beacon_chain/eth2_network.nim#L109-L115
proc initAddress(T: type MultiAddress, str: string): T =
let address = MultiAddress.init(str)
if IPFS.match(address) and matchPartial(multiaddress.TCP, address):
result = address
else:
raise newException(MultiAddressError,
"Invalid bootstrap node multi-address")
proc dialPeer(p: ChatProto, address: string) {.async.} =
let multiAddr = MultiAddress.initAddress(address);
let parts = address.split("/")
let remotePeer = PeerInfo.init(parts[^1],
[multiAddr])
echo &"dialing peer: {multiAddr}"
p.conn = await p.switch.dial(remotePeer, ChatCodec)
p.connected = true
proc readAndPrint(p: ChatProto) {.async.} =
while true:
while p.connected:
echo cast[string](await p.conn.readLp(1024))
await sleepAsync(100.millis)
proc writeAndPrint(p: ChatProto) {.async.} =
while true:
if not p.connected:
echo "type an address or wait for a connection:"
echo "type /[help|?] for help"
let line = await p.transp.readLine()
if line.startsWith("/help") or line.startsWith("/?") or not p.started:
echo Help
continue
if line.startsWith("/disconnect"):
echo "Ending current session"
if p.connected and p.conn.closed.not:
await p.conn.close()
p.connected = false
elif line.startsWith("/connect"):
if p.connected:
var yesno = "N"
echo "a session is already in progress, do you want end it [y/N]?"
yesno = await p.transp.readLine()
if yesno.cmpIgnoreCase("y") == 0:
await p.conn.close()
p.connected = false
elif yesno.cmpIgnoreCase("n") == 0:
continue
else:
echo "unrecognized response"
continue
echo "enter address of remote peer"
let address = await p.transp.readLine()
if address.len > 0:
await p.dialPeer(address)
elif line.startsWith("/exit"):
if p.connected and p.conn.closed.not:
await p.conn.close()
p.connected = false
await p.switch.stop()
echo "quitting..."
quit(0)
else:
if p.connected:
await p.conn.writeLp(line)
else:
try:
if line.startsWith("/") and "ipfs" in line:
await p.dialPeer(line)
except:
echo &"unable to dial remote peer {line}"
echo getCurrentExceptionMsg()
proc readWriteLoop(p: ChatProto) {.async.} =
asyncCheck p.writeAndPrint() # execute the async function but does not block
asyncCheck p.readAndPrint()
proc processInput(rfd: AsyncFD) {.async.} =
let transp = fromPipe(rfd)
while true:
let a = await transp.readLine()
echo "You just entered: " & a
proc readInput(wfd: AsyncFD) {.thread.} =
## This procedure performs reading from `stdin` and sends data over
## pipe to main thread.
let transp = fromPipe(wfd)
while true:
let line = stdin.readLine()
discard waitFor transp.write(line & "\r\n")
proc main() {.async.} =
let (rfd, wfd) = createAsyncPipe()
if rfd == asyncInvalidPipe or wfd == asyncInvalidPipe:
raise newException(ValueError, "Could not initialize pipe!")
var thread: Thread[AsyncFD]
thread.createThread(readInput, wfd)
await processInput(rfd)
when isMainModule: # isMainModule = true when the module is compiled as the main file
waitFor(main())

View File

@@ -1,39 +0,0 @@
when not(compileOption("threads")):
{.fatal: "Please, compile this program with the --threads:on option!".}
import chronos # an efficient library for async
proc processInput(rfd: AsyncFD) {.async.} =
echo "Type something below to see if the multithread IO works:\nType 'exit' to exit."
let transp = fromPipe(rfd)
while true:
let a = await transp.readLine()
if a == "exit":
quit(0);
echo "You just entered: " & a
proc readInput(wfd: AsyncFD) {.thread.} =
## This procedure performs reading from `stdin` and sends data over
## pipe to main thread.
let transp = fromPipe(wfd)
while true:
let line = stdin.readLine()
discard waitFor transp.write(line & "\r\n")
proc main() {.async.} =
let (rfd, wfd) = createAsyncPipe()
if rfd == asyncInvalidPipe or wfd == asyncInvalidPipe:
raise newException(ValueError, "Could not initialize pipe!")
var thread: Thread[AsyncFD]
thread.createThread(readInput, wfd)
await processInput(rfd)
when isMainModule: # isMainModule = true when the module is compiled as the main file
waitFor(main())

5
examples/README.md Normal file
View File

@@ -0,0 +1,5 @@
# nim-libp2p examples
In this folder, you'll find the sources of the [nim-libp2p website](https://status-im.github.io/nim-libp2p/docs/)
We recommand to follow the tutorials on the website, but feel free to grok the sources here!

93
examples/circuitrelay.nim Normal file
View File

@@ -0,0 +1,93 @@
## # Circuit Relay example
##
## Circuit Relay can be used when a node cannot reach another node
## directly, but can reach it through another node (the Relay).
##
## That may happen because of NAT, Firewalls, or incompatible transports.
##
## More informations [here](https://docs.libp2p.io/concepts/circuit-relay/).
import chronos, stew/byteutils
import libp2p, libp2p/protocols/connectivity/relay/[relay, client]
# Helper to create a circuit relay node
proc createCircuitRelaySwitch(r: Relay): Switch =
SwitchBuilder
.new()
.withRng(newRng())
.withAddresses(@[MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet()])
.withTcpTransport()
.withMplex()
.withNoise()
.withCircuitRelay(r)
.build()
proc main() {.async.} =
# Create a custom protocol
let customProtoCodec = "/test"
var proto = new LPProtocol
proto.codec = customProtoCodec
proto.handler = proc(
conn: Connection, proto: string
) {.async: (raises: [CancelledError]).} =
try:
var msg = string.fromBytes(await conn.readLp(1024))
echo "1 - Dst Received: ", msg
assert "test1" == msg
await conn.writeLp("test2")
msg = string.fromBytes(await conn.readLp(1024))
echo "2 - Dst Received: ", msg
assert "test3" == msg
await conn.writeLp("test4")
except CancelledError as e:
raise e
except CatchableError as e:
echo "exception in handler", e.msg
let
relay = Relay.new()
clSrc = RelayClient.new()
clDst = RelayClient.new()
# Create three hosts, enable relay client on two of them.
# The third one can relay connections for other peers.
# RelayClient can use a relay, Relay is a relay.
swRel = createCircuitRelaySwitch(relay)
swSrc = createCircuitRelaySwitch(clSrc)
swDst = createCircuitRelaySwitch(clDst)
swDst.mount(proto)
await swRel.start()
await swSrc.start()
await swDst.start()
let
# Create a relay address to swDst using swRel as the relay
addrs = MultiAddress
.init(
$swRel.peerInfo.addrs[0] & "/p2p/" & $swRel.peerInfo.peerId & "/p2p-circuit"
)
.get()
# Connect Dst to the relay
await swDst.connect(swRel.peerInfo.peerId, swRel.peerInfo.addrs)
# Dst reserve a slot on the relay.
let rsvp = await clDst.reserve(swRel.peerInfo.peerId, swRel.peerInfo.addrs)
# Src dial Dst using the relay
let conn = await swSrc.dial(swDst.peerInfo.peerId, @[addrs], customProtoCodec)
await conn.writeLp("test1")
var msg = string.fromBytes(await conn.readLp(1024))
echo "1 - Src Received: ", msg
assert "test2" == msg
await conn.writeLp("test3")
msg = string.fromBytes(await conn.readLp(1024))
echo "2 - Src Received: ", msg
assert "test4" == msg
await relay.stop()
await allFutures(swSrc.stop(), swDst.stop(), swRel.stop())
waitFor(main())

View File

@@ -1,146 +1,147 @@
when not(compileOption("threads")):
when not (compileOption("threads")):
{.fatal: "Please, compile this program with the --threads:on option!".}
import tables, strformat, strutils, bearssl
import chronos # an efficient library for async
import ../libp2p/[switch, # manage transports, a single entry point for dialing and listening
multistream, # tag stream with short header to identify it
crypto/crypto, # cryptographic functions
errors, # error handling utilities
protocols/identify, # identify the peer info of a peer
stream/connection, # create and close stream read / write connections
transports/transport, # listen and dial to other peers using p2p protocol
transports/tcptransport, # listen and dial to other peers using client-server protocol
multiaddress, # encode different addressing schemes. For example, /ip4/7.7.7.7/tcp/6543 means it is using IPv4 protocol and TCP
peerinfo, # manage the information of a peer, such as peer ID and public / private key
peerid, # Implement how peers interact
protocols/protocol, # define the protocol base type
protocols/secure/secure, # define the protocol of secure connection
protocols/secure/secio, # define the protocol of secure input / output, allows encrypted communication that uses public keys to validate signed messages instead of a certificate authority like in TLS
muxers/muxer, # define an interface for stream multiplexing, allowing peers to offer many protocols over a single connection
muxers/mplex/mplex] # define some contants and message types for stream multiplexing
import strformat, strutils, stew/byteutils, chronos, libp2p
const ChatCodec = "/nim-libp2p/chat/1.0.0"
const DefaultAddr = "/ip4/127.0.0.1/tcp/55505"
const DefaultAddr = "/ip4/127.0.0.1/tcp/0"
const Help = """
Commands: /[?|hep|connect|disconnect|exit]
const Help =
"""
Commands: /[?|help|connect|disconnect|exit]
help: Prints this help
connect: dials a remote peer
disconnect: ends current session
exit: closes the chat
"""
type Chat = ref object
switch: Switch # a single entry point for dialing and listening to peer
stdinReader: StreamTransport # transport streams between read & write file descriptor
conn: Connection # connection to the other peer
connected: bool # if the node is connected to another peer
##
# Stdout helpers, to write the prompt
##
proc writePrompt(c: Chat) =
if c.connected:
stdout.write '\r' & $c.switch.peerInfo.peerId & ": "
stdout.flushFile()
proc writeStdout(c: Chat, str: string) =
echo '\r' & str
c.writePrompt()
##
# Chat Protocol
##
const ChatCodec = "/nim-libp2p/chat/1.0.0"
type ChatProto = ref object of LPProtocol
switch: Switch # a single entry point for dialing and listening to peer
transp: StreamTransport # transport streams between read & write file descriptor
conn: Connection # create and close read & write stream
connected: bool # if the node is connected to another peer
started: bool # if the node has started
proc new(T: typedesc[ChatProto], c: Chat): T =
let chatproto = T()
proc initAddress(T: type MultiAddress, str: string): T =
let address = MultiAddress.init(str).tryGet()
if IPFS.match(address) and matchPartial(multiaddress.TCP, address):
result = address
else:
raise newException(ValueError,
"Invalid bootstrap node multi-address")
# create handler for incoming connection
proc handle(stream: Connection, proto: string) {.async: (raises: [CancelledError]).} =
try:
if c.connected and not c.conn.closed:
c.writeStdout "a chat session is already in progress - refusing incoming peer!"
else:
await c.handlePeer(stream)
except CancelledError as e:
raise e
except CatchableError as e:
echo "exception in handler", e.msg
finally:
await stream.close()
proc dialPeer(p: ChatProto, address: string) {.async.} =
let multiAddr = MultiAddress.initAddress(address);
let parts = address.split("/")
let remotePeer = PeerInfo.init(parts[^1],
[multiAddr])
# assign the new handler
chatproto.handler = handle
chatproto.codec = ChatCodec
return chatproto
##
# Chat application
##
proc handlePeer(c: Chat, conn: Connection) {.async.} =
# Handle a peer (incoming or outgoing)
try:
c.conn = conn
c.connected = true
c.writeStdout $conn.peerId & " connected"
# Read loop
while true:
let
strData = await conn.readLp(1024)
str = string.fromBytes(strData)
c.writeStdout $conn.peerId & ": " & $str
except LPStreamEOFError:
defer:
c.writeStdout $conn.peerId & " disconnected"
await c.conn.close()
c.connected = false
proc dialPeer(c: Chat, address: string) {.async.} =
# Parse and dial address
let
multiAddr = MultiAddress.init(address).tryGet()
# split the peerId part /p2p/...
peerIdBytes = multiAddr[multiCodec("p2p")].tryGet().protoAddress().tryGet()
remotePeer = PeerId.init(peerIdBytes).tryGet()
# split the wire address
ip4Addr = multiAddr[multiCodec("ip4")].tryGet()
tcpAddr = multiAddr[multiCodec("tcp")].tryGet()
wireAddr = ip4Addr & tcpAddr
echo &"dialing peer: {multiAddr}"
p.conn = await p.switch.dial(remotePeer, ChatCodec)
p.connected = true
asyncSpawn c.handlePeer(await c.switch.dial(remotePeer, @[wireAddr], ChatCodec))
proc readAndPrint(p: ChatProto) {.async.} =
proc readLoop(c: Chat) {.async.} =
while true:
while p.connected:
# TODO: echo &"{p.id} -> "
echo cast[string](await p.conn.readLp(1024))
await sleepAsync(100.millis)
proc writeAndPrint(p: ChatProto) {.async.} =
while true:
if not p.connected:
if not c.connected:
echo "type an address or wait for a connection:"
echo "type /[help|?] for help"
let line = await p.transp.readLine()
if line.startsWith("/help") or line.startsWith("/?") or not p.started:
c.writePrompt()
let line = await c.stdinReader.readLine()
if line.startsWith("/help") or line.startsWith("/?"):
echo Help
continue
if line.startsWith("/disconnect"):
echo "Ending current session"
if p.connected and p.conn.closed.not:
await p.conn.close()
p.connected = false
c.writeStdout "Ending current session"
if c.connected and c.conn.closed.not:
await c.conn.close()
c.connected = false
elif line.startsWith("/connect"):
if p.connected:
var yesno = "N"
echo "a session is already in progress, do you want end it [y/N]?"
yesno = await p.transp.readLine()
if yesno.cmpIgnoreCase("y") == 0:
await p.conn.close()
p.connected = false
elif yesno.cmpIgnoreCase("n") == 0:
continue
else:
echo "unrecognized response"
continue
echo "enter address of remote peer"
let address = await p.transp.readLine()
c.writeStdout "enter address of remote peer"
let address = await c.stdinReader.readLine()
if address.len > 0:
await p.dialPeer(address)
await c.dialPeer(address)
elif line.startsWith("/exit"):
if p.connected and p.conn.closed.not:
await p.conn.close()
p.connected = false
if c.connected and c.conn.closed.not:
await c.conn.close()
c.connected = false
await p.switch.stop()
echo "quitting..."
quit(0)
await c.switch.stop()
c.writeStdout "quitting..."
return
else:
if p.connected:
await p.conn.writeLp(line)
if c.connected:
await c.conn.writeLp(line)
else:
try:
if line.startsWith("/") and "ipfs" in line:
await p.dialPeer(line)
except:
if line.startsWith("/") and "p2p" in line:
await c.dialPeer(line)
except CatchableError as exc:
echo &"unable to dial remote peer {line}"
echo getCurrentExceptionMsg()
proc readWriteLoop(p: ChatProto) {.async.} =
asyncCheck p.writeAndPrint() # execute the async function but does not block
asyncCheck p.readAndPrint()
proc newChatProto(switch: Switch, transp: StreamTransport): ChatProto =
var chatproto = ChatProto(switch: switch, transp: transp, codecs: @[ChatCodec])
# create handler for incoming connection
proc handle(stream: Connection, proto: string) {.async.} =
if chatproto.connected and not chatproto.conn.closed:
echo "a chat session is already in progress - disconnecting!"
await stream.close()
else:
chatproto.conn = stream
chatproto.connected = true
# assign the new handler
chatproto.handler = handle
return chatproto
echo exc.msg
proc readInput(wfd: AsyncFD) {.thread.} =
## This procedure performs reading from `stdin` and sends data over
## This thread performs reading from `stdin` and sends data over
## pipe to main thread.
let transp = fromPipe(wfd)
@@ -148,66 +149,48 @@ proc readInput(wfd: AsyncFD) {.thread.} =
let line = stdin.readLine()
discard waitFor transp.write(line & "\r\n")
proc processInput(rfd: AsyncFD, rng: ref BrHmacDrbgContext) {.async.} =
let transp = fromPipe(rfd)
let seckey = PrivateKey.random(RSA, rng[]).get()
let peerInfo = PeerInfo.init(seckey)
var localAddress = DefaultAddr
while true:
echo &"Type an address to bind to or Enter to use the default {DefaultAddr}"
let a = await transp.readLine()
try:
if a.len > 0:
peerInfo.addrs.add(Multiaddress.init(a).tryGet())
break
peerInfo.addrs.add(Multiaddress.init(localAddress).tryGet())
break
except:
echo "invalid address"
localAddress = DefaultAddr
continue
# a constructor for building different multiplexers under various connections
proc createMplex(conn: Connection): Muxer =
result = Mplex.init(conn)
let mplexProvider = newMuxerProvider(createMplex, MplexCodec)
let transports = @[Transport(TcpTransport.init())]
let muxers = [(MplexCodec, mplexProvider)].toTable()
let identify = newIdentify(peerInfo)
let secureManagers = [Secure(newSecio(rng, seckey))]
let switch = newSwitch(peerInfo,
transports,
identify,
muxers,
secureManagers)
let chatProto = newChatProto(switch, transp)
switch.mount(chatProto)
let libp2pFuts = await switch.start()
chatProto.started = true
let id = $peerInfo.peerId
echo "PeerID: " & id
echo "listening on: "
for a in peerInfo.addrs:
echo &"{a}/ipfs/{id}"
await chatProto.readWriteLoop()
await allFuturesThrowing(libp2pFuts)
proc main() {.async.} =
let rng = newRng() # Singe random number source for the whole application
let (rfd, wfd) = createAsyncPipe()
if rfd == asyncInvalidPipe or wfd == asyncInvalidPipe:
raise newException(ValueError, "Could not initialize pipe!")
let
rng = newRng() # Single random number source for the whole application
# Pipe to read stdin from main thread
(rfd, wfd) = createAsyncPipe()
stdinReader = fromPipe(rfd)
var thread: Thread[AsyncFD]
thread.createThread(readInput, wfd)
try:
thread.createThread(readInput, wfd)
except Exception as exc:
quit("Failed to create thread: " & exc.msg)
await processInput(rfd, rng)
var localAddress = MultiAddress.init(DefaultAddr).tryGet()
when isMainModule: # isMainModule = true when the module is compiled as the main file
waitFor(main())
var switch = SwitchBuilder
.new()
.withRng(rng)
# Give the application RNG
.withAddress(localAddress)
.withTcpTransport()
# Use TCP as transport
.withMplex()
# Use Mplex as muxer
.withNoise()
# Use Noise as secure manager
.build()
let chat = Chat(switch: switch, stdinReader: stdinReader)
switch.mount(ChatProto.new(chat))
await switch.start()
let id = $switch.peerInfo.peerId
echo "PeerId: " & id
echo "listening on: "
for a in switch.peerInfo.addrs:
echo &"{a}/p2p/{id}"
await chat.readLoop()
quit(0)
waitFor(main())

View File

@@ -0,0 +1,3 @@
{.used.}
import directchat, tutorial_6_game

View File

@@ -0,0 +1,5 @@
{.used.}
import
helloworld, circuitrelay, tutorial_1_connect, tutorial_2_customproto,
tutorial_3_protobuf, tutorial_4_gossipsub, tutorial_5_discovery

View File

@@ -2,8 +2,7 @@ import chronos, nimcrypto, strutils
import ../../libp2p/daemon/daemonapi
import ../hexdump
const
PubSubTopic = "test-net"
const PubSubTopic = "test-net"
proc dumpSubscribedPeers(api: DaemonAPI) {.async.} =
var peers = await api.pubsubListPeers(PubSubTopic)
@@ -35,14 +34,14 @@ proc main() {.async.} =
if item.protoCode() == mcip4 or item.protoCode() == mcip6:
echo $item & "/ipfs/" & id.peer.pretty()
asyncCheck monitor(api)
asyncSpawn monitor(api)
proc pubsubLogger(api: DaemonAPI,
ticket: PubsubTicket,
message: PubSubMessage): Future[bool] {.async.} =
proc pubsubLogger(
api: DaemonAPI, ticket: PubsubTicket, message: PubSubMessage
): Future[bool] {.async.} =
let msglen = len(message.data)
echo "= Recieved pubsub message with length ", msglen,
" bytes from peer ", message.peer.pretty()
echo "= Recieved pubsub message with length ",
msglen, " bytes from peer ", message.peer.pretty()
echo dumpHex(message.data)
await api.dumpSubscribedPeers()
result = true

View File

@@ -2,24 +2,22 @@ import chronos, nimcrypto, strutils
import ../../libp2p/daemon/daemonapi
## nim c -r --threads:on chat.nim
when not(compileOption("threads")):
when not (compileOption("threads")):
{.fatal: "Please, compile this program with the --threads:on option!".}
const
ServerProtocols = @["/test-chat-stream"]
const ServerProtocols = @["/test-chat-stream"]
type
CustomData = ref object
api: DaemonAPI
remotes: seq[StreamTransport]
consoleFd: AsyncFD
serveFut: Future[void]
type CustomData = ref object
api: DaemonAPI
remotes: seq[StreamTransport]
consoleFd: AsyncFD
serveFut: Future[void]
proc threadMain(wfd: AsyncFD) {.thread.} =
## This procedure performs reading from `stdin` and sends data over
## pipe to main thread.
var transp = fromPipe(wfd)
while true:
var line = stdin.readLine()
let res = waitFor transp.write(line & "\r\n")
@@ -41,7 +39,7 @@ proc serveThread(udata: CustomData) {.async.} =
if line.startsWith("/connect"):
var parts = line.split(" ")
if len(parts) == 2:
var peerId = PeerID.init(parts[1])
var peerId = PeerId.init(parts[1])
var address = MultiAddress.init(multiCodec("p2p-circuit"))
address &= MultiAddress.init(multiCodec("p2p"), peerId)
echo "= Searching for peer ", peerId.pretty()
@@ -55,11 +53,11 @@ proc serveThread(udata: CustomData) {.async.} =
var stream = await udata.api.openStream(peerId, ServerProtocols)
udata.remotes.add(stream.transp)
echo "= Connected to peer chat ", parts[1]
asyncCheck remoteReader(stream.transp)
asyncSpawn remoteReader(stream.transp)
elif line.startsWith("/search"):
var parts = line.split(" ")
if len(parts) == 2:
var peerId = PeerID.init(parts[1])
var peerId = PeerId.init(parts[1])
echo "= Searching for peer ", peerId.pretty()
var id = await udata.api.dhtFindPeer(peerId)
echo "= Peer " & parts[1] & " found at addresses:"
@@ -68,7 +66,7 @@ proc serveThread(udata: CustomData) {.async.} =
elif line.startsWith("/consearch"):
var parts = line.split(" ")
if len(parts) == 2:
var peerId = PeerID.init(parts[1])
var peerId = PeerId.init(parts[1])
echo "= Searching for peers connected to peer ", parts[1]
var peers = await udata.api.dhtFindPeersConnectedToPeer(peerId)
echo "= Found ", len(peers), " connected to peer ", parts[1]
@@ -82,7 +80,7 @@ proc serveThread(udata: CustomData) {.async.} =
relay = true
break
if relay:
echo peer.pretty(), " * ", " [", addresses.join(", "), "]"
echo peer.pretty(), " * ", " [", addresses.join(", "), "]"
else:
echo peer.pretty(), " [", addresses.join(", "), "]"
elif line.startsWith("/exit"):
@@ -95,8 +93,8 @@ proc serveThread(udata: CustomData) {.async.} =
pending.add(item.write(msg))
if len(pending) > 0:
var results = await all(pending)
except:
echo getCurrentException().msg
except CatchableError as err:
echo err.msg
proc main() {.async.} =
var data = new CustomData
@@ -127,7 +125,7 @@ proc main() {.async.} =
echo ">> ", line
await data.api.addHandler(ServerProtocols, streamHandler)
echo "= Your PeerID is ", id.peer.pretty()
echo "= Your PeerId is ", id.peer.pretty()
await data.serveFut
when isMainModule:

View File

@@ -1,6 +1,8 @@
# Table of Contents
- [Introduction](#introduction)
- [Prerequisites](#prerequisites)
- [Installation](#installation)
- [Script](#script)
- [Usage](#usage)
- [Example](#example)
- [Getting Started](#getting-started)
@@ -8,26 +10,29 @@
# Introduction
This is a libp2p-backed daemon wrapping the functionalities of go-libp2p for use in Nim. <br>
For more information about the go daemon, check out [this repository](https://github.com/libp2p/go-libp2p-daemon).
> **Required only** for running the tests.
# Prerequisites
Go with version `1.16.0`.
> You will *likely* be able to build `go-libp2p-daemon` with different Go versions, but **they haven't been tested**.
# Installation
Follow one of the methods below:
## Script
Run the build script while having the `go` command pointing to the correct Go version.
We recommend using `1.16.0`, as previously stated.
```sh
# clone and install dependencies
git clone https://github.com/status-im/nim-libp2p
cd nim-libp2p
nimble install
# perform unit tests
nimble test
# update the git submodule to install the go daemon
git submodule update --init --recursive
go version
git clone https://github.com/libp2p/go-libp2p-daemon
cd go-libp2p-daemon
git checkout v0.0.1
go install ./...
cd ..
./scripts/build_p2pd.sh
```
If everything goes correctly, the binary (`p2pd`) should be built and placed in the correct directory.
If you find any issues, please head into our discord and ask for our assistance.
After successfully building the binary, remember to add it to your path so it can be found. You can do that by running:
```sh
export PATH="$PATH:$HOME/go/bin"
```
> **Tip:** To make this change permanent, add the command above to your `.bashrc` file.
# Usage
@@ -38,13 +43,13 @@ Examples can be found in the [examples folder](https://github.com/status-im/nim-
Try out the chat example. Full code can be found [here](https://github.com/status-im/nim-libp2p/blob/master/examples/chat.nim):
```bash
nim c -r --threads:on examples\chat.nim
nim c -r --threads:on examples/directchat.nim
```
This will output a peer ID such as `QmbmHfVvouKammmQDJck4hz33WvVktNEe7pasxz2HgseRu` which you can use in another instance to connect to it.
```bash
./example/chat
./examples/directchat
/connect QmbmHfVvouKammmQDJck4hz33WvVktNEe7pasxz2HgseRu
```

View File

@@ -1,26 +1,25 @@
import chronos, nimcrypto, strutils, os
import ../../libp2p/daemon/daemonapi
const
PubSubTopic = "test-net"
const PubSubTopic = "test-net"
proc main(bn: string) {.async.} =
echo "= Starting P2P node"
var bootnodes = bn.split(",")
var api = await newDaemonApi({DHTFull, PSGossipSub, WaitBootstrap},
bootstrapNodes = bootnodes,
peersRequired = 1)
var api = await newDaemonApi(
{DHTFull, PSGossipSub, WaitBootstrap}, bootstrapNodes = bootnodes, peersRequired = 1
)
var id = await api.identity()
echo "= P2P node ", id.peer.pretty(), " started:"
for item in id.addresses:
echo item
proc pubsubLogger(api: DaemonAPI,
ticket: PubsubTicket,
message: PubSubMessage): Future[bool] {.async.} =
proc pubsubLogger(
api: DaemonAPI, ticket: PubsubTicket, message: PubSubMessage
): Future[bool] {.async.} =
let msglen = len(message.data)
echo "= Recieved pubsub message with length ", msglen,
" bytes from peer ", message.peer.pretty(), ": "
echo "= Recieved pubsub message with length ",
msglen, " bytes from peer ", message.peer.pretty(), ": "
var strdata = cast[string](message.data)
echo strdata
result = true

99
examples/helloworld.nim Normal file
View File

@@ -0,0 +1,99 @@
import chronos # an efficient library for async
import stew/byteutils # various utils
import libp2p
##
# Create our custom protocol
##
const TestCodec = "/test/proto/1.0.0" # custom protocol string identifier
type TestProto = ref object of LPProtocol # declare a custom protocol
proc new(T: typedesc[TestProto]): T =
# every incoming connections will be in handled in this closure
proc handle(conn: Connection, proto: string) {.async: (raises: [CancelledError]).} =
try:
echo "Got from remote - ", string.fromBytes(await conn.readLp(1024))
await conn.writeLp("Roger p2p!")
except CancelledError as e:
raise e
except CatchableError as e:
echo "exception in handler", e.msg
finally:
# We must close the connections ourselves when we're done with it
await conn.close()
return T.new(codecs = @[TestCodec], handler = handle)
##
# Helper to create a switch/node
##
proc createSwitch(ma: MultiAddress, rng: ref HmacDrbgContext): Switch =
var switch = SwitchBuilder
.new()
.withRng(rng)
# Give the application RNG
.withAddress(ma)
# Our local address(es)
.withTcpTransport()
# Use TCP as transport
.withMplex()
# Use Mplex as muxer
.withNoise()
# Use Noise as secure manager
.build()
result = switch
##
# The actual application
##
proc main() {.async.} =
let
rng = newRng() # Single random number source for the whole application
# port 0 will take a random available port
# `tryGet` will throw an exception if the MultiAddress failed
# (for instance, if the address is not well formatted)
ma1 = MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet()
ma2 = MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet()
# setup the custom proto
let testProto = TestProto.new()
# setup the two nodes
let
switch1 = createSwitch(ma1, rng) #Create the two switches
switch2 = createSwitch(ma2, rng)
# mount the proto on switch1
# the node will now listen for this proto
# and call the handler everytime a client request it
switch1.mount(testProto)
# Start the nodes. This will start the transports
# and listen on each local addresses
await switch1.start()
await switch2.start()
# the node addrs is populated with it's
# actual port during the start
# use the second node to dial the first node
# using the first node peerid and address
# and specify our custom protocol codec
let conn =
await switch2.dial(switch1.peerInfo.peerId, switch1.peerInfo.addrs, TestCodec)
# conn is now a fully setup connection, we talk directly to the node1 custom protocol handler
await conn.writeLp("Hello p2p!") # writeLp send a length prefixed buffer over the wire
# readLp reads length prefixed bytes and returns a buffer without the prefix
echo "Remote responded with - ", string.fromBytes(await conn.readLp(1024))
# We must close the connection ourselves when we're done with it
await conn.close()
await allFutures(switch1.stop(), switch2.stop())
# close connections and shutdown all transports
waitFor(main())

View File

@@ -35,26 +35,24 @@ proc dumpHex*(pbytes: pointer, nbytes: int, items = 1, ascii = true): string =
var asciiText = ""
while i < nbytes:
if i %% 16 == 0:
result = result & toHex(cast[BiggestInt](slider),
sizeof(BiggestInt) * 2) & ": "
result = result & toHex(cast[BiggestInt](slider), sizeof(BiggestInt) * 2) & ": "
var k = 0
while k < items:
var ch = cast[ptr char](cast[uint](slider) + k.uint)[]
if ord(ch) > 31 and ord(ch) < 127: asciiText &= ch else: asciiText &= "."
if ord(ch) > 31 and ord(ch) < 127:
asciiText &= ch
else:
asciiText &= "."
inc(k)
case items:
case items
of 1:
result = result & toHex(cast[BiggestInt](cast[ptr uint8](slider)[]),
hexSize)
result = result & toHex(cast[BiggestInt](cast[ptr uint8](slider)[]), hexSize)
of 2:
result = result & toHex(cast[BiggestInt](cast[ptr uint16](slider)[]),
hexSize)
result = result & toHex(cast[BiggestInt](cast[ptr uint16](slider)[]), hexSize)
of 4:
result = result & toHex(cast[BiggestInt](cast[ptr uint32](slider)[]),
hexSize)
result = result & toHex(cast[BiggestInt](cast[ptr uint32](slider)[]), hexSize)
of 8:
result = result & toHex(cast[BiggestInt](cast[ptr uint64](slider)[]),
hexSize)
result = result & toHex(cast[BiggestInt](cast[ptr uint64](slider)[]), hexSize)
else:
raise newException(ValueError, "Wrong items size!")
result = result & " "
@@ -71,8 +69,8 @@ proc dumpHex*(pbytes: pointer, nbytes: int, items = 1, ascii = true): string =
result = result & asciiText
result = result & "\n"
proc dumpHex*[T](v: openarray[T], items: int = 0, ascii = true): string =
## Return hexadecimal memory dump representation of openarray[T] ``v``.
proc dumpHex*[T](v: openArray[T], items: int = 0, ascii = true): string =
## Return hexadecimal memory dump representation of openArray[T] ``v``.
## ``items`` - number of bytes in group (supported ``items`` count is
## 0, 1, 2, 4, 8). If ``items`` is ``0`` group size will depend on
## ``sizeof(T)``.

6
examples/index.md Normal file
View File

@@ -0,0 +1,6 @@
# nim-libp2p documentation
Welcome to the nim-libp2p documentation!
Here, you'll find [tutorials](tutorial_1_connect.md) to help you get started, as well as
the [full reference](https://status-im.github.io/nim-libp2p/master/libp2p.html).

View File

@@ -0,0 +1,102 @@
## # Simple ping tutorial
##
## Hi all, welcome to the first nim-libp2p tutorial!
##
## !!! tips ""
## This tutorial is for everyone who is interested in building peer-to-peer applications. No Nim programming experience is needed.
##
## To give you a quick overview, **Nim** is the programming language we are using and **nim-libp2p** is the Nim implementation of [libp2p](https://libp2p.io/), a modular library that enables the development of peer-to-peer network applications.
##
## Hope you'll find it helpful in your journey of learning. Happy coding! ;)
##
## ## Before you start
## The only prerequisite here is [Nim](https://nim-lang.org/), the programming language with a Python-like syntax and a performance similar to C. Detailed information can be found [here](https://nim-lang.org/docs/tut1.html).
##
## Install Nim via their [official website](https://nim-lang.org/install.html).
## Check Nim's installation via `nim --version` and its package manager Nimble via `nimble --version`.
##
## You can now install the latest version of `nim-libp2p`:
## ```bash
## nimble install libp2p@#master
## ```
##
## ## A simple ping application
## We'll start by creating a simple application, which is starting two libp2p [switch](https://docs.libp2p.io/concepts/stream-multiplexing/#switch-swarm), and pinging each other using the [Ping](https://docs.libp2p.io/concepts/protocols/#ping) protocol.
##
## !!! tips ""
## You can find the source of this tutorial (and other tutorials) in the [libp2p/examples](https://github.com/status-im/nim-libp2p/tree/master/examples) folder!
##
## Let's create a `part1.nim`, and import our dependencies:
import chronos
import libp2p
import libp2p/protocols/ping
## [chronos](https://github.com/status-im/nim-chronos) the asynchronous framework used by `nim-libp2p`
##
## Next, we'll create a helper procedure to create our switches. A switch needs a bit of configuration, and it will be easier to do this configuration only once:
proc createSwitch(ma: MultiAddress, rng: ref HmacDrbgContext): Switch =
var switch = SwitchBuilder
.new()
.withRng(rng)
# Give the application RNG
.withAddress(ma)
# Our local address(es)
.withTcpTransport()
# Use TCP as transport
.withMplex()
# Use Mplex as muxer
.withNoise()
# Use Noise as secure manager
.build()
return switch
## This will create a switch using [Mplex](https://docs.libp2p.io/concepts/stream-multiplexing/) as a multiplexer, Noise to secure the communication, and TCP as an underlying transport.
##
## You can of course tweak this, to use a different or multiple transport, or tweak the configuration of Mplex and Noise, but this is some sane defaults that we'll use going forward.
##
##
## Let's now start to create our main procedure:
proc main() {.async.} =
let
rng = newRng()
localAddress = MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet()
pingProtocol = Ping.new(rng = rng)
## We created some variables that we'll need for the rest of the application: the global `rng` instance, our `localAddress`, and an instance of the `Ping` protocol.
## The address is in the [MultiAddress](https://github.com/multiformats/multiaddr) format. The port `0` means "take any port available".
##
## `tryGet` is procedure which is part of [nim-result](https://github.com/arnetheduck/nim-result/), that will throw an exception if the supplied MultiAddress is invalid.
##
## We can now create our two switches:
let
switch1 = createSwitch(localAddress, rng)
switch2 = createSwitch(localAddress, rng)
switch1.mount(pingProtocol)
await switch1.start()
await switch2.start()
## We've **mounted** the `pingProtocol` on our first switch. This means that the first switch will actually listen for any ping requests coming in, and handle them accordingly.
##
## Now that we've started the nodes, they are listening for incoming peers.
## We can find out which port was attributed, and the resulting local addresses, by using `switch1.peerInfo.addrs`.
##
## We'll **dial** the first switch from the second one, by specifying its **Peer ID**, its **MultiAddress** and the **`Ping` protocol codec**:
let conn =
await switch2.dial(switch1.peerInfo.peerId, switch1.peerInfo.addrs, PingCodec)
## We now have a `Ping` connection setup between the second and the first switch, we can use it to actually ping the node:
# ping the other node and echo the ping duration
echo "ping: ", await pingProtocol.ping(conn)
# We must close the connection ourselves when we're done with it
await conn.close()
## And that's it! Just a little bit of cleanup: shutting down the switches, waiting for them to stop, and we'll call our `main` procedure:
await allFutures(switch1.stop(), switch2.stop())
# close connections and shutdown all transports
waitFor(main())
## You can now run this program using `nim c -r part1.nim`, and you should see the dialing sequence, ending with a ping output.
##
## In the [next tutorial](tutorial_2_customproto.md), we'll look at how to create our own custom protocol.

View File

@@ -0,0 +1,81 @@
## # Custom protocol in libp2p
##
## In the [previous tutorial](tutorial_1_connect.md), we've looked at how to create a simple ping program using the `nim-libp2p`.
##
## We'll now look at how to create a custom protocol inside the libp2p
##
## Let's create a `part2.nim`, and import our dependencies:
import chronos
import stew/byteutils
import libp2p
## This is similar to the first tutorial, except we don't need to import the `Ping` protocol.
##
## Next, we'll declare our custom protocol
const TestCodec = "/test/proto/1.0.0"
type TestProto = ref object of LPProtocol
## We've set a [protocol ID](https://docs.libp2p.io/concepts/protocols/#protocol-ids), and created a custom `LPProtocol`. In a more complex protocol, we could use this structure to store interesting variables.
##
## A protocol generally has two parts: a handling/server part, and a dialing/client part.
## These two parts can be identical, but in our trivial protocol, the server will wait for a message from the client, and the client will send a message, so we have to handle the two cases separately.
##
## Let's start with the server part:
proc new(T: typedesc[TestProto]): T =
# every incoming connections will in be handled in this closure
proc handle(conn: Connection, proto: string) {.async: (raises: [CancelledError]).} =
# Read up to 1024 bytes from this connection, and transform them into
# a string
try:
echo "Got from remote - ", string.fromBytes(await conn.readLp(1024))
except CancelledError as e:
raise e
except CatchableError as e:
echo "exception in handler", e.msg
finally:
await conn.close()
return T.new(codecs = @[TestCodec], handler = handle)
## This is a constructor for our `TestProto`, that will specify our `codecs` and a `handler`, which will be called for each incoming peer asking for this protocol.
## In our handle, we simply read a message from the connection and `echo` it.
##
## We can now create our client part:
proc hello(p: TestProto, conn: Connection) {.async.} =
await conn.writeLp("Hello p2p!")
## Again, pretty straightforward, we just send a message on the connection.
##
## We can now create our main procedure:
proc main() {.async.} =
let
rng = newRng()
testProto = TestProto.new()
switch1 = newStandardSwitch(rng = rng)
switch2 = newStandardSwitch(rng = rng)
switch1.mount(testProto)
await switch1.start()
await switch2.start()
let conn =
await switch2.dial(switch1.peerInfo.peerId, switch1.peerInfo.addrs, TestCodec)
await testProto.hello(conn)
# We must close the connection ourselves when we're done with it
await conn.close()
await allFutures(switch1.stop(), switch2.stop())
# close connections and shutdown all transports
## This is very similar to the first tutorial's `main`, the only noteworthy difference is that we use `newStandardSwitch`, which is similar to the `createSwitch` of the first tutorial, but is bundled directly in libp2p
##
## We can now wrap our program by calling our main proc:
waitFor(main())
## And that's it!
## In the [next tutorial](tutorial_3_protobuf.md), we'll create a more complex protocol using Protobuf.

View File

@@ -0,0 +1,171 @@
## # Protobuf usage
##
## In the [previous tutorial](tutorial_2_customproto.md), we created a simple "ping" protocol.
## Most real protocol want their messages to be structured and extensible, which is why
## most real protocols use [protobuf](https://developers.google.com/protocol-buffers) to
## define their message structures.
##
## Here, we'll create a slightly more complex protocol, which parses & generate protobuf
## messages. Let's start by importing our dependencies, as usual:
import chronos
import stew/results # for Opt[T]
import libp2p
## ## Protobuf encoding & decoding
## This will be the structure of our messages:
## ```protobuf
## message MetricList {
## message Metric {
## string name = 1;
## float value = 2;
## }
##
## repeated Metric metrics = 2;
## }
## ```
## We'll create our protobuf types, encoders & decoders, according to this format.
## To create the encoders & decoders, we are going to use minprotobuf
## (included in libp2p).
##
## While more modern technics
## (such as [nim-protobuf-serialization](https://github.com/status-im/nim-protobuf-serialization))
## exists, minprotobuf is currently the recommended method to handle protobuf, since it has
## been used in production extensively, and audited.
type
Metric = object
name: string
value: float
MetricList = object
metrics: seq[Metric]
{.push raises: [].}
proc encode(m: Metric): ProtoBuffer =
result = initProtoBuffer()
result.write(1, m.name)
result.write(2, m.value)
result.finish()
proc decode(_: type Metric, buf: seq[byte]): Result[Metric, ProtoError] =
var res: Metric
let pb = initProtoBuffer(buf)
# "getField" will return a Result[bool, ProtoError].
# The Result will hold an error if the protobuf is invalid.
# The Result will hold "false" if the field is missing
#
# We are just checking the error, and ignoring whether the value
# is present or not (default values are valid).
discard ?pb.getField(1, res.name)
discard ?pb.getField(2, res.value)
ok(res)
proc encode(m: MetricList): ProtoBuffer =
result = initProtoBuffer()
for metric in m.metrics:
result.write(1, metric.encode())
result.finish()
proc decode(_: type MetricList, buf: seq[byte]): Result[MetricList, ProtoError] =
var
res: MetricList
metrics: seq[seq[byte]]
let pb = initProtoBuffer(buf)
discard ?pb.getRepeatedField(1, metrics)
for metric in metrics:
res.metrics &= ?Metric.decode(metric)
ok(res)
## ## Results instead of exceptions
## As you can see, this part of the program also uses Results instead of exceptions for error handling.
## We start by `{.push raises: [].}`, which will prevent every non-async function from raising
## exceptions.
##
## Then, we use [nim-result](https://github.com/arnetheduck/nim-result) to convey
## errors to function callers. A `Result[T, E]` will either hold a valid result of type
## T, or an error of type E.
##
## You can check if the call succeeded by using `res.isOk`, and then get the
## value using `res.value` or the error by using `res.error`.
##
## Another useful tool is `?`, which will unpack a Result if it succeeded,
## or if it failed, exit the current procedure returning the error.
##
## nim-result is packed with other functionalities that you'll find in the
## nim-result repository.
##
## Results and exception are generally interchangeable, but have different semantics
## that you may or may not prefer.
##
## ## Creating the protocol
## We'll next create a protocol, like in the last tutorial, to request these metrics from our host
type
MetricCallback = proc(): Future[MetricList] {.raises: [], gcsafe.}
MetricProto = ref object of LPProtocol
metricGetter: MetricCallback
proc new(_: typedesc[MetricProto], cb: MetricCallback): MetricProto =
var res: MetricProto
proc handle(conn: Connection, proto: string) {.async: (raises: [CancelledError]).} =
try:
let
metrics = await res.metricGetter()
asProtobuf = metrics.encode()
await conn.writeLp(asProtobuf.buffer)
except CancelledError as e:
raise e
except CatchableError as e:
echo "exception in handler", e.msg
finally:
await conn.close()
res = MetricProto.new(@["/metric-getter/1.0.0"], handle)
res.metricGetter = cb
return res
proc fetch(p: MetricProto, conn: Connection): Future[MetricList] {.async.} =
let protobuf = await conn.readLp(2048)
# tryGet will raise an exception if the Result contains an error.
# It's useful to bridge between exception-world and result-world
return MetricList.decode(protobuf).tryGet()
## We can now create our main procedure:
proc main() {.async.} =
let rng = newRng()
proc randomMetricGenerator(): Future[MetricList] {.async.} =
let metricCount = rng[].generate(uint32) mod 16
for i in 0 ..< metricCount + 1:
result.metrics.add(
Metric(name: "metric_" & $i, value: float(rng[].generate(uint16)) / 1000.0)
)
return result
let
metricProto1 = MetricProto.new(randomMetricGenerator)
metricProto2 = MetricProto.new(randomMetricGenerator)
switch1 = newStandardSwitch(rng = rng)
switch2 = newStandardSwitch(rng = rng)
switch1.mount(metricProto1)
await switch1.start()
await switch2.start()
let
conn = await switch2.dial(
switch1.peerInfo.peerId, switch1.peerInfo.addrs, metricProto2.codecs
)
metrics = await metricProto2.fetch(conn)
await conn.close()
for metric in metrics.metrics:
echo metric.name, " = ", metric.value
await allFutures(switch1.stop(), switch2.stop())
# close connections and shutdown all transports
waitFor(main())
## If you run this program, you should see random metrics being sent from the switch1 to the switch2.

View File

@@ -0,0 +1,168 @@
## # GossipSub
##
## In this tutorial, we'll build a simple GossipSub network
## to broadcast the metrics we built in the previous tutorial.
##
## GossipSub is used to broadcast some messages in a network,
## and allows to balance between latency, bandwidth usage,
## privacy and attack resistance.
##
## You'll find a good explanation of how GossipSub works
## [here.](https://docs.libp2p.io/concepts/publish-subscribe/) There are a lot
## of parameters you can tweak to adjust how GossipSub behaves but here we'll
## use the sane defaults shipped with libp2p.
##
## We'll start by creating our metric structure like previously
import chronos
import stew/results
import libp2p
import libp2p/protocols/pubsub/rpc/messages
type
Metric = object
name: string
value: float
MetricList = object
hostname: string
metrics: seq[Metric]
{.push raises: [].}
proc encode(m: Metric): ProtoBuffer =
result = initProtoBuffer()
result.write(1, m.name)
result.write(2, m.value)
result.finish()
proc decode(_: type Metric, buf: seq[byte]): Result[Metric, ProtoError] =
var res: Metric
let pb = initProtoBuffer(buf)
discard ?pb.getField(1, res.name)
discard ?pb.getField(2, res.value)
ok(res)
proc encode(m: MetricList): ProtoBuffer =
result = initProtoBuffer()
for metric in m.metrics:
result.write(1, metric.encode())
result.write(2, m.hostname)
result.finish()
proc decode(_: type MetricList, buf: seq[byte]): Result[MetricList, ProtoError] =
var
res: MetricList
metrics: seq[seq[byte]]
let pb = initProtoBuffer(buf)
discard ?pb.getRepeatedField(1, metrics)
for metric in metrics:
res.metrics &= ?Metric.decode(metric)
?pb.getRequiredField(2, res.hostname)
ok(res)
## This is exactly like the previous structure, except that we added
## a `hostname` to distinguish where the metric is coming from.
##
## Now we'll create a small GossipSub network to broadcast the metrics,
## and collect them on one of the node.
type Node = tuple[switch: Switch, gossip: GossipSub, hostname: string]
proc oneNode(node: Node, rng: ref HmacDrbgContext) {.async.} =
# This procedure will handle one of the node of the network
node.gossip.addValidator(
["metrics"],
proc(topic: string, message: Message): Future[ValidationResult] {.async.} =
let decoded = MetricList.decode(message.data)
if decoded.isErr:
return ValidationResult.Reject
return ValidationResult.Accept,
)
# This "validator" will attach to the `metrics` topic and make sure
# that every message in this topic is valid. This allows us to stop
# propagation of invalid messages quickly in the network, and punish
# peers sending them.
# `John` will be responsible to log the metrics, the rest of the nodes
# will just forward them in the network
if node.hostname == "John":
node.gossip.subscribe(
"metrics",
proc(topic: string, data: seq[byte]) {.async.} =
let m = MetricList.decode(data).expect("metric can be decoded")
echo m
,
)
else:
node.gossip.subscribe("metrics", nil)
# Create random metrics 10 times and broadcast them
for _ in 0 ..< 10:
await sleepAsync(500.milliseconds)
var metricList = MetricList(hostname: node.hostname)
let metricCount = rng[].generate(uint32) mod 4
for i in 0 ..< metricCount + 1:
metricList.metrics.add(
Metric(name: "metric_" & $i, value: float(rng[].generate(uint16)) / 1000.0)
)
discard await node.gossip.publish("metrics", encode(metricList).buffer)
await node.switch.stop()
## For our main procedure, we'll create a few nodes, and connect them together.
## Note that they are not all interconnected, but GossipSub will take care of
## broadcasting to the full network nonetheless.
proc main() {.async.} =
let rng = newRng()
var nodes: seq[Node]
for hostname in ["John", "Walter", "David", "Thuy", "Amy"]:
let
switch = newStandardSwitch(rng = rng)
gossip = GossipSub.init(switch = switch, triggerSelf = true)
switch.mount(gossip)
await switch.start()
nodes.add((switch, gossip, hostname))
for index, node in nodes:
# Connect to a few neighbors
for otherNodeIdx in index - 1 .. index + 2:
if otherNodeIdx notin 0 ..< nodes.len or otherNodeIdx == index:
continue
let otherNode = nodes[otherNodeIdx]
await node.switch.connect(
otherNode.switch.peerInfo.peerId, otherNode.switch.peerInfo.addrs
)
var allFuts: seq[Future[void]]
for node in nodes:
allFuts.add(oneNode(node, rng))
await allFutures(allFuts)
waitFor(main())
## If you run this program, you should see something like:
## ```
## (hostname: "John", metrics: @[(name: "metric_0", value: 42.097), (name: "metric_1", value: 50.99), (name: "metric_2", value: 47.86), (name: "metric_3", value: 5.368)])
## (hostname: "Walter", metrics: @[(name: "metric_0", value: 39.452), (name: "metric_1", value: 15.606), (name: "metric_2", value: 14.059), (name: "metric_3", value: 6.68)])
## (hostname: "David", metrics: @[(name: "metric_0", value: 9.82), (name: "metric_1", value: 2.862), (name: "metric_2", value: 15.514)])
## (hostname: "Thuy", metrics: @[(name: "metric_0", value: 59.038)])
## (hostname: "Amy", metrics: @[(name: "metric_0", value: 55.616), (name: "metric_1", value: 23.52), (name: "metric_2", value: 59.081), (name: "metric_3", value: 2.516)])
## ```
##
## This is John receiving & logging everyone's metrics.
##
## ## Going further
## Building efficient & safe GossipSub networks is a tricky subject. By tweaking the [gossip params](https://vacp2p.github.io/nim-libp2p/master/libp2p/protocols/pubsub/gossipsub/types.html#GossipSubParams)
## and [topic params](https://vacp2p.github.io/nim-libp2p/master/libp2p/protocols/pubsub/gossipsub/types.html#TopicParams),
## you can achieve very different properties.
##
## Also see reports for [GossipSub v1.1](https://gateway.ipfs.io/ipfs/QmRAFP5DBnvNjdYSbWhEhVRJJDFCLpPyvew5GwCCB4VxM4)
##
## If you are interested in broadcasting for your application, you may want to use [Waku](https://waku.org/), which builds on top of GossipSub,
## and adds features such as history, spam protection, and light node friendliness.

View File

@@ -0,0 +1,140 @@
## # Discovery Manager
##
## In the [previous tutorial](tutorial_4_gossipsub.md), we built a custom protocol using [protobuf](https://developers.google.com/protocol-buffers) and
## spread informations (some metrics) on the network using gossipsub.
## For this tutorial, on the other hand, we'll go back to a simple example
## we'll try to discover a specific peers to greet on the network.
##
## First, as usual, we import the dependencies:
import sequtils
import chronos
import stew/byteutils
import libp2p
import libp2p/protocols/rendezvous
import libp2p/discovery/rendezvousinterface
import libp2p/discovery/discoverymngr
## We'll not use newStandardSwitch this time as we need the discovery protocol
## [RendezVous](https://github.com/libp2p/specs/blob/master/rendezvous/README.md) to be mounted on the switch using withRendezVous.
##
## Note that other discovery methods such as [Kademlia](https://github.com/libp2p/specs/blob/master/kad-dht/README.md) or [discv5](https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md) exist.
proc createSwitch(rdv: RendezVous = RendezVous.new()): Switch =
SwitchBuilder
.new()
.withRng(newRng())
.withAddresses(@[MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet()])
.withTcpTransport()
.withYamux()
.withNoise()
.withRendezVous(rdv)
.build()
# Create a really simple protocol to log one message received then close the stream
const DumbCodec = "/dumb/proto/1.0.0"
type DumbProto = ref object of LPProtocol
proc new(T: typedesc[DumbProto], nodeNumber: int): T =
proc handle(conn: Connection, proto: string) {.async: (raises: [CancelledError]).} =
try:
echo "Node", nodeNumber, " received: ", string.fromBytes(await conn.readLp(1024))
except CancelledError as e:
raise e
except CatchableError as e:
echo "exception in handler", e.msg
finally:
await conn.close()
return T.new(codecs = @[DumbCodec], handler = handle)
## ## Bootnodes
## The first time a p2p program is ran, he needs to know how to join
## its network. This is generally done by hard-coding a list of stable
## nodes in the binary, called "bootnodes". These bootnodes are a
## critical part of a p2p network, since they are used by every new
## user to onboard the network.
##
## By using libp2p, we can use any node supporting our discovery protocol
## (rendezvous in this case) as a bootnode. For this example, we'll
## create a bootnode, and then every peer will advertise itself on the
## bootnode, and use it to find other peers
proc main() {.async.} =
let bootNode = createSwitch()
await bootNode.start()
# Create 5 nodes in the network
var
switches: seq[Switch] = @[]
discManagers: seq[DiscoveryManager] = @[]
for i in 0 .. 5:
let rdv = RendezVous.new()
# Create a remote future to await at the end of the program
let switch = createSwitch(rdv)
switch.mount(DumbProto.new(i))
switches.add(switch)
# A discovery manager is a simple tool, you can set it up by adding discovery
# interfaces (such as RendezVousInterface) then you can use it to advertise
# something on the network or to request something from it.
let dm = DiscoveryManager()
# A RendezVousInterface is a RendezVous protocol wrapped to be usable by the
# DiscoveryManager.
dm.add(RendezVousInterface.new(rdv))
discManagers.add(dm)
# We can now start the switch and connect to the bootnode
await switch.start()
await switch.connect(bootNode.peerInfo.peerId, bootNode.peerInfo.addrs)
# Each nodes of the network will advertise on some topics (EvenGang or OddClub)
dm.advertise(RdvNamespace(if i mod 2 == 0: "EvenGang" else: "OddClub"))
## We can now create the newcomer. This peer will connect to the boot node, and use
## it to discover peers & greet them.
let
rdv = RendezVous.new()
newcomer = createSwitch(rdv)
dm = DiscoveryManager()
await newcomer.start()
await newcomer.connect(bootNode.peerInfo.peerId, bootNode.peerInfo.addrs)
dm.add(RendezVousInterface.new(rdv, ttr = 250.milliseconds))
# Use the discovery manager to find peers on the OddClub topic to greet them
let queryOddClub = dm.request(RdvNamespace("OddClub"))
for _ in 0 .. 2:
let
# getPeer give you a PeerAttribute containing informations about the peer.
res = await queryOddClub.getPeer()
# Here we will use the PeerId and the MultiAddress to greet him
conn = await newcomer.dial(res[PeerId], res.getAll(MultiAddress), DumbCodec)
await conn.writeLp("Odd Club suuuucks! Even Gang is better!")
# Uh-oh!
await conn.close()
# Wait for the peer to close the stream
await conn.join()
# Queries will run in a loop, so we must stop them when we are done
queryOddClub.stop()
# Maybe it was because he wanted to join the EvenGang
let queryEvenGang = dm.request(RdvNamespace("EvenGang"))
for _ in 0 .. 2:
let
res = await queryEvenGang.getPeer()
conn = await newcomer.dial(res[PeerId], res.getAll(MultiAddress), DumbCodec)
await conn.writeLp("Even Gang is sooo laaame! Odd Club rocks!")
# Or maybe not...
await conn.close()
await conn.join()
queryEvenGang.stop()
# What can I say, some people just want to watch the world burn... Anyway
# Stop all the discovery managers
for d in discManagers:
d.stop()
dm.stop()
# Stop all the switches
await allFutures(switches.mapIt(it.stop()))
await allFutures(bootNode.stop(), newcomer.stop())
waitFor(main())

View File

@@ -0,0 +1,284 @@
## # Tron example
##
## In this tutorial, we will create a video game based on libp2p, using
## all of the features we talked about in the last tutorials.
##
## We will:
## - Discover peers using the Discovery Manager
## - Use GossipSub to find a play mate
## - Create a custom protocol to play with him
##
## While this may look like a daunting project, it's less than 150 lines of code.
##
## The game will be a simple Tron. We will use [nico](https://github.com/ftsf/nico)
## as a game engine. (you need to run `nimble install nico` to have it available)
##
## ![multiplay](https://user-images.githubusercontent.com/13471753/198852714-b55048e3-f233-4723-900d-2193ad259fe1.gif)
##
## We will start by importing our dependencies and creating our types
import os
import nico, chronos, stew/byteutils, stew/endians2
import libp2p
import libp2p/protocols/rendezvous
import libp2p/discovery/rendezvousinterface
import libp2p/discovery/discoverymngr
const
directions = @[(K_UP, 0, -1), (K_LEFT, -1, 0), (K_DOWN, 0, 1), (K_RIGHT, 1, 0)]
mapSize = 32
tickPeriod = 0.2
type
Player = ref object
x, y: int
currentDir, nextDir: int
lost: bool
color: int
Game = ref object
gameMap: array[mapSize * mapSize, int]
tickTime: float
localPlayer, remotePlayer: Player
peerFound: Future[Connection]
hasCandidate: bool
tickFinished: Future[int]
GameProto = ref object of LPProtocol
proc new(_: type[Game]): Game =
# Default state of a game
result = Game(
tickTime: -3.0, # 3 seconds of "warm-up" time
localPlayer: Player(x: 4, y: 16, currentDir: 3, nextDir: 3, color: 8),
remotePlayer: Player(x: 27, y: 16, currentDir: 1, nextDir: 1, color: 12),
peerFound: newFuture[Connection](),
)
for pos in 0 .. result.gameMap.high:
if pos mod mapSize in [0, mapSize - 1] or pos div mapSize in [0, mapSize - 1]:
result.gameMap[pos] = 7
## ## Game Logic
## The networking during the game will work like this:
##
## * Each player will have `tickPeriod` (0.1) seconds to choose
## a direction that he wants to go to (default to current direction)
## * After `tickPeriod`, we will send our choosen direction to the peer,
## and wait for his direction
## * Once we have both direction, we will "tick" the game, and restart the
## loop, as long as both player are alive.
##
## This is a very simplistic scheme, but creating proper networking for
## video games is an [art](https://developer.valvesoftware.com/wiki/Latency_Compensating_Methods_in_Client/Server_In-game_Protocol_Design_and_Optimization)
##
## The main drawback of this scheme is that the more ping you have with
## the peer, the slower the game will run. Or invertedly, the less ping you
## have, the faster it runs!
proc update(g: Game, dt: float32) =
# Will be called at each frame of the game.
#
# Because both Nico and Chronos have a main loop,
# they must share the control of the main thread.
# This is a hacky way to make this happen
waitFor(sleepAsync(1.milliseconds))
# Don't do anything if we are still waiting for an opponent
if not (g.peerFound.finished()) or isNil(g.tickFinished):
return
g.tickTime += dt
# Update the wanted direction, making sure we can't go backward
for i in 0 .. directions.high:
if i != (g.localPlayer.currentDir + 2 mod 4) and keyp(directions[i][0]):
g.localPlayer.nextDir = i
if g.tickTime > tickPeriod and not g.tickFinished.finished():
# We choosen our next direction, let the networking know
g.localPlayer.currentDir = g.localPlayer.nextDir
g.tickFinished.complete(g.localPlayer.currentDir)
proc tick(g: Game, p: Player) =
# Move player and check if he lost
p.x += directions[p.currentDir][1]
p.y += directions[p.currentDir][2]
if g.gameMap[p.y * mapSize + p.x] != 0:
p.lost = true
g.gameMap[p.y * mapSize + p.x] = p.color
proc mainLoop(g: Game, peer: Connection) {.async.} =
while not (g.localPlayer.lost or g.remotePlayer.lost):
if g.tickTime > 0.0:
g.tickTime = 0
g.tickFinished = newFuture[int]()
# Wait for a choosen direction
let dir = await g.tickFinished
# Send it
await peer.writeLp(toBytes(uint32(dir)))
# Get the one from the peer
g.remotePlayer.currentDir = int uint32.fromBytes(await peer.readLp(8))
# Tick the players & restart
g.tick(g.remotePlayer)
g.tick(g.localPlayer)
## We'll draw the map & put some texts when necessary:
proc draw(g: Game) =
for pos, color in g.gameMap:
setColor(color)
boxFill(pos mod 32 * 4, pos div 32 * 4, 4, 4)
let text =
if not (g.peerFound.finished()):
"Matchmaking.."
elif g.tickTime < -1.5:
"Welcome to Etron"
elif g.tickTime < 0.0:
"- " & $(int(abs(g.tickTime) / 0.5) + 1) & " -"
elif g.remotePlayer.lost and g.localPlayer.lost:
"DEUCE"
elif g.localPlayer.lost:
"YOU LOOSE"
elif g.remotePlayer.lost:
"YOU WON"
else:
""
printc(text, screenWidth div 2, screenHeight div 2)
## ## Matchmaking
## To find an opponent, we will broadcast our address on a
## GossipSub topic, and wait for someone to connect to us.
## We will also listen to that topic, and connect to anyone
## broadcasting his address.
##
## If we are looking for a game, we'll send `ok` to let the
## peer know that we are available, check that he is also available,
## and launch the game.
proc new(T: typedesc[GameProto], g: Game): T =
proc handle(conn: Connection, proto: string) {.async: (raises: [CancelledError]).} =
defer:
await conn.closeWithEof()
try:
if g.peerFound.finished or g.hasCandidate:
await conn.close()
return
g.hasCandidate = true
await conn.writeLp("ok")
if "ok" != string.fromBytes(await conn.readLp(1024)):
g.hasCandidate = false
return
g.peerFound.complete(conn)
# The handler of a protocol must wait for the stream to
# be finished before returning
await conn.join()
except CancelledError as e:
raise e
except CatchableError as e:
echo "exception in handler", e.msg
return T.new(codecs = @["/tron/1.0.0"], handler = handle)
proc networking(g: Game) {.async.} =
# Create our switch, similar to the GossipSub example and
# the Discovery examples combined
let
rdv = RendezVous.new()
switch = SwitchBuilder
.new()
.withRng(newRng())
.withAddresses(@[MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet()])
.withTcpTransport()
.withYamux()
.withNoise()
.withRendezVous(rdv)
.build()
dm = DiscoveryManager()
gameProto = GameProto.new(g)
gossip = GossipSub.init(switch = switch, triggerSelf = false)
dm.add(RendezVousInterface.new(rdv))
switch.mount(gossip)
switch.mount(gameProto)
gossip.subscribe(
"/tron/matchmaking",
proc(topic: string, data: seq[byte]) {.async.} =
# If we are still looking for an opponent,
# try to match anyone broadcasting its address
if g.peerFound.finished or g.hasCandidate:
return
g.hasCandidate = true
try:
let
(peerId, multiAddress) = parseFullAddress(data).tryGet()
stream = await switch.dial(peerId, @[multiAddress], gameProto.codec)
await stream.writeLp("ok")
if (await stream.readLp(10)) != "ok".toBytes:
g.hasCandidate = false
return
g.peerFound.complete(stream)
# We are "player 2"
swap(g.localPlayer, g.remotePlayer)
except CatchableError as exc:
discard,
)
await switch.start()
defer:
await switch.stop()
# As explained in the last tutorial, we need a bootnode to be able
# to find peers. We could use any libp2p running rendezvous (or any
# node running tron). We will take it's MultiAddress from the command
# line parameters
if paramCount() > 0:
let (peerId, multiAddress) = paramStr(1).parseFullAddress().tryGet()
await switch.connect(peerId, @[multiAddress])
else:
echo "No bootnode provided, listening on: ", switch.peerInfo.fullAddrs.tryGet()
# Discover peers from the bootnode, and connect to them
dm.advertise(RdvNamespace("tron"))
let discoveryQuery = dm.request(RdvNamespace("tron"))
discoveryQuery.forEach:
try:
await switch.connect(peer[PeerId], peer.getAll(MultiAddress))
except CatchableError as exc:
echo "Failed to dial a peer: ", exc.msg
# We will try to publish our address multiple times, in case
# it takes time to establish connections with other GossipSub peers
var published = false
while not published:
await sleepAsync(500.milliseconds)
for fullAddr in switch.peerInfo.fullAddrs.tryGet():
if (await gossip.publish("/tron/matchmaking", fullAddr.bytes)) == 0:
published = false
break
published = true
discoveryQuery.stop()
# We now wait for someone to connect to us (or for us to connect to someone)
let peerConn = await g.peerFound
defer:
await peerConn.closeWithEof()
await g.mainLoop(peerConn)
let
game = Game.new()
netFut = networking(game)
nico.init("Status", "Tron")
nico.createWindow("Tron", mapSize * 4, mapSize * 4, 4, false)
nico.run(
proc() =
discard,
proc(dt: float32) =
game.update(dt),
proc() =
game.draw(),
)
waitFor(netFut.cancelAndWait())
## And that's it! If you want to run this code locally, the simplest way is to use the
## first node as a boot node for the second one. But you can also use any rendezvous node

5
funding.json Normal file
View File

@@ -0,0 +1,5 @@
{
"opRetro": {
"projectId": "0xc9561ba3e4eca5483b40f8b1a254a73c91fefe4f8aee32dc20c0d96dcf33fe80"
}
}

View File

@@ -1,17 +1,76 @@
## Nim-LibP2P
## Copyright (c) 2018 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-LibP2P
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
import
libp2p/daemon/[daemonapi, transpool],
libp2p/protobuf/minprotobuf,
libp2p/varint
when defined(nimdoc):
## Welcome to the nim-libp2p reference!
##
## On the left, you'll find a switch that allows you to see private
## procedures. By default, you'll only see the public one (marked with `{.public.}`)
##
## The difference between public and private procedures is that public procedure
## stay backward compatible during the Major version, whereas private ones can
## change at each new Minor version.
##
## If you're new to nim-libp2p, you can find a tutorial `here<https://vacp2p.github.io/nim-libp2p/docs/tutorial_1_connect/>`_
## that can help you get started.
export
daemonapi, transpool, minprotobuf, varint
# Import stuff for doc
import
libp2p/[
protobuf/minprotobuf,
switch,
stream/lpstream,
builders,
transports/tcptransport,
transports/wstransport,
protocols/ping,
protocols/pubsub,
peerid,
peerinfo,
peerstore,
multiaddress,
]
proc dummyPrivateProc*() =
## A private proc example
discard
else:
import
libp2p/[
protobuf/minprotobuf,
muxers/muxer,
muxers/mplex/mplex,
stream/lpstream,
stream/bufferstream,
stream/connection,
transports/transport,
transports/tcptransport,
protocols/secure/noise,
cid,
multihash,
multicodec,
errors,
switch,
peerid,
peerinfo,
multiaddress,
builders,
crypto/crypto,
protocols/pubsub,
]
export
minprotobuf, switch, peerid, peerinfo, connection, multiaddress, crypto, lpstream,
bufferstream, muxer, mplex, transport, tcptransport, noise, errors, cid, multihash,
multicodec, builders, pubsub
when defined(libp2p_quic_support):
import libp2p/transports/quictransport
export quictransport

View File

@@ -1,41 +1,51 @@
mode = ScriptMode.Verbose
packageName = "libp2p"
version = "0.0.2"
author = "Status Research & Development GmbH"
description = "LibP2P implementation"
license = "MIT"
skipDirs = @["tests", "examples", "Nim", "tools", "scripts", "docs"]
packageName = "libp2p"
version = "1.10.1"
author = "Status Research & Development GmbH"
description = "LibP2P implementation"
license = "MIT"
skipDirs = @["tests", "examples", "Nim", "tools", "scripts", "docs"]
# test
requires "nim >= 1.2.0",
"nimcrypto >= 0.4.1",
"bearssl >= 0.1.4",
"chronicles >= 0.7.2",
"chronos >= 2.5.2",
"metrics",
"secp256k1",
"stew#head"
requires "nim >= 1.6.0",
"nimcrypto >= 0.6.0 & < 0.7.0", "dnsclient >= 0.3.0 & < 0.4.0", "bearssl >= 0.2.5",
"chronicles >= 0.10.2", "chronos >= 4.0.3", "metrics", "secp256k1", "stew >= 0.4.0",
"websock", "unittest2", "results",
"https://github.com/status-im/nim-quic.git#a6c30263c95fc5ddb2ef4d197c09b282555c06b0"
proc runTest(filename: string, verify: bool = true, sign: bool = true,
moreoptions: string = "") =
var excstr = "nim c --opt:speed -d:debug --verbosity:0 --hints:off"
excstr.add(" --warning[CaseTransition]:off --warning[ObservableStores]:off --warning[LockLevel]:off")
excstr.add(" -d:libp2p_pubsub_sign=" & $sign)
excstr.add(" -d:libp2p_pubsub_verify=" & $verify)
let nimc = getEnv("NIMC", "nim") # Which nim compiler to use
let lang = getEnv("NIMLANG", "c") # Which backend (c/cpp/js)
let flags = getEnv("NIMFLAGS", "") # Extra flags for the compiler
let verbose = getEnv("V", "") notin ["", "0"]
let cfg =
" --styleCheck:usages --styleCheck:error" &
(if verbose: "" else: " --verbosity:0 --hints:off") & " --skipUserCfg -f" &
" --threads:on --opt:speed"
import hashes, strutils
proc runTest(filename: string, moreoptions: string = "") =
var excstr = nimc & " " & lang & " -d:debug " & cfg & " " & flags
excstr.add(" " & moreoptions & " ")
if verify and sign:
# build it with TRACE and JSON logs
exec excstr & " -d:chronicles_log_level=TRACE -d:chronicles_sinks:json" & " tests/" & filename
# build it again, to run it with less verbose logs
exec excstr & " -d:chronicles_log_level=INFO -r" & " tests/" & filename
if getEnv("CICOV").len > 0:
excstr &= " --nimcache:nimcache/" & filename & "-" & $excstr.hash
exec excstr & " -r " & " tests/" & filename
rmFile "tests/" & filename.toExe
proc buildSample(filename: string) =
var excstr = "nim c --opt:speed --threads:on -d:debug --verbosity:0 --hints:off"
excstr.add(" --warning[CaseTransition]:off --warning[ObservableStores]:off --warning[LockLevel]:off")
proc buildSample(filename: string, run = false, extraFlags = "") =
var excstr = nimc & " " & lang & " " & cfg & " " & flags & " -p:. " & extraFlags
excstr.add(" examples/" & filename)
exec excstr
rmFile "examples" & filename.toExe
if run:
exec "./examples/" & filename.toExe
rmFile "examples/" & filename.toExe
proc tutorialToMd(filename: string) =
let markdown = gorge "cat " & filename & " | " & nimc & " " & lang &
" -r --verbosity:0 --hints:off tools/markdown_builder.nim "
writeFile(filename.replace(".nim", ".md"), markdown)
task testnative, "Runs libp2p native tests":
runTest("testnative")
@@ -47,37 +57,73 @@ task testinterop, "Runs interop tests":
runTest("testinterop")
task testpubsub, "Runs pubsub tests":
runTest("pubsub/testgossipinternal", sign = false, verify = false, moreoptions = "-d:pubsub_internal_testing")
runTest("pubsub/testpubsub")
runTest("pubsub/testpubsub", sign = false, verify = false)
runTest("pubsub/testpubsub", sign = false, verify = false, moreoptions = "-d:libp2p_pubsub_anonymize=true")
task testpubsub_slim, "Runs pubsub tests":
runTest("pubsub/testgossipinternal", sign = false, verify = false, moreoptions = "-d:pubsub_internal_testing")
runTest("pubsub/testpubsub")
task testfilter, "Run PKI filter test":
runTest("testpkifilter",
moreoptions = "-d:libp2p_pki_schemes=\"secp256k1\"")
runTest("testpkifilter",
moreoptions = "-d:libp2p_pki_schemes=\"secp256k1;ed25519\"")
runTest("testpkifilter",
moreoptions = "-d:libp2p_pki_schemes=\"secp256k1;ed25519;ecnist\"")
runTest("testpkifilter",
moreoptions = "-d:libp2p_pki_schemes=")
runTest("testpkifilter")
runTest("testpkifilter", moreoptions = "-d:libp2p_pki_schemes=")
task test, "Runs the test suite":
exec "nimble testnative"
exec "nimble testpubsub"
exec "nimble testdaemon"
exec "nimble testinterop"
runTest("testall")
exec "nimble testfilter"
task test_slim, "Runs the test suite":
exec "nimble testnative"
exec "nimble testpubsub_slim"
exec "nimble testinterop"
exec "nimble testfilter"
task website, "Build the website":
tutorialToMd("examples/tutorial_1_connect.nim")
tutorialToMd("examples/tutorial_2_customproto.nim")
tutorialToMd("examples/tutorial_3_protobuf.nim")
tutorialToMd("examples/tutorial_4_gossipsub.nim")
tutorialToMd("examples/tutorial_5_discovery.nim")
tutorialToMd("examples/tutorial_6_game.nim")
tutorialToMd("examples/circuitrelay.nim")
exec "mkdocs build"
task examples_build, "Build the samples":
buildSample("directchat")
task examples, "Build and run examples":
exec "nimble install -y nimpng"
exec "nimble install -y nico --passNim=--skipParentCfg"
buildSample("examples_build", false, "--styleCheck:off") # build only
buildSample("examples_run", true)
# pin system
# while nimble lockfile
# isn't available
const PinFile = ".pinned"
task pin, "Create a lockfile":
# pinner.nim was originally here
# but you can't read output from
# a command in a nimscript
exec nimc & " c -r tools/pinner.nim"
import sequtils
import os
task install_pinned, "Reads the lockfile":
let toInstall = readFile(PinFile).splitWhitespace().mapIt(
(it.split(";", 1)[0], it.split(";", 1)[1])
)
# [('packageName', 'packageFullUri')]
rmDir("nimbledeps")
mkDir("nimbledeps")
exec "nimble install -y " & toInstall.mapIt(it[1]).join(" ")
# Remove the automatically installed deps
# (inefficient you say?)
let nimblePkgs =
if system.dirExists("nimbledeps/pkgs"): "nimbledeps/pkgs" else: "nimbledeps/pkgs2"
for dependency in listDirs(nimblePkgs):
let
fileName = dependency.extractFilename
fileContent = readFile(dependency & "/nimblemeta.json")
packageName = fileName.split('-')[0]
if toInstall.anyIt(
it[0] == packageName and (
it[1].split('#')[^1] in fileContent or # nimble for nim 2.X
fileName.endsWith(it[1].split('#')[^1]) # nimble for nim 1.X
)
) == false or fileName.split('-')[^1].len < 20: # safegard for nimble for nim 1.X
rmDir(dependency)
task unpin, "Restore global package use":
rmDir("nimbledeps")

383
libp2p/builders.nim Normal file
View File

@@ -0,0 +1,383 @@
# Nim-Libp2p
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
## This module contains a Switch Building helper.
runnableExamples:
let switch = SwitchBuilder.new().withRng(rng).withAddresses(multiaddress)
# etc
.build()
{.push raises: [].}
import options, tables, chronos, chronicles, sequtils
import
switch,
peerid,
peerinfo,
stream/connection,
multiaddress,
crypto/crypto,
transports/[transport, tcptransport, memorytransport],
muxers/[muxer, mplex/mplex, yamux/yamux],
protocols/[identify, secure/secure, secure/noise, rendezvous],
protocols/connectivity/[autonat/server, relay/relay, relay/client, relay/rtransport],
connmanager,
upgrademngrs/muxedupgrade,
observedaddrmanager,
nameresolving/nameresolver,
errors,
utility
import services/wildcardresolverservice
export switch, peerid, peerinfo, connection, multiaddress, crypto, errors
const MemoryAutoAddress* = memorytransport.MemoryAutoAddress
type
TransportProvider* {.public.} =
proc(upgr: Upgrade, privateKey: PrivateKey): Transport {.gcsafe, raises: [].}
SecureProtocol* {.pure.} = enum
Noise
SwitchBuilder* = ref object
privKey: Option[PrivateKey]
addresses: seq[MultiAddress]
secureManagers: seq[SecureProtocol]
muxers: seq[MuxerProvider]
transports: seq[TransportProvider]
rng: ref HmacDrbgContext
maxConnections: int
maxIn: int
sendSignedPeerRecord: bool
maxOut: int
maxConnsPerPeer: int
protoVersion: string
agentVersion: string
nameResolver: NameResolver
peerStoreCapacity: Opt[int]
autonat: bool
circuitRelay: Relay
rdv: RendezVous
services: seq[Service]
observedAddrManager: ObservedAddrManager
enableWildcardResolver: bool
proc new*(T: type[SwitchBuilder]): T {.public.} =
## Creates a SwitchBuilder
let address =
MultiAddress.init("/ip4/127.0.0.1/tcp/0").expect("Should initialize to default")
SwitchBuilder(
privKey: none(PrivateKey),
addresses: @[address],
secureManagers: @[],
maxConnections: MaxConnections,
maxIn: -1,
maxOut: -1,
maxConnsPerPeer: MaxConnectionsPerPeer,
protoVersion: ProtoVersion,
agentVersion: AgentVersion,
enableWildcardResolver: true,
)
proc withPrivateKey*(
b: SwitchBuilder, privateKey: PrivateKey
): SwitchBuilder {.public.} =
## Set the private key of the switch. Will be used to
## generate a PeerId
b.privKey = some(privateKey)
b
proc withAddresses*(
b: SwitchBuilder, addresses: seq[MultiAddress], enableWildcardResolver: bool = true
): SwitchBuilder {.public.} =
## | Set the listening addresses of the switch
## | Calling it multiple time will override the value
b.addresses = addresses
b.enableWildcardResolver = enableWildcardResolver
b
proc withAddress*(
b: SwitchBuilder, address: MultiAddress, enableWildcardResolver: bool = true
): SwitchBuilder {.public.} =
## | Set the listening address of the switch
## | Calling it multiple time will override the value
b.withAddresses(@[address], enableWildcardResolver)
proc withSignedPeerRecord*(b: SwitchBuilder, sendIt = true): SwitchBuilder {.public.} =
b.sendSignedPeerRecord = sendIt
b
proc withMplex*(
b: SwitchBuilder, inTimeout = 5.minutes, outTimeout = 5.minutes, maxChannCount = 200
): SwitchBuilder {.public.} =
## | Uses `Mplex <https://docs.libp2p.io/concepts/stream-multiplexing/#mplex>`_ as a multiplexer
## | `Timeout` is the duration after which a inactive connection will be closed
proc newMuxer(conn: Connection): Muxer =
Mplex.new(conn, inTimeout, outTimeout, maxChannCount)
assert b.muxers.countIt(it.codec == MplexCodec) == 0, "Mplex build multiple times"
b.muxers.add(MuxerProvider.new(newMuxer, MplexCodec))
b
proc withYamux*(
b: SwitchBuilder,
windowSize: int = YamuxDefaultWindowSize,
inTimeout: Duration = 5.minutes,
outTimeout: Duration = 5.minutes,
): SwitchBuilder =
proc newMuxer(conn: Connection): Muxer =
Yamux.new(conn, windowSize, inTimeout = inTimeout, outTimeout = outTimeout)
assert b.muxers.countIt(it.codec == YamuxCodec) == 0, "Yamux build multiple times"
b.muxers.add(MuxerProvider.new(newMuxer, YamuxCodec))
b
proc withNoise*(b: SwitchBuilder): SwitchBuilder {.public.} =
b.secureManagers.add(SecureProtocol.Noise)
b
proc withTransport*(
b: SwitchBuilder, prov: TransportProvider
): SwitchBuilder {.public.} =
## Use a custom transport
runnableExamples:
let switch = SwitchBuilder
.new()
.withTransport(
proc(upgr: Upgrade, privateKey: PrivateKey): Transport =
TcpTransport.new(flags, upgr)
)
.build()
b.transports.add(prov)
b
proc withTcpTransport*(
b: SwitchBuilder, flags: set[ServerFlags] = {}
): SwitchBuilder {.public.} =
b.withTransport(
proc(upgr: Upgrade, privateKey: PrivateKey): Transport =
TcpTransport.new(flags, upgr)
)
when defined(libp2p_quic_support):
import transports/quictransport
proc withQuicTransport*(b: SwitchBuilder): SwitchBuilder {.public.} =
b.withTransport(
proc(upgr: Upgrade, privateKey: PrivateKey): Transport =
QuicTransport.new(upgr, privateKey)
)
proc withMemoryTransport*(b: SwitchBuilder): SwitchBuilder {.public.} =
b.withTransport(
proc(upgr: Upgrade, privateKey: PrivateKey): Transport =
MemoryTransport.new(upgr)
)
proc withRng*(b: SwitchBuilder, rng: ref HmacDrbgContext): SwitchBuilder {.public.} =
b.rng = rng
b
proc withMaxConnections*(
b: SwitchBuilder, maxConnections: int
): SwitchBuilder {.public.} =
## Maximum concurrent connections of the switch. You should either use this, or
## `withMaxIn <#withMaxIn,SwitchBuilder,int>`_ & `withMaxOut<#withMaxOut,SwitchBuilder,int>`_
b.maxConnections = maxConnections
b
proc withMaxIn*(b: SwitchBuilder, maxIn: int): SwitchBuilder {.public.} =
## Maximum concurrent incoming connections. Should be used with `withMaxOut<#withMaxOut,SwitchBuilder,int>`_
b.maxIn = maxIn
b
proc withMaxOut*(b: SwitchBuilder, maxOut: int): SwitchBuilder {.public.} =
## Maximum concurrent outgoing connections. Should be used with `withMaxIn<#withMaxIn,SwitchBuilder,int>`_
b.maxOut = maxOut
b
proc withMaxConnsPerPeer*(
b: SwitchBuilder, maxConnsPerPeer: int
): SwitchBuilder {.public.} =
b.maxConnsPerPeer = maxConnsPerPeer
b
proc withPeerStore*(b: SwitchBuilder, capacity: int): SwitchBuilder {.public.} =
b.peerStoreCapacity = Opt.some(capacity)
b
proc withProtoVersion*(
b: SwitchBuilder, protoVersion: string
): SwitchBuilder {.public.} =
b.protoVersion = protoVersion
b
proc withAgentVersion*(
b: SwitchBuilder, agentVersion: string
): SwitchBuilder {.public.} =
b.agentVersion = agentVersion
b
proc withNameResolver*(
b: SwitchBuilder, nameResolver: NameResolver
): SwitchBuilder {.public.} =
b.nameResolver = nameResolver
b
proc withAutonat*(b: SwitchBuilder): SwitchBuilder =
b.autonat = true
b
proc withCircuitRelay*(b: SwitchBuilder, r: Relay = Relay.new()): SwitchBuilder =
b.circuitRelay = r
b
proc withRendezVous*(
b: SwitchBuilder, rdv: RendezVous = RendezVous.new()
): SwitchBuilder =
b.rdv = rdv
b
proc withServices*(b: SwitchBuilder, services: seq[Service]): SwitchBuilder =
b.services = services
b
proc withObservedAddrManager*(
b: SwitchBuilder, observedAddrManager: ObservedAddrManager
): SwitchBuilder =
b.observedAddrManager = observedAddrManager
b
proc build*(b: SwitchBuilder): Switch {.raises: [LPError], public.} =
if b.rng == nil: # newRng could fail
raise newException(Defect, "Cannot initialize RNG")
let pkRes = PrivateKey.random(b.rng[])
let seckey = b.privKey.get(otherwise = pkRes.expect("Expected default Private Key"))
if b.secureManagers.len == 0:
debug "no secure managers defined. Adding noise by default"
b.secureManagers.add(SecureProtocol.Noise)
var secureManagerInstances: seq[Secure]
if SecureProtocol.Noise in b.secureManagers:
secureManagerInstances.add(Noise.new(b.rng, seckey).Secure)
let peerInfo = PeerInfo.new(
seckey, b.addresses, protoVersion = b.protoVersion, agentVersion = b.agentVersion
)
let identify =
if b.observedAddrManager != nil:
Identify.new(peerInfo, b.sendSignedPeerRecord, b.observedAddrManager)
else:
Identify.new(peerInfo, b.sendSignedPeerRecord)
let
connManager =
ConnManager.new(b.maxConnsPerPeer, b.maxConnections, b.maxIn, b.maxOut)
ms = MultistreamSelect.new()
muxedUpgrade = MuxedUpgrade.new(b.muxers, secureManagerInstances, ms)
let transports = block:
var transports: seq[Transport]
for tProvider in b.transports:
transports.add(tProvider(muxedUpgrade, seckey))
transports
if b.secureManagers.len == 0:
b.secureManagers &= SecureProtocol.Noise
if isNil(b.rng):
b.rng = newRng()
let peerStore = block:
b.peerStoreCapacity.withValue(capacity):
PeerStore.new(identify, capacity)
else:
PeerStore.new(identify)
if b.enableWildcardResolver:
b.services.insert(WildcardAddressResolverService.new(), 0)
let switch = newSwitch(
peerInfo = peerInfo,
transports = transports,
secureManagers = secureManagerInstances,
connManager = connManager,
ms = ms,
nameResolver = b.nameResolver,
peerStore = peerStore,
services = b.services,
)
switch.mount(identify)
if b.autonat:
let autonat = Autonat.new(switch)
switch.mount(autonat)
if not isNil(b.circuitRelay):
if b.circuitRelay of RelayClient:
switch.addTransport(RelayTransport.new(RelayClient(b.circuitRelay), muxedUpgrade))
b.circuitRelay.setup(switch)
switch.mount(b.circuitRelay)
if not isNil(b.rdv):
b.rdv.setup(switch)
switch.mount(b.rdv)
return switch
proc newStandardSwitch*(
privKey = none(PrivateKey),
addrs: MultiAddress | seq[MultiAddress] =
MultiAddress.init("/ip4/127.0.0.1/tcp/0").expect("valid address"),
secureManagers: openArray[SecureProtocol] = [SecureProtocol.Noise],
transportFlags: set[ServerFlags] = {},
rng = newRng(),
inTimeout: Duration = 5.minutes,
outTimeout: Duration = 5.minutes,
maxConnections = MaxConnections,
maxIn = -1,
maxOut = -1,
maxConnsPerPeer = MaxConnectionsPerPeer,
nameResolver: NameResolver = nil,
sendSignedPeerRecord = false,
peerStoreCapacity = 1000,
): Switch {.raises: [LPError], public.} =
## Helper for common switch configurations.
let addrs =
when addrs is MultiAddress:
@[addrs]
else:
addrs
var b = SwitchBuilder
.new()
.withAddresses(addrs)
.withRng(rng)
.withSignedPeerRecord(sendSignedPeerRecord)
.withMaxConnections(maxConnections)
.withMaxIn(maxIn)
.withMaxOut(maxOut)
.withMaxConnsPerPeer(maxConnsPerPeer)
.withPeerStore(capacity = peerStoreCapacity)
.withMplex(inTimeout, outTimeout)
.withTcpTransport(transportFlags)
.withNameResolver(nameResolver)
.withNoise()
privKey.withValue(pkey):
b = b.withPrivateKey(pkey)
b.build()

View File

@@ -1,28 +1,34 @@
## Nim-LibP2P
## Copyright (c) 2018 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-LibP2P
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
## This module implementes CID (Content IDentifier).
{.push raises: [Defect].}
{.push raises: [].}
import tables
import multibase, multicodec, multihash, vbuffer, varint
import stew/[base58, results]
import tables, hashes
import multibase, multicodec, multihash, vbuffer, varint, results
import stew/base58
export results
type
CidError* {.pure.} = enum
Error, Incorrect, Unsupported, Overrun
Error
Incorrect
Unsupported
Overrun
CidVersion* = enum
CIDvIncorrect, CIDv0, CIDv1, CIDvReserved
CIDvIncorrect
CIDv0
CIDv1
CIDvReserved
Cid* = object
cidver*: CidVersion
@@ -30,55 +36,52 @@ type
hpos*: int
data*: VBuffer
const
ContentIdsList = [
multiCodec("raw"),
multiCodec("dag-pb"),
multiCodec("dag-cbor"),
multiCodec("dag-json"),
multiCodec("git-raw"),
multiCodec("eth-block"),
multiCodec("eth-block-list"),
multiCodec("eth-tx-trie"),
multiCodec("eth-tx"),
multiCodec("eth-tx-receipt-trie"),
multiCodec("eth-tx-receipt"),
multiCodec("eth-state-trie"),
multiCodec("eth-account-snapshot"),
multiCodec("eth-storage-trie"),
multiCodec("bitcoin-block"),
multiCodec("bitcoin-tx"),
multiCodec("zcash-block"),
multiCodec("zcash-tx"),
multiCodec("stellar-block"),
multiCodec("stellar-tx"),
multiCodec("decred-block"),
multiCodec("decred-tx"),
multiCodec("dash-block"),
multiCodec("dash-tx"),
multiCodec("torrent-info"),
multiCodec("torrent-file"),
multiCodec("ed25519-pub")
]
const ContentIdsList = [
multiCodec("raw"),
multiCodec("dag-pb"),
multiCodec("dag-cbor"),
multiCodec("dag-json"),
multiCodec("libp2p-key"),
multiCodec("git-raw"),
multiCodec("eth-block"),
multiCodec("eth-block-list"),
multiCodec("eth-tx-trie"),
multiCodec("eth-tx"),
multiCodec("eth-tx-receipt-trie"),
multiCodec("eth-tx-receipt"),
multiCodec("eth-state-trie"),
multiCodec("eth-account-snapshot"),
multiCodec("eth-storage-trie"),
multiCodec("bitcoin-block"),
multiCodec("bitcoin-tx"),
multiCodec("zcash-block"),
multiCodec("zcash-tx"),
multiCodec("stellar-block"),
multiCodec("stellar-tx"),
multiCodec("decred-block"),
multiCodec("decred-tx"),
multiCodec("dash-block"),
multiCodec("dash-tx"),
multiCodec("torrent-info"),
multiCodec("torrent-file"),
multiCodec("ed25519-pub"),
]
proc initCidCodeTable(): Table[int, MultiCodec] {.compileTime.} =
result = initTable[int, MultiCodec]()
for item in ContentIdsList:
result[int(item)] = item
const
CodeContentIds = initCidCodeTable()
const CodeContentIds = initCidCodeTable()
template orError*(exp: untyped, err: untyped): untyped =
(exp.mapErr do (_: auto) -> auto: err)
exp.mapErr do(_: auto) -> auto:
err
proc decode(data: openarray[byte]): Result[Cid, CidError] =
proc decode(data: openArray[byte]): Result[Cid, CidError] =
if len(data) == 34 and data[0] == 0x12'u8 and data[1] == 0x20'u8:
ok(Cid(
cidver: CIDv0,
mcodec: multiCodec("dag-pb"),
hpos: 0,
data: initVBuffer(data)))
ok(
Cid(cidver: CIDv0, mcodec: multiCodec("dag-pb"), hpos: 0, data: initVBuffer(data))
)
else:
var version, codec: uint64
var res, offset: int
@@ -99,23 +102,20 @@ proc decode(data: openarray[byte]): Result[Cid, CidError] =
err(CidError.Incorrect)
else:
offset += res
var mcodec = CodeContentIds.getOrDefault(cast[int](codec),
InvalidMultiCodec)
var mcodec =
CodeContentIds.getOrDefault(cast[int](codec), InvalidMultiCodec)
if mcodec == InvalidMultiCodec:
err(CidError.Incorrect)
else:
if not MultiHash.validate(vb.buffer.toOpenArray(vb.offset,
vb.buffer.high)):
if not MultiHash.validate(
vb.buffer.toOpenArray(vb.offset, vb.buffer.high)
):
err(CidError.Incorrect)
else:
vb.finish()
ok(Cid(
cidver: CIDv1,
mcodec: mcodec,
hpos: offset,
data: vb))
ok(Cid(cidver: CIDv1, mcodec: mcodec, hpos: offset, data: vb))
proc decode(data: openarray[char]): Result[Cid, CidError] =
proc decode(data: openArray[char]): Result[Cid, CidError] =
var buffer: seq[byte]
var plen = 0
if len(data) < 2:
@@ -138,7 +138,7 @@ proc decode(data: openarray[char]): Result[Cid, CidError] =
return err(CidError.Incorrect)
decode(buffer)
proc validate*(ctype: typedesc[Cid], data: openarray[byte]): bool =
proc validate*(ctype: typedesc[Cid], data: openArray[byte]): bool =
## Returns ``true`` is data has valid binary CID representation.
var version, codec: uint64
var res: VarintResult[void]
@@ -173,7 +173,9 @@ proc mhash*(cid: Cid): Result[MultiHash, CidError] =
if cid.cidver notin {CIDv0, CIDv1}:
err(CidError.Incorrect)
else:
MultiHash.init(cid.data.buffer.toOpenArray(cid.hpos, cid.data.high)).orError(CidError.Incorrect)
MultiHash.init(cid.data.buffer.toOpenArray(cid.hpos, cid.data.high)).orError(
CidError.Incorrect
)
proc contentType*(cid: Cid): Result[MultiCodec, CidError] =
## Returns content type part of CID
@@ -186,12 +188,15 @@ proc version*(cid: Cid): CidVersion =
## Returns CID version
result = cid.cidver
proc init*[T: char|byte](ctype: typedesc[Cid], data: openarray[T]): Result[Cid, CidError] =
proc init*[T: char | byte](
ctype: typedesc[Cid], data: openArray[T]
): Result[Cid, CidError] =
## Create new content identifier using array of bytes or string ``data``.
decode(data)
proc init*(ctype: typedesc[Cid], version: CidVersion, content: MultiCodec,
hash: MultiHash): Result[Cid, CidError] =
proc init*(
ctype: typedesc[Cid], version: CidVersion, content: MultiCodec, hash: MultiHash
): Result[Cid, CidError] =
## Create new content identifier using content type ``content`` and
## MultiHash ``hash`` using version ``version``.
##
@@ -199,10 +204,10 @@ proc init*(ctype: typedesc[Cid], version: CidVersion, content: MultiCodec,
## Cid.init(CIDv0, multiCodec("dag-pb"), MultiHash.digest("sha2-256", data))
##
## All other encodings and hashes are not supported by CIDv0.
var res: Cid
res.cidver = version
if version == CIDv0:
if content != multiCodec("dag-pb"):
return err(CidError.Unsupported)
@@ -214,8 +219,7 @@ proc init*(ctype: typedesc[Cid], version: CidVersion, content: MultiCodec,
res.data.finish()
return ok(res)
elif version == CIDv1:
let mcodec = CodeContentIds.getOrDefault(cast[int](content),
InvalidMultiCodec)
let mcodec = CodeContentIds.getOrDefault(cast[int](content), InvalidMultiCodec)
if mcodec == InvalidMultiCodec:
return err(CidError.Incorrect)
res.mcodec = mcodec
@@ -234,11 +238,9 @@ proc `==`*(a: Cid, b: Cid): bool =
## are equal, ``false`` otherwise.
if a.mcodec == b.mcodec:
var ah, bh: MultiHash
if MultiHash.decode(
a.data.buffer.toOpenArray(a.hpos, a.data.high), ah).isErr:
if MultiHash.decode(a.data.buffer.toOpenArray(a.hpos, a.data.high), ah).isErr:
return false
if MultiHash.decode(
b.data.buffer.toOpenArray(b.hpos, b.data.high), bh).isErr:
if MultiHash.decode(b.data.buffer.toOpenArray(b.hpos, b.data.high), bh).isErr:
return false
result = (ah == bh)
@@ -262,21 +264,15 @@ proc write*(vb: var VBuffer, cid: Cid) {.inline.} =
## Write CID value ``cid`` to buffer ``vb``.
vb.writeArray(cid.data.buffer)
proc encode*(mbtype: typedesc[MultiBase], encoding: string,
cid: Cid): string {.inline.} =
## Get MultiBase encoded representation of ``cid`` using encoding
## ``encoding``.
result = MultiBase.encode(encoding, cid.data.buffer).tryGet()
proc hash*(cid: Cid): Hash {.inline.} =
hash(cid.data.buffer)
proc `$`*(cid: Cid): string =
## Return official string representation of content identifier ``cid``.
if cid.cidver == CIDv0:
BTCBase58.encode(cid.data.buffer)
elif cid.cidver == CIDv1:
let res = Multibase.encode("base58btc", cid.data.buffer)
if res.isOk():
res.get()
else:
""
let res = MultiBase.encode("base58btc", cid.data.buffer)
res.get("")
else:
""

View File

@@ -1,19 +1,17 @@
## Nim-LibP2P
## Copyright (c) 2020 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-LibP2P
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
import std/[options, tables, sequtils, sets]
import chronos, chronicles, metrics
import peerinfo,
stream/connection,
muxers/muxer,
utils/semaphore,
errors
{.push raises: [].}
import std/[tables, sequtils, sets]
import pkg/[chronos, chronicles, metrics]
import peerinfo, peerstore, stream/connection, muxers/muxer, utils/semaphore, errors
logScope:
topics = "libp2p connmanager"
@@ -22,20 +20,20 @@ declareGauge(libp2p_peers, "total connected peers")
const
MaxConnections* = 50
MaxConnectionsPerPeer* = 5
MaxConnectionsPerPeer* = 1
type
TooManyConnectionsError* = object of CatchableError
ConnProvider* = proc(): Future[Connection] {.gcsafe, closure.}
TooManyConnectionsError* = object of LPError
AlreadyExpectingConnectionError* = object of LPError
ConnEventKind* {.pure.} = enum
Connected, # A connection was made and securely upgraded - there may be
# more than one concurrent connection thus more than one upgrade
# event per peer.
Disconnected # Peer disconnected - this event is fired once per upgrade
# when the associated connection is terminated.
Connected
# A connection was made and securely upgraded - there may be
# more than one concurrent connection thus more than one upgrade
# event per peer.
Disconnected
# Peer disconnected - this event is fired once per upgrade
# when the associated connection is terminated.
ConnEvent* = object
case kind*: ConnEventKind
@@ -44,44 +42,50 @@ type
else:
discard
ConnEventHandler* =
proc(peerId: PeerID, event: ConnEvent): Future[void] {.gcsafe.}
ConnEventHandler* = proc(peerId: PeerId, event: ConnEvent): Future[void] {.
gcsafe, async: (raises: [CancelledError])
.}
PeerEventKind* {.pure.} = enum
Left,
Left
Joined
Identified
PeerEvent* = object
case kind*: PeerEventKind
of PeerEventKind.Joined:
initiator*: bool
else:
discard
of PeerEventKind.Joined, PeerEventKind.Identified:
initiator*: bool
else:
discard
PeerEventHandler* =
proc(peerId: PeerID, event: PeerEvent): Future[void] {.gcsafe.}
MuxerHolder = object
muxer: Muxer
handle: Future[void]
PeerEventHandler* = proc(peerId: PeerId, event: PeerEvent): Future[void] {.
gcsafe, async: (raises: [CancelledError])
.}
ConnManager* = ref object of RootObj
maxConnsPerPeer: int
inSema*: AsyncSemaphore
outSema*: AsyncSemaphore
conns: Table[PeerID, HashSet[Connection]]
muxed: Table[Connection, MuxerHolder]
connEvents: Table[ConnEventKind, OrderedSet[ConnEventHandler]]
peerEvents: Table[PeerEventKind, OrderedSet[PeerEventHandler]]
muxed: Table[PeerId, seq[Muxer]]
connEvents: array[ConnEventKind, OrderedSet[ConnEventHandler]]
peerEvents: array[PeerEventKind, OrderedSet[PeerEventHandler]]
expectedConnectionsOverLimit*: Table[(PeerId, Direction), Future[Muxer]]
peerStore*: PeerStore
ConnectionSlot* = object
connManager: ConnManager
direction: Direction
proc newTooManyConnectionsError(): ref TooManyConnectionsError {.inline.} =
result = newException(TooManyConnectionsError, "Too many connections")
proc init*(C: type ConnManager,
maxConnsPerPeer = MaxConnectionsPerPeer,
maxConnections = MaxConnections,
maxIn = -1,
maxOut = -1): ConnManager =
proc new*(
C: type ConnManager,
maxConnsPerPeer = MaxConnectionsPerPeer,
maxConnections = MaxConnections,
maxIn = -1,
maxOut = -1,
): ConnManager =
var inSema, outSema: AsyncSemaphore
if maxIn > 0 or maxOut > 0:
inSema = newAsyncSemaphore(maxIn)
@@ -92,37 +96,41 @@ proc init*(C: type ConnManager,
else:
raiseAssert "Invalid connection counts!"
C(maxConnsPerPeer: maxConnsPerPeer,
conns: initTable[PeerID, HashSet[Connection]](),
muxed: initTable[Connection, MuxerHolder](),
inSema: inSema,
outSema: outSema)
C(maxConnsPerPeer: maxConnsPerPeer, inSema: inSema, outSema: outSema)
proc connCount*(c: ConnManager, peerId: PeerID): int =
c.conns.getOrDefault(peerId).len
proc connCount*(c: ConnManager, peerId: PeerId): int =
c.muxed.getOrDefault(peerId).len
proc addConnEventHandler*(c: ConnManager,
handler: ConnEventHandler,
kind: ConnEventKind) =
proc connectedPeers*(c: ConnManager, dir: Direction): seq[PeerId] =
var peers = newSeq[PeerId]()
for peerId, mux in c.muxed:
if mux.anyIt(it.connection.dir == dir):
peers.add(peerId)
return peers
proc getConnections*(c: ConnManager): Table[PeerId, seq[Muxer]] =
return c.muxed
proc addConnEventHandler*(
c: ConnManager, handler: ConnEventHandler, kind: ConnEventKind
) =
## Add peer event handler - handlers must not raise exceptions!
##
if isNil(handler):
return
c.connEvents[kind].incl(handler)
if isNil(handler): return
c.connEvents.mgetOrPut(kind,
initOrderedSet[ConnEventHandler]()).incl(handler)
proc removeConnEventHandler*(
c: ConnManager, handler: ConnEventHandler, kind: ConnEventKind
) =
c.connEvents[kind].excl(handler)
proc removeConnEventHandler*(c: ConnManager,
handler: ConnEventHandler,
kind: ConnEventKind) =
c.connEvents.withValue(kind, handlers) do:
handlers[].excl(handler)
proc triggerConnEvent*(c: ConnManager,
peerId: PeerID,
event: ConnEvent) {.async, gcsafe.} =
proc triggerConnEvent*(
c: ConnManager, peerId: PeerId, event: ConnEvent
) {.async: (raises: [CancelledError]).} =
try:
trace "About to trigger connection events", peer = peerId
if event.kind in c.connEvents:
if c.connEvents[event.kind].len() > 0:
trace "triggering connection events", peer = peerId, event = $event.kind
var connEvents: seq[Future[void]]
for h in c.connEvents[event.kind]:
@@ -133,42 +141,32 @@ proc triggerConnEvent*(c: ConnManager,
raise exc
except CatchableError as exc:
warn "Exception in triggerConnEvents",
msg = exc.msg, peerId, event = $event
description = exc.msg, peer = peerId, event = $event
proc addPeerEventHandler*(c: ConnManager,
handler: PeerEventHandler,
kind: PeerEventKind) =
proc addPeerEventHandler*(
c: ConnManager, handler: PeerEventHandler, kind: PeerEventKind
) =
## Add peer event handler - handlers must not raise exceptions!
##
if isNil(handler): return
c.peerEvents.mgetOrPut(kind,
initOrderedSet[PeerEventHandler]()).incl(handler)
if isNil(handler):
return
c.peerEvents[kind].incl(handler)
proc removePeerEventHandler*(c: ConnManager,
handler: PeerEventHandler,
kind: PeerEventKind) =
c.peerEvents.withValue(kind, handlers) do:
handlers[].excl(handler)
proc triggerPeerEvents*(c: ConnManager,
peerId: PeerID,
event: PeerEvent) {.async, gcsafe.} =
proc removePeerEventHandler*(
c: ConnManager, handler: PeerEventHandler, kind: PeerEventKind
) =
c.peerEvents[kind].excl(handler)
proc triggerPeerEvents*(
c: ConnManager, peerId: PeerId, event: PeerEvent
) {.async: (raises: [CancelledError]).} =
trace "About to trigger peer events", peer = peerId
if event.kind notin c.peerEvents:
if c.peerEvents[event.kind].len == 0:
return
try:
let count = c.connCount(peerId)
if event.kind == PeerEventKind.Joined and count != 1:
trace "peer already joined", peerId, event = $event
return
elif event.kind == PeerEventKind.Left and count != 0:
trace "peer still connected or already left", peerId, event = $event
return
trace "triggering peer events", peerId, event = $event
trace "triggering peer events", peer = peerId, event = $event
var peerEvents: seq[Future[void]]
for h in c.peerEvents[event.kind]:
@@ -178,23 +176,29 @@ proc triggerPeerEvents*(c: ConnManager,
except CancelledError as exc:
raise exc
except CatchableError as exc: # handlers should not raise!
warn "Exception in triggerPeerEvents", exc = exc.msg, peerId
warn "Exception in triggerPeerEvents", description = exc.msg, peer = peerId
proc contains*(c: ConnManager, conn: Connection): bool =
## checks if a connection is being tracked by the
## connection manager
##
proc expectConnection*(
c: ConnManager, p: PeerId, dir: Direction
): Future[Muxer] {.async: (raises: [AlreadyExpectingConnectionError, CancelledError]).} =
## Wait for a peer to connect to us. This will bypass the `MaxConnectionsPerPeer`
let key = (p, dir)
if key in c.expectedConnectionsOverLimit:
raise newException(
AlreadyExpectingConnectionError,
"Already expecting an incoming connection from that peer",
)
if isNil(conn):
return
let future = Future[Muxer].Raising([CancelledError]).init()
c.expectedConnectionsOverLimit[key] = future
if isNil(conn.peerInfo):
return
try:
return await future
finally:
c.expectedConnectionsOverLimit.del(key)
return conn in c.conns.getOrDefault(conn.peerInfo.peerId)
proc contains*(c: ConnManager, peerId: PeerID): bool =
peerId in c.conns
proc contains*(c: ConnManager, peerId: PeerId): bool =
peerId in c.muxed
proc contains*(c: ConnManager, muxer: Muxer): bool =
## checks if a muxer is being tracked by the connection
@@ -202,361 +206,230 @@ proc contains*(c: ConnManager, muxer: Muxer): bool =
##
if isNil(muxer):
return
return false
let conn = muxer.connection
if conn notin c:
return
return muxer in c.muxed.getOrDefault(conn.peerId)
if conn notin c.muxed:
return
proc closeMuxer(muxer: Muxer) {.async: (raises: [CancelledError]).} =
trace "Cleaning up muxer", m = muxer
return muxer == c.muxed[conn].muxer
proc closeMuxerHolder(muxerHolder: MuxerHolder) {.async.} =
trace "Cleaning up muxer", m = muxerHolder.muxer
await muxerHolder.muxer.close()
if not(isNil(muxerHolder.handle)):
await muxer.close()
if not (isNil(muxer.handler)):
try:
await muxerHolder.handle # TODO noraises?
await muxer.handler
except CatchableError as exc:
trace "Exception in close muxer handler", exc = exc.msg
trace "Cleaned up muxer", m = muxerHolder.muxer
proc delConn(c: ConnManager, conn: Connection) =
let peerId = conn.peerInfo.peerId
if peerId in c.conns:
c.conns[peerId].excl(conn)
if c.conns[peerId].len == 0:
c.conns.del(peerId)
libp2p_peers.set(c.conns.len.int64)
trace "Removed connection", conn
proc cleanupConn(c: ConnManager, conn: Connection) {.async.} =
## clean connection's resources such as muxers and streams
if isNil(conn):
trace "Wont cleanup a nil connection"
return
if isNil(conn.peerInfo):
trace "No peer info for connection"
return
# Remove connection from all tables without async breaks
var muxer = some(MuxerHolder())
if not c.muxed.pop(conn, muxer.get()):
muxer = none(MuxerHolder)
delConn(c, conn)
trace "Exception in close muxer handler", description = exc.msg
trace "Cleaned up muxer", m = muxer
proc muxCleanup(c: ConnManager, mux: Muxer) {.async: (raises: []).} =
try:
if muxer.isSome:
await closeMuxerHolder(muxer.get())
finally:
await conn.close()
trace "Triggering disconnect events", mux
let peerId = mux.connection.peerId
trace "Connection cleaned up", conn
let muxers = c.muxed.getOrDefault(peerId).filterIt(it != mux)
if muxers.len > 0:
c.muxed[peerId] = muxers
else:
c.muxed.del(peerId)
libp2p_peers.set(c.muxed.len.int64)
await c.triggerPeerEvents(peerId, PeerEvent(kind: PeerEventKind.Left))
proc onConnUpgraded(c: ConnManager, conn: Connection) {.async.} =
try:
trace "Triggering connect events", conn
conn.upgrade()
if not (c.peerStore.isNil):
c.peerStore.cleanup(peerId)
let peerId = conn.peerInfo.peerId
await c.triggerPeerEvents(
peerId, PeerEvent(kind: PeerEventKind.Joined, initiator: conn.dir == Direction.Out))
await c.triggerConnEvent(
peerId, ConnEvent(kind: ConnEventKind.Connected, incoming: conn.dir == Direction.In))
await c.triggerConnEvent(peerId, ConnEvent(kind: ConnEventKind.Disconnected))
except CatchableError as exc:
# This is top-level procedure which will work as separate task, so it
# do not need to propagate CancelledError and should handle other errors
warn "Unexpected exception in switch peer connection cleanup",
conn, msg = exc.msg
warn "Unexpected exception peer cleanup handler", mux, description = exc.msg
proc peerCleanup(c: ConnManager, conn: Connection) {.async.} =
try:
trace "Triggering disconnect events", conn
let peerId = conn.peerInfo.peerId
await c.triggerConnEvent(
peerId, ConnEvent(kind: ConnEventKind.Disconnected))
await c.triggerPeerEvents(peerId, PeerEvent(kind: PeerEventKind.Left))
except CatchableError as exc:
# This is top-level procedure which will work as separate task, so it
# do not need to propagate CancelledError and should handle other errors
warn "Unexpected exception peer cleanup handler",
conn, msg = exc.msg
proc onClose(c: ConnManager, conn: Connection) {.async.} =
proc onClose(c: ConnManager, mux: Muxer) {.async: (raises: []).} =
## connection close even handler
##
## triggers the connections resource cleanup
##
try:
await conn.join()
trace "Connection closed, cleaning up", conn
await c.cleanupConn(conn)
except CancelledError:
# This is top-level procedure which will work as separate task, so it
# do not need to propagate CancelledError.
debug "Unexpected cancellation in connection manager's cleanup", conn
await mux.connection.join()
trace "Connection closed, cleaning up", mux
except CatchableError as exc:
debug "Unexpected exception in connection manager's cleanup",
errMsg = exc.msg, conn
description = exc.msg, mux
finally:
trace "Triggering peerCleanup", conn
asyncSpawn c.peerCleanup(conn)
await c.muxCleanup(mux)
proc selectConn*(c: ConnManager,
peerId: PeerID,
dir: Direction): Connection =
proc selectMuxer*(c: ConnManager, peerId: PeerId, dir: Direction): Muxer =
## Select a connection for the provided peer and direction
##
let conns = toSeq(
c.conns.getOrDefault(peerId))
.filterIt( it.dir == dir )
let conns = toSeq(c.muxed.getOrDefault(peerId)).filterIt(it.connection.dir == dir)
if conns.len > 0:
return conns[0]
proc selectConn*(c: ConnManager, peerId: PeerID): Connection =
proc selectMuxer*(c: ConnManager, peerId: PeerId): Muxer =
## Select a connection for the provided giving priority
## to outgoing connections
##
var conn = c.selectConn(peerId, Direction.Out)
if isNil(conn):
conn = c.selectConn(peerId, Direction.In)
if isNil(conn):
var mux = c.selectMuxer(peerId, Direction.Out)
if isNil(mux):
mux = c.selectMuxer(peerId, Direction.In)
if isNil(mux):
trace "connection not found", peerId
return mux
return conn
proc selectMuxer*(c: ConnManager, conn: Connection): Muxer =
## select the muxer for the provided connection
##
if isNil(conn):
return
if conn in c.muxed:
return c.muxed[conn].muxer
else:
debug "no muxer for connection", conn
proc storeConn*(c: ConnManager, conn: Connection) =
## store a connection
##
if isNil(conn):
raise newException(CatchableError, "Connection cannot be nil")
if conn.closed or conn.atEof:
raise newException(CatchableError, "Connection closed or EOF")
if isNil(conn.peerInfo):
raise newException(CatchableError, "Empty peer info")
let peerId = conn.peerInfo.peerId
if c.conns.getOrDefault(peerId).len > c.maxConnsPerPeer:
debug "Too many connections for peer",
conn, conns = c.conns.getOrDefault(peerId).len
raise newTooManyConnectionsError()
if peerId notin c.conns:
c.conns[peerId] = initHashSet[Connection]()
c.conns[peerId].incl(conn)
libp2p_peers.set(c.conns.len.int64)
# Launch on close listener
# All the errors are handled inside `onClose()` procedure.
asyncSpawn c.onClose(conn)
trace "Stored connection",
conn, direction = $conn.dir, connections = c.conns.len
proc trackConn(c: ConnManager,
provider: ConnProvider,
sema: AsyncSemaphore):
Future[Connection] {.async.} =
var conn: Connection
try:
conn = await provider()
if isNil(conn):
return
trace "Got connection", conn
proc semaphoreMonitor() {.async.} =
try:
await conn.join()
except CatchableError as exc:
trace "Exception in semaphore monitor, ignoring", exc = exc.msg
sema.release()
asyncSpawn semaphoreMonitor()
except CatchableError as exc:
trace "Exception tracking connection", exc = exc.msg
if not isNil(conn):
await conn.close()
raise exc
return conn
proc trackIncomingConn*(c: ConnManager,
provider: ConnProvider):
Future[Connection] {.async.} =
## await for a connection slot before attempting
## to call the connection provider
##
var conn: Connection
try:
trace "Tracking incoming connection"
await c.inSema.acquire()
conn = await c.trackConn(provider, c.inSema)
if isNil(conn):
trace "Couldn't acquire connection, releasing semaphore slot", dir = $Direction.In
c.inSema.release()
return conn
except CatchableError as exc:
trace "Exception tracking connection", exc = exc.msg
c.inSema.release()
raise exc
proc trackOutgoingConn*(c: ConnManager,
provider: ConnProvider):
Future[Connection] {.async.} =
## try acquiring a connection if all slots
## are already taken, raise TooManyConnectionsError
## exception
##
trace "Tracking outgoing connection", count = c.outSema.count,
max = c.outSema.size
if not c.outSema.tryAcquire():
trace "Too many outgoing connections!", count = c.outSema.count,
max = c.outSema.size
raise newTooManyConnectionsError()
var conn: Connection
try:
conn = await c.trackConn(provider, c.outSema)
if isNil(conn):
trace "Couldn't acquire connection, releasing semaphore slot", dir = $Direction.Out
c.outSema.release()
return conn
except CatchableError as exc:
trace "Exception tracking connection", exc = exc.msg
c.outSema.release()
raise exc
proc storeMuxer*(c: ConnManager,
muxer: Muxer,
handle: Future[void] = nil) =
proc storeMuxer*(c: ConnManager, muxer: Muxer) {.raises: [LPError].} =
## store the connection and muxer
##
if isNil(muxer):
raise newException(CatchableError, "muxer cannot be nil")
raise newException(LPError, "muxer cannot be nil")
if isNil(muxer.connection):
raise newException(CatchableError, "muxer's connection cannot be nil")
raise newException(LPError, "muxer's connection cannot be nil")
if muxer.connection notin c:
raise newException(CatchableError, "cant add muxer for untracked connection")
if muxer.connection.closed or muxer.connection.atEof:
raise newException(LPError, "Connection closed or EOF")
c.muxed[muxer.connection] = MuxerHolder(
muxer: muxer,
handle: handle)
let
peerId = muxer.connection.peerId
dir = muxer.connection.dir
trace "Stored muxer",
muxer, handle = not handle.isNil, connections = c.conns.len
# we use getOrDefault in the if below instead of [] to avoid the KeyError
if c.muxed.getOrDefault(peerId).len > c.maxConnsPerPeer:
let key = (peerId, dir)
let expectedConn = c.expectedConnectionsOverLimit.getOrDefault(key)
if expectedConn != nil and not expectedConn.finished:
expectedConn.complete(muxer)
else:
debug "Too many connections for peer",
conns = c.muxed.getOrDefault(peerId).len, peerId, dir
asyncSpawn c.onConnUpgraded(muxer.connection)
raise newTooManyConnectionsError()
proc getStream*(c: ConnManager,
peerId: PeerID,
dir: Direction): Future[Connection] {.async, gcsafe.} =
## get a muxed stream for the provided peer
## with the given direction
var newPeer = false
c.muxed.withValue(peerId, muxers):
doAssert muxers[].len > 0
doAssert muxer notin muxers[]
muxers[].add(muxer)
do:
c.muxed[peerId] = @[muxer]
newPeer = true
libp2p_peers.set(c.muxed.len.int64)
asyncSpawn c.triggerConnEvent(
peerId, ConnEvent(kind: ConnEventKind.Connected, incoming: dir == Direction.In)
)
if newPeer:
asyncSpawn c.triggerPeerEvents(
peerId, PeerEvent(kind: PeerEventKind.Joined, initiator: dir == Direction.Out)
)
asyncSpawn c.onClose(muxer)
trace "Stored muxer", muxer, direction = $muxer.connection.dir, peers = c.muxed.len
proc getIncomingSlot*(
c: ConnManager
): Future[ConnectionSlot] {.async: (raises: [CancelledError]).} =
await c.inSema.acquire()
return ConnectionSlot(connManager: c, direction: In)
proc getOutgoingSlot*(
c: ConnManager, forceDial = false
): ConnectionSlot {.raises: [TooManyConnectionsError].} =
if forceDial:
c.outSema.forceAcquire()
elif not c.outSema.tryAcquire():
trace "Too many outgoing connections!",
available = c.outSema.count, max = c.outSema.size
raise newTooManyConnectionsError()
return ConnectionSlot(connManager: c, direction: Out)
func semaphore(c: ConnManager, dir: Direction): AsyncSemaphore {.inline.} =
return if dir == In: c.inSema else: c.outSema
proc slotsAvailable*(c: ConnManager, dir: Direction): int =
return semaphore(c, dir).count
proc release*(cs: ConnectionSlot) =
semaphore(cs.connManager, cs.direction).release()
proc trackConnection*(cs: ConnectionSlot, conn: Connection) =
if isNil(conn):
cs.release()
return
proc semaphoreMonitor() {.async: (raises: [CancelledError]).} =
try:
await conn.join()
except CatchableError as exc:
trace "Exception in semaphore monitor, ignoring", description = exc.msg
cs.release()
asyncSpawn semaphoreMonitor()
proc trackMuxer*(cs: ConnectionSlot, mux: Muxer) =
if isNil(mux):
cs.release()
return
cs.trackConnection(mux.connection)
proc getStream*(
c: ConnManager, muxer: Muxer
): Future[Connection] {.async: (raises: [LPStreamError, MuxerError, CancelledError]).} =
## get a muxed stream for the passed muxer
##
let muxer = c.selectMuxer(c.selectConn(peerId, dir))
if not(isNil(muxer)):
if not (isNil(muxer)):
return await muxer.newStream()
proc getStream*(c: ConnManager,
peerId: PeerID): Future[Connection] {.async, gcsafe.} =
proc getStream*(
c: ConnManager, peerId: PeerId
): Future[Connection] {.async: (raises: [LPStreamError, MuxerError, CancelledError]).} =
## get a muxed stream for the passed peer from any connection
##
let muxer = c.selectMuxer(c.selectConn(peerId))
if not(isNil(muxer)):
return await muxer.newStream()
return await c.getStream(c.selectMuxer(peerId))
proc getStream*(c: ConnManager,
conn: Connection): Future[Connection] {.async, gcsafe.} =
## get a muxed stream for the passed connection
proc getStream*(
c: ConnManager, peerId: PeerId, dir: Direction
): Future[Connection] {.async: (raises: [LPStreamError, MuxerError, CancelledError]).} =
## get a muxed stream for the passed peer from a connection with `dir`
##
let muxer = c.selectMuxer(conn)
if not(isNil(muxer)):
return await muxer.newStream()
return await c.getStream(c.selectMuxer(peerId, dir))
proc dropPeer*(c: ConnManager, peerId: PeerID) {.async.} =
proc dropPeer*(c: ConnManager, peerId: PeerId) {.async: (raises: [CancelledError]).} =
## drop connections and cleanup resources for peer
##
trace "Dropping peer", peerId
let conns = c.conns.getOrDefault(peerId)
for conn in conns:
trace "Removing connection", conn
delConn(c, conn)
var muxers: seq[MuxerHolder]
for conn in conns:
if conn in c.muxed:
muxers.add c.muxed[conn]
c.muxed.del(conn)
let muxers = c.muxed.getOrDefault(peerId)
for muxer in muxers:
await closeMuxerHolder(muxer)
for conn in conns:
await conn.close()
trace "Dropped peer", peerId
await closeMuxer(muxer)
trace "Peer dropped", peerId
proc close*(c: ConnManager) {.async.} =
proc close*(c: ConnManager) {.async: (raises: [CancelledError]).} =
## cleanup resources for the connection
## manager
##
trace "Closing ConnManager"
let conns = c.conns
c.conns.clear()
let muxed = c.muxed
c.muxed.clear()
for _, muxer in muxed:
await closeMuxerHolder(muxer)
let expected = c.expectedConnectionsOverLimit
c.expectedConnectionsOverLimit.clear()
for _, conns2 in conns:
for conn in conns2:
await conn.close()
for _, fut in expected:
await fut.cancelAndWait()
for _, muxers in muxed:
for mux in muxers:
await closeMuxer(mux)
trace "Closed ConnManager"

View File

@@ -1,11 +1,11 @@
## Nim-Libp2p
## Copyright (c) 2020 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-Libp2p
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
## This module integrates BearSSL ChaCha20+Poly1305
##
@@ -15,16 +15,11 @@
# RFC @ https://tools.ietf.org/html/rfc7539
{.push raises: [Defect].}
{.push raises: [].}
import bearssl
# have to do this due to a nim bug and raises[] on callbacks
# https://github.com/nim-lang/Nim/issues/13905
proc ourPoly1305CtmulRun*(key: pointer; iv: pointer; data: pointer; len: int;
aad: pointer; aadLen: int; tag: pointer; ichacha: pointer;
encrypt: cint) {.cdecl, importc: "br_poly1305_ctmul_run",
header: "bearssl_block.h".}
import bearssl/blockx
from stew/assign2 import assign
from stew/ptrops import baseAddr
const
ChaChaPolyKeySize = 32
@@ -37,64 +32,72 @@ type
ChaChaPolyNonce* = array[ChaChaPolyNonceSize, byte]
ChaChaPolyTag* = array[ChaChaPolyTagSize, byte]
proc intoChaChaPolyKey*(s: openarray[byte]): ChaChaPolyKey =
proc intoChaChaPolyKey*(s: openArray[byte]): ChaChaPolyKey =
assert s.len == ChaChaPolyKeySize
copyMem(addr result[0], unsafeaddr s[0], ChaChaPolyKeySize)
assign(result, s)
proc intoChaChaPolyNonce*(s: openarray[byte]): ChaChaPolyNonce =
proc intoChaChaPolyNonce*(s: openArray[byte]): ChaChaPolyNonce =
assert s.len == ChaChaPolyNonceSize
copyMem(addr result[0], unsafeaddr s[0], ChaChaPolyNonceSize)
assign(result, s)
proc intoChaChaPolyTag*(s: openarray[byte]): ChaChaPolyTag =
proc intoChaChaPolyTag*(s: openArray[byte]): ChaChaPolyTag =
assert s.len == ChaChaPolyTagSize
copyMem(addr result[0], unsafeaddr s[0], ChaChaPolyTagSize)
assign(result, s)
# bearssl allows us to use optimized versions
# this is reconciled at runtime
# we do this in the global scope / module init
proc encrypt*(_: type[ChaChaPoly],
key: ChaChaPolyKey,
nonce: ChaChaPolyNonce,
tag: var ChaChaPolyTag,
data: var openarray[byte],
aad: openarray[byte]) =
let
ad = if aad.len > 0:
unsafeaddr aad[0]
else:
nil
proc encrypt*(
_: type[ChaChaPoly],
key: ChaChaPolyKey,
nonce: ChaChaPolyNonce,
tag: var ChaChaPolyTag,
data: var openArray[byte],
aad: openArray[byte],
) =
let ad =
if aad.len > 0:
unsafeAddr aad[0]
else:
nil
ourPoly1305CtmulRun(
unsafeaddr key[0],
unsafeaddr nonce[0],
addr data[0],
data.len,
poly1305CtmulRun(
unsafeAddr key[0],
unsafeAddr nonce[0],
baseAddr(data),
uint(data.len),
ad,
aad.len,
addr tag[0],
chacha20CtRun,
#[encrypt]# 1.cint)
uint(aad.len),
baseAddr(tag),
# cast is required to workaround https://github.com/nim-lang/Nim/issues/13905
cast[Chacha20Run](chacha20CtRun), #[encrypt]#
1.cint,
)
proc decrypt*(_: type[ChaChaPoly],
key: ChaChaPolyKey,
nonce: ChaChaPolyNonce,
tag: var ChaChaPolyTag,
data: var openarray[byte],
aad: openarray[byte]) =
let
ad = if aad.len > 0:
unsafeaddr aad[0]
else:
nil
proc decrypt*(
_: type[ChaChaPoly],
key: ChaChaPolyKey,
nonce: ChaChaPolyNonce,
tag: var ChaChaPolyTag,
data: var openArray[byte],
aad: openArray[byte],
) =
let ad =
if aad.len > 0:
unsafeAddr aad[0]
else:
nil
ourPoly1305CtmulRun(
unsafeaddr key[0],
unsafeaddr nonce[0],
addr data[0],
data.len,
poly1305CtmulRun(
unsafeAddr key[0],
unsafeAddr nonce[0],
baseAddr(data),
uint(data.len),
ad,
aad.len,
addr tag[0],
chacha20CtRun,
#[decrypt]# 0.cint)
uint(aad.len),
baseAddr(tag),
# cast is required to workaround https://github.com/nim-lang/Nim/issues/13905
cast[Chacha20Run](chacha20CtRun), #[decrypt]#
0.cint,
)

View File

@@ -1,24 +1,24 @@
## Nim-Libp2p
## Copyright (c) 2018 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-Libp2p
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
## This module implements Public Key and Private Key interface for libp2p.
{.push raises: [Defect].}
{.push raises: [].}
from strutils import split, strip, cmpIgnoreCase
const libp2p_pki_schemes* {.strdefine.} = "rsa,ed25519,secp256k1,ecnist"
type
PKScheme* = enum
RSA = 0,
Ed25519,
Secp256k1,
ECDSA
type PKScheme* = enum
RSA = 0
Ed25519
Secp256k1
ECDSA
proc initSupportedSchemes(list: static string): set[PKScheme] =
var res: set[PKScheme]
@@ -64,31 +64,29 @@ when supported(PKScheme.Ed25519):
import ed25519/ed25519
when supported(PKScheme.Secp256k1):
import secp
when supported(PKScheme.ECDSA):
import ecnist
# We are still importing `ecnist` because, it is used for SECIO handshake,
# but it will be impossible to create ECNIST keys or import ECNIST keys.
# These used to be declared in `crypto` itself
export ecnist.ephemeral, ecnist.ECDHEScheme
import ecnist, bearssl
import bearssl/rand, bearssl/hash as bhash
import ../protobuf/minprotobuf, ../vbuffer, ../multihash, ../multicodec
import nimcrypto/[rijndael, twofish, sha2, hash, hmac]
# We use `ncrutils` for constant-time hexadecimal encoding/decoding procedures.
import nimcrypto/utils as ncrutils
import ../utility
import stew/results
export results
import results
export results, utility
# This is workaround for Nim's `import` bug
export rijndael, twofish, sha2, hash, hmac, ncrutils
from strutils import split
export rijndael, twofish, sha2, hash, hmac, ncrutils, rand
type
DigestSheme* = enum
Sha256,
Sha256
Sha512
ECDHEScheme* = EcCurveKind
PublicKey* = object
case scheme*: PKScheme
of PKScheme.RSA:
@@ -129,8 +127,8 @@ type
skkey*: SkPrivateKey
else:
discard
of PKSCheme.ECDSA:
when supported(PKSCheme.ECDSA):
of PKScheme.ECDSA:
when supported(PKScheme.ECDSA):
eckey*: ecnist.EcPrivateKey
else:
discard
@@ -149,33 +147,50 @@ type
data*: seq[byte]
CryptoError* = enum
KeyError,
SigError,
HashError,
KeyError
SigError
HashError
SchemeError
CryptoResult*[T] = Result[T, CryptoError]
template orError*(exp: untyped, err: untyped): untyped =
(exp.mapErr do (_: auto) -> auto: err)
exp.mapErr do(_: auto) -> auto:
err
proc newRng*(): ref BrHmacDrbgContext =
proc newRng*(): ref HmacDrbgContext =
# You should only create one instance of the RNG per application / library
# Ref is used so that it can be shared between components
# TODO consider moving to bearssl
var seeder = brPrngSeederSystem(nil)
var seeder = prngSeederSystem(nil)
if seeder == nil:
return nil
var rng = (ref BrHmacDrbgContext)()
brHmacDrbgInit(addr rng[], addr sha256Vtable, nil, 0)
var rng = (ref HmacDrbgContext)()
hmacDrbgInit(rng[], addr sha256Vtable, nil, 0)
if seeder(addr rng.vtable) == 0:
return nil
rng
proc random*(T: typedesc[PrivateKey], scheme: PKScheme,
rng: var BrHmacDrbgContext,
bits = RsaDefaultKeySize): CryptoResult[PrivateKey] =
proc shuffle*[T](rng: ref HmacDrbgContext, x: var openArray[T]) =
if x.len == 0:
return
var randValues = newSeqUninitialized[byte](len(x) * 2)
hmacDrbgGenerate(rng[], randValues)
for i in countdown(x.high, 1):
let
rand = randValues[i * 2].int32 or (randValues[i * 2 + 1].int32 shl 8)
y = rand mod i
swap(x[i], x[y])
proc random*(
T: typedesc[PrivateKey],
scheme: PKScheme,
rng: var HmacDrbgContext,
bits = RsaDefaultKeySize,
): CryptoResult[PrivateKey] =
## Generate random private key for scheme ``scheme``.
##
## ``bits`` is number of bits for RSA key, ``bits`` value must be in
@@ -183,7 +198,7 @@ proc random*(T: typedesc[PrivateKey], scheme: PKScheme,
case scheme
of PKScheme.RSA:
when supported(PKScheme.RSA):
let rsakey = ? RsaPrivateKey.random(rng, bits).orError(KeyError)
let rsakey = ?RsaPrivateKey.random(rng, bits).orError(CryptoError.KeyError)
ok(PrivateKey(scheme: scheme, rsakey: rsakey))
else:
err(SchemeError)
@@ -195,7 +210,8 @@ proc random*(T: typedesc[PrivateKey], scheme: PKScheme,
err(SchemeError)
of PKScheme.ECDSA:
when supported(PKScheme.ECDSA):
let eckey = ? ecnist.EcPrivateKey.random(Secp256r1, rng).orError(KeyError)
let eckey =
?ecnist.EcPrivateKey.random(Secp256r1, rng).orError(CryptoError.KeyError)
ok(PrivateKey(scheme: scheme, eckey: eckey))
else:
err(SchemeError)
@@ -206,8 +222,9 @@ proc random*(T: typedesc[PrivateKey], scheme: PKScheme,
else:
err(SchemeError)
proc random*(T: typedesc[PrivateKey], rng: var BrHmacDrbgContext,
bits = RsaDefaultKeySize): CryptoResult[PrivateKey] =
proc random*(
T: typedesc[PrivateKey], rng: var HmacDrbgContext, bits = RsaDefaultKeySize
): CryptoResult[PrivateKey] =
## Generate random private key using default public-key cryptography scheme.
##
## Default public-key cryptography schemes are following order:
@@ -221,17 +238,21 @@ proc random*(T: typedesc[PrivateKey], rng: var BrHmacDrbgContext,
let skkey = SkPrivateKey.random(rng)
ok(PrivateKey(scheme: PKScheme.Secp256k1, skkey: skkey))
elif supported(PKScheme.RSA):
let rsakey = ? RsaPrivateKey.random(rng, bits).orError(KeyError)
let rsakey = ?RsaPrivateKey.random(rng, bits).orError(CryptoError.KeyError)
ok(PrivateKey(scheme: PKScheme.RSA, rsakey: rsakey))
elif supported(PKScheme.ECDSA):
let eckey = ? ecnist.EcPrivateKey.random(Secp256r1, rng).orError(KeyError)
let eckey =
?ecnist.EcPrivateKey.random(Secp256r1, rng).orError(CryptoError.KeyError)
ok(PrivateKey(scheme: PKScheme.ECDSA, eckey: eckey))
else:
err(SchemeError)
proc random*(T: typedesc[KeyPair], scheme: PKScheme,
rng: var BrHmacDrbgContext,
bits = RsaDefaultKeySize): CryptoResult[KeyPair] =
proc random*(
T: typedesc[KeyPair],
scheme: PKScheme,
rng: var HmacDrbgContext,
bits = RsaDefaultKeySize,
): CryptoResult[KeyPair] =
## Generate random key pair for scheme ``scheme``.
##
## ``bits`` is number of bits for RSA key, ``bits`` value must be in
@@ -239,39 +260,52 @@ proc random*(T: typedesc[KeyPair], scheme: PKScheme,
case scheme
of PKScheme.RSA:
when supported(PKScheme.RSA):
let pair = ? RsaKeyPair.random(rng, bits).orError(KeyError)
ok(KeyPair(
seckey: PrivateKey(scheme: scheme, rsakey: pair.seckey),
pubkey: PublicKey(scheme: scheme, rsakey: pair.pubkey)))
let pair = ?RsaKeyPair.random(rng, bits).orError(CryptoError.KeyError)
ok(
KeyPair(
seckey: PrivateKey(scheme: scheme, rsakey: pair.seckey),
pubkey: PublicKey(scheme: scheme, rsakey: pair.pubkey),
)
)
else:
err(SchemeError)
of PKScheme.Ed25519:
when supported(PKScheme.Ed25519):
let pair = EdKeyPair.random(rng)
ok(KeyPair(
seckey: PrivateKey(scheme: scheme, edkey: pair.seckey),
pubkey: PublicKey(scheme: scheme, edkey: pair.pubkey)))
ok(
KeyPair(
seckey: PrivateKey(scheme: scheme, edkey: pair.seckey),
pubkey: PublicKey(scheme: scheme, edkey: pair.pubkey),
)
)
else:
err(SchemeError)
of PKScheme.ECDSA:
when supported(PKScheme.ECDSA):
let pair = ? EcKeyPair.random(Secp256r1, rng).orError(KeyError)
ok(KeyPair(
seckey: PrivateKey(scheme: scheme, eckey: pair.seckey),
pubkey: PublicKey(scheme: scheme, eckey: pair.pubkey)))
let pair = ?EcKeyPair.random(Secp256r1, rng).orError(CryptoError.KeyError)
ok(
KeyPair(
seckey: PrivateKey(scheme: scheme, eckey: pair.seckey),
pubkey: PublicKey(scheme: scheme, eckey: pair.pubkey),
)
)
else:
err(SchemeError)
of PKScheme.Secp256k1:
when supported(PKScheme.Secp256k1):
let pair = SkKeyPair.random(rng)
ok(KeyPair(
seckey: PrivateKey(scheme: scheme, skkey: pair.seckey),
pubkey: PublicKey(scheme: scheme, skkey: pair.pubkey)))
ok(
KeyPair(
seckey: PrivateKey(scheme: scheme, skkey: pair.seckey),
pubkey: PublicKey(scheme: scheme, skkey: pair.pubkey),
)
)
else:
err(SchemeError)
proc random*(T: typedesc[KeyPair], rng: var BrHmacDrbgContext,
bits = RsaDefaultKeySize): CryptoResult[KeyPair] =
proc random*(
T: typedesc[KeyPair], rng: var HmacDrbgContext, bits = RsaDefaultKeySize
): CryptoResult[KeyPair] =
## Generate random private pair of keys using default public-key cryptography
## scheme.
##
@@ -281,57 +315,70 @@ proc random*(T: typedesc[KeyPair], rng: var BrHmacDrbgContext,
## So will be used first available (supported) method.
when supported(PKScheme.Ed25519):
let pair = EdKeyPair.random(rng)
ok(KeyPair(
seckey: PrivateKey(scheme: PKScheme.Ed25519, edkey: pair.seckey),
pubkey: PublicKey(scheme: PKScheme.Ed25519, edkey: pair.pubkey)))
ok(
KeyPair(
seckey: PrivateKey(scheme: PKScheme.Ed25519, edkey: pair.seckey),
pubkey: PublicKey(scheme: PKScheme.Ed25519, edkey: pair.pubkey),
)
)
elif supported(PKScheme.Secp256k1):
let pair = SkKeyPair.random(rng)
ok(KeyPair(
seckey: PrivateKey(scheme: PKScheme.Secp256k1, skkey: pair.seckey),
pubkey: PublicKey(scheme: PKScheme.Secp256k1, skkey: pair.pubkey)))
ok(
KeyPair(
seckey: PrivateKey(scheme: PKScheme.Secp256k1, skkey: pair.seckey),
pubkey: PublicKey(scheme: PKScheme.Secp256k1, skkey: pair.pubkey),
)
)
elif supported(PKScheme.RSA):
let pair = ? RsaKeyPair.random(rng, bits).orError(KeyError)
ok(KeyPair(
seckey: PrivateKey(scheme: PKScheme.RSA, rsakey: pair.seckey),
pubkey: PublicKey(scheme: PKScheme.RSA, rsakey: pair.pubkey)))
let pair = ?RsaKeyPair.random(rng, bits).orError(KeyError)
ok(
KeyPair(
seckey: PrivateKey(scheme: PKScheme.RSA, rsakey: pair.seckey),
pubkey: PublicKey(scheme: PKScheme.RSA, rsakey: pair.pubkey),
)
)
elif supported(PKScheme.ECDSA):
let pair = ? EcKeyPair.random(Secp256r1, rng).orError(KeyError)
ok(KeyPair(
seckey: PrivateKey(scheme: PKScheme.ECDSA, eckey: pair.seckey),
pubkey: PublicKey(scheme: PKScheme.ECDSA, eckey: pair.pubkey)))
let pair = ?EcKeyPair.random(Secp256r1, rng).orError(KeyError)
ok(
KeyPair(
seckey: PrivateKey(scheme: PKScheme.ECDSA, eckey: pair.seckey),
pubkey: PublicKey(scheme: PKScheme.ECDSA, eckey: pair.pubkey),
)
)
else:
err(SchemeError)
proc getKey*(key: PrivateKey): CryptoResult[PublicKey] =
proc getPublicKey*(key: PrivateKey): CryptoResult[PublicKey] =
## Get public key from corresponding private key ``key``.
case key.scheme
of PKScheme.RSA:
when supported(PKScheme.RSA):
let rsakey = key.rsakey.getKey()
let rsakey = key.rsakey.getPublicKey()
ok(PublicKey(scheme: RSA, rsakey: rsakey))
else:
err(SchemeError)
of PKScheme.Ed25519:
when supported(PKScheme.Ed25519):
let edkey = key.edkey.getKey()
let edkey = key.edkey.getPublicKey()
ok(PublicKey(scheme: Ed25519, edkey: edkey))
else:
err(SchemeError)
of PKScheme.ECDSA:
when supported(PKScheme.ECDSA):
let eckey = ? key.eckey.getKey().orError(KeyError)
let eckey = ?key.eckey.getPublicKey().orError(KeyError)
ok(PublicKey(scheme: ECDSA, eckey: eckey))
else:
err(SchemeError)
of PKScheme.Secp256k1:
when supported(PKScheme.Secp256k1):
let skkey = key.skkey.getKey()
let skkey = key.skkey.getPublicKey()
ok(PublicKey(scheme: Secp256k1, skkey: skkey))
else:
err(SchemeError)
proc toRawBytes*(key: PrivateKey | PublicKey,
data: var openarray[byte]): CryptoResult[int] =
proc toRawBytes*(
key: PrivateKey | PublicKey, data: var openArray[byte]
): CryptoResult[int] =
## Serialize private key ``key`` (using scheme's own serialization) and store
## it to ``data``.
##
@@ -383,35 +430,35 @@ proc getRawBytes*(key: PrivateKey | PublicKey): CryptoResult[seq[byte]] =
else:
err(SchemeError)
proc toBytes*(key: PrivateKey, data: var openarray[byte]): CryptoResult[int] =
proc toBytes*(key: PrivateKey, data: var openArray[byte]): CryptoResult[int] =
## Serialize private key ``key`` (using libp2p protobuf scheme) and store
## it to ``data``.
##
## Returns number of bytes (octets) needed to store private key ``key``.
var msg = initProtoBuffer()
msg.write(1, uint64(key.scheme))
msg.write(2, ? key.getRawBytes())
msg.write(2, ?key.getRawBytes())
msg.finish()
var blen = len(msg.buffer)
if len(data) >= blen:
copyMem(addr data[0], addr msg.buffer[0], blen)
ok(blen)
proc toBytes*(key: PublicKey, data: var openarray[byte]): CryptoResult[int] =
proc toBytes*(key: PublicKey, data: var openArray[byte]): CryptoResult[int] =
## Serialize public key ``key`` (using libp2p protobuf scheme) and store
## it to ``data``.
##
## Returns number of bytes (octets) needed to store public key ``key``.
var msg = initProtoBuffer()
msg.write(1, uint64(key.scheme))
msg.write(2, ? key.getRawBytes())
msg.write(2, ?key.getRawBytes())
msg.finish()
var blen = len(msg.buffer)
if len(data) >= blen and blen > 0:
copyMem(addr data[0], addr msg.buffer[0], blen)
ok(blen)
proc toBytes*(sig: Signature, data: var openarray[byte]): int =
proc toBytes*(sig: Signature, data: var openArray[byte]): int =
## Serialize signature ``sig`` and store it to ``data``.
##
## Returns number of bytes (octets) needed to store signature ``sig``.
@@ -424,7 +471,7 @@ proc getBytes*(key: PrivateKey): CryptoResult[seq[byte]] =
## serialization).
var msg = initProtoBuffer()
msg.write(1, uint64(key.scheme))
msg.write(2, ? key.getRawBytes())
msg.write(2, ?key.getRawBytes())
msg.finish()
ok(msg.buffer)
@@ -433,7 +480,7 @@ proc getBytes*(key: PublicKey): CryptoResult[seq[byte]] =
## serialization).
var msg = initProtoBuffer()
msg.write(1, uint64(key.scheme))
msg.write(2, ? key.getRawBytes())
msg.write(2, ?key.getRawBytes())
msg.finish()
ok(msg.buffer)
@@ -441,7 +488,7 @@ proc getBytes*(sig: Signature): seq[byte] =
## Return signature ``sig`` in binary form.
result = sig.data
proc init*[T: PrivateKey|PublicKey](key: var T, data: openarray[byte]): bool =
template initImpl[T: PrivateKey | PublicKey](key: var T, data: openArray[byte]): bool =
## Initialize private key ``key`` from libp2p's protobuf serialized raw
## binary form.
##
@@ -454,7 +501,7 @@ proc init*[T: PrivateKey|PublicKey](key: var T, data: openarray[byte]): bool =
var pb = initProtoBuffer(@data)
let r1 = pb.getField(1, id)
let r2 = pb.getField(2, buffer)
if not(r1.isOk() and r1.get() and r2.isOk() and r2.get()):
if not (r1.get(false) and r2.get(false)):
false
else:
if cast[int8](id) notin SupportedSchemesInt or len(buffer) <= 0:
@@ -465,7 +512,7 @@ proc init*[T: PrivateKey|PublicKey](key: var T, data: openarray[byte]): bool =
var nkey = PrivateKey(scheme: scheme)
else:
var nkey = PublicKey(scheme: scheme)
case scheme:
case scheme
of PKScheme.RSA:
when supported(PKScheme.RSA):
if init(nkey.rsakey, buffer).isOk:
@@ -503,7 +550,16 @@ proc init*[T: PrivateKey|PublicKey](key: var T, data: openarray[byte]): bool =
else:
false
proc init*(sig: var Signature, data: openarray[byte]): bool =
{.push warning[ProveField]: off.} # https://github.com/nim-lang/Nim/issues/22060
proc init*(key: var PrivateKey, data: openArray[byte]): bool =
initImpl(key, data)
proc init*(key: var PublicKey, data: openArray[byte]): bool =
initImpl(key, data)
{.pop.}
proc init*(sig: var Signature, data: openArray[byte]): bool =
## Initialize signature ``sig`` from raw binary form.
##
## Returns ``true`` on success.
@@ -511,7 +567,7 @@ proc init*(sig: var Signature, data: openarray[byte]): bool =
sig.data = @data
result = true
proc init*[T: PrivateKey|PublicKey](key: var T, data: string): bool =
proc init*[T: PrivateKey | PublicKey](key: var T, data: string): bool =
## Initialize private/public key ``key`` from libp2p's protobuf serialized
## hexadecimal string representation.
##
@@ -525,26 +581,23 @@ proc init*(sig: var Signature, data: string): bool =
## Returns ``true`` on success.
sig.init(ncrutils.fromHex(data))
proc init*(t: typedesc[PrivateKey],
data: openarray[byte]): CryptoResult[PrivateKey] =
proc init*(t: typedesc[PrivateKey], data: openArray[byte]): CryptoResult[PrivateKey] =
## Create new private key from libp2p's protobuf serialized binary form.
var res: t
if not res.init(data):
err(KeyError)
err(CryptoError.KeyError)
else:
ok(res)
proc init*(t: typedesc[PublicKey],
data: openarray[byte]): CryptoResult[PublicKey] =
proc init*(t: typedesc[PublicKey], data: openArray[byte]): CryptoResult[PublicKey] =
## Create new public key from libp2p's protobuf serialized binary form.
var res: t
if not res.init(data):
err(KeyError)
err(CryptoError.KeyError)
else:
ok(res)
proc init*(t: typedesc[Signature],
data: openarray[byte]): CryptoResult[Signature] =
proc init*(t: typedesc[Signature], data: openArray[byte]): CryptoResult[Signature] =
## Create new public key from libp2p's protobuf serialized binary form.
var res: t
if not res.init(data):
@@ -560,24 +613,28 @@ proc init*(t: typedesc[PrivateKey], data: string): CryptoResult[PrivateKey] =
when supported(PKScheme.RSA):
proc init*(t: typedesc[PrivateKey], key: rsa.RsaPrivateKey): PrivateKey =
PrivateKey(scheme: RSA, rsakey: key)
proc init*(t: typedesc[PublicKey], key: rsa.RsaPublicKey): PublicKey =
PublicKey(scheme: RSA, rsakey: key)
when supported(PKScheme.Ed25519):
proc init*(t: typedesc[PrivateKey], key: EdPrivateKey): PrivateKey =
PrivateKey(scheme: Ed25519, edkey: key)
proc init*(t: typedesc[PublicKey], key: EdPublicKey): PublicKey =
PublicKey(scheme: Ed25519, edkey: key)
when supported(PKScheme.Secp256k1):
proc init*(t: typedesc[PrivateKey], key: SkPrivateKey): PrivateKey =
PrivateKey(scheme: Secp256k1, skkey: key)
proc init*(t: typedesc[PublicKey], key: SkPublicKey): PublicKey =
PublicKey(scheme: Secp256k1, skkey: key)
when supported(PKScheme.ECDSA):
proc init*(t: typedesc[PrivateKey], key: ecnist.EcPrivateKey): PrivateKey =
PrivateKey(scheme: ECDSA, eckey: key)
proc init*(t: typedesc[PublicKey], key: ecnist.EcPublicKey): PublicKey =
PublicKey(scheme: ECDSA, eckey: key)
@@ -646,9 +703,9 @@ proc `==`*(key1, key2: PrivateKey): bool =
else:
false
proc `$`*(key: PrivateKey|PublicKey): string =
proc `$`*(key: PrivateKey | PublicKey): string =
## Get string representation of private/public key ``key``.
case key.scheme:
case key.scheme
of PKScheme.RSA:
when supported(PKScheme.RSA):
$(key.rsakey)
@@ -670,9 +727,9 @@ proc `$`*(key: PrivateKey|PublicKey): string =
else:
"unsupported secp256k1 key"
func shortLog*(key: PrivateKey|PublicKey): string =
func shortLog*(key: PrivateKey | PublicKey): string =
## Get short string representation of private/public key ``key``.
case key.scheme:
case key.scheme
of PKScheme.RSA:
when supported(PKScheme.RSA):
($key.rsakey).shortLog
@@ -698,16 +755,15 @@ proc `$`*(sig: Signature): string =
## Get string representation of signature ``sig``.
result = ncrutils.toHex(sig.data)
proc sign*(key: PrivateKey,
data: openarray[byte]): CryptoResult[Signature] {.gcsafe.} =
proc sign*(key: PrivateKey, data: openArray[byte]): CryptoResult[Signature] {.gcsafe.} =
## Sign message ``data`` using private key ``key`` and return generated
## signature in raw binary form.
var res: Signature
case key.scheme:
case key.scheme
of PKScheme.RSA:
when supported(PKScheme.RSA):
let sig = ? key.rsakey.sign(data).orError(SigError)
res.data = ? sig.getBytes().orError(SigError)
let sig = ?key.rsakey.sign(data).orError(SigError)
res.data = ?sig.getBytes().orError(SigError)
ok(res)
else:
err(SchemeError)
@@ -720,8 +776,8 @@ proc sign*(key: PrivateKey,
err(SchemeError)
of PKScheme.ECDSA:
when supported(PKScheme.ECDSA):
let sig = ? key.eckey.sign(data).orError(SigError)
res.data = ? sig.getBytes().orError(SigError)
let sig = ?key.eckey.sign(data).orError(SigError)
res.data = ?sig.getBytes().orError(SigError)
ok(res)
else:
err(SchemeError)
@@ -733,10 +789,10 @@ proc sign*(key: PrivateKey,
else:
err(SchemeError)
proc verify*(sig: Signature, message: openarray[byte], key: PublicKey): bool =
proc verify*(sig: Signature, message: openArray[byte], key: PublicKey): bool =
## Verify signature ``sig`` using message ``message`` and public key ``key``.
## Return ``true`` if message signature is valid.
case key.scheme:
case key.scheme
of PKScheme.RSA:
when supported(PKScheme.RSA):
var signature: RsaSignature
@@ -774,12 +830,12 @@ proc verify*(sig: Signature, message: openarray[byte], key: PublicKey): bool =
else:
false
template makeSecret(buffer, hmactype, secret, seed: untyped) {.dirty.}=
template makeSecret(buffer, hmactype, secret, seed: untyped) {.dirty.} =
var ctx: hmactype
var j = 0
# We need to strip leading zeros, because Go bigint serialization do it.
var offset = 0
for i in 0..<len(secret):
for i in 0 ..< len(secret):
if secret[i] != 0x00'u8:
break
inc(offset)
@@ -800,8 +856,9 @@ template makeSecret(buffer, hmactype, secret, seed: untyped) {.dirty.}=
ctx.update(a.data)
a = ctx.finish()
proc stretchKeys*(cipherType: string, hashType: string,
sharedSecret: seq[byte]): Secret =
proc stretchKeys*(
cipherType: string, hashType: string, sharedSecret: seq[byte]
): Secret =
## Expand shared secret to cryptographic keys.
if cipherType == "AES-128":
result.ivsize = aes128.sizeBlock
@@ -827,65 +884,57 @@ template goffset*(secret, id, o: untyped): untyped =
id * (len(secret.data) shr 1) + o
template ivOpenArray*(secret: Secret, id: int): untyped =
toOpenArray(secret.data, goffset(secret, id, 0),
goffset(secret, id, secret.ivsize - 1))
toOpenArray(
secret.data, goffset(secret, id, 0), goffset(secret, id, secret.ivsize - 1)
)
template keyOpenArray*(secret: Secret, id: int): untyped =
toOpenArray(secret.data, goffset(secret, id, secret.ivsize),
goffset(secret, id, secret.ivsize + secret.keysize - 1))
toOpenArray(
secret.data,
goffset(secret, id, secret.ivsize),
goffset(secret, id, secret.ivsize + secret.keysize - 1),
)
template macOpenArray*(secret: Secret, id: int): untyped =
toOpenArray(secret.data, goffset(secret, id, secret.ivsize + secret.keysize),
goffset(secret, id, secret.ivsize + secret.keysize + secret.macsize - 1))
toOpenArray(
secret.data,
goffset(secret, id, secret.ivsize + secret.keysize),
goffset(secret, id, secret.ivsize + secret.keysize + secret.macsize - 1),
)
proc iv*(secret: Secret, id: int): seq[byte] {.inline.} =
## Get array of bytes with with initial vector.
result = newSeq[byte](secret.ivsize)
var offset = if id == 0: 0 else: (len(secret.data) div 2)
var offset =
if id == 0:
0
else:
(len(secret.data) div 2)
copyMem(addr result[0], unsafeAddr secret.data[offset], secret.ivsize)
proc key*(secret: Secret, id: int): seq[byte] {.inline.} =
result = newSeq[byte](secret.keysize)
var offset = if id == 0: 0 else: (len(secret.data) div 2)
var offset =
if id == 0:
0
else:
(len(secret.data) div 2)
offset += secret.ivsize
copyMem(addr result[0], unsafeAddr secret.data[offset], secret.keysize)
proc mac*(secret: Secret, id: int): seq[byte] {.inline.} =
result = newSeq[byte](secret.macsize)
var offset = if id == 0: 0 else: (len(secret.data) div 2)
var offset =
if id == 0:
0
else:
(len(secret.data) div 2)
offset += secret.ivsize + secret.keysize
copyMem(addr result[0], unsafeAddr secret.data[offset], secret.macsize)
proc ephemeral*(
scheme: ECDHEScheme,
rng: var BrHmacDrbgContext): CryptoResult[EcKeyPair] =
## Generate ephemeral keys used to perform ECDHE.
var keypair: EcKeyPair
if scheme == Secp256r1:
keypair = ? EcKeyPair.random(Secp256r1, rng).orError(KeyError)
elif scheme == Secp384r1:
keypair = ? EcKeyPair.random(Secp384r1, rng).orError(KeyError)
elif scheme == Secp521r1:
keypair = ? EcKeyPair.random(Secp521r1, rng).orError(KeyError)
ok(EcKeyPair(keypair))
proc ephemeral*(
scheme: string, rng: var BrHmacDrbgContext): CryptoResult[EcKeyPair] =
## Generate ephemeral keys used to perform ECDHE using string encoding.
##
## Currently supported encoding strings are P-256, P-384, P-521, if encoding
## string is not supported P-521 key will be generated.
if scheme == "P-256":
ephemeral(Secp256r1, rng)
elif scheme == "P-384":
ephemeral(Secp384r1, rng)
elif scheme == "P-521":
ephemeral(Secp521r1, rng)
else:
ephemeral(Secp521r1, rng)
proc getOrder*(remotePubkey, localNonce: openarray[byte],
localPubkey, remoteNonce: openarray[byte]): CryptoResult[int] =
proc getOrder*(
remotePubkey, localNonce: openArray[byte], localPubkey, remoteNonce: openArray[byte]
): CryptoResult[int] =
## Compare values and calculate `order` parameter.
var ctx: sha256
ctx.init()
@@ -896,9 +945,9 @@ proc getOrder*(remotePubkey, localNonce: openarray[byte],
ctx.update(localPubkey)
ctx.update(remoteNonce)
var digest2 = ctx.finish()
var mh1 = ? MultiHash.init(multiCodec("sha2-256"), digest1).orError(HashError)
var mh2 = ? MultiHash.init(multiCodec("sha2-256"), digest2).orError(HashError)
var res = 0;
var mh1 = ?MultiHash.init(multiCodec("sha2-256"), digest1).orError(HashError)
var mh2 = ?MultiHash.init(multiCodec("sha2-256"), digest2).orError(HashError)
var res = 0
for i in 0 ..< len(mh1.data.buffer):
res = int(mh1.data.buffer[i]) - int(mh2.data.buffer[i])
if res != 0:
@@ -929,129 +978,45 @@ proc selectBest*(order: int, p1, p2: string): string =
if felement == selement:
return felement
proc createProposal*(nonce, pubkey: openarray[byte],
exchanges, ciphers, hashes: string): seq[byte] =
## Create SecIO proposal message using random ``nonce``, local public key
## ``pubkey``, comma-delimieted list of supported exchange schemes
## ``exchanges``, comma-delimeted list of supported ciphers ``ciphers`` and
## comma-delimeted list of supported hashes ``hashes``.
var msg = initProtoBuffer({WithUint32BeLength})
msg.write(1, nonce)
msg.write(2, pubkey)
msg.write(3, exchanges)
msg.write(4, ciphers)
msg.write(5, hashes)
msg.finish()
msg.buffer
proc decodeProposal*(message: seq[byte], nonce, pubkey: var seq[byte],
exchanges, ciphers, hashes: var string): bool =
## Parse incoming proposal message and decode remote random nonce ``nonce``,
## remote public key ``pubkey``, comma-delimieted list of supported exchange
## schemes ``exchanges``, comma-delimeted list of supported ciphers
## ``ciphers`` and comma-delimeted list of supported hashes ``hashes``.
##
## Procedure returns ``true`` on success and ``false`` on error.
var pb = initProtoBuffer(message)
let r1 = pb.getField(1, nonce)
let r2 = pb.getField(2, pubkey)
let r3 = pb.getField(3, exchanges)
let r4 = pb.getField(4, ciphers)
let r5 = pb.getField(5, hashes)
r1.isOk() and r1.get() and r2.isOk() and r2.get() and
r3.isOk() and r3.get() and r4.isOk() and r4.get() and
r5.isOk() and r5.get()
proc createExchange*(epubkey, signature: openarray[byte]): seq[byte] =
## Create SecIO exchange message using ephemeral public key ``epubkey`` and
## signature of proposal blocks ``signature``.
var msg = initProtoBuffer({WithUint32BeLength})
msg.write(1, epubkey)
msg.write(2, signature)
msg.finish()
msg.buffer
proc decodeExchange*(message: seq[byte],
pubkey, signature: var seq[byte]): bool =
## Parse incoming exchange message and decode remote ephemeral public key
## ``pubkey`` and signature ``signature``.
##
## Procedure returns ``true`` on success and ``false`` on error.
var pb = initProtoBuffer(message)
let r1 = pb.getField(1, pubkey)
let r2 = pb.getField(2, signature)
r1.isOk() and r1.get() and r2.isOk() and r2.get()
## Serialization/Deserialization helpers
proc write*(vb: var VBuffer, pubkey: PublicKey) {.
inline, raises: [Defect, ResultError[CryptoError]].} =
proc write*(
vb: var VBuffer, pubkey: PublicKey
) {.inline, raises: [ResultError[CryptoError]].} =
## Write PublicKey value ``pubkey`` to buffer ``vb``.
vb.writeSeq(pubkey.getBytes().tryGet())
proc write*(vb: var VBuffer, seckey: PrivateKey) {.
inline, raises: [Defect, ResultError[CryptoError]].} =
proc write*(
vb: var VBuffer, seckey: PrivateKey
) {.inline, raises: [ResultError[CryptoError]].} =
## Write PrivateKey value ``seckey`` to buffer ``vb``.
vb.writeSeq(seckey.getBytes().tryGet())
proc write*(vb: var VBuffer, sig: PrivateKey) {.
inline, raises: [Defect, ResultError[CryptoError]].} =
proc write*(
vb: var VBuffer, sig: PrivateKey
) {.inline, raises: [ResultError[CryptoError]].} =
## Write Signature value ``sig`` to buffer ``vb``.
vb.writeSeq(sig.getBytes().tryGet())
proc write*[T: PublicKey|PrivateKey](pb: var ProtoBuffer, field: int,
key: T) {.
inline, raises: [Defect, ResultError[CryptoError]].} =
proc write*[T: PublicKey | PrivateKey](
pb: var ProtoBuffer, field: int, key: T
) {.inline, raises: [ResultError[CryptoError]].} =
write(pb, field, key.getBytes().tryGet())
proc write*(pb: var ProtoBuffer, field: int, sig: Signature) {.
inline, raises: [Defect, ResultError[CryptoError]].} =
proc write*(pb: var ProtoBuffer, field: int, sig: Signature) {.inline, raises: [].} =
write(pb, field, sig.getBytes())
proc initProtoField*(index: int, key: PublicKey|PrivateKey): ProtoField {.
deprecated, raises: [Defect, ResultError[CryptoError]].} =
## Initialize ProtoField with PublicKey/PrivateKey ``key``.
result = initProtoField(index, key.getBytes().tryGet())
proc initProtoField*(index: int, sig: Signature): ProtoField {.deprecated.} =
## Initialize ProtoField with Signature ``sig``.
result = initProtoField(index, sig.getBytes())
proc getValue*[T: PublicKey|PrivateKey](data: var ProtoBuffer, field: int,
value: var T): int {.deprecated.} =
## Read PublicKey/PrivateKey from ProtoBuf's message and validate it.
var buf: seq[byte]
var key: PublicKey
result = getLengthValue(data, field, buf)
if result > 0:
if not key.init(buf):
result = -1
else:
value = key
proc getValue*(data: var ProtoBuffer, field: int, value: var Signature): int {.
deprecated.} =
## Read ``Signature`` from ProtoBuf's message and validate it.
var buf: seq[byte]
var sig: Signature
result = getLengthValue(data, field, buf)
if result > 0:
if not sig.init(buf):
result = -1
else:
value = sig
proc getField*[T: PublicKey|PrivateKey](pb: ProtoBuffer, field: int,
value: var T): ProtoResult[bool] =
proc getField*[T: PublicKey | PrivateKey](
pb: ProtoBuffer, field: int, value: var T
): ProtoResult[bool] =
## Deserialize public/private key from protobuf's message ``pb`` using field
## index ``field``.
##
## On success deserialized key will be stored in ``value``.
var buffer: seq[byte]
var key: T
let res = ? pb.getField(field, buffer)
if not(res):
let res = ?pb.getField(field, buffer)
if not (res):
ok(false)
else:
if key.init(buffer):
@@ -1060,16 +1025,15 @@ proc getField*[T: PublicKey|PrivateKey](pb: ProtoBuffer, field: int,
else:
err(ProtoError.IncorrectBlob)
proc getField*(pb: ProtoBuffer, field: int,
value: var Signature): ProtoResult[bool] =
proc getField*(pb: ProtoBuffer, field: int, value: var Signature): ProtoResult[bool] =
## Deserialize signature from protobuf's message ``pb`` using field index
## ``field``.
##
## On success deserialized signature will be stored in ``value``.
var buffer: seq[byte]
var sig: Signature
let res = ? pb.getField(field, buffer)
if not(res):
let res = ?pb.getField(field, buffer)
if not (res):
ok(false)
else:
if sig.init(buffer):

View File

@@ -1,11 +1,11 @@
## Nim-Libp2p
## Copyright (c) 2020 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-Libp2p
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
## This module integrates BearSSL Cyrve25519 mul and mulgen
##
@@ -15,75 +15,69 @@
# RFC @ https://tools.ietf.org/html/rfc7748
{.push raises: [Defect].}
{.push raises: [].}
import bearssl
import stew/results
import bearssl/[ec, rand]
import results
from stew/assign2 import assign
export results
const
Curve25519KeySize* = 32
const Curve25519KeySize* = 32
type
Curve25519* = object
Curve25519Key* = array[Curve25519KeySize, byte]
pcuchar = ptr cuchar
Curve25519Error* = enum
Curver25519GenError
proc intoCurve25519Key*(s: openarray[byte]): Curve25519Key =
proc intoCurve25519Key*(s: openArray[byte]): Curve25519Key =
assert s.len == Curve25519KeySize
copyMem(addr result[0], unsafeaddr s[0], Curve25519KeySize)
assign(result, s)
proc getBytes*(key: Curve25519Key): seq[byte] = @key
proc getBytes*(key: Curve25519Key): seq[byte] =
@key
proc byteswap(buf: var Curve25519Key) {.inline.} =
for i in 0..<16:
let
x = buf[i]
for i in 0 ..< 16:
let x = buf[i]
buf[i] = buf[31 - i]
buf[31 - i] = x
proc mul*(_: type[Curve25519], point: var Curve25519Key, multiplier: Curve25519Key) =
let defaultBrEc = brEcGetDefault()
let defaultBrEc = ecGetDefault()
# multiplier needs to be big-endian
var
multiplierBs = multiplier
var multiplierBs = multiplier
multiplierBs.byteswap()
let
res = defaultBrEc.mul(
cast[pcuchar](addr point[0]),
Curve25519KeySize,
cast[pcuchar](addr multiplierBs[0]),
Curve25519KeySize,
EC_curve25519)
let res = defaultBrEc.mul(
addr point[0],
Curve25519KeySize,
addr multiplierBs[0],
Curve25519KeySize,
EC_curve25519,
)
assert res == 1
proc mulgen(_: type[Curve25519], dst: var Curve25519Key, point: Curve25519Key) =
let defaultBrEc = brEcGetDefault()
let defaultBrEc = ecGetDefault()
var
rpoint = point
var rpoint = point
rpoint.byteswap()
let
size = defaultBrEc.mulgen(
cast[pcuchar](addr dst[0]),
cast[pcuchar](addr rpoint[0]),
Curve25519KeySize,
EC_curve25519)
let size =
defaultBrEc.mulgen(addr dst[0], addr rpoint[0], Curve25519KeySize, EC_curve25519)
assert size == Curve25519KeySize
proc public*(private: Curve25519Key): Curve25519Key =
Curve25519.mulgen(result, private)
proc random*(_: type[Curve25519Key], rng: var BrHmacDrbgContext): Curve25519Key =
proc random*(_: type[Curve25519Key], rng: var HmacDrbgContext): Curve25519Key =
var res: Curve25519Key
let defaultBrEc = brEcGetDefault()
let len = brEcKeygen(
addr rng.vtable, defaultBrEc, nil, addr res[0], EC_curve25519)
let defaultBrEc = ecGetDefault()
let len = ecKeygen(
PrngClassPointerConst(addr rng.vtable), defaultBrEc, nil, addr res[0], EC_curve25519
)
# Per bearssl documentation, the keygen only fails if the curve is
# unrecognised -
doAssert len == Curve25519KeySize, "Could not generate curve"

View File

@@ -1,11 +1,11 @@
## Nim-Libp2p
## Copyright (c) 2018 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-Libp2p
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
## This module implements constant-time ECDSA and ECDHE for NIST elliptic
## curves secp256r1, secp384r1 and secp521r1.
@@ -14,14 +14,18 @@
## BearSSL library <https://bearssl.org/>
## Copyright(C) 2018 Thomas Pornin <pornin@bolet.org>.
{.push raises: [Defect].}
{.push raises: [].}
import bearssl
import bearssl/[ec, rand, hash]
# We use `ncrutils` for constant-time hexadecimal encoding/decoding procedures.
import nimcrypto/utils as ncrutils
import minasn1
export minasn1.Asn1Error
import stew/[results, ctops]
import stew/ctops
import results
import ../utility
export results
const
@@ -40,12 +44,12 @@ const
type
EcPrivateKey* = ref object
buffer*: array[BR_EC_KBUF_PRIV_MAX_SIZE, byte]
key*: BrEcPrivateKey
buffer*: array[EC_KBUF_PRIV_MAX_SIZE, byte]
key*: ec.EcPrivateKey
EcPublicKey* = ref object
buffer*: array[BR_EC_KBUF_PUB_MAX_SIZE, byte]
key*: BrEcPublicKey
buffer*: array[EC_KBUF_PUB_MAX_SIZE, byte]
key*: ec.EcPublicKey
EcKeyPair* = object
seckey*: EcPrivateKey
@@ -55,23 +59,22 @@ type
buffer*: seq[byte]
EcCurveKind* = enum
Secp256r1 = BR_EC_SECP256R1,
Secp384r1 = BR_EC_SECP384R1,
Secp521r1 = BR_EC_SECP521R1
Secp256r1 = EC_secp256r1
Secp384r1 = EC_secp384r1
Secp521r1 = EC_secp521r1
EcPKI* = EcPrivateKey | EcPublicKey | EcSignature
EcError* = enum
EcRngError,
EcKeyGenError,
EcPublicKeyError,
EcKeyIncorrectError,
EcRngError
EcKeyGenError
EcPublicKeyError
EcKeyIncorrectError
EcSignatureError
EcResult*[T] = Result[T, EcError]
const
EcSupportedCurvesCint* = {cint(Secp256r1), cint(Secp384r1), cint(Secp521r1)}
const EcSupportedCurvesCint* = @[cint(Secp256r1), cint(Secp384r1), cint(Secp521r1)]
proc `-`(x: uint32): uint32 {.inline.} =
result = (0xFFFF_FFFF'u32 - x) + 1'u32
@@ -85,7 +88,7 @@ proc CMP(x, y: uint32): int32 {.inline.} =
proc EQ0(x: int32): uint32 {.inline.} =
var q = cast[uint32](x)
result = not(q or -q) shr 31
result = not (q or -q) shr 31
proc NEQ(x, y: uint32): uint32 {.inline.} =
var q = cast[uint32](x xor y)
@@ -94,37 +97,36 @@ proc NEQ(x, y: uint32): uint32 {.inline.} =
proc LT0(x: int32): uint32 {.inline.} =
result = cast[uint32](x) shr 31
proc checkScalar(scalar: openarray[byte], curve: cint): uint32 =
proc checkScalar(scalar: openArray[byte], curve: cint): uint32 =
## Return ``1`` if all of the following hold:
## - len(``scalar``) <= ``orderlen``
## - ``scalar`` != 0
## - ``scalar`` is lower than the curve ``order``.
##
## Otherwise, return ``0``.
var impl = brEcGetDefault()
var orderlen = 0
var order = cast[ptr UncheckedArray[byte]](impl.order(curve, addr orderlen))
var impl = ecGetDefault()
var orderlen: uint = 0
var order = cast[ptr UncheckedArray[byte]](impl.order(curve, orderlen))
var z = 0'u32
var c = 0'i32
for u in scalar:
z = z or u
if len(scalar) == orderlen:
for i in 0..<len(scalar):
if len(scalar) == int(orderlen):
for i in 0 ..< len(scalar):
c = c or (-(cast[int32](EQ0(c))) and CMP(scalar[i], order[i]))
else:
c = -1
result = NEQ(z, 0'u32) and LT0(c)
proc checkPublic(key: openarray[byte], curve: cint): uint32 =
proc checkPublic(key: openArray[byte], curve: cint): uint32 =
## Return ``1`` if public key ``key`` is on curve.
var ckey = @key
var x = [0x00'u8, 0x01'u8]
var impl = brEcGetDefault()
var orderlen = 0
discard impl.order(curve, addr orderlen)
result = impl.mul(cast[ptr cuchar](unsafeAddr ckey[0]), len(ckey),
cast[ptr cuchar](addr x[0]), len(x), curve)
var x = [byte 0x00, 0x01]
var impl = ecGetDefault()
var orderlen: uint = 0
discard impl.order(curve, orderlen)
result = impl.mul(unsafeAddr ckey[0], uint(len(ckey)), addr x[0], uint(len(x)), curve)
proc getOffset(pubkey: EcPublicKey): int {.inline.} =
let o = cast[uint](pubkey.key.q) - cast[uint](unsafeAddr pubkey.buffer[0])
@@ -142,21 +144,15 @@ proc getOffset(seckey: EcPrivateKey): int {.inline.} =
template getPublicKeyLength*(curve: EcCurveKind): int =
case curve
of Secp256r1:
PubKey256Length
of Secp384r1:
PubKey384Length
of Secp521r1:
PubKey521Length
of Secp256r1: PubKey256Length
of Secp384r1: PubKey384Length
of Secp521r1: PubKey521Length
template getPrivateKeyLength*(curve: EcCurveKind): int =
case curve
of Secp256r1:
SecKey256Length
of Secp384r1:
SecKey384Length
of Secp521r1:
SecKey521Length
of Secp256r1: SecKey256Length
of Secp384r1: SecKey384Length
of Secp521r1: SecKey521Length
proc copy*[T: EcPKI](dst: var T, src: T): bool =
## Copy EC `private key`, `public key` or `signature` ``src`` to ``dst``.
@@ -174,7 +170,7 @@ proc copy*[T: EcPKI](dst: var T, src: T): bool =
dst.buffer = src.buffer
dst.key.curve = src.key.curve
dst.key.xlen = length
dst.key.x = cast[ptr cuchar](addr dst.buffer[offset])
dst.key.x = addr dst.buffer[offset]
result = true
elif T is EcPublicKey:
let length = src.key.qlen
@@ -184,7 +180,7 @@ proc copy*[T: EcPKI](dst: var T, src: T): bool =
dst.buffer = src.buffer
dst.key.curve = src.key.curve
dst.key.qlen = length
dst.key.q = cast[ptr cuchar](addr dst.buffer[offset])
dst.key.q = addr dst.buffer[offset]
result = true
else:
let length = len(src.buffer)
@@ -198,7 +194,7 @@ proc copy*[T: EcPKI](src: T): T {.inline.} =
if not copy(result, src):
raise newException(EcKeyIncorrectError, "Incorrect key or signature")
proc clear*[T: EcPKI|EcKeyPair](pki: var T) =
proc clear*[T: EcPKI | EcKeyPair](pki: var T) =
## Wipe and clear EC `private key`, `public key` or `signature` object.
doAssert(not isNil(pki))
when T is EcPrivateKey:
@@ -229,33 +225,36 @@ proc clear*[T: EcPKI|EcKeyPair](pki: var T) =
pki.pubkey.key.curve = 0
proc random*(
T: typedesc[EcPrivateKey], kind: EcCurveKind,
rng: var BrHmacDrbgContext): EcResult[EcPrivateKey] =
T: typedesc[EcPrivateKey], kind: EcCurveKind, rng: var HmacDrbgContext
): EcResult[EcPrivateKey] =
## Generate new random EC private key using BearSSL's HMAC-SHA256-DRBG
## algorithm.
##
## ``kind`` elliptic curve kind of your choice (secp256r1, secp384r1 or
## secp521r1).
var ecimp = brEcGetDefault()
var ecimp = ecGetDefault()
var res = new EcPrivateKey
if brEcKeygen(addr rng.vtable, ecimp,
addr res.key, addr res.buffer[0],
cast[cint](kind)) == 0:
if ecKeygen(
PrngClassPointerConst(addr rng.vtable),
ecimp,
addr res.key,
addr res.buffer[0],
safeConvert[cint](kind),
) == 0:
err(EcKeyGenError)
else:
ok(res)
proc getKey*(seckey: EcPrivateKey): EcResult[EcPublicKey] =
proc getPublicKey*(seckey: EcPrivateKey): EcResult[EcPublicKey] =
## Calculate and return EC public key from private key ``seckey``.
if isNil(seckey):
return err(EcKeyIncorrectError)
var ecimp = brEcGetDefault()
var ecimp = ecGetDefault()
if seckey.key.curve in EcSupportedCurvesCint:
var length = getPublicKeyLength(cast[EcCurveKind](seckey.key.curve))
var res = new EcPublicKey
if brEcComputePublicKey(ecimp, addr res.key,
addr res.buffer[0], unsafeAddr seckey.key) == 0:
assert res.buffer.len > getPublicKeyLength(cast[EcCurveKind](seckey.key.curve))
if ecComputePub(ecimp, addr res.key, addr res.buffer[0], unsafeAddr seckey.key) == 0:
err(EcKeyIncorrectError)
else:
ok(res)
@@ -263,23 +262,23 @@ proc getKey*(seckey: EcPrivateKey): EcResult[EcPublicKey] =
err(EcKeyIncorrectError)
proc random*(
T: typedesc[EcKeyPair], kind: EcCurveKind,
rng: var BrHmacDrbgContext): EcResult[T] =
T: typedesc[EcKeyPair], kind: EcCurveKind, rng: var HmacDrbgContext
): EcResult[T] =
## Generate new random EC private and public keypair using BearSSL's
## HMAC-SHA256-DRBG algorithm.
##
## ``kind`` elliptic curve kind of your choice (secp256r1, secp384r1 or
## secp521r1).
let
seckey = ? EcPrivateKey.random(kind, rng)
pubkey = ? seckey.getKey()
seckey = ?EcPrivateKey.random(kind, rng)
pubkey = ?seckey.getPublicKey()
key = EcKeyPair(seckey: seckey, pubkey: pubkey)
ok(key)
proc `$`*(seckey: EcPrivateKey): string =
## Return string representation of EC private key.
if isNil(seckey) or seckey.key.curve == 0 or seckey.key.xlen == 0 or
len(seckey.buffer) == 0:
len(seckey.buffer) == 0:
result = "Empty or uninitialized ECNIST key"
else:
if seckey.key.curve notin EcSupportedCurvesCint:
@@ -295,7 +294,7 @@ proc `$`*(seckey: EcPrivateKey): string =
proc `$`*(pubkey: EcPublicKey): string =
## Return string representation of EC public key.
if isNil(pubkey) or pubkey.key.curve == 0 or pubkey.key.qlen == 0 or
len(pubkey.buffer) == 0:
len(pubkey.buffer) == 0:
result = "Empty or uninitialized ECNIST key"
else:
if pubkey.key.curve notin EcSupportedCurvesCint:
@@ -315,7 +314,7 @@ proc `$`*(sig: EcSignature): string =
else:
result = ncrutils.toHex(sig.buffer)
proc toRawBytes*(seckey: EcPrivateKey, data: var openarray[byte]): EcResult[int] =
proc toRawBytes*(seckey: EcPrivateKey, data: var openArray[byte]): EcResult[int] =
## Serialize EC private key ``seckey`` to raw binary form and store it
## to ``data``.
##
@@ -331,7 +330,7 @@ proc toRawBytes*(seckey: EcPrivateKey, data: var openarray[byte]): EcResult[int]
else:
err(EcKeyIncorrectError)
proc toRawBytes*(pubkey: EcPublicKey, data: var openarray[byte]): EcResult[int] =
proc toRawBytes*(pubkey: EcPublicKey, data: var openArray[byte]): EcResult[int] =
## Serialize EC public key ``pubkey`` to uncompressed form specified in
## section 4.3.6 of ANSI X9.62.
##
@@ -347,7 +346,7 @@ proc toRawBytes*(pubkey: EcPublicKey, data: var openarray[byte]): EcResult[int]
else:
err(EcKeyIncorrectError)
proc toRawBytes*(sig: EcSignature, data: var openarray[byte]): int =
proc toRawBytes*(sig: EcSignature, data: var openArray[byte]): int =
## Serialize EC signature ``sig`` to raw binary form and store it to ``data``.
##
## Returns number of bytes (octets) needed to store EC signature, or `0`
@@ -358,7 +357,7 @@ proc toRawBytes*(sig: EcSignature, data: var openarray[byte]): int =
if len(sig.buffer) > 0:
copyMem(addr data[0], unsafeAddr sig.buffer[0], len(sig.buffer))
proc toBytes*(seckey: EcPrivateKey, data: var openarray[byte]): EcResult[int] =
proc toBytes*(seckey: EcPrivateKey, data: var openArray[byte]): EcResult[int] =
## Serialize EC private key ``seckey`` to ASN.1 DER binary form and store it
## to ``data``.
##
@@ -368,28 +367,30 @@ proc toBytes*(seckey: EcPrivateKey, data: var openarray[byte]): EcResult[int] =
return err(EcKeyIncorrectError)
if seckey.key.curve in EcSupportedCurvesCint:
var offset, length: int
var pubkey = ? seckey.getKey()
var pubkey = ?seckey.getPublicKey()
var b = Asn1Buffer.init()
var p = Asn1Composite.init(Asn1Tag.Sequence)
var c0 = Asn1Composite.init(0)
var c1 = Asn1Composite.init(1)
if seckey.key.curve == BR_EC_SECP256R1:
if seckey.key.curve == EC_secp256r1:
c0.write(Asn1Tag.Oid, Asn1OidSecp256r1)
elif seckey.key.curve == BR_EC_SECP384R1:
elif seckey.key.curve == EC_secp384r1:
c0.write(Asn1Tag.Oid, Asn1OidSecp384r1)
elif seckey.key.curve == BR_EC_SECP521R1:
elif seckey.key.curve == EC_secp521r1:
c0.write(Asn1Tag.Oid, Asn1OidSecp521r1)
c0.finish()
offset = pubkey.getOffset()
length = pubkey.key.qlen
c1.write(Asn1Tag.BitString,
pubkey.buffer.toOpenArray(offset, offset + length - 1))
if offset < 0:
return err(EcKeyIncorrectError)
length = int(pubkey.key.qlen)
c1.write(Asn1Tag.BitString, pubkey.buffer.toOpenArray(offset, offset + length - 1))
c1.finish()
offset = seckey.getOffset()
length = seckey.key.xlen
if offset < 0:
return err(EcKeyIncorrectError)
length = int(seckey.key.xlen)
p.write(1'u64)
p.write(Asn1Tag.OctetString,
seckey.buffer.toOpenArray(offset, offset + length - 1))
p.write(Asn1Tag.OctetString, seckey.buffer.toOpenArray(offset, offset + length - 1))
p.write(c0)
p.write(c1)
p.finish()
@@ -403,8 +404,7 @@ proc toBytes*(seckey: EcPrivateKey, data: var openarray[byte]): EcResult[int] =
else:
err(EcKeyIncorrectError)
proc toBytes*(pubkey: EcPublicKey, data: var openarray[byte]): EcResult[int] =
proc toBytes*(pubkey: EcPublicKey, data: var openArray[byte]): EcResult[int] =
## Serialize EC public key ``pubkey`` to ASN.1 DER binary form and store it
## to ``data``.
##
@@ -417,18 +417,19 @@ proc toBytes*(pubkey: EcPublicKey, data: var openarray[byte]): EcResult[int] =
var p = Asn1Composite.init(Asn1Tag.Sequence)
var c = Asn1Composite.init(Asn1Tag.Sequence)
c.write(Asn1Tag.Oid, Asn1OidEcPublicKey)
if pubkey.key.curve == BR_EC_SECP256R1:
if pubkey.key.curve == EC_secp256r1:
c.write(Asn1Tag.Oid, Asn1OidSecp256r1)
elif pubkey.key.curve == BR_EC_SECP384R1:
elif pubkey.key.curve == EC_secp384r1:
c.write(Asn1Tag.Oid, Asn1OidSecp384r1)
elif pubkey.key.curve == BR_EC_SECP521R1:
elif pubkey.key.curve == EC_secp521r1:
c.write(Asn1Tag.Oid, Asn1OidSecp521r1)
c.finish()
p.write(c)
let offset = getOffset(pubkey)
let length = pubkey.key.qlen
p.write(Asn1Tag.BitString,
pubkey.buffer.toOpenArray(offset, offset + length - 1))
if offset < 0:
return err(EcKeyIncorrectError)
let length = int(pubkey.key.qlen)
p.write(Asn1Tag.BitString, pubkey.buffer.toOpenArray(offset, offset + length - 1))
p.finish()
b.write(p)
b.finish()
@@ -439,7 +440,7 @@ proc toBytes*(pubkey: EcPublicKey, data: var openarray[byte]): EcResult[int] =
else:
err(EcKeyIncorrectError)
proc toBytes*(sig: EcSignature, data: var openarray[byte]): EcResult[int] =
proc toBytes*(sig: EcSignature, data: var openArray[byte]): EcResult[int] =
## Serialize EC signature ``sig`` to ASN.1 DER binary form and store it
## to ``data``.
##
@@ -458,9 +459,9 @@ proc getBytes*(seckey: EcPrivateKey): EcResult[seq[byte]] =
return err(EcKeyIncorrectError)
if seckey.key.curve in EcSupportedCurvesCint:
var res = newSeq[byte]()
let length = ? seckey.toBytes(res)
let length = ?seckey.toBytes(res)
res.setLen(length)
discard ? seckey.toBytes(res)
discard ?seckey.toBytes(res)
ok(res)
else:
err(EcKeyIncorrectError)
@@ -471,9 +472,9 @@ proc getBytes*(pubkey: EcPublicKey): EcResult[seq[byte]] =
return err(EcKeyIncorrectError)
if pubkey.key.curve in EcSupportedCurvesCint:
var res = newSeq[byte]()
let length = ? pubkey.toBytes(res)
let length = ?pubkey.toBytes(res)
res.setLen(length)
discard ? pubkey.toBytes(res)
discard ?pubkey.toBytes(res)
ok(res)
else:
err(EcKeyIncorrectError)
@@ -483,9 +484,9 @@ proc getBytes*(sig: EcSignature): EcResult[seq[byte]] =
if isNil(sig):
return err(EcSignatureError)
var res = newSeq[byte]()
let length = ? sig.toBytes(res)
let length = ?sig.toBytes(res)
res.setLen(length)
discard ? sig.toBytes(res)
discard ?sig.toBytes(res)
ok(res)
proc getRawBytes*(seckey: EcPrivateKey): EcResult[seq[byte]] =
@@ -494,9 +495,9 @@ proc getRawBytes*(seckey: EcPrivateKey): EcResult[seq[byte]] =
return err(EcKeyIncorrectError)
if seckey.key.curve in EcSupportedCurvesCint:
var res = newSeq[byte]()
let length = ? seckey.toRawBytes(res)
let length = ?seckey.toRawBytes(res)
res.setLen(length)
discard ? seckey.toRawBytes(res)
discard ?seckey.toRawBytes(res)
ok(res)
else:
err(EcKeyIncorrectError)
@@ -507,9 +508,9 @@ proc getRawBytes*(pubkey: EcPublicKey): EcResult[seq[byte]] =
return err(EcKeyIncorrectError)
if pubkey.key.curve in EcSupportedCurvesCint:
var res = newSeq[byte]()
let length = ? pubkey.toRawBytes(res)
let length = ?pubkey.toRawBytes(res)
res.setLen(length)
discard ? pubkey.toRawBytes(res)
discard ?pubkey.toRawBytes(res)
return ok(res)
else:
return err(EcKeyIncorrectError)
@@ -519,9 +520,9 @@ proc getRawBytes*(sig: EcSignature): EcResult[seq[byte]] =
if isNil(sig):
return err(EcSignatureError)
var res = newSeq[byte]()
let length = ? sig.toBytes(res)
let length = ?sig.toBytes(res)
res.setLen(length)
discard ? sig.toBytes(res)
discard ?sig.toBytes(res)
ok(res)
proc `==`*(pubkey1, pubkey2: EcPublicKey): bool =
@@ -541,8 +542,10 @@ proc `==`*(pubkey1, pubkey2: EcPublicKey): bool =
let op2 = pubkey2.getOffset()
if op1 == -1 or op2 == -1:
return false
return CT.isEqual(pubkey1.buffer.toOpenArray(op1, pubkey1.key.qlen - 1),
pubkey2.buffer.toOpenArray(op2, pubkey2.key.qlen - 1))
return CT.isEqual(
pubkey1.buffer.toOpenArray(op1, pubkey1.key.qlen - 1),
pubkey2.buffer.toOpenArray(op2, pubkey2.key.qlen - 1),
)
proc `==`*(seckey1, seckey2: EcPrivateKey): bool =
## Returns ``true`` if both keys ``seckey1`` and ``seckey2`` are equal.
@@ -561,8 +564,10 @@ proc `==`*(seckey1, seckey2: EcPrivateKey): bool =
let op2 = seckey2.getOffset()
if op1 == -1 or op2 == -1:
return false
return CT.isEqual(seckey1.buffer.toOpenArray(op1, seckey1.key.xlen - 1),
seckey2.buffer.toOpenArray(op2, seckey2.key.xlen - 1))
return CT.isEqual(
seckey1.buffer.toOpenArray(op1, seckey1.key.xlen - 1),
seckey2.buffer.toOpenArray(op2, seckey2.key.xlen - 1),
)
proc `==`*(a, b: EcSignature): bool =
## Return ``true`` if both signatures ``sig1`` and ``sig2`` are equal.
@@ -586,7 +591,7 @@ proc `==`*(a, b: EcSignature): bool =
else:
CT.isEqual(a.buffer, b.buffer)
proc init*(key: var EcPrivateKey, data: openarray[byte]): Result[void, Asn1Error] =
proc init*(key: var EcPrivateKey, data: openArray[byte]): Result[void, Asn1Error] =
## Initialize EC `private key` or `signature` ``key`` from ASN.1 DER binary
## representation ``data``.
##
@@ -596,50 +601,50 @@ proc init*(key: var EcPrivateKey, data: openarray[byte]): Result[void, Asn1Error
var ab = Asn1Buffer.init(data)
field = ? ab.read()
field = ?ab.read()
if field.kind != Asn1Tag.Sequence:
return err(Asn1Error.Incorrect)
var ib = field.getBuffer()
field = ? ib.read()
field = ?ib.read()
if field.kind != Asn1Tag.Integer:
return err(Asn1Error.Incorrect)
if field.vint != 1'u64:
return err(Asn1Error.Incorrect)
raw = ? ib.read()
raw = ?ib.read()
if raw.kind != Asn1Tag.OctetString:
return err(Asn1Error.Incorrect)
oid = ? ib.read()
oid = ?ib.read()
if oid.kind != Asn1Tag.Oid:
return err(Asn1Error.Incorrect)
if oid == Asn1OidSecp256r1:
curve = cast[cint](Secp256r1)
curve = safeConvert[cint](Secp256r1)
elif oid == Asn1OidSecp384r1:
curve = cast[cint](Secp384r1)
curve = safeConvert[cint](Secp384r1)
elif oid == Asn1OidSecp521r1:
curve = cast[cint](Secp521r1)
curve = safeConvert[cint](Secp521r1)
else:
return err(Asn1Error.Incorrect)
if checkScalar(raw.toOpenArray(), curve) == 1'u32:
key = new EcPrivateKey
copyMem(addr key.buffer[0], addr raw.buffer[raw.offset], raw.length)
key.key.x = cast[ptr cuchar](addr key.buffer[0])
key.key.xlen = raw.length
key.key.x = addr key.buffer[0]
key.key.xlen = uint(raw.length)
key.key.curve = curve
ok()
else:
err(Asn1Error.Incorrect)
proc init*(pubkey: var EcPublicKey, data: openarray[byte]): Result[void, Asn1Error] =
proc init*(pubkey: var EcPublicKey, data: openArray[byte]): Result[void, Asn1Error] =
## Initialize EC public key ``pubkey`` from ASN.1 DER binary representation
## ``data``.
##
@@ -649,19 +654,19 @@ proc init*(pubkey: var EcPublicKey, data: openarray[byte]): Result[void, Asn1Err
var ab = Asn1Buffer.init(data)
field = ? ab.read()
field = ?ab.read()
if field.kind != Asn1Tag.Sequence:
return err(Asn1Error.Incorrect)
var ib = field.getBuffer()
field = ? ib.read()
field = ?ib.read()
if field.kind != Asn1Tag.Sequence:
return err(Asn1Error.Incorrect)
var ob = field.getBuffer()
oid = ? ob.read()
oid = ?ob.read()
if oid.kind != Asn1Tag.Oid:
return err(Asn1Error.Incorrect)
@@ -669,21 +674,21 @@ proc init*(pubkey: var EcPublicKey, data: openarray[byte]): Result[void, Asn1Err
if oid != Asn1OidEcPublicKey:
return err(Asn1Error.Incorrect)
oid = ? ob.read()
oid = ?ob.read()
if oid.kind != Asn1Tag.Oid:
return err(Asn1Error.Incorrect)
if oid == Asn1OidSecp256r1:
curve = cast[cint](Secp256r1)
curve = safeConvert[cint](Secp256r1)
elif oid == Asn1OidSecp384r1:
curve = cast[cint](Secp384r1)
curve = safeConvert[cint](Secp384r1)
elif oid == Asn1OidSecp521r1:
curve = cast[cint](Secp521r1)
curve = safeConvert[cint](Secp521r1)
else:
return err(Asn1Error.Incorrect)
raw = ? ib.read()
raw = ?ib.read()
if raw.kind != Asn1Tag.BitString:
return err(Asn1Error.Incorrect)
@@ -691,14 +696,14 @@ proc init*(pubkey: var EcPublicKey, data: openarray[byte]): Result[void, Asn1Err
if checkPublic(raw.toOpenArray(), curve) != 0:
pubkey = new EcPublicKey
copyMem(addr pubkey.buffer[0], addr raw.buffer[raw.offset], raw.length)
pubkey.key.q = cast[ptr cuchar](addr pubkey.buffer[0])
pubkey.key.qlen = raw.length
pubkey.key.q = addr pubkey.buffer[0]
pubkey.key.qlen = uint(raw.length)
pubkey.key.curve = curve
ok()
else:
err(Asn1Error.Incorrect)
proc init*(sig: var EcSignature, data: openarray[byte]): Result[void, Asn1Error] =
proc init*(sig: var EcSignature, data: openArray[byte]): Result[void, Asn1Error] =
## Initialize EC signature ``sig`` from raw binary representation ``data``.
##
## Procedure returns ``Result[void, Asn1Error]``.
@@ -709,16 +714,14 @@ proc init*(sig: var EcSignature, data: openarray[byte]): Result[void, Asn1Error]
else:
err(Asn1Error.Incorrect)
proc init*[T: EcPKI](sospk: var T,
data: string): Result[void, Asn1Error] {.inline.} =
proc init*[T: EcPKI](sospk: var T, data: string): Result[void, Asn1Error] {.inline.} =
## Initialize EC `private key`, `public key` or `signature` ``sospk`` from
## ASN.1 DER hexadecimal string representation ``data``.
##
## Procedure returns ``Asn1Status``.
sospk.init(ncrutils.fromHex(data))
proc init*(t: typedesc[EcPrivateKey],
data: openarray[byte]): EcResult[EcPrivateKey] =
proc init*(t: typedesc[EcPrivateKey], data: openArray[byte]): EcResult[EcPrivateKey] =
## Initialize EC private key from ASN.1 DER binary representation ``data`` and
## return constructed object.
var key: EcPrivateKey
@@ -728,8 +731,7 @@ proc init*(t: typedesc[EcPrivateKey],
else:
ok(key)
proc init*(t: typedesc[EcPublicKey],
data: openarray[byte]): EcResult[EcPublicKey] =
proc init*(t: typedesc[EcPublicKey], data: openArray[byte]): EcResult[EcPublicKey] =
## Initialize EC public key from ASN.1 DER binary representation ``data`` and
## return constructed object.
var key: EcPublicKey
@@ -739,8 +741,7 @@ proc init*(t: typedesc[EcPublicKey],
else:
ok(key)
proc init*(t: typedesc[EcSignature],
data: openarray[byte]): EcResult[EcSignature] =
proc init*(t: typedesc[EcSignature], data: openArray[byte]): EcResult[EcSignature] =
## Initialize EC signature from raw binary representation ``data`` and
## return constructed object.
var sig: EcSignature
@@ -755,7 +756,7 @@ proc init*[T: EcPKI](t: typedesc[T], data: string): EcResult[T] =
## string representation ``data`` and return constructed object.
t.init(ncrutils.fromHex(data))
proc initRaw*(key: var EcPrivateKey, data: openarray[byte]): bool =
proc initRaw*(key: var EcPrivateKey, data: openArray[byte]): bool =
## Initialize EC `private key` or `scalar` ``key`` from raw binary
## representation ``data``.
##
@@ -765,13 +766,13 @@ proc initRaw*(key: var EcPrivateKey, data: openarray[byte]): bool =
## Procedure returns ``true`` on success, ``false`` otherwise.
var curve: cint
if len(data) == SecKey256Length:
curve = cast[cint](Secp256r1)
curve = safeConvert[cint](Secp256r1)
result = true
elif len(data) == SecKey384Length:
curve = cast[cint](Secp384r1)
curve = safeConvert[cint](Secp384r1)
result = true
elif len(data) == SecKey521Length:
curve = cast[cint](Secp521r1)
curve = safeConvert[cint](Secp521r1)
result = true
if result:
result = false
@@ -779,12 +780,12 @@ proc initRaw*(key: var EcPrivateKey, data: openarray[byte]): bool =
let length = len(data)
key = new EcPrivateKey
copyMem(addr key.buffer[0], unsafeAddr data[0], length)
key.key.x = cast[ptr cuchar](addr key.buffer[0])
key.key.xlen = length
key.key.x = addr key.buffer[0]
key.key.xlen = uint(length)
key.key.curve = curve
result = true
proc initRaw*(pubkey: var EcPublicKey, data: openarray[byte]): bool =
proc initRaw*(pubkey: var EcPublicKey, data: openArray[byte]): bool =
## Initialize EC public key ``pubkey`` from raw binary representation
## ``data``.
##
@@ -796,13 +797,13 @@ proc initRaw*(pubkey: var EcPublicKey, data: openarray[byte]): bool =
if len(data) > 0:
if data[0] == 0x04'u8:
if len(data) == PubKey256Length:
curve = cast[cint](Secp256r1)
curve = safeConvert[cint](Secp256r1)
result = true
elif len(data) == PubKey384Length:
curve = cast[cint](Secp384r1)
curve = safeConvert[cint](Secp384r1)
result = true
elif len(data) == PubKey521Length:
curve = cast[cint](Secp521r1)
curve = safeConvert[cint](Secp521r1)
result = true
if result:
result = false
@@ -810,12 +811,12 @@ proc initRaw*(pubkey: var EcPublicKey, data: openarray[byte]): bool =
let length = len(data)
pubkey = new EcPublicKey
copyMem(addr pubkey.buffer[0], unsafeAddr data[0], length)
pubkey.key.q = cast[ptr cuchar](addr pubkey.buffer[0])
pubkey.key.qlen = length
pubkey.key.q = addr pubkey.buffer[0]
pubkey.key.qlen = uint(length)
pubkey.key.curve = curve
result = true
proc initRaw*(sig: var EcSignature, data: openarray[byte]): bool =
proc initRaw*(sig: var EcSignature, data: openArray[byte]): bool =
## Initialize EC signature ``sig`` from raw binary representation ``data``.
##
## Length of ``data`` array must be ``Sig256Length``, ``Sig384Length``
@@ -823,8 +824,7 @@ proc initRaw*(sig: var EcSignature, data: openarray[byte]): bool =
##
## Procedure returns ``true`` on success, ``false`` otherwise.
let length = len(data)
if (length == Sig256Length) or (length == Sig384Length) or
(length == Sig521Length):
if (length == Sig256Length) or (length == Sig384Length) or (length == Sig521Length):
result = true
if result:
sig = new EcSignature
@@ -837,8 +837,9 @@ proc initRaw*[T: EcPKI](sospk: var T, data: string): bool {.inline.} =
## Procedure returns ``true`` on success, ``false`` otherwise.
result = sospk.initRaw(ncrutils.fromHex(data))
proc initRaw*(t: typedesc[EcPrivateKey],
data: openarray[byte]): EcResult[EcPrivateKey] =
proc initRaw*(
t: typedesc[EcPrivateKey], data: openArray[byte]
): EcResult[EcPrivateKey] =
## Initialize EC private key from raw binary representation ``data`` and
## return constructed object.
var res: EcPrivateKey
@@ -847,8 +848,7 @@ proc initRaw*(t: typedesc[EcPrivateKey],
else:
ok(res)
proc initRaw*(t: typedesc[EcPublicKey],
data: openarray[byte]): EcResult[EcPublicKey] =
proc initRaw*(t: typedesc[EcPublicKey], data: openArray[byte]): EcResult[EcPublicKey] =
## Initialize EC public key from raw binary representation ``data`` and
## return constructed object.
var res: EcPublicKey
@@ -857,8 +857,7 @@ proc initRaw*(t: typedesc[EcPublicKey],
else:
ok(res)
proc initRaw*(t: typedesc[EcSignature],
data: openarray[byte]): EcResult[EcSignature] =
proc initRaw*(t: typedesc[EcSignature], data: openArray[byte]): EcResult[EcSignature] =
## Initialize EC signature from raw binary representation ``data`` and
## return constructed object.
var res: EcSignature
@@ -877,7 +876,7 @@ proc scalarMul*(pub: EcPublicKey, sec: EcPrivateKey): EcPublicKey =
##
## Returns point in curve as ``pub * sec`` or ``nil`` otherwise.
doAssert((not isNil(pub)) and (not isNil(sec)))
var impl = brEcGetDefault()
var impl = ecGetDefault()
if sec.key.curve in EcSupportedCurvesCint:
if pub.key.curve == sec.key.curve:
var key = new EcPublicKey
@@ -885,16 +884,19 @@ proc scalarMul*(pub: EcPublicKey, sec: EcPrivateKey): EcPublicKey =
let poffset = key.getOffset()
let soffset = sec.getOffset()
if poffset >= 0 and soffset >= 0:
let res = impl.mul(cast[ptr cuchar](addr key.buffer[poffset]),
key.key.qlen,
cast[ptr cuchar](unsafeAddr sec.buffer[soffset]),
sec.key.xlen,
key.key.curve)
let res = impl.mul(
addr key.buffer[poffset],
key.key.qlen,
unsafeAddr sec.buffer[soffset],
sec.key.xlen,
key.key.curve,
)
if res != 0:
result = key
proc toSecret*(pubkey: EcPublicKey, seckey: EcPrivateKey,
data: var openarray[byte]): int =
proc toSecret*(
pubkey: EcPublicKey, seckey: EcPrivateKey, data: var openArray[byte]
): int =
## Calculate ECDHE shared secret using Go's elliptic/curve approach, using
## remote public key ``pubkey`` and local private key ``seckey`` and store
## shared secret to ``data``.
@@ -907,11 +909,11 @@ proc toSecret*(pubkey: EcPublicKey, seckey: EcPrivateKey,
doAssert((not isNil(pubkey)) and (not isNil(seckey)))
var mult = scalarMul(pubkey, seckey)
if not isNil(mult):
if seckey.key.curve == BR_EC_SECP256R1:
if seckey.key.curve == EC_secp256r1:
result = Secret256Length
elif seckey.key.curve == BR_EC_SECP384R1:
elif seckey.key.curve == EC_secp384r1:
result = Secret384Length
elif seckey.key.curve == BR_EC_SECP521R1:
elif seckey.key.curve == EC_secp521r1:
result = Secret521Length
if len(data) >= result:
var qplus1 = cast[pointer](cast[uint](mult.key.q) + 1'u)
@@ -930,26 +932,27 @@ proc getSecret*(pubkey: EcPublicKey, seckey: EcPrivateKey): seq[byte] =
result = newSeq[byte](res)
copyMem(addr result[0], addr data[0], res)
proc sign*[T: byte|char](seckey: EcPrivateKey,
message: openarray[T]): EcResult[EcSignature] {.gcsafe.} =
proc sign*[T: byte | char](
seckey: EcPrivateKey, message: openArray[T]
): EcResult[EcSignature] {.gcsafe.} =
## Get ECDSA signature of data ``message`` using private key ``seckey``.
if isNil(seckey):
return err(EcKeyIncorrectError)
var hc: BrHashCompatContext
var hc: HashCompatContext
var hash: array[32, byte]
var impl = brEcGetDefault()
var impl = ecGetDefault()
if seckey.key.curve in EcSupportedCurvesCint:
var sig = new EcSignature
sig.buffer = newSeq[byte](256)
var kv = addr sha256Vtable
kv.init(addr hc.vtable)
if len(message) > 0:
kv.update(addr hc.vtable, unsafeAddr message[0], len(message))
kv.update(addr hc.vtable, unsafeAddr message[0], uint(len(message)))
else:
kv.update(addr hc.vtable, nil, 0)
kv.output(addr hc.vtable, addr hash[0])
let res = brEcdsaSignAsn1(impl, kv, addr hash[0], addr seckey.key,
addr sig.buffer[0])
kv.out(addr hc.vtable, addr hash[0])
let res =
ecdsaI31SignAsn1(impl, kv, addr hash[0], addr seckey.key, addr sig.buffer[0])
# Clear context with initial value
kv.init(addr hc.vtable)
if res != 0:
@@ -960,28 +963,61 @@ proc sign*[T: byte|char](seckey: EcPrivateKey,
else:
err(EcKeyIncorrectError)
proc verify*[T: byte|char](sig: EcSignature, message: openarray[T],
pubkey: EcPublicKey): bool {.inline.} =
proc verify*[T: byte | char](
sig: EcSignature, message: openArray[T], pubkey: EcPublicKey
): bool {.inline.} =
## Verify ECDSA signature ``sig`` using public key ``pubkey`` and data
## ``message``.
##
## Return ``true`` if message verification succeeded, ``false`` if
## verification failed.
doAssert((not isNil(sig)) and (not isNil(pubkey)))
var hc: BrHashCompatContext
var hc: HashCompatContext
var hash: array[32, byte]
var impl = brEcGetDefault()
var impl = ecGetDefault()
if pubkey.key.curve in EcSupportedCurvesCint:
var kv = addr sha256Vtable
kv.init(addr hc.vtable)
if len(message) > 0:
kv.update(addr hc.vtable, unsafeAddr message[0], len(message))
kv.update(addr hc.vtable, unsafeAddr message[0], uint(len(message)))
else:
kv.update(addr hc.vtable, nil, 0)
kv.output(addr hc.vtable, addr hash[0])
let res = brEcdsaVerifyAsn1(impl, addr hash[0], len(hash),
unsafeAddr pubkey.key,
addr sig.buffer[0], len(sig.buffer))
kv.out(addr hc.vtable, addr hash[0])
let res = ecdsaI31VrfyAsn1(
impl,
addr hash[0],
uint(len(hash)),
unsafeAddr pubkey.key,
addr sig.buffer[0],
uint(len(sig.buffer)),
)
# Clear context with initial value
kv.init(addr hc.vtable)
result = (res == 1)
type ECDHEScheme* = EcCurveKind
proc ephemeral*(scheme: ECDHEScheme, rng: var HmacDrbgContext): EcResult[EcKeyPair] =
## Generate ephemeral keys used to perform ECDHE.
var keypair: EcKeyPair
if scheme == Secp256r1:
keypair = ?EcKeyPair.random(Secp256r1, rng)
elif scheme == Secp384r1:
keypair = ?EcKeyPair.random(Secp384r1, rng)
elif scheme == Secp521r1:
keypair = ?EcKeyPair.random(Secp521r1, rng)
ok(keypair)
proc ephemeral*(scheme: string, rng: var HmacDrbgContext): EcResult[EcKeyPair] =
## Generate ephemeral keys used to perform ECDHE using string encoding.
##
## Currently supported encoding strings are P-256, P-384, P-521, if encoding
## string is not supported P-521 key will be generated.
if scheme == "P-256":
ephemeral(Secp256r1, rng)
elif scheme == "P-384":
ephemeral(Secp384r1, rng)
elif scheme == "P-521":
ephemeral(Secp521r1, rng)
else:
ephemeral(Secp521r1, rng)

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -1,40 +1,53 @@
## Nim-LibP2P
## Copyright (c) 2020 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-LibP2P
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
# https://tools.ietf.org/html/rfc5869
{.push raises: [Defect].}
{.push raises: [].}
import nimcrypto
import bearssl
import bearssl/[kdf, hash]
type
BearHKDFContext {.importc: "br_hkdf_context", header: "bearssl_kdf.h".} = object
HKDFResult*[len: static int] = array[len, byte]
type HkdfResult*[len: static int] = array[len, byte]
proc br_hkdf_init(ctx: ptr BearHKDFContext; hashClass: ptr HashClass; salt: pointer; len: csize_t) {.importc: "br_hkdf_init", header: "bearssl_kdf.h", raises: [].}
proc br_hkdf_inject(ctx: ptr BearHKDFContext; ikm: pointer; len: csize_t) {.importc: "br_hkdf_inject", header: "bearssl_kdf.h", raises: [].}
proc br_hkdf_flip(ctx: ptr BearHKDFContext) {.importc: "br_hkdf_flip", header: "bearssl_kdf.h", raises: [].}
proc br_hkdf_produce(ctx: ptr BearHKDFContext; info: pointer; infoLen: csize_t; output: pointer; outputLen: csize_t) {.importc: "br_hkdf_produce", header: "bearssl_kdf.h", raises: [].}
proc hkdf*[T: sha256; len: static int](_: type[T]; salt, ikm, info: openarray[byte]; outputs: var openarray[HKDFResult[len]]) =
var
ctx: BearHKDFContext
br_hkdf_init(
addr ctx, addr sha256Vtable,
if salt.len > 0: unsafeaddr salt[0] else: nil, csize_t(salt.len))
br_hkdf_inject(
addr ctx, if ikm.len > 0: unsafeaddr ikm[0] else: nil, csize_t(ikm.len))
br_hkdf_flip(addr ctx)
for i in 0..outputs.high:
br_hkdf_produce(
addr ctx,
if info.len > 0: unsafeaddr info[0]
else: nil, csize_t(info.len),
addr outputs[i][0], csize_t(outputs[i].len))
proc hkdf*[T: sha256, len: static int](
_: type[T],
salt, ikm, info: openArray[byte],
outputs: var openArray[HkdfResult[len]],
) =
var ctx: HkdfContext
hkdfInit(
ctx,
addr sha256Vtable,
if salt.len > 0:
unsafeAddr salt[0]
else:
nil,
csize_t(salt.len),
)
hkdfInject(
ctx,
if ikm.len > 0:
unsafeAddr ikm[0]
else:
nil,
csize_t(ikm.len),
)
hkdfFlip(ctx)
for i in 0 .. outputs.high:
discard hkdfProduce(
ctx,
if info.len > 0:
unsafeAddr info[0]
else:
nil,
csize_t(info.len),
addr outputs[i][0],
csize_t(outputs[i].len),
)

View File

@@ -1,52 +1,53 @@
## Nim-Libp2p
## Copyright (c) 2018 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-Libp2p
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
## This module implements minimal ASN.1 encoding/decoding primitives.
{.push raises: [Defect].}
{.push raises: [].}
import stew/[endians2, results, ctops]
import stew/[endians2, ctops]
import results
export results
# We use `ncrutils` for constant-time hexadecimal encoding/decoding procedures.
import nimcrypto/utils as ncrutils
import ../utility
type
Asn1Error* {.pure.} = enum
Overflow,
Incomplete,
Indefinite,
Incorrect,
NoSupport,
Overflow
Incomplete
Indefinite
Incorrect
NoSupport
Overrun
Asn1Result*[T] = Result[T, Asn1Error]
Asn1Class* {.pure.} = enum
Universal = 0x00,
Universal = 0x00
Application = 0x01
ContextSpecific = 0x02
Private = 0x03
Asn1Tag* {.pure.} = enum
## Protobuf's field types enum
NoSupport,
Boolean,
Integer,
BitString,
OctetString,
Null,
Oid,
Sequence,
NoSupport
Boolean
Integer
BitString
OctetString
Null
Oid
Sequence
Context
Asn1Buffer* = object of RootObj
## ASN.1's message representation object
Asn1Buffer* = object of RootObj ## ASN.1's message representation object
buffer*: seq[byte]
offset*: int
length*: int
@@ -72,37 +73,23 @@ type
idx*: int
const
Asn1OidSecp256r1* = [
0x2A'u8, 0x86'u8, 0x48'u8, 0xCE'u8, 0x3D'u8, 0x03'u8, 0x01'u8, 0x07'u8
]
Asn1OidSecp256r1* =
[0x2A'u8, 0x86'u8, 0x48'u8, 0xCE'u8, 0x3D'u8, 0x03'u8, 0x01'u8, 0x07'u8]
## Encoded OID for `secp256r1` curve (1.2.840.10045.3.1.7)
Asn1OidSecp384r1* = [
0x2B'u8, 0x81'u8, 0x04'u8, 0x00'u8, 0x22'u8
]
Asn1OidSecp384r1* = [0x2B'u8, 0x81'u8, 0x04'u8, 0x00'u8, 0x22'u8]
## Encoded OID for `secp384r1` curve (1.3.132.0.34)
Asn1OidSecp521r1* = [
0x2B'u8, 0x81'u8, 0x04'u8, 0x00'u8, 0x23'u8
]
Asn1OidSecp521r1* = [0x2B'u8, 0x81'u8, 0x04'u8, 0x00'u8, 0x23'u8]
## Encoded OID for `secp521r1` curve (1.3.132.0.35)
Asn1OidSecp256k1* = [
0x2B'u8, 0x81'u8, 0x04'u8, 0x00'u8, 0x0A'u8
]
Asn1OidSecp256k1* = [0x2B'u8, 0x81'u8, 0x04'u8, 0x00'u8, 0x0A'u8]
## Encoded OID for `secp256k1` curve (1.3.132.0.10)
Asn1OidEcPublicKey* = [
0x2A'u8, 0x86'u8, 0x48'u8, 0xCE'u8, 0x3D'u8, 0x02'u8, 0x01'u8
]
Asn1OidEcPublicKey* = [0x2A'u8, 0x86'u8, 0x48'u8, 0xCE'u8, 0x3D'u8, 0x02'u8, 0x01'u8]
## Encoded OID for Elliptic Curve Public Key (1.2.840.10045.2.1)
Asn1OidRsaEncryption* = [
0x2A'u8, 0x86'u8, 0x48'u8, 0x86'u8, 0xF7'u8, 0x0D'u8, 0x01'u8,
0x01'u8, 0x01'u8
]
Asn1OidRsaEncryption* =
[0x2A'u8, 0x86'u8, 0x48'u8, 0x86'u8, 0xF7'u8, 0x0D'u8, 0x01'u8, 0x01'u8, 0x01'u8]
## Encoded OID for RSA Encryption (1.2.840.113549.1.1.1)
Asn1True* = [0x01'u8, 0x01'u8, 0xFF'u8]
## Encoded boolean ``TRUE``.
Asn1False* = [0x01'u8, 0x01'u8, 0x00'u8]
## Encoded boolean ``FALSE``.
Asn1Null* = [0x05'u8, 0x00'u8]
## Encoded ``NULL`` value.
Asn1True* = [0x01'u8, 0x01'u8, 0xFF'u8] ## Encoded boolean ``TRUE``.
Asn1False* = [0x01'u8, 0x01'u8, 0x00'u8] ## Encoded boolean ``FALSE``.
Asn1Null* = [0x05'u8, 0x00'u8] ## Encoded ``NULL`` value.
template toOpenArray*(ab: Asn1Buffer): untyped =
toOpenArray(ab.buffer, ab.offset, ab.buffer.high)
@@ -116,10 +103,10 @@ template toOpenArray*(af: Asn1Field): untyped =
template isEmpty*(ab: Asn1Buffer): bool =
ab.offset >= len(ab.buffer)
template isEnough*(ab: Asn1Buffer, length: int): bool =
template isEnough*(ab: Asn1Buffer, length: int64): bool =
len(ab.buffer) >= ab.offset + length
proc len*[T: Asn1Buffer|Asn1Composite](abc: T): int {.inline.} =
proc len*[T: Asn1Buffer | Asn1Composite](abc: T): int {.inline.} =
len(abc.buffer) - abc.offset
proc len*(field: Asn1Field): int {.inline.} =
@@ -128,33 +115,24 @@ proc len*(field: Asn1Field): int {.inline.} =
template getPtr*(field: untyped): pointer =
cast[pointer](unsafeAddr field.buffer[field.offset])
proc extend*[T: Asn1Buffer|Asn1Composite](abc: var T, length: int) {.inline.} =
proc extend*[T: Asn1Buffer | Asn1Composite](abc: var T, length: int) {.inline.} =
## Extend buffer or composite's internal buffer by ``length`` octets.
abc.buffer.setLen(len(abc.buffer) + length)
proc code*(tag: Asn1Tag): byte {.inline.} =
## Converts Nim ``tag`` enum to ASN.1 tag code.
case tag:
of Asn1Tag.NoSupport:
0x00'u8
of Asn1Tag.Boolean:
0x01'u8
of Asn1Tag.Integer:
0x02'u8
of Asn1Tag.BitString:
0x03'u8
of Asn1Tag.OctetString:
0x04'u8
of Asn1Tag.Null:
0x05'u8
of Asn1Tag.Oid:
0x06'u8
of Asn1Tag.Sequence:
0x30'u8
of Asn1Tag.Context:
0xA0'u8
case tag
of Asn1Tag.NoSupport: 0x00'u8
of Asn1Tag.Boolean: 0x01'u8
of Asn1Tag.Integer: 0x02'u8
of Asn1Tag.BitString: 0x03'u8
of Asn1Tag.OctetString: 0x04'u8
of Asn1Tag.Null: 0x05'u8
of Asn1Tag.Oid: 0x06'u8
of Asn1Tag.Sequence: 0x30'u8
of Asn1Tag.Context: 0xA0'u8
proc asn1EncodeLength*(dest: var openarray[byte], length: uint64): int =
proc asn1EncodeLength*(dest: var openArray[byte], length: uint64): int =
## Encode ASN.1 DER length part of TLV triple and return number of bytes
## (octets) used.
##
@@ -181,8 +159,7 @@ proc asn1EncodeLength*(dest: var openarray[byte], length: uint64): int =
# then 9, so it is safe to convert it to `int`.
int(res)
proc asn1EncodeInteger*(dest: var openarray[byte],
value: openarray[byte]): int =
proc asn1EncodeInteger*(dest: var openArray[byte], value: openArray[byte]): int =
## Encode big-endian binary representation of integer as ASN.1 DER `INTEGER`
## and return number of bytes (octets) used.
##
@@ -192,17 +169,16 @@ proc asn1EncodeInteger*(dest: var openarray[byte],
var buffer: array[16, byte]
var lenlen = 0
let offset =
block:
var o = 0
for i in 0 ..< len(value):
if value[o] != 0x00:
break
inc(o)
if o < len(value):
o
else:
o - 1
let offset = block:
var o = 0
for i in 0 ..< len(value):
if value[o] != 0x00:
break
inc(o)
if o < len(value):
o
else:
o - 1
let destlen =
if len(value) > 0:
@@ -224,12 +200,10 @@ proc asn1EncodeInteger*(dest: var openarray[byte],
if value[offset] >= 0x80'u8:
dest[1 + lenlen] = 0x00'u8
shift = 2
copyMem(addr dest[shift + lenlen], unsafeAddr value[offset],
len(value) - offset)
copyMem(addr dest[shift + lenlen], unsafeAddr value[offset], len(value) - offset)
destlen
proc asn1EncodeInteger*[T: SomeUnsignedInt](dest: var openarray[byte],
value: T): int =
proc asn1EncodeInteger*[T: SomeUnsignedInt](dest: var openArray[byte], value: T): int =
## Encode Nim's unsigned integer as ASN.1 DER `INTEGER` and return number of
## bytes (octets) used.
##
@@ -238,7 +212,7 @@ proc asn1EncodeInteger*[T: SomeUnsignedInt](dest: var openarray[byte],
## but number of bytes (octets) required will be returned.
dest.asn1EncodeInteger(value.toBytesBE())
proc asn1EncodeBoolean*(dest: var openarray[byte], value: bool): int =
proc asn1EncodeBoolean*(dest: var openArray[byte], value: bool): int =
## Encode Nim's boolean as ASN.1 DER `BOOLEAN` and return number of bytes
## (octets) used.
##
@@ -252,7 +226,7 @@ proc asn1EncodeBoolean*(dest: var openarray[byte], value: bool): int =
dest[2] = if value: 0xFF'u8 else: 0x00'u8
res
proc asn1EncodeNull*(dest: var openarray[byte]): int =
proc asn1EncodeNull*(dest: var openArray[byte]): int =
## Encode ASN.1 DER `NULL` and return number of bytes (octets) used.
##
## If length of ``dest`` is less then number of required bytes to encode
@@ -264,8 +238,7 @@ proc asn1EncodeNull*(dest: var openarray[byte]): int =
dest[1] = 0x00'u8
res
proc asn1EncodeOctetString*(dest: var openarray[byte],
value: openarray[byte]): int =
proc asn1EncodeOctetString*(dest: var openArray[byte], value: openArray[byte]): int =
## Encode array of bytes as ASN.1 DER `OCTET STRING` and return number of
## bytes (octets) used.
##
@@ -282,8 +255,9 @@ proc asn1EncodeOctetString*(dest: var openarray[byte],
copyMem(addr dest[1 + lenlen], unsafeAddr value[0], len(value))
res
proc asn1EncodeBitString*(dest: var openarray[byte],
value: openarray[byte], bits = 0): int =
proc asn1EncodeBitString*(
dest: var openArray[byte], value: openArray[byte], bits = 0
): int =
## Encode array of bytes as ASN.1 DER `BIT STRING` and return number of bytes
## (octets) used.
##
@@ -304,7 +278,7 @@ proc asn1EncodeBitString*(dest: var openarray[byte],
let bytelen = (bitlen + 7) shr 3
# Number of unused bits
let unused = (8 - (bitlen and 7)) and 7
let mask = not((1'u8 shl unused) - 1'u8)
let mask = not ((1'u8 shl unused) - 1'u8)
var lenlen = asn1EncodeLength(buffer, uint64(bytelen + 1))
let res = 1 + lenlen + 1 + len(value)
if len(dest) >= res:
@@ -318,56 +292,7 @@ proc asn1EncodeBitString*(dest: var openarray[byte],
dest[2 + lenlen + bytelen - 1] = lastbyte and mask
res
proc asn1EncodeTag[T: SomeUnsignedInt](dest: var openarray[byte],
value: T): int =
var v = value
if value <= cast[T](0x7F):
if len(dest) >= 1:
dest[0] = cast[byte](value)
1
else:
var s = 0
var res = 0
while v != 0:
v = v shr 7
s += 7
inc(res)
if len(dest) >= res:
var k = 0
while s != 0:
s -= 7
dest[k] = cast[byte](((value shr s) and cast[T](0x7F)) or cast[T](0x80))
inc(k)
dest[k - 1] = dest[k - 1] and 0x7F'u8
res
proc asn1EncodeOid*(dest: var openarray[byte], value: openarray[int]): int =
## Encode array of integers ``value`` as ASN.1 DER `OBJECT IDENTIFIER` and
## return number of bytes (octets) used.
##
## If length of ``dest`` is less then number of required bytes to encode
## ``value``, then result of encoding will not be stored in ``dest``
## but number of bytes (octets) required will be returned.
var buffer: array[16, byte]
var res = 1
var oidlen = 1
for i in 2..<len(value):
oidlen += asn1EncodeTag(buffer, cast[uint64](value[i]))
res += asn1EncodeLength(buffer, uint64(oidlen))
res += oidlen
if len(dest) >= res:
let last = dest.high
var offset = 1
dest[0] = Asn1Tag.Oid.code()
offset += asn1EncodeLength(dest.toOpenArray(offset, last), uint64(oidlen))
dest[offset] = cast[byte](value[0] * 40 + value[1])
offset += 1
for i in 2..<len(value):
offset += asn1EncodeTag(dest.toOpenArray(offset, last),
cast[uint64](value[i]))
res
proc asn1EncodeOid*(dest: var openarray[byte], value: openarray[byte]): int =
proc asn1EncodeOid*(dest: var openArray[byte], value: openArray[byte]): int =
## Encode array of bytes ``value`` as ASN.1 DER `OBJECT IDENTIFIER` and return
## number of bytes (octets) used.
##
@@ -386,8 +311,7 @@ proc asn1EncodeOid*(dest: var openarray[byte], value: openarray[byte]): int =
copyMem(addr dest[1 + lenlen], unsafeAddr value[0], len(value))
res
proc asn1EncodeSequence*(dest: var openarray[byte],
value: openarray[byte]): int =
proc asn1EncodeSequence*(dest: var openArray[byte], value: openArray[byte]): int =
## Encode ``value`` as ASN.1 DER `SEQUENCE` and return number of bytes
## (octets) used.
##
@@ -403,8 +327,7 @@ proc asn1EncodeSequence*(dest: var openarray[byte],
copyMem(addr dest[1 + lenlen], unsafeAddr value[0], len(value))
res
proc asn1EncodeComposite*(dest: var openarray[byte],
value: Asn1Composite): int =
proc asn1EncodeComposite*(dest: var openArray[byte], value: Asn1Composite): int =
## Encode composite value and return number of bytes (octets) used.
##
## If length of ``dest`` is less then number of required bytes to encode
@@ -416,12 +339,12 @@ proc asn1EncodeComposite*(dest: var openarray[byte],
if len(dest) >= res:
dest[0] = value.tag.code()
copyMem(addr dest[1], addr buffer[0], lenlen)
copyMem(addr dest[1 + lenlen], unsafeAddr value.buffer[0],
len(value.buffer))
copyMem(addr dest[1 + lenlen], unsafeAddr value.buffer[0], len(value.buffer))
res
proc asn1EncodeContextTag*(dest: var openarray[byte], value: openarray[byte],
tag: int): int =
proc asn1EncodeContextTag*(
dest: var openArray[byte], value: openArray[byte], tag: int
): int =
## Encode ASN.1 DER `CONTEXT SPECIFIC TAG` ``tag`` for value ``value`` and
## return number of bytes (octets) used.
##
@@ -440,26 +363,29 @@ proc asn1EncodeContextTag*(dest: var openarray[byte], value: openarray[byte],
copyMem(addr dest[1 + lenlen], unsafeAddr value[0], len(value))
res
proc getLength(ab: var Asn1Buffer): Asn1Result[uint64] =
proc getLength(ab: var Asn1Buffer): Asn1Result[int] =
## Decode length part of ASN.1 TLV triplet.
if not ab.isEmpty():
let b = ab.buffer[ab.offset]
if (b and 0x80'u8) == 0x00'u8:
let length = cast[uint64](b)
let length = safeConvert[int](b)
ab.offset += 1
return ok(length)
if b == 0x80'u8:
return err(Asn1Error.Indefinite)
if b == 0xFF'u8:
return err(Asn1Error.Incorrect)
let octets = cast[uint64](b and 0x7F'u8)
if octets > 8'u64:
let octets = safeConvert[int](b and 0x7F'u8)
if octets > 8:
return err(Asn1Error.Overflow)
if ab.isEnough(int(octets)):
var length: uint64 = 0
for i in 0..<int(octets):
length = (length shl 8) or cast[uint64](ab.buffer[ab.offset + i + 1])
ab.offset = ab.offset + int(octets) + 1
if ab.isEnough(octets):
var lengthU: uint64 = 0
for i in 0 ..< octets:
lengthU = (lengthU shl 8) or safeConvert[uint64](ab.buffer[ab.offset + i + 1])
if lengthU > uint64(int64.high):
return err(Asn1Error.Overflow)
let length = int(lengthU)
ab.offset = ab.offset + octets + 1
return ok(length)
else:
return err(Asn1Error.Incomplete)
@@ -471,8 +397,8 @@ proc getTag(ab: var Asn1Buffer, tag: var int): Asn1Result[Asn1Class] =
if not ab.isEmpty():
let
b = ab.buffer[ab.offset]
c = int((b and 0xC0'u8) shr 6)
tag = int(b and 0x3F)
c = safeConvert[int]((b and 0xC0'u8) shr 6)
tag = safeConvert[int](b and 0x3F)
ab.offset += 1
if c >= 0 and c < 4:
ok(cast[Asn1Class](c))
@@ -486,14 +412,14 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] =
var
field: Asn1Field
tag, ttag, offset: int
length, tlength: uint64
length, tlength: int
aclass: Asn1Class
inclass: bool
inclass = false
while true:
offset = ab.offset
aclass = ? ab.getTag(tag)
aclass = ?ab.getTag(tag)
case aclass
of Asn1Class.ContextSpecific:
@@ -502,9 +428,9 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] =
else:
inclass = true
ttag = tag
tlength = ? ab.getLength()
tlength = ?ab.getLength()
of Asn1Class.Universal:
length = ? ab.getLength()
length = ?ab.getLength()
if inclass:
if length >= tlength:
@@ -516,32 +442,35 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] =
if length != 1:
return err(Asn1Error.Incorrect)
if not ab.isEnough(int(length)):
if not ab.isEnough(length):
return err(Asn1Error.Incomplete)
let b = ab.buffer[ab.offset]
if b != 0xFF'u8 and b != 0x00'u8:
return err(Asn1Error.Incorrect)
return err(Asn1Error.Incorrect)
field = Asn1Field(kind: Asn1Tag.Boolean, klass: aclass,
index: ttag, offset: int(ab.offset),
length: 1)
shallowCopy(field.buffer, ab.buffer)
field = Asn1Field(
kind: Asn1Tag.Boolean,
klass: aclass,
index: ttag,
offset: ab.offset,
length: 1,
buffer: ab.buffer,
)
field.vbool = (b == 0xFF'u8)
ab.offset += 1
return ok(field)
of Asn1Tag.Integer.code():
# INTEGER
if length == 0:
return err(Asn1Error.Incorrect)
if not ab.isEnough(int(length)):
return err(Asn1Error.Incomplete)
if not ab.isEnough(length):
return err(Asn1Error.Incomplete)
# Count number of leading zeroes
var zc = 0
while (zc < int(length)) and (ab.buffer[ab.offset + zc] == 0x00'u8):
while (zc < length) and (ab.buffer[ab.offset + zc] == 0x00'u8):
inc(zc)
if zc > 1:
@@ -549,71 +478,87 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] =
if zc == 0:
# Negative or Positive integer
field = Asn1Field(kind: Asn1Tag.Integer, klass: aclass,
index: ttag, offset: int(ab.offset),
length: int(length))
shallowCopy(field.buffer, ab.buffer)
field = Asn1Field(
kind: Asn1Tag.Integer,
klass: aclass,
index: ttag,
offset: ab.offset,
length: length,
buffer: ab.buffer,
)
if (ab.buffer[ab.offset] and 0x80'u8) == 0x80'u8:
# Negative integer
if length <= 8:
# We need this transformation because our field.vint is uint64.
for i in 0 ..< 8:
if i < 8 - int(length):
if i < 8 - length:
field.vint = (field.vint shl 8) or 0xFF'u64
else:
let offset = ab.offset + i - (8 - int(length))
field.vint = (field.vint shl 8) or uint64(ab.buffer[offset])
let offset = ab.offset + i - (8 - length)
field.vint =
(field.vint shl 8) or safeConvert[uint64](ab.buffer[offset])
else:
# Positive integer
if length <= 8:
for i in 0 ..< int(length):
field.vint = (field.vint shl 8) or
uint64(ab.buffer[ab.offset + i])
ab.offset += int(length)
for i in 0 ..< length:
field.vint =
(field.vint shl 8) or safeConvert[uint64](ab.buffer[ab.offset + i])
ab.offset += length
return ok(field)
else:
if length == 1:
# Zero value integer
field = Asn1Field(kind: Asn1Tag.Integer, klass: aclass,
index: ttag, offset: int(ab.offset),
length: int(length), vint: 0'u64)
shallowCopy(field.buffer, ab.buffer)
ab.offset += int(length)
field = Asn1Field(
kind: Asn1Tag.Integer,
klass: aclass,
index: ttag,
offset: ab.offset,
length: length,
vint: 0'u64,
buffer: ab.buffer,
)
ab.offset += length
return ok(field)
else:
# Positive integer with leading zero
field = Asn1Field(kind: Asn1Tag.Integer, klass: aclass,
index: ttag, offset: int(ab.offset) + 1,
length: int(length) - 1)
shallowCopy(field.buffer, ab.buffer)
field = Asn1Field(
kind: Asn1Tag.Integer,
klass: aclass,
index: ttag,
offset: ab.offset + 1,
length: length - 1,
buffer: ab.buffer,
)
if length <= 9:
for i in 1 ..< int(length):
field.vint = (field.vint shl 8) or
uint64(ab.buffer[ab.offset + i])
ab.offset += int(length)
for i in 1 ..< length:
field.vint =
(field.vint shl 8) or safeConvert[uint64](ab.buffer[ab.offset + i])
ab.offset += length
return ok(field)
of Asn1Tag.BitString.code():
# BIT STRING
if length == 0:
# BIT STRING should include `unused` bits field, so length should be
# bigger then 1.
return err(Asn1Error.Incorrect)
elif length == 1:
if ab.buffer[ab.offset] != 0x00'u8:
return err(Asn1Error.Incorrect)
else:
# Zero-length BIT STRING.
field = Asn1Field(kind: Asn1Tag.BitString, klass: aclass,
index: ttag, offset: int(ab.offset + 1),
length: 0, ubits: 0)
shallowCopy(field.buffer, ab.buffer)
ab.offset += int(length)
field = Asn1Field(
kind: Asn1Tag.BitString,
klass: aclass,
index: ttag,
offset: ab.offset + 1,
length: 0,
ubits: 0,
buffer: ab.buffer,
)
ab.offset += length
return ok(field)
else:
if not ab.isEnough(int(length)):
if not ab.isEnough(length):
return err(Asn1Error.Incomplete)
let unused = ab.buffer[ab.offset]
@@ -621,70 +566,84 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] =
# Number of unused bits should not be bigger then `7`.
return err(Asn1Error.Incorrect)
let mask = (1'u8 shl int(unused)) - 1'u8
if (ab.buffer[ab.offset + int(length) - 1] and mask) != 0x00'u8:
let mask = (1'u8 shl safeConvert[int](unused)) - 1'u8
if (ab.buffer[ab.offset + length - 1] and mask) != 0x00'u8:
## All unused bits should be set to `0`.
return err(Asn1Error.Incorrect)
field = Asn1Field(kind: Asn1Tag.BitString, klass: aclass,
index: ttag, offset: int(ab.offset + 1),
length: int(length - 1), ubits: int(unused))
shallowCopy(field.buffer, ab.buffer)
ab.offset += int(length)
field = Asn1Field(
kind: Asn1Tag.BitString,
klass: aclass,
index: ttag,
offset: ab.offset + 1,
length: length - 1,
ubits: safeConvert[int](unused),
buffer: ab.buffer,
)
ab.offset += length
return ok(field)
of Asn1Tag.OctetString.code():
# OCTET STRING
if not ab.isEnough(int(length)):
if not ab.isEnough(length):
return err(Asn1Error.Incomplete)
field = Asn1Field(kind: Asn1Tag.OctetString, klass: aclass,
index: ttag, offset: int(ab.offset),
length: int(length))
shallowCopy(field.buffer, ab.buffer)
ab.offset += int(length)
field = Asn1Field(
kind: Asn1Tag.OctetString,
klass: aclass,
index: ttag,
offset: ab.offset,
length: length,
buffer: ab.buffer,
)
ab.offset += length
return ok(field)
of Asn1Tag.Null.code():
# NULL
if length != 0:
return err(Asn1Error.Incorrect)
field = Asn1Field(kind: Asn1Tag.Null, klass: aclass, index: ttag,
offset: int(ab.offset), length: 0)
shallowCopy(field.buffer, ab.buffer)
ab.offset += int(length)
field = Asn1Field(
kind: Asn1Tag.Null,
klass: aclass,
index: ttag,
offset: ab.offset,
length: 0,
buffer: ab.buffer,
)
ab.offset += length
return ok(field)
of Asn1Tag.Oid.code():
# OID
if not ab.isEnough(int(length)):
if not ab.isEnough(length):
return err(Asn1Error.Incomplete)
field = Asn1Field(kind: Asn1Tag.Oid, klass: aclass,
index: ttag, offset: int(ab.offset),
length: int(length))
shallowCopy(field.buffer, ab.buffer)
ab.offset += int(length)
field = Asn1Field(
kind: Asn1Tag.Oid,
klass: aclass,
index: ttag,
offset: ab.offset,
length: length,
buffer: ab.buffer,
)
ab.offset += length
return ok(field)
of Asn1Tag.Sequence.code():
# SEQUENCE
if not ab.isEnough(int(length)):
if not ab.isEnough(length):
return err(Asn1Error.Incomplete)
field = Asn1Field(kind: Asn1Tag.Sequence, klass: aclass,
index: ttag, offset: int(ab.offset),
length: int(length))
shallowCopy(field.buffer, ab.buffer)
ab.offset += int(length)
field = Asn1Field(
kind: Asn1Tag.Sequence,
klass: aclass,
index: ttag,
offset: ab.offset,
length: length,
buffer: ab.buffer,
)
ab.offset += length
return ok(field)
else:
return err(Asn1Error.NoSupport)
inclass = false
ttag = 0
else:
return err(Asn1Error.NoSupport)
@@ -692,7 +651,7 @@ proc getBuffer*(field: Asn1Field): Asn1Buffer {.inline.} =
## Return ``field`` as Asn1Buffer to enter composite types.
Asn1Buffer(buffer: field.buffer, offset: field.offset, length: field.length)
proc `==`*(field: Asn1Field, data: openarray[byte]): bool =
proc `==`*(field: Asn1Field, data: openArray[byte]): bool =
## Compares field ``field`` data with ``data`` and returns ``true`` if both
## buffers are equal.
let length = len(field.buffer)
@@ -702,15 +661,15 @@ proc `==`*(field: Asn1Field, data: openarray[byte]): bool =
if length > 0:
if field.length == len(data):
CT.isEqual(
field.buffer.toOpenArray(field.offset,
field.offset + field.length - 1),
data.toOpenArray(0, field.length - 1))
field.buffer.toOpenArray(field.offset, field.offset + field.length - 1),
data.toOpenArray(0, field.length - 1),
)
else:
false
else:
false
proc init*(t: typedesc[Asn1Buffer], data: openarray[byte]): Asn1Buffer =
proc init*(t: typedesc[Asn1Buffer], data: openArray[byte]): Asn1Buffer =
## Initialize ``Asn1Buffer`` from array of bytes ``data``.
Asn1Buffer(buffer: @data)
@@ -782,13 +741,14 @@ proc `$`*(field: Asn1Field): string =
res.add(ncrutils.toHex(field.toOpenArray()))
res
proc write*[T: Asn1Buffer|Asn1Composite](abc: var T, tag: Asn1Tag) =
proc write*[T: Asn1Buffer | Asn1Composite](abc: var T, tag: Asn1Tag) =
## Write empty value to buffer or composite with ``tag``.
##
## This procedure must be used to write `NULL`, `0` or empty `BIT STRING`,
## `OCTET STRING` types.
doAssert(tag in {Asn1Tag.Null, Asn1Tag.Integer, Asn1Tag.BitString,
Asn1Tag.OctetString})
doAssert(
tag in {Asn1Tag.Null, Asn1Tag.Integer, Asn1Tag.BitString, Asn1Tag.OctetString}
)
var length: int
if tag == Asn1Tag.Null:
length = asn1EncodeNull(abc.toOpenArray())
@@ -810,22 +770,23 @@ proc write*[T: Asn1Buffer|Asn1Composite](abc: var T, tag: Asn1Tag) =
discard asn1EncodeOctetString(abc.toOpenArray(), tmp.toOpenArray(0, -1))
abc.offset += length
proc write*[T: Asn1Buffer|Asn1Composite](abc: var T, value: uint64) =
proc write*[T: Asn1Buffer | Asn1Composite](abc: var T, value: uint64) =
## Write uint64 ``value`` to buffer or composite as ASN.1 `INTEGER`.
let length = asn1EncodeInteger(abc.toOpenArray(), value)
abc.extend(length)
discard asn1EncodeInteger(abc.toOpenArray(), value)
abc.offset += length
proc write*[T: Asn1Buffer|Asn1Composite](abc: var T, value: bool) =
proc write*[T: Asn1Buffer | Asn1Composite](abc: var T, value: bool) =
## Write bool ``value`` to buffer or composite as ASN.1 `BOOLEAN`.
let length = asn1EncodeBoolean(abc.toOpenArray(), value)
abc.extend(length)
discard asn1EncodeBoolean(abc.toOpenArray(), value)
abc.offset += length
proc write*[T: Asn1Buffer|Asn1Composite](abc: var T, tag: Asn1Tag,
value: openarray[byte], bits = 0) =
proc write*[T: Asn1Buffer | Asn1Composite](
abc: var T, tag: Asn1Tag, value: openArray[byte], bits = 0
) =
## Write array ``value`` using ``tag``.
##
## This procedure is used to write ASN.1 `INTEGER`, `OCTET STRING`,
@@ -833,8 +794,9 @@ proc write*[T: Asn1Buffer|Asn1Composite](abc: var T, tag: Asn1Tag,
##
## For `BIT STRING` you can use ``bits`` argument to specify number of used
## bits.
doAssert(tag in {Asn1Tag.Integer, Asn1Tag.OctetString, Asn1Tag.BitString,
Asn1Tag.Oid})
doAssert(
tag in {Asn1Tag.Integer, Asn1Tag.OctetString, Asn1Tag.BitString, Asn1Tag.Oid}
)
var length: int
if tag == Asn1Tag.Integer:
length = asn1EncodeInteger(abc.toOpenArray(), value)
@@ -854,7 +816,7 @@ proc write*[T: Asn1Buffer|Asn1Composite](abc: var T, tag: Asn1Tag,
discard asn1EncodeOid(abc.toOpenArray(), value)
abc.offset += length
proc write*[T: Asn1Buffer|Asn1Composite](abc: var T, value: Asn1Composite) =
proc write*[T: Asn1Buffer | Asn1Composite](abc: var T, value: Asn1Composite) =
doAssert(len(value) > 0, "Composite value not finished")
var length: int
if value.tag == Asn1Tag.Sequence:
@@ -871,6 +833,6 @@ proc write*[T: Asn1Buffer|Asn1Composite](abc: var T, value: Asn1Composite) =
discard asn1EncodeContextTag(abc.toOpenArray(), value.buffer, value.idx)
abc.offset += length
proc finish*[T: Asn1Buffer|Asn1Composite](abc: var T) {.inline.} =
proc finish*[T: Asn1Buffer | Asn1Composite](abc: var T) {.inline.} =
## Finishes buffer or composite and prepares it for writing.
abc.offset = 0

View File

@@ -1,11 +1,11 @@
## Nim-Libp2p
## Copyright (c) 2018 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-Libp2p
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
## This module implements constant-time RSA PKCS#1.5 DSA.
##
@@ -13,10 +13,12 @@
## BearSSL library <https://bearssl.org/>
## Copyright(C) 2018 Thomas Pornin <pornin@bolet.org>.
{.push raises: Defect.}
import bearssl
{.push raises: [].}
import bearssl/[rsa, rand, hash]
import minasn1
import stew/[results, ctops]
import results
import stew/ctops
# We use `ncrutils` for constant-time hexadecimal encoding/decoding procedures.
import nimcrypto/utils as ncrutils
@@ -29,45 +31,30 @@ const
MinKeySize* = 2048
## Minimal allowed RSA key size in bits.
## https://github.com/libp2p/go-libp2p-core/blob/master/crypto/rsa_common.go#L13
DefaultKeySize* = 3072
## Default RSA key size in bits.
DefaultKeySize* = 3072 ## Default RSA key size in bits.
RsaOidSha1* = [
0x05'u8, 0x2B'u8, 0x0E'u8, 0x03'u8, 0x02'u8, 0x1A'u8
]
RsaOidSha1* = [byte 0x05, 0x2B, 0x0E, 0x03, 0x02, 0x1A]
## RSA PKCS#1.5 SHA-1 hash object identifier.
RsaOidSha224* = [
0x09'u8, 0x60'u8, 0x86'u8, 0x48'u8, 0x01'u8, 0x65'u8, 0x03'u8, 0x04'u8,
0x02'u8, 0x04'u8
]
RsaOidSha224* = [byte 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x04]
## RSA PKCS#1.5 SHA-224 hash object identifier.
RsaOidSha256* = [
0x09'u8, 0x60'u8, 0x86'u8, 0x48'u8, 0x01'u8, 0x65'u8, 0x03'u8, 0x04'u8,
0x02'u8, 0x01'u8
]
RsaOidSha256* = [byte 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x01]
## RSA PKCS#1.5 SHA-256 hash object identifier.
RsaOidSha384* = [
0x09'u8, 0x60'u8, 0x86'u8, 0x48'u8, 0x01'u8, 0x65'u8, 0x03'u8, 0x04'u8,
0x02'u8, 0x02'u8
]
RsaOidSha384* = [byte 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x02]
## RSA PKCS#1.5 SHA-384 hash object identifier.
RsaOidSha512* = [
0x09'u8, 0x60'u8, 0x86'u8, 0x48'u8, 0x01'u8, 0x65'u8, 0x03'u8, 0x04'u8,
0x02'u8, 0x03'u8
]
RsaOidSha512* = [byte 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x03]
## RSA PKCS#1.5 SHA-512 hash object identifier.
type
RsaPrivateKey* = ref object
buffer*: seq[byte]
seck*: BrRsaPrivateKey
pubk*: BrRsaPublicKey
pexp*: ptr cuchar
pexplen*: int
seck*: rsa.RsaPrivateKey
pubk*: rsa.RsaPublicKey
pexp*: ptr byte
pexplen*: uint
RsaPublicKey* = ref object
buffer*: seq[byte]
key*: BrRsaPublicKey
key*: rsa.RsaPublicKey
RsaKeyPair* = RsaPrivateKey
@@ -78,9 +65,9 @@ type
RsaKP* = RsaPrivateKey | RsaKeyPair
RsaError* = enum
RsaGenError,
RsaKeyIncorrectError,
RsaSignatureError,
RsaGenError
RsaKeyIncorrectError
RsaSignatureError
RsaLowSecurityError
RsaResult*[T] = Result[T, RsaError]
@@ -99,8 +86,8 @@ template getFinish(bs, os, ls: untyped): untyped =
var eo = -1
if p >= s:
let so = cast[int](p - s)
if so + ls <= len(bs):
eo = so + ls - 1
if so + int(ls) <= len(bs):
eo = so + int(ls) - 1
eo
template getArray*(bs, os, ls: untyped): untyped =
@@ -108,15 +95,18 @@ template getArray*(bs, os, ls: untyped): untyped =
template trimZeroes(b: seq[byte], pt, ptlen: untyped) =
var length = ptlen
for i in 0..<length:
if pt[] != cast[cuchar](0x00'u8):
for i in 0 ..< length:
if pt[] != byte(0x00):
break
pt = cast[ptr cuchar](cast[uint](pt) + 1)
pt = cast[ptr byte](cast[uint](pt) + 1)
ptlen -= 1
proc random*[T: RsaKP](t: typedesc[T], rng: var BrHmacDrbgContext,
bits = DefaultKeySize,
pubexp = DefaultPublicExponent): RsaResult[T] =
proc random*[T: RsaKP](
t: typedesc[T],
rng: var HmacDrbgContext,
bits = DefaultKeySize,
pubexp = DefaultPublicExponent,
): RsaResult[T] =
## Generate new random RSA private key using BearSSL's HMAC-SHA256-DRBG
## algorithm.
##
@@ -129,28 +119,33 @@ proc random*[T: RsaKP](t: typedesc[T], rng: var BrHmacDrbgContext,
let
sko = 0
pko = brRsaPrivateKeyBufferSize(bits)
eko = pko + brRsaPublicKeyBufferSize(bits)
pko = rsaKbufPrivSize(bits)
eko = pko + rsaKbufPubSize(bits)
length = eko + ((bits + 7) shr 3)
let res = new T
res.buffer = newSeq[byte](length)
var keygen = brRsaKeygenGetDefault()
var keygen = rsaKeygenGetDefault()
if keygen(addr rng.vtable,
addr res.seck, addr res.buffer[sko],
addr res.pubk, addr res.buffer[pko],
cuint(bits), pubexp) == 0:
if keygen(
addr rng.vtable,
addr res.seck,
addr res.buffer[sko],
addr res.pubk,
addr res.buffer[pko],
cuint(bits),
pubexp,
) == 0:
return err(RsaGenError)
let
compute = brRsaComputePrivexpGetDefault()
compute = rsaComputePrivexpGetDefault()
computed = compute(addr res.buffer[eko], addr res.seck, pubexp)
if computed == 0:
return err(RsaGenError)
res.pexp = cast[ptr cuchar](addr res.buffer[eko])
res.pexp = addr res.buffer[eko]
res.pexplen = computed
trimZeroes(res.buffer, res.seck.p, res.seck.plen)
@@ -169,12 +164,13 @@ proc copy*[T: RsaPKI](key: T): T =
doAssert(not isNil(key))
when T is RsaPrivateKey:
if len(key.buffer) > 0:
let length = key.seck.plen + key.seck.qlen + key.seck.dplen +
key.seck.dqlen + key.seck.iqlen + key.pubk.nlen +
key.pubk.elen + key.pexplen
let length =
key.seck.plen.uint + key.seck.qlen.uint + key.seck.dplen.uint +
key.seck.dqlen.uint + key.seck.iqlen.uint + key.pubk.nlen.uint +
key.pubk.elen.uint + key.pexplen.uint
result = new RsaPrivateKey
result.buffer = newSeq[byte](length)
let po = 0
let po: uint = 0
let qo = po + key.seck.plen
let dpo = qo + key.seck.qlen
let dqo = dpo + key.seck.dplen
@@ -190,14 +186,14 @@ proc copy*[T: RsaPKI](key: T): T =
copyMem(addr result.buffer[no], key.pubk.n, key.pubk.nlen)
copyMem(addr result.buffer[eo], key.pubk.e, key.pubk.elen)
copyMem(addr result.buffer[peo], key.pexp, key.pexplen)
result.seck.p = cast[ptr cuchar](addr result.buffer[po])
result.seck.q = cast[ptr cuchar](addr result.buffer[qo])
result.seck.dp = cast[ptr cuchar](addr result.buffer[dpo])
result.seck.dq = cast[ptr cuchar](addr result.buffer[dqo])
result.seck.iq = cast[ptr cuchar](addr result.buffer[iqo])
result.pubk.n = cast[ptr cuchar](addr result.buffer[no])
result.pubk.e = cast[ptr cuchar](addr result.buffer[eo])
result.pexp = cast[ptr cuchar](addr result.buffer[peo])
result.seck.p = addr result.buffer[po]
result.seck.q = addr result.buffer[qo]
result.seck.dp = addr result.buffer[dpo]
result.seck.dq = addr result.buffer[dqo]
result.seck.iq = addr result.buffer[iqo]
result.pubk.n = addr result.buffer[no]
result.pubk.e = addr result.buffer[eo]
result.pexp = addr result.buffer[peo]
result.seck.plen = key.seck.plen
result.seck.qlen = key.seck.qlen
result.seck.dplen = key.seck.dplen
@@ -216,8 +212,8 @@ proc copy*[T: RsaPKI](key: T): T =
let eo = no + key.key.nlen
copyMem(addr result.buffer[no], key.key.n, key.key.nlen)
copyMem(addr result.buffer[eo], key.key.e, key.key.elen)
result.key.n = cast[ptr cuchar](addr result.buffer[no])
result.key.e = cast[ptr cuchar](addr result.buffer[eo])
result.key.n = cast[ptr char](addr result.buffer[no])
result.key.e = cast[ptr char](addr result.buffer[eo])
result.key.nlen = key.key.nlen
result.key.elen = key.key.elen
elif T is RsaSignature:
@@ -225,17 +221,16 @@ proc copy*[T: RsaPKI](key: T): T =
result = new RsaSignature
result.buffer = key.buffer
proc getKey*(key: RsaPrivateKey): RsaPublicKey =
proc getPublicKey*(key: RsaPrivateKey): RsaPublicKey =
## Get RSA public key from RSA private key.
doAssert(not isNil(key))
let length = key.pubk.nlen + key.pubk.elen
result = new RsaPublicKey
result.buffer = newSeq[byte](length)
result.key.n = cast[ptr cuchar](addr result.buffer[0])
result.key.e = cast[ptr cuchar](addr result.buffer[key.pubk.nlen])
result.key.n = addr result.buffer[0]
result.key.e = addr result.buffer[key.pubk.nlen]
copyMem(addr result.buffer[0], cast[pointer](key.pubk.n), key.pubk.nlen)
copyMem(addr result.buffer[key.pubk.nlen], cast[pointer](key.pubk.e),
key.pubk.elen)
copyMem(addr result.buffer[key.pubk.nlen], cast[pointer](key.pubk.e), key.pubk.elen)
result.key.nlen = key.pubk.nlen
result.key.elen = key.pubk.elen
@@ -245,9 +240,9 @@ proc seckey*(pair: RsaKeyPair): RsaPrivateKey {.inline.} =
proc pubkey*(pair: RsaKeyPair): RsaPublicKey {.inline.} =
## Get RSA public key from pair ``pair``.
result = cast[RsaPrivateKey](pair).getKey()
result = cast[RsaPrivateKey](pair).getPublicKey()
proc clear*[T: RsaPKI|RsaKeyPair](pki: var T) =
proc clear*[T: RsaPKI | RsaKeyPair](pki: var T) =
## Wipe and clear EC private key, public key or scalar object.
doAssert(not isNil(pki))
when T is RsaPrivateKey:
@@ -279,7 +274,7 @@ proc clear*[T: RsaPKI|RsaKeyPair](pki: var T) =
burnMem(pki.buffer)
pki.buffer.setLen(0)
proc toBytes*(key: RsaPrivateKey, data: var openarray[byte]): RsaResult[int] =
proc toBytes*(key: RsaPrivateKey, data: var openArray[byte]): RsaResult[int] =
## Serialize RSA private key ``key`` to ASN.1 DER binary form and store it
## to ``data``.
##
@@ -291,21 +286,14 @@ proc toBytes*(key: RsaPrivateKey, data: var openarray[byte]): RsaResult[int] =
var b = Asn1Buffer.init()
var p = Asn1Composite.init(Asn1Tag.Sequence)
p.write(0'u64)
p.write(Asn1Tag.Integer, getArray(key.buffer, key.pubk.n,
key.pubk.nlen))
p.write(Asn1Tag.Integer, getArray(key.buffer, key.pubk.e,
key.pubk.elen))
p.write(Asn1Tag.Integer, getArray(key.buffer, key.pubk.n, key.pubk.nlen))
p.write(Asn1Tag.Integer, getArray(key.buffer, key.pubk.e, key.pubk.elen))
p.write(Asn1Tag.Integer, getArray(key.buffer, key.pexp, key.pexplen))
p.write(Asn1Tag.Integer, getArray(key.buffer, key.seck.p,
key.seck.plen))
p.write(Asn1Tag.Integer, getArray(key.buffer, key.seck.q,
key.seck.qlen))
p.write(Asn1Tag.Integer, getArray(key.buffer, key.seck.dp,
key.seck.dplen))
p.write(Asn1Tag.Integer, getArray(key.buffer, key.seck.dq,
key.seck.dqlen))
p.write(Asn1Tag.Integer, getArray(key.buffer, key.seck.iq,
key.seck.iqlen))
p.write(Asn1Tag.Integer, getArray(key.buffer, key.seck.p, key.seck.plen))
p.write(Asn1Tag.Integer, getArray(key.buffer, key.seck.q, key.seck.qlen))
p.write(Asn1Tag.Integer, getArray(key.buffer, key.seck.dp, key.seck.dplen))
p.write(Asn1Tag.Integer, getArray(key.buffer, key.seck.dq, key.seck.dqlen))
p.write(Asn1Tag.Integer, getArray(key.buffer, key.seck.iq, key.seck.iqlen))
p.finish()
b.write(p)
b.finish()
@@ -316,7 +304,7 @@ proc toBytes*(key: RsaPrivateKey, data: var openarray[byte]): RsaResult[int] =
else:
err(RsaKeyIncorrectError)
proc toBytes*(key: RsaPublicKey, data: var openarray[byte]): RsaResult[int] =
proc toBytes*(key: RsaPublicKey, data: var openArray[byte]): RsaResult[int] =
## Serialize RSA public key ``key`` to ASN.1 DER binary form and store it
## to ``data``.
##
@@ -350,7 +338,7 @@ proc toBytes*(key: RsaPublicKey, data: var openarray[byte]): RsaResult[int] =
else:
err(RsaKeyIncorrectError)
proc toBytes*(sig: RsaSignature, data: var openarray[byte]): RSaResult[int] =
proc toBytes*(sig: RsaSignature, data: var openArray[byte]): RsaResult[int] =
## Serialize RSA signature ``sig`` to raw binary form and store it
## to ``data``.
##
@@ -370,7 +358,7 @@ proc getBytes*(key: RsaPrivateKey): RsaResult[seq[byte]] =
if isNil(key):
return err(RsaKeyIncorrectError)
var res = newSeq[byte](4096)
let length = ? key.toBytes(res)
let length = ?key.toBytes(res)
if length > 0:
res.setLen(length)
ok(res)
@@ -383,7 +371,7 @@ proc getBytes*(key: RsaPublicKey): RsaResult[seq[byte]] =
if isNil(key):
return err(RsaKeyIncorrectError)
var res = newSeq[byte](4096)
let length = ? key.toBytes(res)
let length = ?key.toBytes(res)
if length > 0:
res.setLen(length)
ok(res)
@@ -395,32 +383,31 @@ proc getBytes*(sig: RsaSignature): RsaResult[seq[byte]] =
if isNil(sig):
return err(RsaSignatureError)
var res = newSeq[byte](4096)
let length = ? sig.toBytes(res)
let length = ?sig.toBytes(res)
if length > 0:
res.setLen(length)
ok(res)
else:
err(RsaSignatureError)
proc init*(key: var RsaPrivateKey, data: openarray[byte]): Result[void, Asn1Error] =
proc init*(key: var RsaPrivateKey, data: openArray[byte]): Result[void, Asn1Error] =
## Initialize RSA private key ``key`` from ASN.1 DER binary representation
## ``data``.
##
## Procedure returns ``Asn1Status``.
var
field, rawn, rawpube, rawprie, rawp, rawq, rawdp, rawdq, rawiq: Asn1Field
var field, rawn, rawpube, rawprie, rawp, rawq, rawdp, rawdq, rawiq: Asn1Field
# Asn1Field is not trivial so avoid too much Result
var ab = Asn1Buffer.init(data)
field = ? ab.read()
field = ?ab.read()
if field.kind != Asn1Tag.Sequence:
return err(Asn1Error.Incorrect)
var ib = field.getBuffer()
field = ? ib.read()
field = ?ib.read()
if field.kind != Asn1Tag.Integer:
return err(Asn1Error.Incorrect)
@@ -428,72 +415,72 @@ proc init*(key: var RsaPrivateKey, data: openarray[byte]): Result[void, Asn1Erro
if field.vint != 0'u64:
return err(Asn1Error.Incorrect)
rawn = ? ib.read()
rawn = ?ib.read()
if rawn.kind != Asn1Tag.Integer:
return err(Asn1Error.Incorrect)
rawpube = ? ib.read()
rawpube = ?ib.read()
if rawpube.kind != Asn1Tag.Integer:
return err(Asn1Error.Incorrect)
rawprie = ? ib.read()
rawprie = ?ib.read()
if rawprie.kind != Asn1Tag.Integer:
return err(Asn1Error.Incorrect)
rawp = ? ib.read()
rawp = ?ib.read()
if rawp.kind != Asn1Tag.Integer:
return err(Asn1Error.Incorrect)
rawq = ? ib.read()
rawq = ?ib.read()
if rawq.kind != Asn1Tag.Integer:
return err(Asn1Error.Incorrect)
rawdp = ? ib.read()
rawdp = ?ib.read()
if rawdp.kind != Asn1Tag.Integer:
return err(Asn1Error.Incorrect)
rawdq = ? ib.read()
rawdq = ?ib.read()
if rawdq.kind != Asn1Tag.Integer:
return err(Asn1Error.Incorrect)
rawiq = ? ib.read()
rawiq = ?ib.read()
if rawiq.kind != Asn1Tag.Integer:
return err(Asn1Error.Incorrect)
if len(rawn) >= (MinKeySize shr 3) and len(rawp) > 0 and len(rawq) > 0 and
len(rawdp) > 0 and len(rawdq) > 0 and len(rawiq) > 0:
len(rawdp) > 0 and len(rawdq) > 0 and len(rawiq) > 0:
key = new RsaPrivateKey
key.buffer = @data
key.pubk.n = cast[ptr cuchar](addr key.buffer[rawn.offset])
key.pubk.e = cast[ptr cuchar](addr key.buffer[rawpube.offset])
key.seck.p = cast[ptr cuchar](addr key.buffer[rawp.offset])
key.seck.q = cast[ptr cuchar](addr key.buffer[rawq.offset])
key.seck.dp = cast[ptr cuchar](addr key.buffer[rawdp.offset])
key.seck.dq = cast[ptr cuchar](addr key.buffer[rawdq.offset])
key.seck.iq = cast[ptr cuchar](addr key.buffer[rawiq.offset])
key.pexp = cast[ptr cuchar](addr key.buffer[rawprie.offset])
key.pubk.nlen = len(rawn)
key.pubk.elen = len(rawpube)
key.seck.plen = len(rawp)
key.seck.qlen = len(rawq)
key.seck.dplen = len(rawdp)
key.seck.dqlen = len(rawdq)
key.seck.iqlen = len(rawiq)
key.pexplen = len(rawprie)
key.pubk.n = addr key.buffer[rawn.offset]
key.pubk.e = addr key.buffer[rawpube.offset]
key.seck.p = addr key.buffer[rawp.offset]
key.seck.q = addr key.buffer[rawq.offset]
key.seck.dp = addr key.buffer[rawdp.offset]
key.seck.dq = addr key.buffer[rawdq.offset]
key.seck.iq = addr key.buffer[rawiq.offset]
key.pexp = addr key.buffer[rawprie.offset]
key.pubk.nlen = uint(len(rawn))
key.pubk.elen = uint(len(rawpube))
key.seck.plen = uint(len(rawp))
key.seck.qlen = uint(len(rawq))
key.seck.dplen = uint(len(rawdp))
key.seck.dqlen = uint(len(rawdq))
key.seck.iqlen = uint(len(rawiq))
key.pexplen = uint(len(rawprie))
key.seck.nBitlen = cast[uint32](len(rawn) shl 3)
ok()
else:
err(Asn1Error.Incorrect)
proc init*(key: var RsaPublicKey, data: openarray[byte]): Result[void, Asn1Error] =
proc init*(key: var RsaPublicKey, data: openArray[byte]): Result[void, Asn1Error] =
## Initialize RSA public key ``key`` from ASN.1 DER binary representation
## ``data``.
##
@@ -501,52 +488,52 @@ proc init*(key: var RsaPublicKey, data: openarray[byte]): Result[void, Asn1Error
var field, rawn, rawe: Asn1Field
var ab = Asn1Buffer.init(data)
field = ? ab.read()
field = ?ab.read()
if field.kind != Asn1Tag.Sequence:
return err(Asn1Error.Incorrect)
var ib = field.getBuffer()
field = ? ib.read()
field = ?ib.read()
if field.kind != Asn1Tag.Sequence:
return err(Asn1Error.Incorrect)
var ob = field.getBuffer()
field = ? ob.read()
field = ?ob.read()
if field.kind != Asn1Tag.Oid:
return err(Asn1Error.Incorrect)
elif field != Asn1OidRsaEncryption:
return err(Asn1Error.Incorrect)
field = ? ob.read()
field = ?ob.read()
if field.kind != Asn1Tag.Null:
return err(Asn1Error.Incorrect)
field = ? ib.read()
field = ?ib.read()
if field.kind != Asn1Tag.BitString:
return err(Asn1Error.Incorrect)
var vb = field.getBuffer()
field = ? vb.read()
field = ?vb.read()
if field.kind != Asn1Tag.Sequence:
return err(Asn1Error.Incorrect)
var sb = field.getBuffer()
rawn = ? sb.read()
rawn = ?sb.read()
if rawn.kind != Asn1Tag.Integer:
return err(Asn1Error.Incorrect)
rawe = ? sb.read()
rawe = ?sb.read()
if rawe.kind != Asn1Tag.Integer:
return err(Asn1Error.Incorrect)
@@ -554,15 +541,15 @@ proc init*(key: var RsaPublicKey, data: openarray[byte]): Result[void, Asn1Error
if len(rawn) >= (MinKeySize shr 3) and len(rawe) > 0:
key = new RsaPublicKey
key.buffer = @data
key.key.n = cast[ptr cuchar](addr key.buffer[rawn.offset])
key.key.e = cast[ptr cuchar](addr key.buffer[rawe.offset])
key.key.nlen = len(rawn)
key.key.elen = len(rawe)
key.key.n = addr key.buffer[rawn.offset]
key.key.e = addr key.buffer[rawe.offset]
key.key.nlen = uint(len(rawn))
key.key.elen = uint(len(rawe))
ok()
else:
err(Asn1Error.Incorrect)
proc init*(sig: var RsaSignature, data: openarray[byte]): Result[void, Asn1Error] =
proc init*(sig: var RsaSignature, data: openArray[byte]): Result[void, Asn1Error] =
## Initialize RSA signature ``sig`` from ASN.1 DER binary representation
## ``data``.
##
@@ -574,16 +561,16 @@ proc init*(sig: var RsaSignature, data: openarray[byte]): Result[void, Asn1Error
else:
err(Asn1Error.Incorrect)
proc init*[T: RsaPKI](sospk: var T,
data: string): Result[void, Asn1Error] {.inline.} =
proc init*[T: RsaPKI](sospk: var T, data: string): Result[void, Asn1Error] {.inline.} =
## Initialize EC `private key`, `public key` or `scalar` ``sospk`` from
## hexadecimal string representation ``data``.
##
## Procedure returns ``Result[void, Asn1Status]``.
sospk.init(ncrutils.fromHex(data))
proc init*(t: typedesc[RsaPrivateKey],
data: openarray[byte]): RsaResult[RsaPrivateKey] =
proc init*(
t: typedesc[RsaPrivateKey], data: openArray[byte]
): RsaResult[RsaPrivateKey] =
## Initialize RSA private key from ASN.1 DER binary representation ``data``
## and return constructed object.
var res: RsaPrivateKey
@@ -592,8 +579,7 @@ proc init*(t: typedesc[RsaPrivateKey],
else:
ok(res)
proc init*(t: typedesc[RsaPublicKey],
data: openarray[byte]): RsaResult[RsaPublicKey] =
proc init*(t: typedesc[RsaPublicKey], data: openArray[byte]): RsaResult[RsaPublicKey] =
## Initialize RSA public key from ASN.1 DER binary representation ``data``
## and return constructed object.
var res: RsaPublicKey
@@ -602,8 +588,7 @@ proc init*(t: typedesc[RsaPublicKey],
else:
ok(res)
proc init*(t: typedesc[RsaSignature],
data: openarray[byte]): RsaResult[RsaSignature] =
proc init*(t: typedesc[RsaSignature], data: openArray[byte]): RsaResult[RsaSignature] =
## Initialize RSA signature from raw binary representation ``data`` and
## return constructed object.
var res: RsaSignature
@@ -630,14 +615,11 @@ proc `$`*(key: RsaPrivateKey): string =
result.add("\nq = ")
result.add(ncrutils.toHex(getArray(key.buffer, key.seck.q, key.seck.qlen)))
result.add("\ndp = ")
result.add(ncrutils.toHex(getArray(key.buffer, key.seck.dp,
key.seck.dplen)))
result.add(ncrutils.toHex(getArray(key.buffer, key.seck.dp, key.seck.dplen)))
result.add("\ndq = ")
result.add(ncrutils.toHex(getArray(key.buffer, key.seck.dq,
key.seck.dqlen)))
result.add(ncrutils.toHex(getArray(key.buffer, key.seck.dq, key.seck.dqlen)))
result.add("\niq = ")
result.add(ncrutils.toHex(getArray(key.buffer, key.seck.iq,
key.seck.iqlen)))
result.add(ncrutils.toHex(getArray(key.buffer, key.seck.iq, key.seck.iqlen)))
result.add("\npre = ")
result.add(ncrutils.toHex(getArray(key.buffer, key.pexp, key.pexplen)))
result.add("\nm = ")
@@ -682,23 +664,38 @@ proc `==`*(a, b: RsaPrivateKey): bool =
false
else:
if a.seck.nBitlen == b.seck.nBitlen:
if cast[int](a.seck.nBitlen) > 0:
let r1 = CT.isEqual(getArray(a.buffer, a.seck.p, a.seck.plen),
getArray(b.buffer, b.seck.p, b.seck.plen))
let r2 = CT.isEqual(getArray(a.buffer, a.seck.q, a.seck.qlen),
getArray(b.buffer, b.seck.q, b.seck.qlen))
let r3 = CT.isEqual(getArray(a.buffer, a.seck.dp, a.seck.dplen),
getArray(b.buffer, b.seck.dp, b.seck.dplen))
let r4 = CT.isEqual(getArray(a.buffer, a.seck.dq, a.seck.dqlen),
getArray(b.buffer, b.seck.dq, b.seck.dqlen))
let r5 = CT.isEqual(getArray(a.buffer, a.seck.iq, a.seck.iqlen),
getArray(b.buffer, b.seck.iq, b.seck.iqlen))
let r6 = CT.isEqual(getArray(a.buffer, a.pexp, a.pexplen),
getArray(b.buffer, b.pexp, b.pexplen))
let r7 = CT.isEqual(getArray(a.buffer, a.pubk.n, a.pubk.nlen),
getArray(b.buffer, b.pubk.n, b.pubk.nlen))
let r8 = CT.isEqual(getArray(a.buffer, a.pubk.e, a.pubk.elen),
getArray(b.buffer, b.pubk.e, b.pubk.elen))
if a.seck.nBitlen > 0'u:
let r1 = CT.isEqual(
getArray(a.buffer, a.seck.p, a.seck.plen),
getArray(b.buffer, b.seck.p, b.seck.plen),
)
let r2 = CT.isEqual(
getArray(a.buffer, a.seck.q, a.seck.qlen),
getArray(b.buffer, b.seck.q, b.seck.qlen),
)
let r3 = CT.isEqual(
getArray(a.buffer, a.seck.dp, a.seck.dplen),
getArray(b.buffer, b.seck.dp, b.seck.dplen),
)
let r4 = CT.isEqual(
getArray(a.buffer, a.seck.dq, a.seck.dqlen),
getArray(b.buffer, b.seck.dq, b.seck.dqlen),
)
let r5 = CT.isEqual(
getArray(a.buffer, a.seck.iq, a.seck.iqlen),
getArray(b.buffer, b.seck.iq, b.seck.iqlen),
)
let r6 = CT.isEqual(
getArray(a.buffer, a.pexp, a.pexplen), getArray(b.buffer, b.pexp, b.pexplen)
)
let r7 = CT.isEqual(
getArray(a.buffer, a.pubk.n, a.pubk.nlen),
getArray(b.buffer, b.pubk.n, b.pubk.nlen),
)
let r8 = CT.isEqual(
getArray(a.buffer, a.pubk.e, a.pubk.elen),
getArray(b.buffer, b.pubk.e, b.pubk.elen),
)
r1 and r2 and r3 and r4 and r5 and r6 and r7 and r8
else:
true
@@ -736,42 +733,45 @@ proc `==`*(a, b: RsaPublicKey): bool =
elif isNil(b) and (not isNil(a)):
false
else:
let r1 = CT.isEqual(getArray(a.buffer, a.key.n, a.key.nlen),
getArray(b.buffer, b.key.n, b.key.nlen))
let r2 = CT.isEqual(getArray(a.buffer, a.key.e, a.key.elen),
getArray(b.buffer, b.key.e, b.key.elen))
let r1 = CT.isEqual(
getArray(a.buffer, a.key.n, a.key.nlen), getArray(b.buffer, b.key.n, b.key.nlen)
)
let r2 = CT.isEqual(
getArray(a.buffer, a.key.e, a.key.elen), getArray(b.buffer, b.key.e, b.key.elen)
)
(r1 and r2)
proc sign*[T: byte|char](key: RsaPrivateKey,
message: openarray[T]): RsaResult[RsaSignature] {.gcsafe.} =
proc sign*[T: byte | char](
key: RsaPrivateKey, message: openArray[T]
): RsaResult[RsaSignature] {.gcsafe.} =
## Get RSA PKCS1.5 signature of data ``message`` using SHA256 and private
## key ``key``.
if isNil(key):
return err(RsaKeyIncorrectError)
var hc: BrHashCompatContext
var hc: HashCompatContext
var hash: array[32, byte]
let impl = BrRsaPkcs1SignGetDefault()
let impl = rsaPkcs1SignGetDefault()
var res = new RsaSignature
res.buffer = newSeq[byte]((key.seck.nBitlen + 7) shr 3)
var kv = addr sha256Vtable
kv.init(addr hc.vtable)
if len(message) > 0:
kv.update(addr hc.vtable, unsafeAddr message[0], len(message))
kv.update(addr hc.vtable, unsafeAddr message[0], uint(len(message)))
else:
kv.update(addr hc.vtable, nil, 0)
kv.output(addr hc.vtable, addr hash[0])
kv.out(addr hc.vtable, addr hash[0])
var oid = RsaOidSha256
let implRes = impl(cast[ptr cuchar](addr oid[0]),
cast[ptr cuchar](addr hash[0]), len(hash),
addr key.seck, cast[ptr cuchar](addr res.buffer[0]))
let implRes =
impl(addr oid[0], addr hash[0], uint(len(hash)), addr key.seck, addr res.buffer[0])
if implRes == 0:
err(RsaSignatureError)
else:
ok(res)
proc verify*[T: byte|char](sig: RsaSignature, message: openarray[T],
pubkey: RsaPublicKey): bool {.inline.} =
proc verify*[T: byte | char](
sig: RsaSignature, message: openArray[T], pubkey: RsaPublicKey
): bool {.inline.} =
## Verify RSA signature ``sig`` using public key ``pubkey`` and data
## ``message``.
##
@@ -779,20 +779,25 @@ proc verify*[T: byte|char](sig: RsaSignature, message: openarray[T],
## verification failed.
doAssert((not isNil(sig)) and (not isNil(pubkey)))
if len(sig.buffer) > 0:
var hc: BrHashCompatContext
var hc: HashCompatContext
var hash: array[32, byte]
var check: array[32, byte]
var impl = BrRsaPkcs1VrfyGetDefault()
var impl = rsaPkcs1VrfyGetDefault()
var kv = addr sha256Vtable
kv.init(addr hc.vtable)
if len(message) > 0:
kv.update(addr hc.vtable, unsafeAddr message[0], len(message))
kv.update(addr hc.vtable, unsafeAddr message[0], uint(len(message)))
else:
kv.update(addr hc.vtable, nil, 0)
kv.output(addr hc.vtable, addr hash[0])
kv.out(addr hc.vtable, addr hash[0])
var oid = RsaOidSha256
let res = impl(cast[ptr cuchar](addr sig.buffer[0]), len(sig.buffer),
cast[ptr cuchar](addr oid[0]),
len(check), addr pubkey.key, cast[ptr cuchar](addr check[0]))
let res = impl(
addr sig.buffer[0],
uint(len(sig.buffer)),
addr oid[0],
uint(len(check)),
addr pubkey.key,
addr check[0],
)
if res == 1:
result = equalMem(addr check[0], addr hash[0], len(hash))

View File

@@ -1,28 +1,24 @@
## Nim-Libp2p
## Copyright (c) 2018 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-Libp2p
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
{.push raises: [Defect].}
{.push raises: [].}
import
secp256k1, bearssl,
stew/[byteutils, results],
nimcrypto/[hash, sha2]
import bearssl/rand
import secp256k1, results, stew/byteutils, nimcrypto/[hash, sha2]
export sha2, results
export sha2, results, rand
const
SkRawPrivateKeySize* = 256 div 8
## Size of private key in octets (bytes)
SkRawPrivateKeySize* = 256 div 8 ## Size of private key in octets (bytes)
SkRawSignatureSize* = SkRawPrivateKeySize * 2 + 1
## Size of signature in octets (bytes)
SkRawPublicKeySize* = SkRawPrivateKeySize + 1
## Size of public key in octets (bytes)
SkRawPublicKeySize* = SkRawPrivateKeySize + 1 ## Size of public key in octets (bytes)
# This is extremely confusing but it's to avoid.. confusion between Eth standard and Secp standard
type
@@ -31,20 +27,18 @@ type
SkSignature* = distinct secp256k1.SkSignature
SkKeyPair* = distinct secp256k1.SkKeyPair
template pubkey*(v: SkKeyPair): SkPublicKey = SkPublicKey(secp256k1.SkKeyPair(v).pubkey)
template seckey*(v: SkKeyPair): SkPrivateKey = SkPrivateKey(secp256k1.SkKeyPair(v).seckey)
proc random*(t: typedesc[SkPrivateKey], rng: var BrHmacDrbgContext): SkPrivateKey =
let rngPtr = unsafeAddr rng # doesn't escape
proc random*(t: typedesc[SkPrivateKey], rng: var HmacDrbgContext): SkPrivateKey =
#TODO is there a better way?
var rngPtr = addr rng
proc callRng(data: var openArray[byte]) =
brHmacDrbgGenerate(rngPtr[], data)
hmacDrbgGenerate(rngPtr[], data)
SkPrivateKey(SkSecretKey.random(callRng))
proc random*(t: typedesc[SkKeyPair], rng: var BrHmacDrbgContext): SkKeyPair =
let rngPtr = unsafeAddr rng # doesn't escape
proc random*(t: typedesc[SkKeyPair], rng: var HmacDrbgContext): SkKeyPair =
let rngPtr = addr rng
proc callRng(data: var openArray[byte]) =
brHmacDrbgGenerate(rngPtr[], data)
hmacDrbgGenerate(rngPtr[], data)
SkKeyPair(secp256k1.SkKeyPair.random(callRng))
@@ -54,34 +48,34 @@ template seckey*(v: SkKeyPair): SkPrivateKey =
template pubkey*(v: SkKeyPair): SkPublicKey =
SkPublicKey(secp256k1.SkKeyPair(v).pubkey)
proc init*(key: var SkPrivateKey, data: openarray[byte]): SkResult[void] =
proc init*(key: var SkPrivateKey, data: openArray[byte]): SkResult[void] =
## Initialize Secp256k1 `private key` ``key`` from raw binary
## representation ``data``.
key = SkPrivateKey(? secp256k1.SkSecretKey.fromRaw(data))
key = SkPrivateKey(?secp256k1.SkSecretKey.fromRaw(data))
ok()
proc init*(key: var SkPrivateKey, data: string): SkResult[void] =
## Initialize Secp256k1 `private key` ``key`` from hexadecimal string
## representation ``data``.
key = SkPrivateKey(? secp256k1.SkSecretKey.fromHex(data))
key = SkPrivateKey(?secp256k1.SkSecretKey.fromHex(data))
ok()
proc init*(key: var SkPublicKey, data: openarray[byte]): SkResult[void] =
proc init*(key: var SkPublicKey, data: openArray[byte]): SkResult[void] =
## Initialize Secp256k1 `public key` ``key`` from raw binary
## representation ``data``.
key = SkPublicKey(? secp256k1.SkPublicKey.fromRaw(data))
key = SkPublicKey(?secp256k1.SkPublicKey.fromRaw(data))
ok()
proc init*(key: var SkPublicKey, data: string): SkResult[void] =
## Initialize Secp256k1 `public key` ``key`` from hexadecimal string
## representation ``data``.
key = SkPublicKey(? secp256k1.SkPublicKey.fromHex(data))
key = SkPublicKey(?secp256k1.SkPublicKey.fromHex(data))
ok()
proc init*(sig: var SkSignature, data: openarray[byte]): SkResult[void] =
proc init*(sig: var SkSignature, data: openArray[byte]): SkResult[void] =
## Initialize Secp256k1 `signature` ``sig`` from raw binary
## representation ``data``.
sig = SkSignature(? secp256k1.SkSignature.fromDer(data))
sig = SkSignature(?secp256k1.SkSignature.fromDer(data))
ok()
proc init*(sig: var SkSignature, data: string): SkResult[void] =
@@ -95,7 +89,7 @@ proc init*(sig: var SkSignature, data: string): SkResult[void] =
return err("secp: Hex to bytes failed")
init(sig, buffer)
proc init*(t: typedesc[SkPrivateKey], data: openarray[byte]): SkResult[SkPrivateKey] =
proc init*(t: typedesc[SkPrivateKey], data: openArray[byte]): SkResult[SkPrivateKey] =
## Initialize Secp256k1 `private key` from raw binary
## representation ``data``.
##
@@ -109,7 +103,7 @@ proc init*(t: typedesc[SkPrivateKey], data: string): SkResult[SkPrivateKey] =
## Procedure returns `private key` on success.
SkSecretKey.fromHex(data).mapConvert(SkPrivateKey)
proc init*(t: typedesc[SkPublicKey], data: openarray[byte]): SkResult[SkPublicKey] =
proc init*(t: typedesc[SkPublicKey], data: openArray[byte]): SkResult[SkPublicKey] =
## Initialize Secp256k1 `public key` from raw binary
## representation ``data``.
##
@@ -125,7 +119,7 @@ proc init*(t: typedesc[SkPublicKey], data: string): SkResult[SkPublicKey] =
var key: SkPublicKey
key.init(data) and ok(key)
proc init*(t: typedesc[SkSignature], data: openarray[byte]): SkResult[SkSignature] =
proc init*(t: typedesc[SkSignature], data: openArray[byte]): SkResult[SkSignature] =
## Initialize Secp256k1 `signature` from raw binary
## representation ``data``.
##
@@ -141,35 +135,35 @@ proc init*(t: typedesc[SkSignature], data: string): SkResult[SkSignature] =
var sig: SkSignature
sig.init(data) and ok(sig)
proc getKey*(key: SkPrivateKey): SkPublicKey =
proc getPublicKey*(key: SkPrivateKey): SkPublicKey =
## Calculate and return Secp256k1 `public key` from `private key` ``key``.
SkPublicKey(SkSecretKey(key).toPublicKey())
proc toBytes*(key: SkPrivateKey, data: var openarray[byte]): SkResult[int] =
proc toBytes*(key: SkPrivateKey, data: var openArray[byte]): SkResult[int] =
## Serialize Secp256k1 `private key` ``key`` to raw binary form and store it
## to ``data``.
##
## Procedure returns number of bytes (octets) needed to store
## Secp256k1 private key.
if len(data) >= SkRawPrivateKeySize:
data[0..<SkRawPrivateKeySize] = SkSecretKey(key).toRaw()
data[0 ..< SkRawPrivateKeySize] = SkSecretKey(key).toRaw()
ok(SkRawPrivateKeySize)
else:
err("secp: Not enough bytes")
proc toBytes*(key: SkPublicKey, data: var openarray[byte]): SkResult[int] =
proc toBytes*(key: SkPublicKey, data: var openArray[byte]): SkResult[int] =
## Serialize Secp256k1 `public key` ``key`` to raw binary form and store it
## to ``data``.
##
## Procedure returns number of bytes (octets) needed to store
## Secp256k1 public key.
if len(data) >= SkRawPublicKeySize:
data[0..<SkRawPublicKeySize] = secp256k1.SkPublicKey(key).toRawCompressed()
data[0 ..< SkRawPublicKeySize] = secp256k1.SkPublicKey(key).toRawCompressed()
ok(SkRawPublicKeySize)
else:
err("secp: Not enough bytes")
proc toBytes*(sig: SkSignature, data: var openarray[byte]): int =
proc toBytes*(sig: SkSignature, data: var openArray[byte]): int =
## Serialize Secp256k1 `signature` ``sig`` to raw binary form and store it
## to ``data``.
##
@@ -191,24 +185,34 @@ proc getBytes*(sig: SkSignature): seq[byte] {.inline.} =
let length = toBytes(sig, result)
result.setLen(length)
proc sign*[T: byte|char](key: SkPrivateKey, msg: openarray[T]): SkSignature =
proc sign*[T: byte | char](key: SkPrivateKey, msg: openArray[T]): SkSignature =
## Sign message `msg` using private key `key` and return signature object.
let h = sha256.digest(msg)
SkSignature(sign(SkSecretKey(key), SkMessage(h.data)))
proc verify*[T: byte|char](sig: SkSignature, msg: openarray[T],
key: SkPublicKey): bool =
proc verify*[T: byte | char](
sig: SkSignature, msg: openArray[T], key: SkPublicKey
): bool =
let h = sha256.digest(msg)
verify(secp256k1.SkSignature(sig), SkMessage(h.data), secp256k1.SkPublicKey(key))
func clear*(key: var SkPrivateKey) {.borrow.}
func clear*(key: var SkPrivateKey) =
clear(secp256k1.SkSecretKey(key))
proc `$`*(key: SkPrivateKey): string {.borrow.}
proc `$`*(key: SkPublicKey): string {.borrow.}
proc `$`*(key: SkSignature): string {.borrow.}
proc `$`*(key: SkKeyPair): string {.borrow.}
func `$`*(key: SkPrivateKey): string =
$secp256k1.SkSecretKey(key)
func `$`*(key: SkPublicKey): string =
$secp256k1.SkPublicKey(key)
func `$`*(key: SkSignature): string =
$secp256k1.SkSignature(key)
func `$`*(key: SkKeyPair): string =
$secp256k1.SkKeyPair(key)
proc `==`*(a, b: SkPrivateKey): bool {.borrow.}
proc `==`*(a, b: SkPublicKey): bool {.borrow.}
proc `==`*(a, b: SkSignature): bool {.borrow.}
proc `==`*(a, b: SkKeyPair): bool {.borrow.}
func `==`*(a, b: SkPrivateKey): bool =
secp256k1.SkSecretKey(a) == secp256k1.SkSecretKey(b)
func `==`*(a, b: SkPublicKey): bool =
secp256k1.SkPublicKey(a) == secp256k1.SkPublicKey(b)
func `==`*(a, b: SkSignature): bool =
secp256k1.SkSignature(a) == secp256k1.SkSignature(b)
func `==`*(a, b: SkKeyPair): bool =
secp256k1.SkKeyPair(a) == secp256k1.SkKeyPair(b)

File diff suppressed because it is too large Load Diff

View File

@@ -1,32 +1,35 @@
## Nim-Libp2p
## Copyright (c) 2018 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-Libp2p
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
{.push raises: [].}
## This module implements Pool of StreamTransport.
import chronos
const
DefaultPoolSize* = 8
## Default pool size
const DefaultPoolSize* = 8 ## Default pool size
type
ConnectionFlags = enum
None, Busy
None
Busy
PoolItem = object
transp*: StreamTransport
flags*: set[ConnectionFlags]
PoolState = enum
Connecting, Connected, Closing, Closed
Connecting
Connected
Closing
Closed
TransportPool* = ref object
## Transports pool object
TransportPool* = ref object ## Transports pool object
transports: seq[PoolItem]
busyCount: int
state: PoolState
@@ -43,13 +46,16 @@ proc waitAll[T](futs: seq[Future[T]]): Future[void] =
dec(counter)
if counter == 0:
retFuture.complete()
for fut in futs:
fut.addCallback(cb)
return retFuture
proc newPool*(address: TransportAddress, poolsize: int = DefaultPoolSize,
bufferSize = DefaultStreamBufferSize,
): Future[TransportPool] {.async.} =
proc newPool*(
address: TransportAddress,
poolsize: int = DefaultPoolSize,
bufferSize = DefaultStreamBufferSize,
): Future[TransportPool] {.async: (raises: [CancelledError]).} =
## Establish pool of connections to address ``address`` with size
## ``poolsize``.
var pool = new TransportPool
@@ -57,12 +63,12 @@ proc newPool*(address: TransportAddress, poolsize: int = DefaultPoolSize,
pool.transports = newSeq[PoolItem](poolsize)
var conns = newSeq[Future[StreamTransport]](poolsize)
pool.state = Connecting
for i in 0..<poolsize:
for i in 0 ..< poolsize:
conns[i] = connect(address, bufferSize)
# Waiting for all connections to be established.
await waitAll(conns)
# Checking connections and preparing pool.
for i in 0..<poolsize:
for i in 0 ..< poolsize:
if conns[i].failed:
raise conns[i].error
else:
@@ -74,7 +80,9 @@ proc newPool*(address: TransportAddress, poolsize: int = DefaultPoolSize,
pool.state = Connected
result = pool
proc acquire*(pool: TransportPool): Future[StreamTransport] {.async.} =
proc acquire*(
pool: TransportPool
): Future[StreamTransport] {.async: (raises: [CancelledError, TransportPoolError]).} =
## Acquire non-busy connection from pool ``pool``.
var transp: StreamTransport
if pool.state in {Connected}:
@@ -96,7 +104,9 @@ proc acquire*(pool: TransportPool): Future[StreamTransport] {.async.} =
raise newException(TransportPoolError, "Pool is not ready!")
result = transp
proc release*(pool: TransportPool, transp: StreamTransport) =
proc release*(
pool: TransportPool, transp: StreamTransport
) {.async: (raises: [TransportPoolError]).} =
## Release connection ``transp`` back to pool ``pool``.
if pool.state in {Connected, Closing}:
var found = false
@@ -112,7 +122,9 @@ proc release*(pool: TransportPool, transp: StreamTransport) =
else:
raise newException(TransportPoolError, "Pool is not ready!")
proc join*(pool: TransportPool) {.async.} =
proc join*(
pool: TransportPool
) {.async: (raises: [TransportPoolError, CancelledError]).} =
## Waiting for all connection to become available.
if pool.state in {Connected, Closing}:
while true:
@@ -124,7 +136,9 @@ proc join*(pool: TransportPool) {.async.} =
elif pool.state == Connecting:
raise newException(TransportPoolError, "Pool is not ready!")
proc close*(pool: TransportPool) {.async.} =
proc close*(
pool: TransportPool
) {.async: (raises: [TransportPoolError, CancelledError]).} =
## Closes transports pool ``pool`` and release all resources.
if pool.state == Connected:
pool.state = Closing
@@ -132,7 +146,7 @@ proc close*(pool: TransportPool) {.async.} =
await pool.join()
# Closing all transports
var pending = newSeq[Future[void]](len(pool.transports))
for i in 0..<len(pool.transports):
for i in 0 ..< len(pool.transports):
let transp = pool.transports[i].transp
transp.close()
pending[i] = transp.join()

View File

@@ -1,19 +1,19 @@
## Nim-LibP2P
## Copyright (c) 2018 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-LibP2P
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
## To enable dump of all incoming and outgoing unencrypted messages you need
## to compile project with ``-d:libp2p_dump`` compile-time option. When this
## option enabled ``nim-libp2p`` will create dumps of unencrypted messages for
## every peer libp2p communicates.
##
## Every file is created with name "<PeerID>.pbcap". One file represents
## all the communication with peer which identified by ``PeerID``.
## Every file is created with name "<PeerId>.pbcap". One file represents
## all the communication with peer which identified by ``PeerId``.
##
## File can have multiple protobuf encoded messages of this format:
##
@@ -25,34 +25,40 @@
## 5. LocalAddress: optional bytes
## 6. RemoteAddress: optional bytes
## 7. Message: required bytes
import os, options
import os
import nimcrypto/utils, stew/endians2
import protobuf/minprotobuf, stream/connection, protocols/secure/secure,
multiaddress, peerid, varint, muxers/mplex/coder
import
protobuf/minprotobuf,
stream/connection,
protocols/secure/secure,
multiaddress,
peerid,
varint,
muxers/mplex/coder
from times import getTime, toUnix, fromUnix, nanosecond, format, Time,
NanosecondRange, initTime
from times import
getTime, toUnix, fromUnix, nanosecond, format, Time, NanosecondRange, initTime
from strutils import toHex, repeat
export peerid, options, multiaddress
export peerid, multiaddress
type
FlowDirection* = enum
Outgoing, Incoming
Outgoing
Incoming
ProtoMessage* = object
timestamp*: uint64
direction*: FlowDirection
message*: seq[byte]
seqID*: Option[uint64]
mtype*: Option[uint64]
local*: Option[MultiAddress]
remote*: Option[MultiAddress]
seqID*: Opt[uint64]
mtype*: Opt[uint64]
local*: Opt[MultiAddress]
remote*: Opt[MultiAddress]
const
libp2p_dump_dir* {.strdefine.} = "nim-libp2p"
## default directory where all the dumps will be stored, if the path
## relative it will be created in home directory. You can overload this path
## using ``-d:libp2p_dump_dir=<otherpath>``.
const libp2p_dump_dir* {.strdefine.} = "nim-libp2p"
## default directory where all the dumps will be stored, if the path
## relative it will be created in home directory. You can overload this path
## using ``-d:libp2p_dump_dir=<otherpath>``.
proc getTimestamp(): uint64 =
## This procedure is present because `stdlib.times` missing it.
@@ -65,15 +71,14 @@ proc getTimedate(value: uint64): string =
let time = initTime(int64(value div 1_000_000_000), value mod 1_000_000_000)
time.format("yyyy-MM-dd HH:mm:ss'.'fffzzz")
proc dumpMessage*(conn: SecureConn, direction: FlowDirection,
data: openArray[byte]) =
proc dumpMessage*(conn: SecureConn, direction: FlowDirection, data: openArray[byte]) =
## Store unencrypted message ``data`` to dump file, all the metadata will be
## extracted from ``conn`` instance.
var pb = initProtoBuffer(options = {WithVarintLength})
pb.write(2, getTimestamp())
pb.write(4, uint64(direction))
if len(conn.peerInfo.addrs) > 0:
pb.write(6, conn.peerInfo.addrs[0])
conn.observedAddr.withValue(oaddr):
pb.write(6, oaddr)
pb.write(7, data)
pb.finish()
@@ -83,11 +88,11 @@ proc dumpMessage*(conn: SecureConn, direction: FlowDirection,
else:
getHomeDir() / libp2p_dump_dir
let fileName = $(conn.peerInfo.peerId) & ".pbcap"
let fileName = $(conn.peerId) & ".pbcap"
# This is debugging procedure so it should not generate any exceptions,
# and we going to return at every possible OS error.
if not(dirExists(dirName)):
if not (dirExists(dirName)):
try:
createDir(dirName)
except CatchableError:
@@ -101,7 +106,7 @@ proc dumpMessage*(conn: SecureConn, direction: FlowDirection,
finally:
close(handle)
proc decodeDumpMessage*(data: openArray[byte]): Option[ProtoMessage] =
proc decodeDumpMessage*(data: openArray[byte]): Opt[ProtoMessage] =
## Decode protobuf's message ProtoMessage from array of bytes ``data``.
var
pb = initProtoBuffer(data)
@@ -109,13 +114,12 @@ proc decodeDumpMessage*(data: openArray[byte]): Option[ProtoMessage] =
ma1, ma2: MultiAddress
pmsg: ProtoMessage
let res2 = pb.getField(2, pmsg.timestamp)
if res2.isErr() or not(res2.get()):
return none[ProtoMessage]()
let res4 = pb.getField(4, value)
if res4.isErr() or not(res4.get()):
return none[ProtoMessage]()
let
r2 = pb.getField(2, pmsg.timestamp)
r4 = pb.getField(4, value)
r7 = pb.getField(7, pmsg.message)
if not r2.get(false) or not r4.get(false) or not r7.get(false):
return Opt.none(ProtoMessage)
# `case` statement could not work here with an error "selector must be of an
# ordinal type, float or string"
@@ -125,30 +129,27 @@ proc decodeDumpMessage*(data: openArray[byte]): Option[ProtoMessage] =
elif value == uint64(Incoming):
Incoming
else:
return none[ProtoMessage]()
return Opt.none(ProtoMessage)
let res7 = pb.getField(7, pmsg.message)
if res7.isErr() or not(res7.get()):
return none[ProtoMessage]()
let r1 = pb.getField(1, value)
if r1.get(false):
pmsg.seqID = Opt.some(value)
value = 0'u64
let res1 = pb.getField(1, value)
if res1.isOk() and res1.get():
pmsg.seqID = some(value)
value = 0'u64
let res3 = pb.getField(3, value)
if res3.isOk() and res3.get():
pmsg.mtype = some(value)
let res5 = pb.getField(5, ma1)
if res5.isOk() and res5.get():
pmsg.local = some(ma1)
let res6 = pb.getField(6, ma2)
if res6.isOk() and res6.get():
pmsg.remote = some(ma2)
let r3 = pb.getField(3, value)
if r3.get(false):
pmsg.mtype = Opt.some(value)
some(pmsg)
let
r5 = pb.getField(5, ma1)
r6 = pb.getField(6, ma2)
if r5.get(false):
pmsg.local = Opt.some(ma1)
if r6.get(false):
pmsg.remote = Opt.some(ma2)
iterator messages*(data: seq[byte]): Option[ProtoMessage] =
Opt.some(pmsg)
iterator messages*(data: seq[byte]): Opt[ProtoMessage] =
## Iterate over sequence of bytes and decode all the ``ProtoMessage``
## messages we found.
var value: uint64
@@ -157,13 +158,11 @@ iterator messages*(data: seq[byte]): Option[ProtoMessage] =
while offset < len(data):
value = 0
size = 0
let res = PB.getUVarint(data.toOpenArray(offset, len(data) - 1),
size, value)
let res = PB.getUVarint(data.toOpenArray(offset, len(data) - 1), size, value)
if res.isOk():
if (value > 0'u64) and (value < uint64(len(data) - offset)):
offset += size
yield decodeDumpMessage(data.toOpenArray(offset,
offset + int(value) - 1))
yield decodeDumpMessage(data.toOpenArray(offset, offset + int(value) - 1))
# value is previously checked to be less then len(data) which is `int`.
offset += int(value)
else:
@@ -171,7 +170,7 @@ iterator messages*(data: seq[byte]): Option[ProtoMessage] =
else:
break
proc dumpHex*(pbytes: openarray[byte], groupBy = 1, ascii = true): string =
proc dumpHex*(pbytes: openArray[byte], groupBy = 1, ascii = true): string =
## Get hexadecimal dump of memory for array ``pbytes``.
var res = ""
var offset = 0
@@ -183,10 +182,15 @@ proc dumpHex*(pbytes: openarray[byte], groupBy = 1, ascii = true): string =
for k in 0 ..< groupBy:
let ch = pbytes[offset + k]
ascii.add(if ord(ch) > 31 and ord(ch) < 127: char(ch) else: '.')
ascii.add(
if ord(ch) > 31 and ord(ch) < 127:
char(ch)
else:
'.'
)
let item =
case groupBy:
case groupBy
of 1:
toHex(pbytes[offset])
of 2:
@@ -208,8 +212,7 @@ proc dumpHex*(pbytes: openarray[byte], groupBy = 1, ascii = true): string =
res.add("\p")
if (offset mod 16) != 0:
let spacesCount = ((16 - (offset mod 16)) div groupBy) *
(groupBy * 2 + 1) + 1
let spacesCount = ((16 - (offset mod 16)) div groupBy) * (groupBy * 2 + 1) + 1
res = res & repeat(' ', spacesCount)
res = res & ascii
@@ -237,31 +240,28 @@ proc toString*(msg: ProtoMessage, dump = true): string =
var res = getTimedate(msg.timestamp)
let direction =
case msg.direction
of Incoming:
" << "
of Outgoing:
" >> "
let address =
block:
let local =
if msg.local.isSome():
"[" & $(msg.local.get()) & "]"
else:
"[LOCAL]"
let remote =
if msg.remote.isSome():
"[" & $(msg.remote.get()) & "]"
else:
"[REMOTE]"
local & direction & remote
let seqid =
if msg.seqID.isSome():
"seqID = " & $(msg.seqID.get()) & " "
of Incoming: " << "
of Outgoing: " >> "
let address = block:
let local = block:
msg.local.withValue(loc):
"[" & $loc & "]"
else:
"[LOCAL]"
let remote = block:
msg.remote.withValue(rem):
"[" & $rem & "]"
else:
"[REMOTE]"
local & direction & remote
let seqid = block:
msg.seqID.withValue(seqid):
"seqID = " & $seqid & " "
else:
""
let mtype =
if msg.mtype.isSome():
"type = " & $(msg.mtype.get()) & " "
let mtype = block:
msg.mtype.withValue(typ):
"type = " & $typ & " "
else:
""
res.add(" ")

73
libp2p/dial.nim Normal file
View File

@@ -0,0 +1,73 @@
# Nim-LibP2P
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
{.push raises: [].}
import chronos
import results
import peerid, stream/connection, transports/transport
export results
type
Dial* = ref object of RootObj
DialFailedError* = object of LPError
method connect*(
self: Dial,
peerId: PeerId,
addrs: seq[MultiAddress],
forceDial = false,
reuseConnection = true,
dir = Direction.Out,
) {.base, async: (raises: [DialFailedError, CancelledError]).} =
## connect remote peer without negotiating
## a protocol
##
doAssert(false, "[Dial.connect] abstract method not implemented!")
method connect*(
self: Dial, address: MultiAddress, allowUnknownPeerId = false
): Future[PeerId] {.base, async: (raises: [DialFailedError, CancelledError]).} =
## Connects to a peer and retrieve its PeerId
doAssert(false, "[Dial.connect] abstract method not implemented!")
method dial*(
self: Dial, peerId: PeerId, protos: seq[string]
): Future[Connection] {.base, async: (raises: [DialFailedError, CancelledError]).} =
## create a protocol stream over an
## existing connection
##
doAssert(false, "[Dial.dial] abstract method not implemented!")
method dial*(
self: Dial,
peerId: PeerId,
addrs: seq[MultiAddress],
protos: seq[string],
forceDial = false,
): Future[Connection] {.base, async: (raises: [DialFailedError, CancelledError]).} =
## create a protocol stream and establish
## a connection if one doesn't exist already
##
doAssert(false, "[Dial.dial] abstract method not implemented!")
method addTransport*(self: Dial, transport: Transport) {.base.} =
doAssert(false, "[Dial.addTransport] abstract method not implemented!")
method tryDial*(
self: Dial, peerId: PeerId, addrs: seq[MultiAddress]
): Future[Opt[MultiAddress]] {.
base, async: (raises: [DialFailedError, CancelledError])
.} =
doAssert(false, "[Dial.tryDial] abstract method not implemented!")

379
libp2p/dialer.nim Normal file
View File

@@ -0,0 +1,379 @@
# Nim-LibP2P
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
import std/tables
import pkg/[chronos, chronicles, metrics, results]
import
dial,
peerid,
peerinfo,
peerstore,
multicodec,
muxers/muxer,
multistream,
connmanager,
stream/connection,
transports/transport,
nameresolving/nameresolver,
upgrademngrs/upgrade,
errors
export dial, errors, results
logScope:
topics = "libp2p dialer"
declareCounter(libp2p_total_dial_attempts, "total attempted dials")
declareCounter(libp2p_successful_dials, "dialed successful peers")
declareCounter(libp2p_failed_dials, "failed dials")
type Dialer* = ref object of Dial
localPeerId*: PeerId
connManager: ConnManager
dialLock: Table[PeerId, AsyncLock]
transports: seq[Transport]
peerStore: PeerStore
nameResolver: NameResolver
proc dialAndUpgrade(
self: Dialer,
peerId: Opt[PeerId],
hostname: string,
address: MultiAddress,
dir = Direction.Out,
): Future[Muxer] {.async: (raises: [CancelledError]).} =
for transport in self.transports: # for each transport
if transport.handles(address): # check if it can dial it
trace "Dialing address", address, peerId = peerId.get(default(PeerId)), hostname
let dialed =
try:
libp2p_total_dial_attempts.inc()
await transport.dial(hostname, address, peerId)
except CancelledError as exc:
trace "Dialing canceled",
description = exc.msg, peerId = peerId.get(default(PeerId))
raise exc
except CatchableError as exc:
debug "Dialing failed",
description = exc.msg, peerId = peerId.get(default(PeerId))
libp2p_failed_dials.inc()
return nil # Try the next address
libp2p_successful_dials.inc()
let mux =
try:
# This is for the very specific case of a simultaneous dial during DCUtR. In this case, both sides will have
# an Outbound direction at the transport level. Therefore we update the DCUtR initiator transport direction to Inbound.
# The if below is more general and might handle other use cases in the future.
if dialed.dir != dir:
dialed.dir = dir
await transport.upgrade(dialed, peerId)
except CancelledError as exc:
await dialed.close()
raise exc
except CatchableError as exc:
# If we failed to establish the connection through one transport,
# we won't succeeded through another - no use in trying again
await dialed.close()
debug "Connection upgrade failed",
description = exc.msg, peerId = peerId.get(default(PeerId))
if dialed.dir == Direction.Out:
libp2p_failed_upgrades_outgoing.inc()
else:
libp2p_failed_upgrades_incoming.inc()
# Try other address
return nil
doAssert not isNil(mux), "connection died after upgrade " & $dialed.dir
debug "Dial successful", peerId = mux.connection.peerId
return mux
return nil
proc expandDnsAddr(
self: Dialer, peerId: Opt[PeerId], address: MultiAddress
): Future[seq[(MultiAddress, Opt[PeerId])]] {.
async: (raises: [CancelledError, MaError, TransportAddressError, LPError])
.} =
if not DNSADDR.matchPartial(address):
return @[(address, peerId)]
if isNil(self.nameResolver):
info "Can't resolve DNSADDR without NameResolver", ma = address
return @[]
let
toResolve =
if peerId.isSome:
try:
address & MultiAddress.init(multiCodec("p2p"), peerId.tryGet()).tryGet()
except ResultError[void]:
raiseAssert "checked with if"
else:
address
resolved = await self.nameResolver.resolveDnsAddr(toResolve)
for resolvedAddress in resolved:
let lastPart = resolvedAddress[^1].tryGet()
if lastPart.protoCode == Result[MultiCodec, string].ok(multiCodec("p2p")):
var peerIdBytes: seq[byte]
try:
peerIdBytes = lastPart.protoArgument().tryGet()
except ResultError[string]:
raiseAssert "expandDnsAddr failed in protoArgument: " & getCurrentExceptionMsg()
let addrPeerId = PeerId.init(peerIdBytes).tryGet()
result.add((resolvedAddress[0 ..^ 2].tryGet(), Opt.some(addrPeerId)))
else:
result.add((resolvedAddress, peerId))
proc dialAndUpgrade(
self: Dialer, peerId: Opt[PeerId], addrs: seq[MultiAddress], dir = Direction.Out
): Future[Muxer] {.
async: (raises: [CancelledError, MaError, TransportAddressError, LPError])
.} =
debug "Dialing peer", peerId = peerId.get(default(PeerId)), addrs
for rawAddress in addrs:
# resolve potential dnsaddr
let addresses = await self.expandDnsAddr(peerId, rawAddress)
for (expandedAddress, addrPeerId) in addresses:
# DNS resolution
let
hostname = expandedAddress.getHostname()
resolvedAddresses =
if isNil(self.nameResolver):
@[expandedAddress]
else:
await self.nameResolver.resolveMAddress(expandedAddress)
for resolvedAddress in resolvedAddresses:
result = await self.dialAndUpgrade(addrPeerId, hostname, resolvedAddress, dir)
if not isNil(result):
return result
proc tryReusingConnection(self: Dialer, peerId: PeerId): Opt[Muxer] =
let muxer = self.connManager.selectMuxer(peerId)
if muxer == nil:
return Opt.none(Muxer)
trace "Reusing existing connection", muxer, direction = $muxer.connection.dir
return Opt.some(muxer)
proc internalConnect(
self: Dialer,
peerId: Opt[PeerId],
addrs: seq[MultiAddress],
forceDial: bool,
reuseConnection = true,
dir = Direction.Out,
): Future[Muxer] {.async: (raises: [DialFailedError, CancelledError]).} =
if Opt.some(self.localPeerId) == peerId:
raise newException(DialFailedError, "can't dial self!")
# Ensure there's only one in-flight attempt per peer
let lock = self.dialLock.mgetOrPut(peerId.get(default(PeerId)), newAsyncLock())
await lock.acquire()
defer:
try:
lock.release()
except AsyncLockError:
raiseAssert "lock must have been acquired in line above"
if reuseConnection:
peerId.withValue(peerId):
self.tryReusingConnection(peerId).withValue(mux):
return mux
let slot =
try:
self.connManager.getOutgoingSlot(forceDial)
except TooManyConnectionsError as exc:
raise newException(DialFailedError, exc.msg)
let muxed =
try:
await self.dialAndUpgrade(peerId, addrs, dir)
except CancelledError as exc:
slot.release()
raise exc
except CatchableError as exc:
slot.release()
raise newException(DialFailedError, exc.msg)
slot.trackMuxer(muxed)
if isNil(muxed): # None of the addresses connected
raise newException(DialFailedError, "Unable to establish outgoing link")
try:
self.connManager.storeMuxer(muxed)
await self.peerStore.identify(muxed)
await self.connManager.triggerPeerEvents(
muxed.connection.peerId,
PeerEvent(kind: PeerEventKind.Identified, initiator: true),
)
return muxed
except CancelledError as exc:
await muxed.close()
raise exc
except CatchableError as exc:
trace "Failed to finish outgoing upgrade", description = exc.msg
await muxed.close()
raise newException(DialFailedError, "Failed to finish outgoing upgrade")
method connect*(
self: Dialer,
peerId: PeerId,
addrs: seq[MultiAddress],
forceDial = false,
reuseConnection = true,
dir = Direction.Out,
) {.async: (raises: [DialFailedError, CancelledError]).} =
## connect remote peer without negotiating
## a protocol
##
if self.connManager.connCount(peerId) > 0 and reuseConnection:
return
discard
await self.internalConnect(Opt.some(peerId), addrs, forceDial, reuseConnection, dir)
method connect*(
self: Dialer, address: MultiAddress, allowUnknownPeerId = false
): Future[PeerId] {.async: (raises: [DialFailedError, CancelledError]).} =
## Connects to a peer and retrieve its PeerId
parseFullAddress(address).toOpt().withValue(fullAddress):
return (
await self.internalConnect(Opt.some(fullAddress[0]), @[fullAddress[1]], false)
).connection.peerId
if allowUnknownPeerId == false:
raise newException(
DialFailedError, "Address without PeerID and unknown peer id disabled!"
)
return
(await self.internalConnect(Opt.none(PeerId), @[address], false)).connection.peerId
proc negotiateStream(
self: Dialer, conn: Connection, protos: seq[string]
): Future[Connection] {.async: (raises: [CatchableError]).} =
trace "Negotiating stream", conn, protos
let selected = await MultistreamSelect.select(conn, protos)
if not protos.contains(selected):
await conn.closeWithEOF()
raise newException(DialFailedError, "Unable to select sub-protocol " & $protos)
return conn
method tryDial*(
self: Dialer, peerId: PeerId, addrs: seq[MultiAddress]
): Future[Opt[MultiAddress]] {.async: (raises: [DialFailedError, CancelledError]).} =
## Create a protocol stream in order to check
## if a connection is possible.
## Doesn't use the Connection Manager to save it.
##
trace "Check if it can dial", peerId, addrs
try:
let mux = await self.dialAndUpgrade(Opt.some(peerId), addrs)
if mux.isNil():
raise newException(DialFailedError, "No valid multiaddress")
await mux.close()
return mux.connection.observedAddr
except CancelledError as exc:
raise exc
except CatchableError as exc:
raise newException(DialFailedError, exc.msg)
method dial*(
self: Dialer, peerId: PeerId, protos: seq[string]
): Future[Connection] {.async: (raises: [DialFailedError, CancelledError]).} =
## create a protocol stream over an
## existing connection
##
trace "Dialing (existing)", peerId, protos
try:
let stream = await self.connManager.getStream(peerId)
if stream.isNil:
raise newException(DialFailedError, "Couldn't get muxed stream")
return await self.negotiateStream(stream, protos)
except CancelledError as exc:
trace "Dial canceled"
raise exc
except CatchableError as exc:
trace "Error dialing", description = exc.msg
raise newException(DialFailedError, exc.msg)
method dial*(
self: Dialer,
peerId: PeerId,
addrs: seq[MultiAddress],
protos: seq[string],
forceDial = false,
): Future[Connection] {.async: (raises: [DialFailedError, CancelledError]).} =
## create a protocol stream and establish
## a connection if one doesn't exist already
##
var
conn: Muxer
stream: Connection
proc cleanup() {.async: (raises: []).} =
if not (isNil(stream)):
await stream.closeWithEOF()
if not (isNil(conn)):
await conn.close()
try:
trace "Dialing (new)", peerId, protos
conn = await self.internalConnect(Opt.some(peerId), addrs, forceDial)
trace "Opening stream", conn
stream = await self.connManager.getStream(conn)
if isNil(stream):
raise newException(DialFailedError, "Couldn't get muxed stream")
return await self.negotiateStream(stream, protos)
except CancelledError as exc:
trace "Dial canceled", conn
await cleanup()
raise exc
except CatchableError as exc:
debug "Error dialing", conn, description = exc.msg
await cleanup()
raise newException(DialFailedError, exc.msg)
method addTransport*(self: Dialer, t: Transport) =
self.transports &= t
proc new*(
T: type Dialer,
localPeerId: PeerId,
connManager: ConnManager,
peerStore: PeerStore,
transports: seq[Transport],
nameResolver: NameResolver = nil,
): Dialer =
T(
localPeerId: localPeerId,
connManager: connManager,
transports: transports,
peerStore: peerStore,
nameResolver: nameResolver,
)

View File

@@ -0,0 +1,190 @@
# Nim-LibP2P
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
{.push raises: [].}
import std/sequtils
import chronos, chronicles, results
import ../errors
type
BaseAttr = ref object of RootObj
comparator: proc(f, c: BaseAttr): bool {.gcsafe, raises: [].}
Attribute[T] = ref object of BaseAttr
value: T
PeerAttributes* = object
attributes: seq[BaseAttr]
DiscoveryService* = distinct string
proc `==`*(a, b: DiscoveryService): bool {.borrow.}
proc ofType*[T](f: BaseAttr, _: type[T]): bool =
return f of Attribute[T]
proc to*[T](f: BaseAttr, _: type[T]): T =
Attribute[T](f).value
proc add*[T](pa: var PeerAttributes, value: T) =
pa.attributes.add(
Attribute[T](
value: value,
comparator: proc(f: BaseAttr, c: BaseAttr): bool =
f.ofType(T) and c.ofType(T) and f.to(T) == c.to(T),
)
)
iterator items*(pa: PeerAttributes): BaseAttr =
for f in pa.attributes:
yield f
proc getAll*[T](pa: PeerAttributes, t: typedesc[T]): seq[T] =
for f in pa.attributes:
if f.ofType(T):
result.add(f.to(T))
proc `{}`*[T](pa: PeerAttributes, t: typedesc[T]): Opt[T] =
for f in pa.attributes:
if f.ofType(T):
return Opt.some(f.to(T))
Opt.none(T)
proc `[]`*[T](pa: PeerAttributes, t: typedesc[T]): T {.raises: [KeyError].} =
pa{T}.valueOr:
raise newException(KeyError, "Attribute not found")
proc match*(pa, candidate: PeerAttributes): bool =
for f in pa.attributes:
block oneAttribute:
for field in candidate.attributes:
if field.comparator(field, f):
break oneAttribute
return false
return true
type
PeerFoundCallback* = proc(pa: PeerAttributes) {.raises: [], gcsafe.}
DiscoveryInterface* = ref object of RootObj
onPeerFound*: PeerFoundCallback
toAdvertise*: PeerAttributes
advertisementUpdated*: AsyncEvent
advertiseLoop*: Future[void]
DiscoveryError* = object of LPError
DiscoveryFinished* = object of LPError
AdvertiseError* = object of DiscoveryError
method request*(
self: DiscoveryInterface, pa: PeerAttributes
) {.base, async: (raises: [DiscoveryError, CancelledError]).} =
doAssert(false, "[DiscoveryInterface.request] abstract method not implemented!")
method advertise*(
self: DiscoveryInterface
) {.base, async: (raises: [CancelledError, AdvertiseError]).} =
doAssert(false, "[DiscoveryInterface.advertise] abstract method not implemented!")
type
DiscoveryQuery* = ref object
attr: PeerAttributes
peers: AsyncQueue[PeerAttributes]
finished: bool
futs: seq[Future[void]]
DiscoveryManager* = ref object
interfaces: seq[DiscoveryInterface]
queries: seq[DiscoveryQuery]
proc add*(dm: DiscoveryManager, di: DiscoveryInterface) =
dm.interfaces &= di
di.onPeerFound = proc(pa: PeerAttributes) =
for query in dm.queries:
if query.attr.match(pa):
try:
query.peers.putNoWait(pa)
except AsyncQueueFullError as exc:
debug "Cannot push discovered peer to queue"
proc request*(dm: DiscoveryManager, pa: PeerAttributes): DiscoveryQuery =
var query = DiscoveryQuery(attr: pa, peers: newAsyncQueue[PeerAttributes]())
for i in dm.interfaces:
query.futs.add(i.request(pa))
dm.queries.add(query)
dm.queries.keepItIf(it.futs.anyIt(not it.finished()))
return query
proc request*[T](dm: DiscoveryManager, value: T): DiscoveryQuery =
var pa: PeerAttributes
pa.add(value)
return dm.request(pa)
proc advertise*[T](dm: DiscoveryManager, value: T) =
for i in dm.interfaces:
i.toAdvertise.add(value)
if i.advertiseLoop.isNil:
i.advertisementUpdated = newAsyncEvent()
i.advertiseLoop = i.advertise()
else:
i.advertisementUpdated.fire()
template forEach*(query: DiscoveryQuery, code: untyped) =
## Will execute `code` for each discovered peer. The
## peer attritubtes are available through the variable
## `peer`
proc forEachInternal(
q: DiscoveryQuery
) {.async: (raises: [CancelledError, DiscoveryError]).} =
while true:
let peer {.inject.} =
try:
await q.getPeer()
except DiscoveryFinished:
return
code
asyncSpawn forEachInternal(query)
proc stop*(query: DiscoveryQuery) =
query.finished = true
for r in query.futs:
if not r.finished():
r.cancel()
proc stop*(dm: DiscoveryManager) =
for q in dm.queries:
q.stop()
for i in dm.interfaces:
if isNil(i.advertiseLoop):
continue
i.advertiseLoop.cancel()
proc getPeer*(
query: DiscoveryQuery
): Future[PeerAttributes] {.
async: (raises: [CancelledError, DiscoveryError, DiscoveryFinished])
.} =
let getter = query.peers.popFirst()
try:
await getter or allFinished(query.futs)
except CancelledError as exc:
getter.cancel()
raise exc
if not finished(getter):
if query.finished:
raise newException(DiscoveryFinished, "Discovery query stopped")
# discovery loops only finish when they don't handle the query
raise newException(DiscoveryError, "Unable to find any peer matching this request")
return await getter

View File

@@ -0,0 +1,82 @@
# Nim-LibP2P
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
{.push raises: [].}
import chronos
import ./discoverymngr, ../protocols/rendezvous, ../peerid
type
RendezVousInterface* = ref object of DiscoveryInterface
rdv*: RendezVous
timeToRequest: Duration
timeToAdvertise: Duration
ttl: Duration
RdvNamespace* = distinct string
proc `==`*(a, b: RdvNamespace): bool {.borrow.}
method request*(
self: RendezVousInterface, pa: PeerAttributes
) {.async: (raises: [DiscoveryError, CancelledError]).} =
var namespace = Opt.none(string)
for attr in pa:
if attr.ofType(RdvNamespace):
namespace = Opt.some(string attr.to(RdvNamespace))
elif attr.ofType(DiscoveryService):
namespace = Opt.some(string attr.to(DiscoveryService))
elif attr.ofType(PeerId):
namespace = Opt.some($attr.to(PeerId))
else:
# unhandled type
return
while true:
for pr in await self.rdv.request(namespace):
var peer: PeerAttributes
peer.add(pr.peerId)
for address in pr.addresses:
peer.add(address.address)
peer.add(DiscoveryService(namespace.get()))
peer.add(RdvNamespace(namespace.get()))
self.onPeerFound(peer)
await sleepAsync(self.timeToRequest)
method advertise*(
self: RendezVousInterface
) {.async: (raises: [CancelledError, AdvertiseError]).} =
while true:
var toAdvertise: seq[string]
for attr in self.toAdvertise:
if attr.ofType(RdvNamespace):
toAdvertise.add string attr.to(RdvNamespace)
elif attr.ofType(DiscoveryService):
toAdvertise.add string attr.to(DiscoveryService)
elif attr.ofType(PeerId):
toAdvertise.add $attr.to(PeerId)
self.advertisementUpdated.clear()
for toAdv in toAdvertise:
try:
await self.rdv.advertise(toAdv, self.ttl)
except CatchableError as error:
debug "RendezVous advertise error: ", description = error.msg
await sleepAsync(self.timeToAdvertise) or self.advertisementUpdated.wait()
proc new*(
T: typedesc[RendezVousInterface],
rdv: RendezVous,
ttr: Duration = 1.minutes,
tta: Duration = 1.minutes,
ttl: Duration = MinimumDuration,
): RendezVousInterface =
T(rdv: rdv, timeToRequest: ttr, timeToAdvertise: tta, ttl: ttl)

View File

@@ -5,63 +5,44 @@ import chronos
import chronicles
import macros
# could not figure how to make it with a simple template
# sadly nim needs more love for hygenic templates
type
# Base exception type for libp2p
LPError* = object of CatchableError
func toException*(e: cstring): ref LPError =
(ref LPError)(msg: $e)
func toException*(e: string): ref LPError =
(ref LPError)(msg: e)
# TODO: could not figure how to make it with a simple template
# sadly nim needs more love for hygienic templates
# so here goes the macro, its based on the proc/template version
# and uses quote do so it's quite readable
macro checkFutures*[T](futs: seq[Future[T]], exclude: untyped = []): untyped =
# TODO https://github.com/nim-lang/Nim/issues/22936
macro checkFutures*[F](futs: seq[F], exclude: untyped = []): untyped =
let nexclude = exclude.len
case nexclude
of 0:
quote do:
quote:
for res in `futs`:
if res.failed:
let exc = res.readError()
# We still don't abort but warn
debug "A future has failed, enable trace logging for details", error = exc.name
trace "Exception message", msg= exc.msg, stack = getStackTrace()
debug "A future has failed, enable trace logging for details",
error = exc.name
trace "Exception message", description = exc.msg, stack = getStackTrace()
else:
quote do:
quote:
for res in `futs`:
block check:
if res.failed:
let exc = res.readError()
for i in 0..<`nexclude`:
for i in 0 ..< `nexclude`:
if exc of `exclude`[i]:
trace "A future has failed", error=exc.name, msg=exc.msg
trace "A future has failed", error = exc.name, description = exc.msg
break check
# We still don't abort but warn
debug "A future has failed, enable trace logging for details", error=exc.name
trace "Exception details", msg=exc.msg
proc allFuturesThrowing*[T](args: varargs[Future[T]]): Future[void] =
var futs: seq[Future[T]]
for fut in args:
futs &= fut
proc call() {.async.} =
var first: ref Exception = nil
futs = await allFinished(futs)
for fut in futs:
if fut.failed:
let err = fut.readError()
if err of Defect:
raise err
else:
if err of CancelledError:
raise err
if isNil(first):
first = err
if not isNil(first):
raise first
return call()
template tryAndWarn*(message: static[string]; body: untyped): untyped =
try:
body
except CancelledError as exc:
raise exc
except CatchableError as exc:
debug "An exception has ocurred, enable trace logging for details", name = exc.name, msg = message
trace "Exception details", exc = exc.msg
debug "A future has failed, enable trace logging for details",
error = exc.name
trace "Exception details", description = exc.msg

File diff suppressed because it is too large Load Diff

View File

@@ -1,11 +1,11 @@
## Nim-Libp2p
## Copyright (c) 2018 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-Libp2p
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
## This module implements MultiBase.
##
@@ -13,342 +13,387 @@
## 1. base32z
##
{.push raises: [Defect].}
{.push raises: [].}
import tables
import stew/[base32, base58, base64, results]
import results
import stew/[base32, base58, base64]
type
MultibaseStatus* {.pure.} = enum
Error, Success, Overrun, Incorrect, BadCodec, NotSupported
MultiBaseStatus* {.pure.} = enum
Error
Success
Overrun
Incorrect
BadCodec
NotSupported
MultiBase* = object
MBCodeSize = proc(length: int): int {.nimcall, gcsafe, noSideEffect, raises: [Defect].}
MBCodeSize = proc(length: int): int {.nimcall, gcsafe, noSideEffect, raises: [].}
MBCodec = object
code: char
name: string
encr: proc(inbytes: openarray[byte],
outbytes: var openarray[char],
outlen: var int): MultibaseStatus {.nimcall, gcsafe, noSideEffect, raises: [Defect].}
decr: proc(inbytes: openarray[char],
outbytes: var openarray[byte],
outlen: var int): MultibaseStatus {.nimcall, gcsafe, noSideEffect, raises: [Defect].}
encr: proc(
inbytes: openArray[byte], outbytes: var openArray[char], outlen: var int
): MultiBaseStatus {.nimcall, gcsafe, noSideEffect, raises: [].}
decr: proc(
inbytes: openArray[char], outbytes: var openArray[byte], outlen: var int
): MultiBaseStatus {.nimcall, gcsafe, noSideEffect, raises: [].}
encl: MBCodeSize
decl: MBCodeSize
proc idd(inbytes: openarray[char], outbytes: var openarray[byte],
outlen: var int): MultibaseStatus =
proc idd(
inbytes: openArray[char], outbytes: var openArray[byte], outlen: var int
): MultiBaseStatus =
let length = len(inbytes)
if length > len(outbytes):
outlen = length
result = MultibaseStatus.Overrun
result = MultiBaseStatus.Overrun
else:
copyMem(addr outbytes[0], unsafeAddr inbytes[0], length)
outlen = length
result = MultibaseStatus.Success
result = MultiBaseStatus.Success
proc ide(inbytes: openarray[byte],
outbytes: var openarray[char],
outlen: var int): MultibaseStatus =
proc ide(
inbytes: openArray[byte], outbytes: var openArray[char], outlen: var int
): MultiBaseStatus =
let length = len(inbytes)
if length > len(outbytes):
outlen = length
result = MultibaseStatus.Overrun
result = MultiBaseStatus.Overrun
else:
copyMem(addr outbytes[0], unsafeAddr inbytes[0], length)
outlen = length
result = MultibaseStatus.Success
result = MultiBaseStatus.Success
proc idel(length: int): int = length
proc iddl(length: int): int = length
proc idel(length: int): int =
length
proc b16d(inbytes: openarray[char],
outbytes: var openarray[byte],
outlen: var int): MultibaseStatus =
proc iddl(length: int): int =
length
proc b16d(
inbytes: openArray[char], outbytes: var openArray[byte], outlen: var int
): MultiBaseStatus =
discard
proc b16e(inbytes: openarray[byte],
outbytes: var openarray[char],
outlen: var int): MultibaseStatus =
proc b16e(
inbytes: openArray[byte], outbytes: var openArray[char], outlen: var int
): MultiBaseStatus =
discard
proc b16ud(inbytes: openarray[char],
outbytes: var openarray[byte],
outlen: var int): MultibaseStatus =
proc b16ud(
inbytes: openArray[char], outbytes: var openArray[byte], outlen: var int
): MultiBaseStatus =
discard
proc b16ue(inbytes: openarray[byte],
outbytes: var openarray[char],
outlen: var int): MultibaseStatus =
proc b16ue(
inbytes: openArray[byte], outbytes: var openArray[char], outlen: var int
): MultiBaseStatus =
discard
proc b16el(length: int): int = length shl 1
proc b16dl(length: int): int = (length + 1) div 2
proc b16el(length: int): int =
length shl 1
proc b32ce(r: Base32Status): MultibaseStatus {.inline.} =
result = MultibaseStatus.Error
proc b16dl(length: int): int =
(length + 1) div 2
proc b32ce(r: Base32Status): MultiBaseStatus {.inline.} =
result = MultiBaseStatus.Error
if r == Base32Status.Incorrect:
result = MultibaseStatus.Incorrect
result = MultiBaseStatus.Incorrect
elif r == Base32Status.Overrun:
result = MultibaseStatus.Overrun
result = MultiBaseStatus.Overrun
elif r == Base32Status.Success:
result = MultibaseStatus.Success
result = MultiBaseStatus.Success
proc b58ce(r: Base58Status): MultibaseStatus {.inline.} =
result = MultibaseStatus.Error
proc b58ce(r: Base58Status): MultiBaseStatus {.inline.} =
result = MultiBaseStatus.Error
if r == Base58Status.Incorrect:
result = MultibaseStatus.Incorrect
result = MultiBaseStatus.Incorrect
elif r == Base58Status.Overrun:
result = MultibaseStatus.Overrun
result = MultiBaseStatus.Overrun
elif r == Base58Status.Success:
result = MultibaseStatus.Success
result = MultiBaseStatus.Success
proc b64ce(r: Base64Status): MultibaseStatus {.inline.} =
proc b64ce(r: Base64Status): MultiBaseStatus {.inline.} =
result = MultiBaseStatus.Error
if r == Base64Status.Incorrect:
result = MultibaseStatus.Incorrect
result = MultiBaseStatus.Incorrect
elif r == Base64Status.Overrun:
result = MultiBaseStatus.Overrun
elif r == Base64Status.Success:
result = MultibaseStatus.Success
result = MultiBaseStatus.Success
proc b32hd(inbytes: openarray[char],
outbytes: var openarray[byte],
outlen: var int): MultibaseStatus =
proc b32hd(
inbytes: openArray[char], outbytes: var openArray[byte], outlen: var int
): MultiBaseStatus =
result = b32ce(HexBase32Lower.decode(inbytes, outbytes, outlen))
proc b32he(inbytes: openarray[byte],
outbytes: var openarray[char],
outlen: var int): MultibaseStatus =
proc b32he(
inbytes: openArray[byte], outbytes: var openArray[char], outlen: var int
): MultiBaseStatus =
result = b32ce(HexBase32Lower.encode(inbytes, outbytes, outlen))
proc b32hud(inbytes: openarray[char],
outbytes: var openarray[byte],
outlen: var int): MultibaseStatus =
proc b32hud(
inbytes: openArray[char], outbytes: var openArray[byte], outlen: var int
): MultiBaseStatus =
result = b32ce(HexBase32Upper.decode(inbytes, outbytes, outlen))
proc b32hue(inbytes: openarray[byte],
outbytes: var openarray[char],
outlen: var int): MultibaseStatus =
proc b32hue(
inbytes: openArray[byte], outbytes: var openArray[char], outlen: var int
): MultiBaseStatus =
result = b32ce(HexBase32Upper.encode(inbytes, outbytes, outlen))
proc b32hpd(inbytes: openarray[char],
outbytes: var openarray[byte],
outlen: var int): MultibaseStatus =
proc b32hpd(
inbytes: openArray[char], outbytes: var openArray[byte], outlen: var int
): MultiBaseStatus =
result = b32ce(HexBase32LowerPad.decode(inbytes, outbytes, outlen))
proc b32hpe(inbytes: openarray[byte],
outbytes: var openarray[char],
outlen: var int): MultibaseStatus =
proc b32hpe(
inbytes: openArray[byte], outbytes: var openArray[char], outlen: var int
): MultiBaseStatus =
result = b32ce(HexBase32LowerPad.encode(inbytes, outbytes, outlen))
proc b32hpud(inbytes: openarray[char],
outbytes: var openarray[byte],
outlen: var int): MultibaseStatus =
proc b32hpud(
inbytes: openArray[char], outbytes: var openArray[byte], outlen: var int
): MultiBaseStatus =
result = b32ce(HexBase32UpperPad.decode(inbytes, outbytes, outlen))
proc b32hpue(inbytes: openarray[byte],
outbytes: var openarray[char],
outlen: var int): MultibaseStatus =
proc b32hpue(
inbytes: openArray[byte], outbytes: var openArray[char], outlen: var int
): MultiBaseStatus =
result = b32ce(HexBase32UpperPad.encode(inbytes, outbytes, outlen))
proc b32d(inbytes: openarray[char],
outbytes: var openarray[byte],
outlen: var int): MultibaseStatus =
proc b32d(
inbytes: openArray[char], outbytes: var openArray[byte], outlen: var int
): MultiBaseStatus =
result = b32ce(Base32Lower.decode(inbytes, outbytes, outlen))
proc b32e(inbytes: openarray[byte],
outbytes: var openarray[char],
outlen: var int): MultibaseStatus =
proc b32e(
inbytes: openArray[byte], outbytes: var openArray[char], outlen: var int
): MultiBaseStatus =
result = b32ce(Base32Lower.encode(inbytes, outbytes, outlen))
proc b32ud(inbytes: openarray[char],
outbytes: var openarray[byte],
outlen: var int): MultibaseStatus =
proc b32ud(
inbytes: openArray[char], outbytes: var openArray[byte], outlen: var int
): MultiBaseStatus =
result = b32ce(Base32Upper.decode(inbytes, outbytes, outlen))
proc b32ue(inbytes: openarray[byte],
outbytes: var openarray[char],
outlen: var int): MultibaseStatus =
proc b32ue(
inbytes: openArray[byte], outbytes: var openArray[char], outlen: var int
): MultiBaseStatus =
result = b32ce(Base32Upper.encode(inbytes, outbytes, outlen))
proc b32pd(inbytes: openarray[char],
outbytes: var openarray[byte],
outlen: var int): MultibaseStatus =
proc b32pd(
inbytes: openArray[char], outbytes: var openArray[byte], outlen: var int
): MultiBaseStatus =
result = b32ce(Base32LowerPad.decode(inbytes, outbytes, outlen))
proc b32pe(inbytes: openarray[byte],
outbytes: var openarray[char],
outlen: var int): MultibaseStatus =
proc b32pe(
inbytes: openArray[byte], outbytes: var openArray[char], outlen: var int
): MultiBaseStatus =
result = b32ce(Base32LowerPad.encode(inbytes, outbytes, outlen))
proc b32pud(inbytes: openarray[char],
outbytes: var openarray[byte],
outlen: var int): MultibaseStatus =
proc b32pud(
inbytes: openArray[char], outbytes: var openArray[byte], outlen: var int
): MultiBaseStatus =
result = b32ce(Base32UpperPad.decode(inbytes, outbytes, outlen))
proc b32pue(inbytes: openarray[byte],
outbytes: var openarray[char],
outlen: var int): MultibaseStatus =
proc b32pue(
inbytes: openArray[byte], outbytes: var openArray[char], outlen: var int
): MultiBaseStatus =
result = b32ce(Base32UpperPad.encode(inbytes, outbytes, outlen))
proc b32el(length: int): int = Base32Lower.encodedLength(length)
proc b32dl(length: int): int = Base32Lower.decodedLength(length)
proc b32pel(length: int): int = Base32LowerPad.encodedLength(length)
proc b32pdl(length: int): int = Base32LowerPad.decodedLength(length)
proc b32el(length: int): int =
Base32Lower.encodedLength(length)
proc b58fd(inbytes: openarray[char],
outbytes: var openarray[byte],
outlen: var int): MultibaseStatus =
proc b32dl(length: int): int =
Base32Lower.decodedLength(length)
proc b32pel(length: int): int =
Base32LowerPad.encodedLength(length)
proc b32pdl(length: int): int =
Base32LowerPad.decodedLength(length)
proc b58fd(
inbytes: openArray[char], outbytes: var openArray[byte], outlen: var int
): MultiBaseStatus =
result = b58ce(FLCBase58.decode(inbytes, outbytes, outlen))
proc b58fe(inbytes: openarray[byte],
outbytes: var openarray[char],
outlen: var int): MultibaseStatus =
proc b58fe(
inbytes: openArray[byte], outbytes: var openArray[char], outlen: var int
): MultiBaseStatus =
result = b58ce(FLCBase58.encode(inbytes, outbytes, outlen))
proc b58bd(inbytes: openarray[char],
outbytes: var openarray[byte],
outlen: var int): MultibaseStatus =
proc b58bd(
inbytes: openArray[char], outbytes: var openArray[byte], outlen: var int
): MultiBaseStatus =
result = b58ce(BTCBase58.decode(inbytes, outbytes, outlen))
proc b58be(inbytes: openarray[byte],
outbytes: var openarray[char],
outlen: var int): MultibaseStatus =
proc b58be(
inbytes: openArray[byte], outbytes: var openArray[char], outlen: var int
): MultiBaseStatus =
result = b58ce(BTCBase58.encode(inbytes, outbytes, outlen))
proc b58el(length: int): int = Base58.encodedLength(length)
proc b58dl(length: int): int = Base58.decodedLength(length)
proc b58el(length: int): int =
Base58.encodedLength(length)
proc b64el(length: int): int = Base64.encodedLength(length)
proc b64dl(length: int): int = Base64.decodedLength(length)
proc b64pel(length: int): int = Base64Pad.encodedLength(length)
proc b64pdl(length: int): int = Base64Pad.decodedLength(length)
proc b58dl(length: int): int =
Base58.decodedLength(length)
proc b64e(inbytes: openarray[byte],
outbytes: var openarray[char],
outlen: var int): MultibaseStatus =
proc b64el(length: int): int =
Base64.encodedLength(length)
proc b64dl(length: int): int =
Base64.decodedLength(length)
proc b64pel(length: int): int =
Base64Pad.encodedLength(length)
proc b64pdl(length: int): int =
Base64Pad.decodedLength(length)
proc b64e(
inbytes: openArray[byte], outbytes: var openArray[char], outlen: var int
): MultiBaseStatus =
result = b64ce(Base64.encode(inbytes, outbytes, outlen))
proc b64d(inbytes: openarray[char],
outbytes: var openarray[byte],
outlen: var int): MultibaseStatus =
proc b64d(
inbytes: openArray[char], outbytes: var openArray[byte], outlen: var int
): MultiBaseStatus =
result = b64ce(Base64.decode(inbytes, outbytes, outlen))
proc b64pe(inbytes: openarray[byte],
outbytes: var openarray[char],
outlen: var int): MultibaseStatus =
proc b64pe(
inbytes: openArray[byte], outbytes: var openArray[char], outlen: var int
): MultiBaseStatus =
result = b64ce(Base64Pad.encode(inbytes, outbytes, outlen))
proc b64pd(inbytes: openarray[char],
outbytes: var openarray[byte],
outlen: var int): MultibaseStatus =
proc b64pd(
inbytes: openArray[char], outbytes: var openArray[byte], outlen: var int
): MultiBaseStatus =
result = b64ce(Base64Pad.decode(inbytes, outbytes, outlen))
proc b64ue(inbytes: openarray[byte],
outbytes: var openarray[char],
outlen: var int): MultibaseStatus =
proc b64ue(
inbytes: openArray[byte], outbytes: var openArray[char], outlen: var int
): MultiBaseStatus =
result = b64ce(Base64Url.encode(inbytes, outbytes, outlen))
proc b64ud(inbytes: openarray[char],
outbytes: var openarray[byte],
outlen: var int): MultibaseStatus =
proc b64ud(
inbytes: openArray[char], outbytes: var openArray[byte], outlen: var int
): MultiBaseStatus =
result = b64ce(Base64Url.decode(inbytes, outbytes, outlen))
proc b64upe(inbytes: openarray[byte],
outbytes: var openarray[char],
outlen: var int): MultibaseStatus =
proc b64upe(
inbytes: openArray[byte], outbytes: var openArray[char], outlen: var int
): MultiBaseStatus =
result = b64ce(Base64UrlPad.encode(inbytes, outbytes, outlen))
proc b64upd(inbytes: openarray[char],
outbytes: var openarray[byte],
outlen: var int): MultibaseStatus =
proc b64upd(
inbytes: openArray[char], outbytes: var openArray[byte], outlen: var int
): MultiBaseStatus =
result = b64ce(Base64UrlPad.decode(inbytes, outbytes, outlen))
const
MultibaseCodecs = [
MBCodec(name: "identity", code: chr(0x00),
decr: idd, encr: ide, decl: iddl, encl: idel
),
MBCodec(name: "base1", code: '1'),
MBCodec(name: "base2", code: '0'),
MBCodec(name: "base8", code: '7'),
MBCodec(name: "base10", code: '9'),
MBCodec(name: "base16", code: 'f',
decr: b16d, encr: b16e, decl: b16dl, encl: b16el
),
MBCodec(name: "base16upper", code: 'F',
decr: b16ud, encr: b16ue, decl: b16dl, encl: b16el
),
MBCodec(name: "base32hex", code: 'v',
decr: b32hd, encr: b32he, decl: b32dl, encl: b32el
),
MBCodec(name: "base32hexupper", code: 'V',
decr: b32hud, encr: b32hue, decl: b32dl, encl: b32el
),
MBCodec(name: "base32hexpad", code: 't',
decr: b32hpd, encr: b32hpe, decl: b32pdl, encl: b32pel
),
MBCodec(name: "base32hexpadupper", code: 'T',
decr: b32hpud, encr: b32hpue, decl: b32pdl, encl: b32pel
),
MBCodec(name: "base32", code: 'b',
decr: b32d, encr: b32e, decl: b32dl, encl: b32el
),
MBCodec(name: "base32upper", code: 'B',
decr: b32ud, encr: b32ue, decl: b32dl, encl: b32el
),
MBCodec(name: "base32pad", code: 'c',
decr: b32pd, encr: b32pe, decl: b32pdl, encl: b32pel
),
MBCodec(name: "base32padupper", code: 'C',
decr: b32pud, encr: b32pue, decl: b32pdl, encl: b32pel
),
MBCodec(name: "base32z", code: 'h'),
MBCodec(name: "base58flickr", code: 'Z',
decr: b58fd, encr: b58fe, decl: b58dl, encl: b58el
),
MBCodec(name: "base58btc", code: 'z',
decr: b58bd, encr: b58be, decl: b58dl, encl: b58el
),
MBCodec(name: "base64", code: 'm',
decr: b64d, encr: b64e, decl: b64dl, encl: b64el
),
MBCodec(name: "base64pad", code: 'M',
decr: b64pd, encr: b64pe, decl: b64pdl, encl: b64pel
),
MBCodec(name: "base64url", code: 'u',
decr: b64ud, encr: b64ue, decl: b64dl, encl: b64el
),
MBCodec(name: "base64urlpad", code: 'U',
decr: b64upd, encr: b64upe, decl: b64pdl, encl: b64pel
)
]
const MultiBaseCodecs = [
MBCodec(
name: "identity", code: chr(0x00), decr: idd, encr: ide, decl: iddl, encl: idel
),
MBCodec(name: "base1", code: '1'),
MBCodec(name: "base2", code: '0'),
MBCodec(name: "base8", code: '7'),
MBCodec(name: "base10", code: '9'),
MBCodec(name: "base16", code: 'f', decr: b16d, encr: b16e, decl: b16dl, encl: b16el),
MBCodec(
name: "base16upper", code: 'F', decr: b16ud, encr: b16ue, decl: b16dl, encl: b16el
),
MBCodec(
name: "base32hex", code: 'v', decr: b32hd, encr: b32he, decl: b32dl, encl: b32el
),
MBCodec(
name: "base32hexupper",
code: 'V',
decr: b32hud,
encr: b32hue,
decl: b32dl,
encl: b32el,
),
MBCodec(
name: "base32hexpad",
code: 't',
decr: b32hpd,
encr: b32hpe,
decl: b32pdl,
encl: b32pel,
),
MBCodec(
name: "base32hexpadupper",
code: 'T',
decr: b32hpud,
encr: b32hpue,
decl: b32pdl,
encl: b32pel,
),
MBCodec(name: "base32", code: 'b', decr: b32d, encr: b32e, decl: b32dl, encl: b32el),
MBCodec(
name: "base32upper", code: 'B', decr: b32ud, encr: b32ue, decl: b32dl, encl: b32el
),
MBCodec(
name: "base32pad", code: 'c', decr: b32pd, encr: b32pe, decl: b32pdl, encl: b32pel
),
MBCodec(
name: "base32padupper",
code: 'C',
decr: b32pud,
encr: b32pue,
decl: b32pdl,
encl: b32pel,
),
MBCodec(name: "base32z", code: 'h'),
MBCodec(
name: "base58flickr", code: 'Z', decr: b58fd, encr: b58fe, decl: b58dl, encl: b58el
),
MBCodec(
name: "base58btc", code: 'z', decr: b58bd, encr: b58be, decl: b58dl, encl: b58el
),
MBCodec(name: "base64", code: 'm', decr: b64d, encr: b64e, decl: b64dl, encl: b64el),
MBCodec(
name: "base64pad", code: 'M', decr: b64pd, encr: b64pe, decl: b64pdl, encl: b64pel
),
MBCodec(
name: "base64url", code: 'u', decr: b64ud, encr: b64ue, decl: b64dl, encl: b64el
),
MBCodec(
name: "base64urlpad",
code: 'U',
decr: b64upd,
encr: b64upe,
decl: b64pdl,
encl: b64pel,
),
]
proc initMultiBaseCodeTable(): Table[char, MBCodec] {.compileTime.} =
result = initTable[char, MBCodec]()
for item in MultibaseCodecs:
for item in MultiBaseCodecs:
result[item.code] = item
proc initMultiBaseNameTable(): Table[string, MBCodec] {.compileTime.} =
result = initTable[string, MBCodec]()
for item in MultibaseCodecs:
for item in MultiBaseCodecs:
result[item.name] = item
const
CodeMultibases = initMultiBaseCodeTable()
NameMultibases = initMultiBaseNameTable()
CodeMultiBases = initMultiBaseCodeTable()
NameMultiBases = initMultiBaseNameTable()
proc encodedLength*(mbtype: typedesc[MultiBase], encoding: string,
length: int): int =
proc encodedLength*(mbtype: typedesc[MultiBase], encoding: string, length: int): int =
## Return estimated size of buffer to store MultiBase encoded value with
## encoding ``encoding`` of length ``length``.
##
## Procedure returns ``-1`` if ``encoding`` scheme is not supported or
## not present.
let mb = NameMultibases.getOrDefault(encoding)
let mb = NameMultiBases.getOrDefault(encoding)
if len(mb.name) == 0 or isNil(mb.encl):
result = -1
else:
@@ -357,11 +402,10 @@ proc encodedLength*(mbtype: typedesc[MultiBase], encoding: string,
else:
result = mb.encl(length) + 1
proc decodedLength*(mbtype: typedesc[MultiBase], encoding: char,
length: int): int =
proc decodedLength*(mbtype: typedesc[MultiBase], encoding: char, length: int): int =
## Return estimated size of buffer to store MultiBase decoded value with
## encoding character ``encoding`` of length ``length``.
let mb = CodeMultibases.getOrDefault(encoding)
let mb = CodeMultiBases.getOrDefault(encoding)
if len(mb.name) == 0 or isNil(mb.decl) or length == 0:
result = -1
else:
@@ -370,9 +414,13 @@ proc decodedLength*(mbtype: typedesc[MultiBase], encoding: char,
else:
result = mb.decl(length - 1)
proc encode*(mbtype: typedesc[MultiBase], encoding: string,
inbytes: openarray[byte], outbytes: var openarray[char],
outlen: var int): MultibaseStatus =
proc encode*(
mbtype: typedesc[MultiBase],
encoding: string,
inbytes: openArray[byte],
outbytes: var openArray[char],
outlen: var int,
): MultiBaseStatus =
## Encode array ``inbytes`` using MultiBase encoding scheme ``encoding`` and
## store encoded value to ``outbytes``.
##
@@ -388,14 +436,13 @@ proc encode*(mbtype: typedesc[MultiBase], encoding: string,
##
## On successfull encoding ``MultiBaseStatus.Success`` will be returned and
## ``outlen`` will be set to number of encoded octets (bytes).
let mb = NameMultibases.getOrDefault(encoding)
let mb = NameMultiBases.getOrDefault(encoding)
if len(mb.name) == 0:
return MultibaseStatus.BadCodec
return MultiBaseStatus.BadCodec
if isNil(mb.encr) or isNil(mb.encl):
return MultibaseStatus.NotSupported
return MultiBaseStatus.NotSupported
if len(outbytes) > 1:
result = mb.encr(inbytes, outbytes.toOpenArray(1, outbytes.high),
outlen)
result = mb.encr(inbytes, outbytes.toOpenArray(1, outbytes.high), outlen)
if result == MultiBaseStatus.Overrun:
outlen += 1
elif result == MultiBaseStatus.Success:
@@ -410,8 +457,12 @@ proc encode*(mbtype: typedesc[MultiBase], encoding: string,
result = MultiBaseStatus.Overrun
outlen = mb.encl(len(inbytes)) + 1
proc decode*(mbtype: typedesc[MultiBase], inbytes: openarray[char],
outbytes: var openarray[byte], outlen: var int): MultibaseStatus =
proc decode*(
mbtype: typedesc[MultiBase],
inbytes: openArray[char],
outbytes: var openArray[byte],
outlen: var int,
): MultiBaseStatus =
## Decode array ``inbytes`` using MultiBase encoding and store decoded value
## to ``outbytes``.
##
@@ -428,24 +479,25 @@ proc decode*(mbtype: typedesc[MultiBase], inbytes: openarray[char],
## ``outlen`` will be set to number of encoded octets (bytes).
let length = len(inbytes)
if length == 0:
return MultibaseStatus.Incorrect
let mb = CodeMultibases.getOrDefault(inbytes[0])
return MultiBaseStatus.Incorrect
let mb = CodeMultiBases.getOrDefault(inbytes[0])
if len(mb.name) == 0:
return MultibaseStatus.BadCodec
return MultiBaseStatus.BadCodec
if isNil(mb.decr) or isNil(mb.decl):
return MultibaseStatus.NotSupported
return MultiBaseStatus.NotSupported
if length == 1:
outlen = 0
result = MultibaseStatus.Success
result = MultiBaseStatus.Success
else:
result = mb.decr(inbytes.toOpenArray(1, length - 1), outbytes, outlen)
proc encode*(mbtype: typedesc[MultiBase], encoding: string,
inbytes: openarray[byte]): Result[string, string] =
proc encode*(
mbtype: typedesc[MultiBase], encoding: string, inbytes: openArray[byte]
): Result[string, string] =
## Encode array ``inbytes`` using MultiBase encoding scheme ``encoding`` and
## return encoded string.
let length = len(inbytes)
let mb = NameMultibases.getOrDefault(encoding)
let mb = NameMultiBases.getOrDefault(encoding)
if len(mb.name) == 0:
return err("multibase: Encoding scheme is incorrect!")
if isNil(mb.encr) or isNil(mb.encl):
@@ -464,13 +516,15 @@ proc encode*(mbtype: typedesc[MultiBase], encoding: string,
buffer[0] = mb.code
ok(buffer)
proc decode*(mbtype: typedesc[MultiBase], inbytes: openarray[char]): Result[seq[byte], string] =
proc decode*(
mbtype: typedesc[MultiBase], inbytes: openArray[char]
): Result[seq[byte], string] =
## Decode MultiBase encoded array ``inbytes`` and return decoded sequence of
## bytes.
let length = len(inbytes)
if length == 0:
return err("multibase: Could not decode zero-length string")
let mb = CodeMultibases.getOrDefault(inbytes[0])
let mb = CodeMultiBases.getOrDefault(inbytes[0])
if len(mb.name) == 0:
return err("multibase: MultiBase scheme is incorrect!")
if isNil(mb.decr) or isNil(mb.decl):
@@ -481,8 +535,7 @@ proc decode*(mbtype: typedesc[MultiBase], inbytes: openarray[char]): Result[seq[
else:
var buffer = newSeq[byte](mb.decl(length - 1))
var outlen = 0
let res = mb.decr(inbytes.toOpenArray(1, length - 1),
buffer, outlen)
let res = mb.decr(inbytes.toOpenArray(1, length - 1), buffer, outlen)
if res != MultiBaseStatus.Success:
err("multibase: Decoding error [" & $res & "]")
else:

View File

@@ -1,23 +1,21 @@
## Nim-Libp2p
## Copyright (c) 2018 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not BE copied, modified, or distributed except according to
## those terms.
# Nim-Libp2p
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not BE copied, modified, or distributed except according to
# those terms.
## This module implements MultiCodec.
{.push raises: [Defect].}
{.push raises: [].}
import tables, hashes
import varint, vbuffer
import stew/results
import vbuffer
import results
export results
{.deadCodeElim: on.}
## List of officially supported codecs can BE found here
## https://github.com/multiformats/multicodec/blob/master/table.csv
const MultiCodecList = [
@@ -51,132 +49,325 @@ const MultiCodecList = [
("keccak-384", 0x1C),
("keccak-512", 0x1D),
("murmur3", 0x22),
("blake2b-8", 0xB201), ("blake2b-16", 0xB202), ("blake2b-24", 0xB203),
("blake2b-32", 0xB204), ("blake2b-40", 0xB205), ("blake2b-48", 0xB206),
("blake2b-56", 0xB207), ("blake2b-64", 0xB208), ("blake2b-72", 0xB209),
("blake2b-80", 0xB20A), ("blake2b-88", 0xB20B), ("blake2b-96", 0xB20C),
("blake2b-104", 0xB20D), ("blake2b-112", 0xB20E), ("blake2b-120", 0xB20F),
("blake2b-128", 0xB210), ("blake2b-136", 0xB211), ("blake2b-144", 0xB212),
("blake2b-152", 0xB213), ("blake2b-160", 0xB214), ("blake2b-168", 0xB215),
("blake2b-176", 0xB216), ("blake2b-184", 0xB217), ("blake2b-192", 0xB218),
("blake2b-200", 0xB219), ("blake2b-208", 0xB21A), ("blake2b-216", 0xB21B),
("blake2b-224", 0xB21C), ("blake2b-232", 0xB21D), ("blake2b-240", 0xB21E),
("blake2b-248", 0xB21F), ("blake2b-256", 0xB220), ("blake2b-264", 0xB221),
("blake2b-272", 0xB222), ("blake2b-280", 0xB223), ("blake2b-288", 0xB224),
("blake2b-296", 0xB225), ("blake2b-304", 0xB226), ("blake2b-312", 0xB227),
("blake2b-320", 0xB228), ("blake2b-328", 0xB229), ("blake2b-336", 0xB22A),
("blake2b-344", 0xB22B), ("blake2b-352", 0xB22C), ("blake2b-360", 0xB22D),
("blake2b-368", 0xB22E), ("blake2b-376", 0xB22F), ("blake2b-384", 0xB230),
("blake2b-392", 0xB231), ("blake2b-400", 0xB232), ("blake2b-408", 0xB233),
("blake2b-416", 0xB234), ("blake2b-424", 0xB235), ("blake2b-432", 0xB236),
("blake2b-440", 0xB237), ("blake2b-448", 0xB238), ("blake2b-456", 0xB239),
("blake2b-464", 0xB23A), ("blake2b-472", 0xB23B), ("blake2b-480", 0xB23C),
("blake2b-488", 0xB23D), ("blake2b-496", 0xB23E), ("blake2b-504", 0xB23F),
("blake2b-512", 0xB240), ("blake2s-8", 0xB241), ("blake2s-16", 0xB242),
("blake2s-24", 0xB243), ("blake2s-32", 0xB244), ("blake2s-40", 0xB245),
("blake2s-48", 0xB246), ("blake2s-56", 0xB247), ("blake2s-64", 0xB248),
("blake2s-72", 0xB249), ("blake2s-80", 0xB24A), ("blake2s-88", 0xB24B),
("blake2s-96", 0xB24C), ("blake2s-104", 0xB24D), ("blake2s-112", 0xB24E),
("blake2s-120", 0xB24F), ("blake2s-128", 0xB250), ("blake2s-136", 0xB251),
("blake2s-144", 0xB252), ("blake2s-152", 0xB253), ("blake2s-160", 0xB254),
("blake2s-168", 0xB255), ("blake2s-176", 0xB256), ("blake2s-184", 0xB257),
("blake2s-192", 0xB258), ("blake2s-200", 0xB259), ("blake2s-208", 0xB25A),
("blake2s-216", 0xB25B), ("blake2s-224", 0xB25C), ("blake2s-232", 0xB25D),
("blake2s-240", 0xB25E), ("blake2s-248", 0xB25F), ("blake2s-256", 0xB260),
("skein256-8", 0xB301), ("skein256-16", 0xB302), ("skein256-24", 0xB303),
("skein256-32", 0xB304), ("skein256-40", 0xB305), ("skein256-48", 0xB306),
("skein256-56", 0xB307), ("skein256-64", 0xB308), ("skein256-72", 0xB309),
("skein256-80", 0xB30A), ("skein256-88", 0xB30B), ("skein256-96", 0xB30C),
("skein256-104", 0xB30D), ("skein256-112", 0xB30E), ("skein256-120", 0xB30F),
("skein256-128", 0xB310), ("skein256-136", 0xB311), ("skein256-144", 0xB312),
("skein256-152", 0xB313), ("skein256-160", 0xB314), ("skein256-168", 0xB315),
("skein256-176", 0xB316), ("skein256-184", 0xB317), ("skein256-192", 0xB318),
("skein256-200", 0xB319), ("skein256-208", 0xB31A), ("skein256-216", 0xB31B),
("skein256-224", 0xB31C), ("skein256-232", 0xB31D), ("skein256-240", 0xB31E),
("skein256-248", 0xB31F), ("skein256-256", 0xB320),
("skein512-8", 0xB321), ("skein512-16", 0xB322), ("skein512-24", 0xB323),
("skein512-32", 0xB324), ("skein512-40", 0xB325), ("skein512-48", 0xB326),
("skein512-56", 0xB327), ("skein512-64", 0xB328), ("skein512-72", 0xB329),
("skein512-80", 0xB32A), ("skein512-88", 0xB32B), ("skein512-96", 0xB32C),
("skein512-104", 0xB32D), ("skein512-112", 0xB32E), ("skein512-120", 0xB32F),
("skein512-128", 0xB330), ("skein512-136", 0xB331), ("skein512-144", 0xB332),
("skein512-152", 0xB333), ("skein512-160", 0xB334), ("skein512-168", 0xB335),
("skein512-176", 0xB336), ("skein512-184", 0xB337), ("skein512-192", 0xB338),
("skein512-200", 0xB339), ("skein512-208", 0xB33A), ("skein512-216", 0xB33B),
("skein512-224", 0xB33C), ("skein512-232", 0xB33D), ("skein512-240", 0xB33E),
("skein512-248", 0xB33F), ("skein512-256", 0xB340), ("skein512-264", 0xB341),
("skein512-272", 0xB342), ("skein512-280", 0xB343), ("skein512-288", 0xB344),
("skein512-296", 0xB345), ("skein512-304", 0xB346), ("skein512-312", 0xB347),
("skein512-320", 0xB348), ("skein512-328", 0xB349), ("skein512-336", 0xB34A),
("skein512-344", 0xB34B), ("skein512-352", 0xB34C), ("skein512-360", 0xB34D),
("skein512-368", 0xB34E), ("skein512-376", 0xB34F), ("skein512-384", 0xB350),
("skein512-392", 0xB351), ("skein512-400", 0xB352), ("skein512-408", 0xB353),
("skein512-416", 0xB354), ("skein512-424", 0xB355), ("skein512-432", 0xB356),
("skein512-440", 0xB357), ("skein512-448", 0xB358), ("skein512-456", 0xB359),
("skein512-464", 0xB35A), ("skein512-472", 0xB35B), ("skein512-480", 0xB35C),
("skein512-488", 0xB35D), ("skein512-496", 0xB35E), ("skein512-504", 0xB35F),
("skein512-512", 0xB360), ("skein1024-8", 0xB361), ("skein1024-16", 0xB362),
("skein1024-24", 0xB363), ("skein1024-32", 0xB364), ("skein1024-40", 0xB365),
("skein1024-48", 0xB366), ("skein1024-56", 0xB367), ("skein1024-64", 0xB368),
("skein1024-72", 0xB369), ("skein1024-80", 0xB36A), ("skein1024-88", 0xB36B),
("skein1024-96", 0xB36C), ("skein1024-104", 0xB36D),
("skein1024-112", 0xB36E), ("skein1024-120", 0xB36F),
("skein1024-128", 0xB370), ("skein1024-136", 0xB371),
("skein1024-144", 0xB372), ("skein1024-152", 0xB373),
("skein1024-160", 0xB374), ("skein1024-168", 0xB375),
("skein1024-176", 0xB376), ("skein1024-184", 0xB377),
("skein1024-192", 0xB378), ("skein1024-200", 0xB379),
("skein1024-208", 0xB37A), ("skein1024-216", 0xB37B),
("skein1024-224", 0xB37C), ("skein1024-232", 0xB37D),
("skein1024-240", 0xB37E), ("skein1024-248", 0xB37F),
("skein1024-256", 0xB380), ("skein1024-264", 0xB381),
("skein1024-272", 0xB382), ("skein1024-280", 0xB383),
("skein1024-288", 0xB384), ("skein1024-296", 0xB385),
("skein1024-304", 0xB386), ("skein1024-312", 0xB387),
("skein1024-320", 0xB388), ("skein1024-328", 0xB389),
("skein1024-336", 0xB38A), ("skein1024-344", 0xB38B),
("skein1024-352", 0xB38C), ("skein1024-360", 0xB38D),
("skein1024-368", 0xB38E), ("skein1024-376", 0xB38F),
("skein1024-384", 0xB390), ("skein1024-392", 0xB391),
("skein1024-400", 0xB392), ("skein1024-408", 0xB393),
("skein1024-416", 0xB394), ("skein1024-424", 0xB395),
("skein1024-432", 0xB396), ("skein1024-440", 0xB397),
("skein1024-448", 0xB398), ("skein1024-456", 0xB399),
("skein1024-464", 0xB39A), ("skein1024-472", 0xB39B),
("skein1024-480", 0xB39C), ("skein1024-488", 0xB39D),
("skein1024-496", 0xB39E), ("skein1024-504", 0xB39F),
("skein1024-512", 0xB3A0), ("skein1024-520", 0xB3A1),
("skein1024-528", 0xB3A2), ("skein1024-536", 0xB3A3),
("skein1024-544", 0xB3A4), ("skein1024-552", 0xB3A5),
("skein1024-560", 0xB3A6), ("skein1024-568", 0xB3A7),
("skein1024-576", 0xB3A8), ("skein1024-584", 0xB3A9),
("skein1024-592", 0xB3AA), ("skein1024-600", 0xB3AB),
("skein1024-608", 0xB3AC), ("skein1024-616", 0xB3AD),
("skein1024-624", 0xB3AE), ("skein1024-632", 0xB3AF),
("skein1024-640", 0xB3B0), ("skein1024-648", 0xB3B1),
("skein1024-656", 0xB3B2), ("skein1024-664", 0xB3B3),
("skein1024-672", 0xB3B4), ("skein1024-680", 0xB3B5),
("skein1024-688", 0xB3B6), ("skein1024-696", 0xB3B7),
("skein1024-704", 0xB3B8), ("skein1024-712", 0xB3B9),
("skein1024-720", 0xB3BA), ("skein1024-728", 0xB3BB),
("skein1024-736", 0xB3BC), ("skein1024-744", 0xB3BD),
("skein1024-752", 0xB3BE), ("skein1024-760", 0xB3BF),
("skein1024-768", 0xB3C0), ("skein1024-776", 0xB3C1),
("skein1024-784", 0xB3C2), ("skein1024-792", 0xB3C3),
("skein1024-800", 0xB3C4), ("skein1024-808", 0xB3C5),
("skein1024-816", 0xB3C6), ("skein1024-824", 0xB3C7),
("skein1024-832", 0xB3C8), ("skein1024-840", 0xB3C9),
("skein1024-848", 0xB3CA), ("skein1024-856", 0xB3CB),
("skein1024-864", 0xB3CC), ("skein1024-872", 0xB3CD),
("skein1024-880", 0xB3CE), ("skein1024-888", 0xB3CF),
("skein1024-896", 0xB3D0), ("skein1024-904", 0xB3D1),
("skein1024-912", 0xB3D2), ("skein1024-920", 0xB3D3),
("skein1024-928", 0xB3D4), ("skein1024-936", 0xB3D5),
("skein1024-944", 0xB3D6), ("skein1024-952", 0xB3D7),
("skein1024-960", 0xB3D8), ("skein1024-968", 0xB3D9),
("skein1024-976", 0xB3DA), ("skein1024-984", 0xB3DB),
("skein1024-992", 0xB3DC), ("skein1024-1000", 0xB3DD),
("skein1024-1008", 0xB3DE), ("skein1024-1016", 0xB3DF),
("blake2b-8", 0xB201),
("blake2b-16", 0xB202),
("blake2b-24", 0xB203),
("blake2b-32", 0xB204),
("blake2b-40", 0xB205),
("blake2b-48", 0xB206),
("blake2b-56", 0xB207),
("blake2b-64", 0xB208),
("blake2b-72", 0xB209),
("blake2b-80", 0xB20A),
("blake2b-88", 0xB20B),
("blake2b-96", 0xB20C),
("blake2b-104", 0xB20D),
("blake2b-112", 0xB20E),
("blake2b-120", 0xB20F),
("blake2b-128", 0xB210),
("blake2b-136", 0xB211),
("blake2b-144", 0xB212),
("blake2b-152", 0xB213),
("blake2b-160", 0xB214),
("blake2b-168", 0xB215),
("blake2b-176", 0xB216),
("blake2b-184", 0xB217),
("blake2b-192", 0xB218),
("blake2b-200", 0xB219),
("blake2b-208", 0xB21A),
("blake2b-216", 0xB21B),
("blake2b-224", 0xB21C),
("blake2b-232", 0xB21D),
("blake2b-240", 0xB21E),
("blake2b-248", 0xB21F),
("blake2b-256", 0xB220),
("blake2b-264", 0xB221),
("blake2b-272", 0xB222),
("blake2b-280", 0xB223),
("blake2b-288", 0xB224),
("blake2b-296", 0xB225),
("blake2b-304", 0xB226),
("blake2b-312", 0xB227),
("blake2b-320", 0xB228),
("blake2b-328", 0xB229),
("blake2b-336", 0xB22A),
("blake2b-344", 0xB22B),
("blake2b-352", 0xB22C),
("blake2b-360", 0xB22D),
("blake2b-368", 0xB22E),
("blake2b-376", 0xB22F),
("blake2b-384", 0xB230),
("blake2b-392", 0xB231),
("blake2b-400", 0xB232),
("blake2b-408", 0xB233),
("blake2b-416", 0xB234),
("blake2b-424", 0xB235),
("blake2b-432", 0xB236),
("blake2b-440", 0xB237),
("blake2b-448", 0xB238),
("blake2b-456", 0xB239),
("blake2b-464", 0xB23A),
("blake2b-472", 0xB23B),
("blake2b-480", 0xB23C),
("blake2b-488", 0xB23D),
("blake2b-496", 0xB23E),
("blake2b-504", 0xB23F),
("blake2b-512", 0xB240),
("blake2s-8", 0xB241),
("blake2s-16", 0xB242),
("blake2s-24", 0xB243),
("blake2s-32", 0xB244),
("blake2s-40", 0xB245),
("blake2s-48", 0xB246),
("blake2s-56", 0xB247),
("blake2s-64", 0xB248),
("blake2s-72", 0xB249),
("blake2s-80", 0xB24A),
("blake2s-88", 0xB24B),
("blake2s-96", 0xB24C),
("blake2s-104", 0xB24D),
("blake2s-112", 0xB24E),
("blake2s-120", 0xB24F),
("blake2s-128", 0xB250),
("blake2s-136", 0xB251),
("blake2s-144", 0xB252),
("blake2s-152", 0xB253),
("blake2s-160", 0xB254),
("blake2s-168", 0xB255),
("blake2s-176", 0xB256),
("blake2s-184", 0xB257),
("blake2s-192", 0xB258),
("blake2s-200", 0xB259),
("blake2s-208", 0xB25A),
("blake2s-216", 0xB25B),
("blake2s-224", 0xB25C),
("blake2s-232", 0xB25D),
("blake2s-240", 0xB25E),
("blake2s-248", 0xB25F),
("blake2s-256", 0xB260),
("skein256-8", 0xB301),
("skein256-16", 0xB302),
("skein256-24", 0xB303),
("skein256-32", 0xB304),
("skein256-40", 0xB305),
("skein256-48", 0xB306),
("skein256-56", 0xB307),
("skein256-64", 0xB308),
("skein256-72", 0xB309),
("skein256-80", 0xB30A),
("skein256-88", 0xB30B),
("skein256-96", 0xB30C),
("skein256-104", 0xB30D),
("skein256-112", 0xB30E),
("skein256-120", 0xB30F),
("skein256-128", 0xB310),
("skein256-136", 0xB311),
("skein256-144", 0xB312),
("skein256-152", 0xB313),
("skein256-160", 0xB314),
("skein256-168", 0xB315),
("skein256-176", 0xB316),
("skein256-184", 0xB317),
("skein256-192", 0xB318),
("skein256-200", 0xB319),
("skein256-208", 0xB31A),
("skein256-216", 0xB31B),
("skein256-224", 0xB31C),
("skein256-232", 0xB31D),
("skein256-240", 0xB31E),
("skein256-248", 0xB31F),
("skein256-256", 0xB320),
("skein512-8", 0xB321),
("skein512-16", 0xB322),
("skein512-24", 0xB323),
("skein512-32", 0xB324),
("skein512-40", 0xB325),
("skein512-48", 0xB326),
("skein512-56", 0xB327),
("skein512-64", 0xB328),
("skein512-72", 0xB329),
("skein512-80", 0xB32A),
("skein512-88", 0xB32B),
("skein512-96", 0xB32C),
("skein512-104", 0xB32D),
("skein512-112", 0xB32E),
("skein512-120", 0xB32F),
("skein512-128", 0xB330),
("skein512-136", 0xB331),
("skein512-144", 0xB332),
("skein512-152", 0xB333),
("skein512-160", 0xB334),
("skein512-168", 0xB335),
("skein512-176", 0xB336),
("skein512-184", 0xB337),
("skein512-192", 0xB338),
("skein512-200", 0xB339),
("skein512-208", 0xB33A),
("skein512-216", 0xB33B),
("skein512-224", 0xB33C),
("skein512-232", 0xB33D),
("skein512-240", 0xB33E),
("skein512-248", 0xB33F),
("skein512-256", 0xB340),
("skein512-264", 0xB341),
("skein512-272", 0xB342),
("skein512-280", 0xB343),
("skein512-288", 0xB344),
("skein512-296", 0xB345),
("skein512-304", 0xB346),
("skein512-312", 0xB347),
("skein512-320", 0xB348),
("skein512-328", 0xB349),
("skein512-336", 0xB34A),
("skein512-344", 0xB34B),
("skein512-352", 0xB34C),
("skein512-360", 0xB34D),
("skein512-368", 0xB34E),
("skein512-376", 0xB34F),
("skein512-384", 0xB350),
("skein512-392", 0xB351),
("skein512-400", 0xB352),
("skein512-408", 0xB353),
("skein512-416", 0xB354),
("skein512-424", 0xB355),
("skein512-432", 0xB356),
("skein512-440", 0xB357),
("skein512-448", 0xB358),
("skein512-456", 0xB359),
("skein512-464", 0xB35A),
("skein512-472", 0xB35B),
("skein512-480", 0xB35C),
("skein512-488", 0xB35D),
("skein512-496", 0xB35E),
("skein512-504", 0xB35F),
("skein512-512", 0xB360),
("skein1024-8", 0xB361),
("skein1024-16", 0xB362),
("skein1024-24", 0xB363),
("skein1024-32", 0xB364),
("skein1024-40", 0xB365),
("skein1024-48", 0xB366),
("skein1024-56", 0xB367),
("skein1024-64", 0xB368),
("skein1024-72", 0xB369),
("skein1024-80", 0xB36A),
("skein1024-88", 0xB36B),
("skein1024-96", 0xB36C),
("skein1024-104", 0xB36D),
("skein1024-112", 0xB36E),
("skein1024-120", 0xB36F),
("skein1024-128", 0xB370),
("skein1024-136", 0xB371),
("skein1024-144", 0xB372),
("skein1024-152", 0xB373),
("skein1024-160", 0xB374),
("skein1024-168", 0xB375),
("skein1024-176", 0xB376),
("skein1024-184", 0xB377),
("skein1024-192", 0xB378),
("skein1024-200", 0xB379),
("skein1024-208", 0xB37A),
("skein1024-216", 0xB37B),
("skein1024-224", 0xB37C),
("skein1024-232", 0xB37D),
("skein1024-240", 0xB37E),
("skein1024-248", 0xB37F),
("skein1024-256", 0xB380),
("skein1024-264", 0xB381),
("skein1024-272", 0xB382),
("skein1024-280", 0xB383),
("skein1024-288", 0xB384),
("skein1024-296", 0xB385),
("skein1024-304", 0xB386),
("skein1024-312", 0xB387),
("skein1024-320", 0xB388),
("skein1024-328", 0xB389),
("skein1024-336", 0xB38A),
("skein1024-344", 0xB38B),
("skein1024-352", 0xB38C),
("skein1024-360", 0xB38D),
("skein1024-368", 0xB38E),
("skein1024-376", 0xB38F),
("skein1024-384", 0xB390),
("skein1024-392", 0xB391),
("skein1024-400", 0xB392),
("skein1024-408", 0xB393),
("skein1024-416", 0xB394),
("skein1024-424", 0xB395),
("skein1024-432", 0xB396),
("skein1024-440", 0xB397),
("skein1024-448", 0xB398),
("skein1024-456", 0xB399),
("skein1024-464", 0xB39A),
("skein1024-472", 0xB39B),
("skein1024-480", 0xB39C),
("skein1024-488", 0xB39D),
("skein1024-496", 0xB39E),
("skein1024-504", 0xB39F),
("skein1024-512", 0xB3A0),
("skein1024-520", 0xB3A1),
("skein1024-528", 0xB3A2),
("skein1024-536", 0xB3A3),
("skein1024-544", 0xB3A4),
("skein1024-552", 0xB3A5),
("skein1024-560", 0xB3A6),
("skein1024-568", 0xB3A7),
("skein1024-576", 0xB3A8),
("skein1024-584", 0xB3A9),
("skein1024-592", 0xB3AA),
("skein1024-600", 0xB3AB),
("skein1024-608", 0xB3AC),
("skein1024-616", 0xB3AD),
("skein1024-624", 0xB3AE),
("skein1024-632", 0xB3AF),
("skein1024-640", 0xB3B0),
("skein1024-648", 0xB3B1),
("skein1024-656", 0xB3B2),
("skein1024-664", 0xB3B3),
("skein1024-672", 0xB3B4),
("skein1024-680", 0xB3B5),
("skein1024-688", 0xB3B6),
("skein1024-696", 0xB3B7),
("skein1024-704", 0xB3B8),
("skein1024-712", 0xB3B9),
("skein1024-720", 0xB3BA),
("skein1024-728", 0xB3BB),
("skein1024-736", 0xB3BC),
("skein1024-744", 0xB3BD),
("skein1024-752", 0xB3BE),
("skein1024-760", 0xB3BF),
("skein1024-768", 0xB3C0),
("skein1024-776", 0xB3C1),
("skein1024-784", 0xB3C2),
("skein1024-792", 0xB3C3),
("skein1024-800", 0xB3C4),
("skein1024-808", 0xB3C5),
("skein1024-816", 0xB3C6),
("skein1024-824", 0xB3C7),
("skein1024-832", 0xB3C8),
("skein1024-840", 0xB3C9),
("skein1024-848", 0xB3CA),
("skein1024-856", 0xB3CB),
("skein1024-864", 0xB3CC),
("skein1024-872", 0xB3CD),
("skein1024-880", 0xB3CE),
("skein1024-888", 0xB3CF),
("skein1024-896", 0xB3D0),
("skein1024-904", 0xB3D1),
("skein1024-912", 0xB3D2),
("skein1024-920", 0xB3D3),
("skein1024-928", 0xB3D4),
("skein1024-936", 0xB3D5),
("skein1024-944", 0xB3D6),
("skein1024-952", 0xB3D7),
("skein1024-960", 0xB3D8),
("skein1024-968", 0xB3D9),
("skein1024-976", 0xB3DA),
("skein1024-984", 0xB3DB),
("skein1024-992", 0xB3DC),
("skein1024-1000", 0xB3DD),
("skein1024-1008", 0xB3DE),
("skein1024-1016", 0xB3DF),
("skein1024-1024", 0xB3E0),
# multiaddrs
("ip4", 0x04),
@@ -193,20 +384,27 @@ const MultiCodecList = [
("p2p", 0x01A5),
("http", 0x01E0),
("https", 0x01BB),
("tls", 0x01C0),
("quic", 0x01CC),
("quic-v1", 0x01CD),
("ws", 0x01DD),
("wss", 0x01DE), # not in multicodec list
("wss", 0x01DE),
("p2p-websocket-star", 0x01DF), # not in multicodec list
("p2p-webrtc-star", 0x0113), # not in multicodec list
("p2p-webrtc-direct", 0x0114), # not in multicodec list
("onion", 0x01BC),
("onion3", 0x01BD),
("p2p-circuit", 0x0122),
("libp2p-peer-record", 0x0301),
("memory", 0x0309),
("dns", 0x35),
("dns4", 0x36),
("dns6", 0x37),
("dnsaddr", 0x38),
# IPLD formats
("dag-pb", 0x70),
("dag-cbor", 0x71),
("libp2p-key", 0x72),
("dag-json", 0x129),
("git-raw", 0x78),
("eth-block", 0x90),
@@ -230,7 +428,7 @@ const MultiCodecList = [
("dash-tx", 0xF1),
("torrent-info", 0x7B),
("torrent-file", 0x7C),
("ed25519-pub", 0xED)
("ed25519-pub", 0xED),
]
type
@@ -238,16 +436,13 @@ type
MultiCodecError* = enum
MultiCodecNotSupported
const
InvalidMultiCodec* = MultiCodec(-1)
const InvalidMultiCodec* = MultiCodec(-1)
proc initMultiCodecNameTable(): Table[string, int] {.compileTime.} =
result = initTable[string, int]()
for item in MultiCodecList:
result[item[0]] = item[1]
proc initMultiCodecCodeTable(): Table[int, string] {.compileTime.} =
result = initTable[int, string]()
for item in MultiCodecList:
result[item[1]] = item[0]
@@ -271,7 +466,7 @@ proc `$`*(mc: MultiCodec): string =
## Returns string representation of MultiCodec ``mc``.
let name = CodeCodecs.getOrDefault(int(mc), "")
doAssert(name != "")
name
name
proc `==`*(mc: MultiCodec, name: string): bool {.inline.} =
## Compares MultiCodec ``mc`` with string ``name``.
@@ -288,10 +483,6 @@ proc `==`*(a, b: MultiCodec): bool =
## Returns ``true`` if MultiCodecs ``a`` and ``b`` are equal.
int(a) == int(b)
proc `!=`*(a, b: MultiCodec): bool =
## Returns ``true`` if MultiCodecs ``a`` and ``b`` are not equal.
int(a) != int(b)
proc hash*(m: MultiCodec): Hash {.inline.} =
## Hash procedure for tables.
hash(int(m))

View File

@@ -1,11 +1,11 @@
## Nim-Libp2p
## Copyright (c) 2018 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-Libp2p
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
## This module implements MultiHash.
## Supported hashes are:
@@ -21,13 +21,13 @@
## 1. SKEIN
## 2. MURMUR
{.push raises: [Defect].}
{.push raises: [].}
import tables
import nimcrypto/[sha, sha2, keccak, blake2, hash, utils]
import varint, vbuffer, multicodec, multibase
import stew/base58
import stew/results
import results
export results
# This is workaround for Nim `import` bug.
export sha, sha2, keccak, blake2, hash, utils
@@ -41,8 +41,9 @@ const
ErrParseError = "Parse error fromHex"
type
MHashCoderProc* = proc(data: openarray[byte],
output: var openarray[byte]) {.nimcall, gcsafe, noSideEffect, raises: [Defect].}
MHashCoderProc* = proc(data: openArray[byte], output: var openArray[byte]) {.
nimcall, gcsafe, noSideEffect, raises: []
.}
MHash* = object
mcodec*: MultiCodec
size*: int
@@ -56,112 +57,157 @@ type
MhResult*[T] = Result[T, cstring]
proc identhash(data: openarray[byte], output: var openarray[byte]) =
proc identhash(data: openArray[byte], output: var openArray[byte]) =
if len(output) > 0:
var length = if len(data) > len(output): len(output)
else: len(data)
var length =
if len(data) > len(output):
len(output)
else:
len(data)
copyMem(addr output[0], unsafeAddr data[0], length)
proc sha1hash(data: openarray[byte], output: var openarray[byte]) =
proc sha1hash(data: openArray[byte], output: var openArray[byte]) =
if len(output) > 0:
var digest = sha1.digest(data)
var length = if sha1.sizeDigest > len(output): len(output)
else: sha1.sizeDigest
var length =
if sha1.sizeDigest > len(output):
len(output)
else:
sha1.sizeDigest
copyMem(addr output[0], addr digest.data[0], length)
proc dblsha2_256hash(data: openarray[byte], output: var openarray[byte]) =
proc dblsha2_256hash(data: openArray[byte], output: var openArray[byte]) =
if len(output) > 0:
var digest1 = sha256.digest(data)
var digest2 = sha256.digest(digest1.data)
var length = if sha256.sizeDigest > len(output): len(output)
else: sha256.sizeDigest
var length =
if sha256.sizeDigest > len(output):
len(output)
else:
sha256.sizeDigest
copyMem(addr output[0], addr digest2.data[0], length)
proc blake2Bhash(data: openarray[byte], output: var openarray[byte]) =
proc blake2Bhash(data: openArray[byte], output: var openArray[byte]) =
if len(output) > 0:
var digest = blake2_512.digest(data)
var length = if blake2_512.sizeDigest > len(output): len(output)
else: blake2_512.sizeDigest
var length =
if blake2_512.sizeDigest > len(output):
len(output)
else:
blake2_512.sizeDigest
copyMem(addr output[0], addr digest.data[0], length)
proc blake2Shash(data: openarray[byte], output: var openarray[byte]) =
proc blake2Shash(data: openArray[byte], output: var openArray[byte]) =
if len(output) > 0:
var digest = blake2_256.digest(data)
var length = if blake2_256.sizeDigest > len(output): len(output)
else: blake2_256.sizeDigest
var length =
if blake2_256.sizeDigest > len(output):
len(output)
else:
blake2_256.sizeDigest
copyMem(addr output[0], addr digest.data[0], length)
proc sha2_256hash(data: openarray[byte], output: var openarray[byte]) =
proc sha2_256hash(data: openArray[byte], output: var openArray[byte]) =
if len(output) > 0:
var digest = sha256.digest(data)
var length = if sha256.sizeDigest > len(output): len(output)
else: sha256.sizeDigest
var length =
if sha256.sizeDigest > len(output):
len(output)
else:
sha256.sizeDigest
copyMem(addr output[0], addr digest.data[0], length)
proc sha2_512hash(data: openarray[byte], output: var openarray[byte]) =
proc sha2_512hash(data: openArray[byte], output: var openArray[byte]) =
if len(output) > 0:
var digest = sha512.digest(data)
var length = if sha512.sizeDigest > len(output): len(output)
else: sha512.sizeDigest
var length =
if sha512.sizeDigest > len(output):
len(output)
else:
sha512.sizeDigest
copyMem(addr output[0], addr digest.data[0], length)
proc sha3_224hash(data: openarray[byte], output: var openarray[byte]) =
proc sha3_224hash(data: openArray[byte], output: var openArray[byte]) =
if len(output) > 0:
var digest = sha3_224.digest(data)
var length = if sha3_224.sizeDigest > len(output): len(output)
else: sha3_224.sizeDigest
var length =
if sha3_224.sizeDigest > len(output):
len(output)
else:
sha3_224.sizeDigest
copyMem(addr output[0], addr digest.data[0], length)
proc sha3_256hash(data: openarray[byte], output: var openarray[byte]) =
proc sha3_256hash(data: openArray[byte], output: var openArray[byte]) =
if len(output) > 0:
var digest = sha3_256.digest(data)
var length = if sha3_256.sizeDigest > len(output): len(output)
else: sha3_256.sizeDigest
var length =
if sha3_256.sizeDigest > len(output):
len(output)
else:
sha3_256.sizeDigest
copyMem(addr output[0], addr digest.data[0], length)
proc sha3_384hash(data: openarray[byte], output: var openarray[byte]) =
proc sha3_384hash(data: openArray[byte], output: var openArray[byte]) =
if len(output) > 0:
var digest = sha3_384.digest(data)
var length = if sha3_384.sizeDigest > len(output): len(output)
else: sha3_384.sizeDigest
var length =
if sha3_384.sizeDigest > len(output):
len(output)
else:
sha3_384.sizeDigest
copyMem(addr output[0], addr digest.data[0], length)
proc sha3_512hash(data: openarray[byte], output: var openarray[byte]) =
proc sha3_512hash(data: openArray[byte], output: var openArray[byte]) =
if len(output) > 0:
var digest = sha3_512.digest(data)
var length = if sha3_512.sizeDigest > len(output): len(output)
else: sha3_512.sizeDigest
var length =
if sha3_512.sizeDigest > len(output):
len(output)
else:
sha3_512.sizeDigest
copyMem(addr output[0], addr digest.data[0], length)
proc keccak_224hash(data: openarray[byte], output: var openarray[byte]) =
proc keccak_224hash(data: openArray[byte], output: var openArray[byte]) =
if len(output) > 0:
var digest = keccak224.digest(data)
var length = if keccak224.sizeDigest > len(output): len(output)
else: keccak224.sizeDigest
var length =
if keccak224.sizeDigest > len(output):
len(output)
else:
keccak224.sizeDigest
copyMem(addr output[0], addr digest.data[0], length)
proc keccak_256hash(data: openarray[byte], output: var openarray[byte]) =
proc keccak_256hash(data: openArray[byte], output: var openArray[byte]) =
if len(output) > 0:
var digest = keccak256.digest(data)
var length = if keccak256.sizeDigest > len(output): len(output)
else: keccak256.sizeDigest
var length =
if keccak256.sizeDigest > len(output):
len(output)
else:
keccak256.sizeDigest
copyMem(addr output[0], addr digest.data[0], length)
proc keccak_384hash(data: openarray[byte], output: var openarray[byte]) =
proc keccak_384hash(data: openArray[byte], output: var openArray[byte]) =
if len(output) > 0:
var digest = keccak384.digest(data)
var length = if keccak384.sizeDigest > len(output): len(output)
else: keccak384.sizeDigest
var length =
if keccak384.sizeDigest > len(output):
len(output)
else:
keccak384.sizeDigest
copyMem(addr output[0], addr digest.data[0], length)
proc keccak_512hash(data: openarray[byte], output: var openarray[byte]) =
proc keccak_512hash(data: openArray[byte], output: var openArray[byte]) =
if len(output) > 0:
var digest = keccak512.digest(data)
var length = if keccak512.sizeDigest > len(output): len(output)
else: keccak512.sizeDigest
var length =
if keccak512.sizeDigest > len(output):
len(output)
else:
keccak512.sizeDigest
copyMem(addr output[0], addr digest.data[0], length)
proc shake_128hash(data: openarray[byte], output: var openarray[byte]) =
proc shake_128hash(data: openArray[byte], output: var openArray[byte]) =
var sctx: shake128
if len(output) > 0:
sctx.init()
@@ -170,7 +216,7 @@ proc shake_128hash(data: openarray[byte], output: var openarray[byte]) =
discard sctx.output(addr output[0], uint(len(output)))
sctx.clear()
proc shake_256hash(data: openarray[byte], output: var openarray[byte]) =
proc shake_256hash(data: openArray[byte], output: var openArray[byte]) =
var sctx: shake256
if len(output) > 0:
sctx.init()
@@ -179,154 +225,137 @@ proc shake_256hash(data: openarray[byte], output: var openarray[byte]) =
discard sctx.output(addr output[0], uint(len(output)))
sctx.clear()
const
HashesList = [
MHash(mcodec: multiCodec("identity"), size: 0,
coder: identhash),
MHash(mcodec: multiCodec("sha1"), size: sha1.sizeDigest,
coder: sha1hash),
MHash(mcodec: multiCodec("dbl-sha2-256"), size: sha256.sizeDigest,
coder: dblsha2_256hash
),
MHash(mcodec: multiCodec("sha2-256"), size: sha256.sizeDigest,
coder: sha2_256hash
),
MHash(mcodec: multiCodec("sha2-512"), size: sha512.sizeDigest,
coder: sha2_512hash
),
MHash(mcodec: multiCodec("sha3-224"), size: sha3_224.sizeDigest,
coder: sha3_224hash
),
MHash(mcodec: multiCodec("sha3-256"), size: sha3_256.sizeDigest,
coder: sha3_256hash
),
MHash(mcodec: multiCodec("sha3-384"), size: sha3_384.sizeDigest,
coder: sha3_384hash
),
MHash(mcodec: multiCodec("sha3-512"), size: sha3_512.sizeDigest,
coder: sha3_512hash
),
MHash(mcodec: multiCodec("shake-128"), size: 32, coder: shake_128hash),
MHash(mcodec: multiCodec("shake-256"), size: 64, coder: shake_256hash),
MHash(mcodec: multiCodec("keccak-224"), size: keccak_224.sizeDigest,
coder: keccak_224hash
),
MHash(mcodec: multiCodec("keccak-256"), size: keccak_256.sizeDigest,
coder: keccak_256hash
),
MHash(mcodec: multiCodec("keccak-384"), size: keccak_384.sizeDigest,
coder: keccak_384hash
),
MHash(mcodec: multiCodec("keccak-512"), size: keccak_512.sizeDigest,
coder: keccak_512hash
),
MHash(mcodec: multiCodec("blake2b-8"), size: 1, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-16"), size: 2, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-24"), size: 3, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-32"), size: 4, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-40"), size: 5, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-48"), size: 6, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-56"), size: 7, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-64"), size: 8, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-72"), size: 9, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-80"), size: 10, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-88"), size: 11, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-96"), size: 12, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-104"), size: 13, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-112"), size: 14, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-120"), size: 15, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-128"), size: 16, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-136"), size: 17, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-144"), size: 18, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-152"), size: 19, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-160"), size: 20, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-168"), size: 21, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-176"), size: 22, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-184"), size: 23, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-192"), size: 24, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-200"), size: 25, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-208"), size: 26, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-216"), size: 27, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-224"), size: 28, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-232"), size: 29, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-240"), size: 30, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-248"), size: 31, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-256"), size: 32, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-264"), size: 33, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-272"), size: 34, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-280"), size: 35, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-288"), size: 36, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-296"), size: 37, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-304"), size: 38, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-312"), size: 39, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-320"), size: 40, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-328"), size: 41, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-336"), size: 42, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-344"), size: 43, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-352"), size: 44, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-360"), size: 45, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-368"), size: 46, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-376"), size: 47, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-384"), size: 48, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-392"), size: 49, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-400"), size: 50, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-408"), size: 51, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-416"), size: 52, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-424"), size: 53, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-432"), size: 54, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-440"), size: 55, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-448"), size: 56, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-456"), size: 57, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-464"), size: 58, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-472"), size: 59, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-480"), size: 60, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-488"), size: 61, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-496"), size: 62, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-504"), size: 63, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-512"), size: 64, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2s-8"), size: 1, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-16"), size: 2, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-24"), size: 3, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-32"), size: 4, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-40"), size: 5, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-48"), size: 6, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-56"), size: 7, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-64"), size: 8, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-72"), size: 9, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-80"), size: 10, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-88"), size: 11, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-96"), size: 12, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-104"), size: 13, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-112"), size: 14, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-120"), size: 15, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-128"), size: 16, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-136"), size: 17, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-144"), size: 18, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-152"), size: 19, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-160"), size: 20, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-168"), size: 21, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-176"), size: 22, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-184"), size: 23, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-192"), size: 24, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-200"), size: 25, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-208"), size: 26, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-216"), size: 27, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-224"), size: 28, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-232"), size: 29, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-240"), size: 30, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-248"), size: 31, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-256"), size: 32, coder: blake2Shash)
]
const HashesList = [
MHash(mcodec: multiCodec("identity"), size: 0, coder: identhash),
MHash(mcodec: multiCodec("sha1"), size: sha1.sizeDigest, coder: sha1hash),
MHash(
mcodec: multiCodec("dbl-sha2-256"), size: sha256.sizeDigest, coder: dblsha2_256hash
),
MHash(mcodec: multiCodec("sha2-256"), size: sha256.sizeDigest, coder: sha2_256hash),
MHash(mcodec: multiCodec("sha2-512"), size: sha512.sizeDigest, coder: sha2_512hash),
MHash(mcodec: multiCodec("sha3-224"), size: sha3_224.sizeDigest, coder: sha3_224hash),
MHash(mcodec: multiCodec("sha3-256"), size: sha3_256.sizeDigest, coder: sha3_256hash),
MHash(mcodec: multiCodec("sha3-384"), size: sha3_384.sizeDigest, coder: sha3_384hash),
MHash(mcodec: multiCodec("sha3-512"), size: sha3_512.sizeDigest, coder: sha3_512hash),
MHash(mcodec: multiCodec("shake-128"), size: 32, coder: shake_128hash),
MHash(mcodec: multiCodec("shake-256"), size: 64, coder: shake_256hash),
MHash(
mcodec: multiCodec("keccak-224"), size: keccak224.sizeDigest, coder: keccak_224hash
),
MHash(
mcodec: multiCodec("keccak-256"), size: keccak256.sizeDigest, coder: keccak_256hash
),
MHash(
mcodec: multiCodec("keccak-384"), size: keccak384.sizeDigest, coder: keccak_384hash
),
MHash(
mcodec: multiCodec("keccak-512"), size: keccak512.sizeDigest, coder: keccak_512hash
),
MHash(mcodec: multiCodec("blake2b-8"), size: 1, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-16"), size: 2, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-24"), size: 3, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-32"), size: 4, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-40"), size: 5, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-48"), size: 6, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-56"), size: 7, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-64"), size: 8, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-72"), size: 9, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-80"), size: 10, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-88"), size: 11, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-96"), size: 12, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-104"), size: 13, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-112"), size: 14, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-120"), size: 15, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-128"), size: 16, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-136"), size: 17, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-144"), size: 18, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-152"), size: 19, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-160"), size: 20, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-168"), size: 21, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-176"), size: 22, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-184"), size: 23, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-192"), size: 24, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-200"), size: 25, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-208"), size: 26, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-216"), size: 27, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-224"), size: 28, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-232"), size: 29, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-240"), size: 30, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-248"), size: 31, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-256"), size: 32, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-264"), size: 33, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-272"), size: 34, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-280"), size: 35, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-288"), size: 36, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-296"), size: 37, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-304"), size: 38, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-312"), size: 39, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-320"), size: 40, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-328"), size: 41, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-336"), size: 42, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-344"), size: 43, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-352"), size: 44, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-360"), size: 45, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-368"), size: 46, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-376"), size: 47, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-384"), size: 48, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-392"), size: 49, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-400"), size: 50, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-408"), size: 51, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-416"), size: 52, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-424"), size: 53, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-432"), size: 54, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-440"), size: 55, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-448"), size: 56, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-456"), size: 57, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-464"), size: 58, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-472"), size: 59, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-480"), size: 60, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-488"), size: 61, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-496"), size: 62, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-504"), size: 63, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2b-512"), size: 64, coder: blake2Bhash),
MHash(mcodec: multiCodec("blake2s-8"), size: 1, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-16"), size: 2, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-24"), size: 3, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-32"), size: 4, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-40"), size: 5, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-48"), size: 6, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-56"), size: 7, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-64"), size: 8, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-72"), size: 9, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-80"), size: 10, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-88"), size: 11, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-96"), size: 12, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-104"), size: 13, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-112"), size: 14, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-120"), size: 15, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-128"), size: 16, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-136"), size: 17, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-144"), size: 18, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-152"), size: 19, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-160"), size: 20, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-168"), size: 21, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-176"), size: 22, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-184"), size: 23, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-192"), size: 24, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-200"), size: 25, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-208"), size: 26, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-216"), size: 27, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-224"), size: 28, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-232"), size: 29, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-240"), size: 30, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-248"), size: 31, coder: blake2Shash),
MHash(mcodec: multiCodec("blake2s-256"), size: 32, coder: blake2Shash),
]
proc initMultiHashCodeTable(): Table[MultiCodec, MHash] {.compileTime.} =
result = initTable[MultiCodec, MHash]()
for item in HashesList:
result[item.mcodec] = item
const
CodeHashes = initMultiHashCodeTable()
const CodeHashes = initMultiHashCodeTable()
proc digestImplWithHash(hash: MHash, data: openarray[byte]): MultiHash =
proc digestImplWithHash(hash: MHash, data: openArray[byte]): MultiHash =
var buffer: array[MaxHashSize, byte]
result.data = initVBuffer()
result.mcodec = hash.mcodec
@@ -344,7 +373,7 @@ proc digestImplWithHash(hash: MHash, data: openarray[byte]): MultiHash =
result.size = hash.size
result.data.finish()
proc digestImplWithoutHash(hash: MHash, data: openarray[byte]): MultiHash =
proc digestImplWithoutHash(hash: MHash, data: openArray[byte]): MultiHash =
result.data = initVBuffer()
result.mcodec = hash.mcodec
result.size = len(data)
@@ -354,8 +383,9 @@ proc digestImplWithoutHash(hash: MHash, data: openarray[byte]): MultiHash =
result.data.writeArray(data)
result.data.finish()
proc digest*(mhtype: typedesc[MultiHash], hashname: string,
data: openarray[byte]): MhResult[MultiHash] {.inline.} =
proc digest*(
mhtype: typedesc[MultiHash], hashname: string, data: openArray[byte]
): MhResult[MultiHash] {.inline.} =
## Perform digest calculation using hash algorithm with name ``hashname`` on
## data array ``data``.
let mc = MultiCodec.codec(hashname)
@@ -368,8 +398,9 @@ proc digest*(mhtype: typedesc[MultiHash], hashname: string,
else:
ok(digestImplWithHash(hash, data))
proc digest*(mhtype: typedesc[MultiHash], hashcode: int,
data: openarray[byte]): MhResult[MultiHash] {.inline.} =
proc digest*(
mhtype: typedesc[MultiHash], hashcode: int, data: openArray[byte]
): MhResult[MultiHash] {.inline.} =
## Perform digest calculation using hash algorithm with code ``hashcode`` on
## data array ``data``.
let hash = CodeHashes.getOrDefault(hashcode)
@@ -378,8 +409,9 @@ proc digest*(mhtype: typedesc[MultiHash], hashcode: int,
else:
ok(digestImplWithHash(hash, data))
proc init*[T](mhtype: typedesc[MultiHash], hashname: string,
mdigest: MDigest[T]): MhResult[MultiHash] {.inline.} =
proc init*[T](
mhtype: typedesc[MultiHash], hashname: string, mdigest: MDigest[T]
): MhResult[MultiHash] {.inline.} =
## Create MultiHash from nimcrypto's `MDigest` object and hash algorithm name
## ``hashname``.
let mc = MultiCodec.codec(hashname)
@@ -394,8 +426,9 @@ proc init*[T](mhtype: typedesc[MultiHash], hashname: string,
else:
ok(digestImplWithoutHash(hash, mdigest.data))
proc init*[T](mhtype: typedesc[MultiHash], hashcode: MultiCodec,
mdigest: MDigest[T]): MhResult[MultiHash] {.inline.} =
proc init*[T](
mhtype: typedesc[MultiHash], hashcode: MultiCodec, mdigest: MDigest[T]
): MhResult[MultiHash] {.inline.} =
## Create MultiHash from nimcrypto's `MDigest` and hash algorithm code
## ``hashcode``.
let hash = CodeHashes.getOrDefault(hashcode)
@@ -406,8 +439,9 @@ proc init*[T](mhtype: typedesc[MultiHash], hashcode: MultiCodec,
else:
ok(digestImplWithoutHash(hash, mdigest.data))
proc init*(mhtype: typedesc[MultiHash], hashname: string,
bdigest: openarray[byte]): MhResult[MultiHash] {.inline.} =
proc init*(
mhtype: typedesc[MultiHash], hashname: string, bdigest: openArray[byte]
): MhResult[MultiHash] {.inline.} =
## Create MultiHash from array of bytes ``bdigest`` and hash algorithm code
## ``hashcode``.
let mc = MultiCodec.codec(hashname)
@@ -422,8 +456,9 @@ proc init*(mhtype: typedesc[MultiHash], hashname: string,
else:
ok(digestImplWithoutHash(hash, bdigest))
proc init*(mhtype: typedesc[MultiHash], hashcode: MultiCodec,
bdigest: openarray[byte]): MhResult[MultiHash] {.inline.} =
proc init*(
mhtype: typedesc[MultiHash], hashcode: MultiCodec, bdigest: openArray[byte]
): MhResult[MultiHash] {.inline.} =
## Create MultiHash from array of bytes ``bdigest`` and hash algorithm code
## ``hashcode``.
let hash = CodeHashes.getOrDefault(hashcode)
@@ -434,8 +469,9 @@ proc init*(mhtype: typedesc[MultiHash], hashcode: MultiCodec,
else:
ok(digestImplWithoutHash(hash, bdigest))
proc decode*(mhtype: typedesc[MultiHash], data: openarray[byte],
mhash: var MultiHash): MhResult[int] =
proc decode*(
mhtype: typedesc[MultiHash], data: openArray[byte], mhash: var MultiHash
): MhResult[int] =
## Decode MultiHash value from array of bytes ``data``.
##
## On success decoded MultiHash will be stored into ``mhash`` and number of
@@ -474,12 +510,13 @@ proc decode*(mhtype: typedesc[MultiHash], data: openarray[byte],
if not vb.isEnough(int(size)):
return err(ErrDecodeError)
mhash = ? MultiHash.init(MultiCodec(code),
vb.buffer.toOpenArray(vb.offset,
vb.offset + int(size) - 1))
mhash =
?MultiHash.init(
MultiCodec(code), vb.buffer.toOpenArray(vb.offset, vb.offset + int(size) - 1)
)
ok(vb.offset + int(size))
proc validate*(mhtype: typedesc[MultiHash], data: openarray[byte]): bool =
proc validate*(mhtype: typedesc[MultiHash], data: openArray[byte]): bool =
## Returns ``true`` if array of bytes ``data`` has correct MultiHash inside.
var code, size: uint64
var res: VarintResult[void]
@@ -509,29 +546,29 @@ proc validate*(mhtype: typedesc[MultiHash], data: openarray[byte]): bool =
return false
result = true
proc init*(mhtype: typedesc[MultiHash],
data: openarray[byte]): MhResult[MultiHash] {.inline.} =
proc init*(
mhtype: typedesc[MultiHash], data: openArray[byte]
): MhResult[MultiHash] {.inline.} =
## Create MultiHash from bytes array ``data``.
var hash: MultiHash
discard ? MultiHash.decode(data, hash)
discard ?MultiHash.decode(data, hash)
ok(hash)
proc init*(mhtype: typedesc[MultiHash], data: string): MhResult[MultiHash] {.inline.} =
## Create MultiHash from hexadecimal string representation ``data``.
var hash: MultiHash
try:
discard ? MultiHash.decode(fromHex(data), hash)
discard ?MultiHash.decode(fromHex(data), hash)
ok(hash)
except ValueError:
err(ErrParseError)
proc init58*(mhtype: typedesc[MultiHash],
data: string): MultiHash {.inline.} =
proc init58*(mhtype: typedesc[MultiHash], data: string): MultiHash {.inline.} =
## Create MultiHash from BASE58 encoded string representation ``data``.
if MultiHash.decode(Base58.decode(data), result) == -1:
raise newException(MultihashError, "Incorrect MultiHash binary format")
proc cmp(a: openarray[byte], b: openarray[byte]): bool {.inline.} =
proc cmp(a: openArray[byte], b: openArray[byte]): bool {.inline.} =
if len(a) != len(b):
return false
var n = len(a)
@@ -539,7 +576,7 @@ proc cmp(a: openarray[byte], b: openarray[byte]): bool {.inline.} =
while n > 0:
dec(n)
diff = int(a[n]) - int(b[n])
res = (res and -not(diff)) or diff
res = (res and - not (diff)) or diff
result = (res == 0)
proc `==`*[T](mh: MultiHash, mdigest: MDigest[T]): bool =
@@ -549,8 +586,10 @@ proc `==`*[T](mh: MultiHash, mdigest: MDigest[T]): bool =
return false
if len(mdigest.data) != mh.size:
return false
result = cmp(mh.data.buffer.toOpenArray(mh.dpos, mh.dpos + mh.size - 1),
mdigest.data.toOpenArray(0, mdigest.data.high))
result = cmp(
mh.data.buffer.toOpenArray(mh.dpos, mh.dpos + mh.size - 1),
mdigest.data.toOpenArray(0, mdigest.data.high),
)
proc `==`*[T](mdigest: MDigest[T], mh: MultiHash): bool {.inline.} =
## Compares MultiHash with nimcrypto's MDigest[T], returns ``true`` if
@@ -566,8 +605,10 @@ proc `==`*(a: MultiHash, b: MultiHash): bool =
return false
if a.size != b.size:
return false
result = cmp(a.data.buffer.toOpenArray(a.dpos, a.dpos + a.size - 1),
b.data.buffer.toOpenArray(b.dpos, b.dpos + b.size - 1))
result = cmp(
a.data.buffer.toOpenArray(a.dpos, a.dpos + a.size - 1),
b.data.buffer.toOpenArray(b.dpos, b.dpos + b.size - 1),
)
proc hex*(value: MultiHash): string =
## Return hexadecimal string representation of MultiHash ``value``.
@@ -579,16 +620,16 @@ proc base58*(value: MultiHash): string =
proc `$`*(mh: MultiHash): string =
## Return string representation of MultiHash ``value``.
let digest = toHex(mh.data.buffer.toOpenArray(mh.dpos,
mh.dpos + mh.size - 1))
let digest = toHex(mh.data.buffer.toOpenArray(mh.dpos, mh.dpos + mh.size - 1))
result = $(mh.mcodec) & "/" & digest
proc write*(vb: var VBuffer, mh: MultiHash) {.inline.} =
## Write MultiHash value ``mh`` to buffer ``vb``.
vb.writeArray(mh.data.buffer)
proc encode*(mbtype: typedesc[MultiBase], encoding: string,
mh: MultiHash): string {.inline.} =
proc encode*(
mbtype: typedesc[MultiBase], encoding: string, mh: MultiHash
): string {.inline.} =
## Get MultiBase encoded representation of ``mh`` using encoding
## ``encoding``.
result = MultiBase.encode(encoding, mh.data.buffer)

View File

@@ -1,58 +1,58 @@
## Nim-LibP2P
## Copyright (c) 2019 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-LibP2P
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
import std/[strutils]
{.push raises: [].}
import std/[strutils, sequtils, tables]
import chronos, chronicles, stew/byteutils
import stream/connection,
vbuffer,
protocols/protocol
import stream/connection, protocols/protocol
logScope:
topics = "libp2p multistream"
const
MsgSize* = 64*1024
Codec* = "/multistream/1.0.0"
MsgSize = 1024
Codec = "/multistream/1.0.0"
MSCodec* = "\x13" & Codec & "\n"
Na* = "\x03na\n"
Ls* = "\x03ls\n"
Na = "na\n"
Ls = "ls\n"
type
Matcher* = proc (proto: string): bool {.gcsafe.}
Matcher* = proc(proto: string): bool {.gcsafe, raises: [].}
HandlerHolder* = object
MultiStreamError* = object of LPError
HandlerHolder* = ref object
protos*: seq[string]
protocol*: LPProtocol
match*: Matcher
openedStreams: CountTable[PeerId]
MultistreamSelect* = ref object of RootObj
handlers*: seq[HandlerHolder]
codec*: string
proc newMultistream*(): MultistreamSelect =
new result
result.codec = MSCodec
proc new*(T: typedesc[MultistreamSelect]): T =
T(codec: Codec)
template validateSuffix(str: string): untyped =
if str.endsWith("\n"):
str.removeSuffix("\n")
else:
raise newException(CatchableError, "MultistreamSelect failed, malformed message")
if str.endsWith("\n"):
str.removeSuffix("\n")
else:
raise (ref MultiStreamError)(msg: "MultistreamSelect failed, malformed message")
proc select*(m: MultistreamSelect,
conn: Connection,
proto: seq[string]):
Future[string] {.async.} =
trace "initiating handshake", conn, codec = m.codec
proc select*(
_: MultistreamSelect | type MultistreamSelect, conn: Connection, proto: seq[string]
): Future[string] {.async: (raises: [CancelledError, LPStreamError, MultiStreamError]).} =
trace "initiating handshake", conn, codec = Codec
## select a remote protocol
await conn.write(m.codec) # write handshake
await conn.writeLp(Codec & "\n") # write handshake
if proto.len() > 0:
trace "selecting proto", conn, proto = proto[0]
await conn.writeLp((proto[0] & "\n")) # select proto
@@ -62,7 +62,7 @@ proc select*(m: MultistreamSelect,
if s != Codec:
notice "handshake failed", conn, codec = s
raise newException(CatchableError, "MultistreamSelect handshake failed")
raise (ref MultiStreamError)(msg: "MultistreamSelect handshake failed")
else:
trace "multistream handshake success", conn
@@ -74,10 +74,11 @@ proc select*(m: MultistreamSelect,
trace "reading first requested proto", conn
if s == proto[0]:
trace "successfully selected ", conn, proto = proto[0]
conn.protocol = proto[0]
return proto[0]
elif proto.len > 1:
# Try to negotiate alternatives
let protos = proto[1..<proto.len()]
let protos = proto[1 ..< proto.len()]
trace "selecting one of several protos", conn, protos = protos
for p in protos:
trace "selecting proto", conn, proto = p
@@ -86,30 +87,38 @@ proc select*(m: MultistreamSelect,
validateSuffix(s)
if s == p:
trace "selected protocol", conn, protocol = s
conn.protocol = s
return s
return ""
else:
# No alternatives, fail
return ""
proc select*(m: MultistreamSelect,
conn: Connection,
proto: string): Future[bool] {.async.} =
proc select*(
_: MultistreamSelect | type MultistreamSelect, conn: Connection, proto: string
): Future[bool] {.async: (raises: [CancelledError, LPStreamError, MultiStreamError]).} =
if proto.len > 0:
return (await m.select(conn, @[proto])) == proto
(await MultistreamSelect.select(conn, @[proto])) == proto
else:
return (await m.select(conn, @[])) == Codec
(await MultistreamSelect.select(conn, @[])) == Codec
proc select*(m: MultistreamSelect, conn: Connection): Future[bool] =
proc select*(
m: MultistreamSelect, conn: Connection
): Future[bool] {.
async: (raises: [CancelledError, LPStreamError, MultiStreamError], raw: true)
.} =
m.select(conn, "")
proc list*(m: MultistreamSelect,
conn: Connection): Future[seq[string]] {.async.} =
proc list*(
m: MultistreamSelect, conn: Connection
): Future[seq[string]] {.
async: (raises: [CancelledError, LPStreamError, MultiStreamError])
.} =
## list remote protos requests on connection
if not await m.select(conn):
return
await conn.write(Ls) # send ls
await conn.writeLp(Ls) # send ls
var list = newSeq[string]()
let ms = string.fromBytes(await conn.readLp(MsgSize))
@@ -119,88 +128,155 @@ proc list*(m: MultistreamSelect,
result = list
proc handle*(m: MultistreamSelect, conn: Connection, active: bool = false) {.async, gcsafe.} =
trace "Starting multistream handler", conn, handshaked = active
proc handle*(
_: type MultistreamSelect,
conn: Connection,
protos: seq[string],
matchers = newSeq[Matcher](),
active: bool = false,
): Future[string] {.async: (raises: [CancelledError, LPStreamError, MultiStreamError]).} =
trace "Starting multistream negotiation", conn, handshaked = active
var handshaked = active
try:
while not conn.atEof:
var ms = string.fromBytes(await conn.readLp(MsgSize))
validateSuffix(ms)
while not conn.atEof:
var ms = string.fromBytes(await conn.readLp(MsgSize))
validateSuffix(ms)
if not handshaked and ms != Codec:
notice "expected handshake message", conn, instead=ms
raise newException(CatchableError,
"MultistreamSelect handling failed, invalid first message")
if not handshaked and ms != Codec:
debug "expected handshake message", conn, instead = ms
raise (ref MultiStreamError)(
msg: "MultistreamSelect handling failed, invalid first message"
)
trace "handle: got request", conn, ms
if ms.len() <= 0:
trace "handle: invalid proto", conn
await conn.write(Na)
trace "handle: got request", conn, ms
if ms.len() <= 0:
trace "handle: invalid proto", conn
await conn.writeLp(Na)
if m.handlers.len() == 0:
trace "handle: sending `na` for protocol", conn, protocol = ms
await conn.write(Na)
continue
case ms:
of "ls":
trace "handle: listing protos", conn
var protos = ""
for h in m.handlers:
for proto in h.protos:
protos &= (proto & "\n")
await conn.writeLp(protos)
of Codec:
if not handshaked:
await conn.write(m.codec)
handshaked = true
else:
trace "handle: sending `na` for duplicate handshake while handshaked",
conn
await conn.write(Na)
case ms
of "ls":
trace "handle: listing protos", conn
#TODO this doens't seem to follow spec, each protocol
# should be length prefixed. Not very important
# since LS is getting deprecated
await conn.writeLp(protos.join("\n") & "\n")
of Codec:
if not handshaked:
await conn.writeLp(Codec & "\n")
handshaked = true
else:
for h in m.handlers:
if (not isNil(h.match) and h.match(ms)) or h.protos.contains(ms):
trace "found handler", conn, protocol = ms
await conn.writeLp(ms & "\n")
await h.protocol.handler(conn, ms)
return
debug "no handlers", conn, protocol = ms
await conn.write(Na)
trace "handle: sending `na` for duplicate handshake while handshaked", conn
await conn.writeLp(Na)
elif ms in protos or matchers.anyIt(it(ms)):
trace "found handler", conn, protocol = ms
await conn.writeLp(ms & "\n")
conn.protocol = ms
return ms
else:
trace "no handlers", conn, protocol = ms
await conn.writeLp(Na)
proc handle*(
m: MultistreamSelect, conn: Connection, active: bool = false
) {.async: (raises: [CancelledError]).} =
trace "Starting multistream handler", conn, handshaked = active
var
protos: seq[string]
matchers: seq[Matcher]
for h in m.handlers:
if h.match != nil:
matchers.add(h.match)
for proto in h.protos:
protos.add(proto)
try:
let ms = await MultistreamSelect.handle(conn, protos, matchers, active)
for h in m.handlers:
if (h.match != nil and h.match(ms)) or h.protos.contains(ms):
trace "found handler", conn, protocol = ms
var protocolHolder = h
let maxIncomingStreams = protocolHolder.protocol.maxIncomingStreams
if protocolHolder.openedStreams.getOrDefault(conn.peerId) >= maxIncomingStreams:
debug "Max streams for protocol reached, blocking new stream",
conn, protocol = ms, maxIncomingStreams
return
protocolHolder.openedStreams.inc(conn.peerId)
try:
await protocolHolder.protocol.handler(conn, ms)
finally:
protocolHolder.openedStreams.inc(conn.peerId, -1)
if protocolHolder.openedStreams[conn.peerId] == 0:
protocolHolder.openedStreams.del(conn.peerId)
return
debug "no handlers", conn, ms
except CancelledError as exc:
raise exc
except CatchableError as exc:
trace "Exception in multistream", conn, msg = exc.msg
trace "Exception in multistream", conn, description = exc.msg
finally:
await conn.close()
trace "Stopped multistream handler", conn
proc addHandler*(m: MultistreamSelect,
codecs: seq[string],
protocol: LPProtocol,
matcher: Matcher = nil) =
proc addHandler*(
m: MultistreamSelect,
codecs: seq[string],
protocol: LPProtocol,
matcher: Matcher = nil,
) =
trace "registering protocols", protos = codecs
m.handlers.add(HandlerHolder(protos: codecs,
protocol: protocol,
match: matcher))
m.handlers.add(HandlerHolder(protos: codecs, protocol: protocol, match: matcher))
proc addHandler*(m: MultistreamSelect,
codec: string,
protocol: LPProtocol,
matcher: Matcher = nil) =
proc addHandler*(
m: MultistreamSelect, codec: string, protocol: LPProtocol, matcher: Matcher = nil
) =
addHandler(m, @[codec], protocol, matcher)
proc addHandler*(m: MultistreamSelect,
codec: string,
handler: LPProtoHandler,
matcher: Matcher = nil) =
proc addHandler*[E](
m: MultistreamSelect,
codec: string,
handler:
LPProtoHandler |
proc(conn: Connection, proto: string): InternalRaisesFuture[void, E],
matcher: Matcher = nil,
) =
## helper to allow registering pure handlers
trace "registering proto handler", proto = codec
let protocol = new LPProtocol
protocol.codec = codec
protocol.handler = handler
m.handlers.add(HandlerHolder(protos: @[codec],
protocol: protocol,
match: matcher))
m.handlers.add(HandlerHolder(protos: @[codec], protocol: protocol, match: matcher))
proc start*(m: MultistreamSelect) {.async: (raises: [CancelledError]).} =
# Nim 1.6.18: Using `mapIt` results in a seq of `.Raising([])`
# TODO https://github.com/nim-lang/Nim/issues/23445
var futs = newSeqOfCap[Future[void].Raising([CancelledError])](m.handlers.len)
for it in m.handlers:
futs.add it.protocol.start()
try:
await allFutures(futs)
for fut in futs:
await fut
except CancelledError as exc:
var pending: seq[Future[void].Raising([])]
doAssert m.handlers.len == futs.len, "Handlers modified while starting"
for i, fut in futs:
if not fut.finished:
pending.add fut.cancelAndWait()
elif fut.completed:
pending.add m.handlers[i].protocol.stop()
else:
static:
doAssert typeof(fut).E is (CancelledError,)
await noCancel allFutures(pending)
raise exc
proc stop*(m: MultistreamSelect) {.async: (raises: []).} =
# Nim 1.6.18: Using `mapIt` results in a seq of `.Raising([CancelledError])`
var futs = newSeqOfCap[Future[void].Raising([])](m.handlers.len)
for it in m.handlers:
futs.add it.protocol.stop()
await noCancel allFutures(futs)
for fut in futs:
await fut

View File

@@ -1,38 +1,33 @@
## Nim-LibP2P
## Copyright (c) 2019 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-LibP2P
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
import chronos
import nimcrypto/utils, chronicles, stew/byteutils
import ../../stream/connection,
../../utility,
../../varint,
../../vbuffer
{.push raises: [].}
import pkg/[chronos, chronicles, stew/byteutils]
import ../../stream/connection, ../../utility, ../../varint, ../../vbuffer, ../muxer
logScope:
topics = "libp2p mplexcoder"
type
MessageType* {.pure.} = enum
New,
MsgIn,
MsgOut,
CloseIn,
CloseOut,
ResetIn,
New
MsgIn
MsgOut
CloseIn
CloseOut
ResetIn
ResetOut
Msg* = tuple
id: uint64
msgType: MessageType
data: seq[byte]
Msg* = tuple[id: uint64, msgType: MessageType, data: seq[byte]]
InvalidMplexMsgType = object of CatchableError
InvalidMplexMsgType* = object of MuxerError
# https://github.com/libp2p/specs/tree/master/mplex#writing-to-a-stream
const MaxMsgSize* = 1 shl 20 # 1mb
@@ -40,7 +35,9 @@ const MaxMsgSize* = 1 shl 20 # 1mb
proc newInvalidMplexMsgType*(): ref InvalidMplexMsgType =
newException(InvalidMplexMsgType, "invalid message type")
proc readMsg*(conn: Connection): Future[Msg] {.async, gcsafe.} =
proc readMsg*(
conn: Connection
): Future[Msg] {.async: (raises: [CancelledError, LPStreamError, MuxerError]).} =
let header = await conn.readVarint()
trace "read header varint", varint = header, conn
@@ -53,10 +50,9 @@ proc readMsg*(conn: Connection): Future[Msg] {.async, gcsafe.} =
return (header shr 3, MessageType(msgType), data)
proc writeMsg*(conn: Connection,
id: uint64,
msgType: MessageType,
data: seq[byte] = @[]): Future[void] =
proc writeMsg*(
conn: Connection, id: uint64, msgType: MessageType, data: seq[byte] = @[]
): Future[void] {.async: (raises: [CancelledError, LPStreamError], raw: true).} =
var
left = data.len
offset = 0
@@ -64,8 +60,11 @@ proc writeMsg*(conn: Connection,
# Split message into length-prefixed chunks
while left > 0 or data.len == 0:
let
chunkSize = if left > MaxMsgSize: MaxMsgSize - 64 else: left
let chunkSize =
if left > MaxMsgSize:
MaxMsgSize - 64
else:
left
buf.writePBVarint(id shl 3 or ord(msgType).uint64)
buf.writeSeq(data.toOpenArray(offset, offset + chunkSize - 1))
@@ -82,8 +81,7 @@ proc writeMsg*(conn: Connection,
# message gets written before some of the chunks
conn.write(buf.buffer)
proc writeMsg*(conn: Connection,
id: uint64,
msgType: MessageType,
data: string): Future[void] =
proc writeMsg*(
conn: Connection, id: uint64, msgType: MessageType, data: string
): Future[void] {.async: (raises: [CancelledError, LPStreamError], raw: true).} =
conn.writeMsg(id, msgType, data.toBytes())

View File

@@ -1,25 +1,35 @@
## Nim-LibP2P
## Copyright (c) 2019 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-LibP2P
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
{.push raises: [].}
import std/[oids, strformat]
import chronos, chronicles, metrics
import ./coder,
../muxer,
nimcrypto/utils,
../../stream/[bufferstream, connection, streamseq],
../../peerinfo
import pkg/[chronos, chronicles, metrics]
import
./coder, ../muxer, ../../stream/[bufferstream, connection, streamseq], ../../peerinfo
export connection
logScope:
topics = "libp2p mplexchannel"
when defined(libp2p_mplex_metrics):
declareHistogram libp2p_mplex_qlen,
"message queue length",
buckets = [0.0, 1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0]
declareCounter libp2p_mplex_qlenclose, "closed because of max queuelen"
declareHistogram libp2p_mplex_qtime, "message queuing time"
when defined(libp2p_network_protocols_metrics):
declareCounter libp2p_protocols_bytes,
"total sent or received bytes", ["protocol", "direction"]
## Channel half-closed states
##
## | State | Closed local | Closed remote
@@ -32,80 +42,90 @@ logScope:
## EOF marker
const
MaxWrites = 1024 ##\
MaxWrites = 1024
##\
## Maximum number of in-flight writes - after this, we disconnect the peer
LPChannelTrackerName* = "LPChannel"
type
LPChannel* = ref object of BufferStream
id*: uint64 # channel id
name*: string # name of the channel (for debugging)
conn*: Connection # wrapped connection used to for writing
initiator*: bool # initiated remotely or locally flag
isOpen*: bool # has channel been opened
closedLocal*: bool # has channel been closed locally
msgCode*: MessageType # cached in/out message code
closeCode*: MessageType # cached in/out close code
resetCode*: MessageType # cached in/out reset code
writes*: int # In-flight writes
type LPChannel* = ref object of BufferStream
id*: uint64 # channel id
name*: string # name of the channel (for debugging)
conn*: Connection # wrapped connection used to for writing
initiator*: bool # initiated remotely or locally flag
isOpen*: bool # has channel been opened
closedLocal*: bool # has channel been closed locally
remoteReset*: bool # has channel been remotely reset
localReset*: bool # has channel been reset locally
msgCode*: MessageType # cached in/out message code
closeCode*: MessageType # cached in/out close code
resetCode*: MessageType # cached in/out reset code
writes*: int # In-flight writes
func shortLog*(s: LPChannel): auto =
if s.isNil: "LPChannel(nil)"
elif s.conn.peerInfo.isNil: $s.oid
elif s.name != $s.oid and s.name.len > 0:
&"{shortLog(s.conn.peerInfo.peerId)}:{s.oid}:{s.name}"
else: &"{shortLog(s.conn.peerInfo.peerId)}:{s.oid}"
chronicles.formatIt(LPChannel): shortLog(it)
try:
if s == nil:
"LPChannel(nil)"
elif s.name != $s.oid and s.name.len > 0:
&"{shortLog(s.conn.peerId)}:{s.oid}:{s.name}"
else:
&"{shortLog(s.conn.peerId)}:{s.oid}"
except ValueError as exc:
raiseAssert(exc.msg)
proc open*(s: LPChannel) {.async, gcsafe.} =
chronicles.formatIt(LPChannel):
shortLog(it)
proc open*(s: LPChannel) {.async: (raises: [CancelledError, LPStreamError]).} =
trace "Opening channel", s, conn = s.conn
if s.conn.isClosed:
return
try:
await s.conn.writeMsg(s.id, MessageType.New, s.name)
s.isOpen = true
except CatchableError as exc:
except CancelledError as exc:
raise exc
except LPStreamError as exc:
await s.conn.close()
raise exc
method closed*(s: LPChannel): bool =
s.closedLocal
proc closeUnderlying(s: LPChannel): Future[void] {.async.} =
proc closeUnderlying(s: LPChannel): Future[void] {.async: (raises: []).} =
## Channels may be closed for reading and writing in any order - we'll close
## the underlying bufferstream when both directions are closed
if s.closedLocal and s.atEof():
await procCall BufferStream(s).close()
proc reset*(s: LPChannel) {.async, gcsafe.} =
proc reset*(s: LPChannel) {.async: (raises: []).} =
if s.isClosed:
trace "Already closed", s
return
s.isClosed = true
s.closedLocal = true
s.localReset = not s.remoteReset
trace "Resetting channel", s, len = s.len
if s.isOpen and not s.conn.isClosed:
# If the connection is still active, notify the other end
proc resetMessage() {.async.} =
proc resetMessage() {.async: (raises: []).} =
try:
trace "sending reset message", s, conn = s.conn
await s.conn.writeMsg(s.id, s.resetCode) # write reset
except CatchableError as exc:
# No cancellations
await noCancel s.conn.writeMsg(s.id, s.resetCode) # write reset
except LPStreamError as exc:
trace "Can't send reset message", s, conn = s.conn, description = exc.msg
await s.conn.close()
trace "Can't send reset message", s, conn = s.conn, msg = exc.msg
asyncSpawn resetMessage()
await s.closeImpl() # noraises, nocancels
await s.closeImpl()
trace "Channel reset", s
method close*(s: LPChannel) {.async, gcsafe.} =
method close*(s: LPChannel) {.async: (raises: []).} =
## Close channel for writing - a message will be sent to the other peer
## informing them that the channel is closed and that we're waiting for
## their acknowledgement.
@@ -119,14 +139,13 @@ method close*(s: LPChannel) {.async, gcsafe.} =
if s.isOpen and not s.conn.isClosed:
try:
await s.conn.writeMsg(s.id, s.closeCode) # write close
except CancelledError as exc:
except CancelledError:
await s.conn.close()
raise exc
except CatchableError as exc:
except LPStreamError as exc:
# It's harmless that close message cannot be sent - the connection is
# likely down already
await s.conn.close()
trace "Cannot send close message", s, id = s.id, msg = exc.msg
trace "Cannot send close message", s, id = s.id, description = exc.msg
await s.closeUnderlying() # maybe already eofed
@@ -134,41 +153,61 @@ method close*(s: LPChannel) {.async, gcsafe.} =
method initStream*(s: LPChannel) =
if s.objName.len == 0:
s.objName = "LPChannel"
s.objName = LPChannelTrackerName
s.timeoutHandler = proc(): Future[void] {.gcsafe.} =
s.timeoutHandler = proc(): Future[void] {.async: (raises: [], raw: true).} =
trace "Idle timeout expired, resetting LPChannel", s
s.reset()
procCall BufferStream(s).initStream()
method readOnce*(s: LPChannel,
pbytes: pointer,
nbytes: int):
Future[int] {.async.} =
method readOnce*(
s: LPChannel, pbytes: pointer, nbytes: int
): Future[int] {.async: (raises: [CancelledError, LPStreamError]).} =
## Mplex relies on reading being done regularly from every channel, or all
## channels are blocked - in particular, this means that reading from one
## channel must not be done from within a callback / read handler of another
## or the reads will lock each other.
if s.remoteReset:
trace "reset stream in readOnce", s
raise newLPStreamResetError()
if s.localReset:
raise newLPStreamClosedError()
if s.atEof():
raise newLPStreamRemoteClosedError()
if s.conn.closed:
raise newLPStreamConnDownError()
try:
let bytes = await procCall BufferStream(s).readOnce(pbytes, nbytes)
when defined(libp2p_network_protocols_metrics):
if s.protocol.len > 0:
libp2p_protocols_bytes.inc(bytes.int64, labelValues = [s.protocol, "in"])
trace "readOnce", s, bytes
if bytes == 0:
await s.closeUnderlying()
return bytes
except CatchableError as exc:
# readOnce in BufferStream generally raises on EOF or cancellation - for
# the former, resetting is harmless, for the latter it's necessary because
# data has been lost in s.readBuf and there's no way to gracefully recover /
# use the channel any more
except CancelledError as exc:
await s.reset()
raise exc
except LPStreamError as exc:
# Resetting is necessary because data has been lost in s.readBuf and
# there's no way to gracefully recover / use the channel any more
await s.reset()
raise newLPStreamConnDownError(exc)
method write*(s: LPChannel, msg: seq[byte]): Future[void] {.async.} =
## Write to mplex channel - there may be up to MaxWrite concurrent writes
## pending after which the peer is disconnected
if s.closedLocal or s.conn.closed:
proc prepareWrite(
s: LPChannel, msg: seq[byte]
): Future[void] {.async: (raises: [CancelledError, LPStreamError]).} =
# prepareWrite is the slow path of writing a message - see conditions in
# write
if s.remoteReset:
trace "stream is reset when prepareWrite", s
raise newLPStreamResetError()
if s.closedLocal:
raise newLPStreamClosedError()
if s.conn.closed:
raise newLPStreamConnDownError()
if msg.len == 0:
return
@@ -176,36 +215,86 @@ method write*(s: LPChannel, msg: seq[byte]): Future[void] {.async.} =
if s.writes >= MaxWrites:
debug "Closing connection, too many in-flight writes on channel",
s, conn = s.conn, writes = s.writes
when defined(libp2p_mplex_metrics):
libp2p_mplex_qlenclose.inc()
await s.reset()
await s.conn.close()
return
s.writes += 1
if not s.isOpen:
await s.open()
await s.conn.writeMsg(s.id, s.msgCode, msg)
proc completeWrite(
s: LPChannel,
fut: Future[void].Raising([CancelledError, LPStreamError]),
msgLen: int,
): Future[void] {.async: (raises: [CancelledError, LPStreamError]).} =
try:
if not s.isOpen:
await s.open()
s.writes += 1
# writes should happen in sequence
trace "write msg", s, conn = s.conn, len = msg.len
when defined(libp2p_mplex_metrics):
libp2p_mplex_qlen.observe(s.writes.int64 - 1)
libp2p_mplex_qtime.time:
await fut
else:
await fut
when defined(libp2p_network_protocols_metrics):
if s.protocol.len > 0:
# This crashes on Nim 2.0.2 with `--mm:orc` during `nimble test`
# https://github.com/status-im/nim-metrics/issues/79
libp2p_protocols_bytes.inc(msgLen.int64, labelValues = [s.protocol, "out"])
await s.conn.writeMsg(s.id, s.msgCode, msg)
s.activity = true
except CatchableError as exc:
trace "exception in lpchannel write handler", s, msg = exc.msg
except CancelledError as exc:
# Chronos may still send the data
raise exc
except LPStreamConnDownError as exc:
await s.reset()
await s.conn.close()
raise exc
except LPStreamEOFError as exc:
raise exc
except LPStreamError as exc:
trace "exception in lpchannel write handler", s, description = exc.msg
await s.reset()
await s.conn.close()
raise newLPStreamConnDownError(exc)
finally:
s.writes -= 1
proc init*(
L: type LPChannel,
id: uint64,
conn: Connection,
initiator: bool,
name: string = "",
timeout: Duration = DefaultChanTimeout): LPChannel =
method write*(
s: LPChannel, msg: seq[byte]
): Future[void] {.async: (raises: [CancelledError, LPStreamError], raw: true).} =
## Write to mplex channel - there may be up to MaxWrite concurrent writes
## pending after which the peer is disconnected
let closed = s.closedLocal or s.conn.closed
let fut =
if (not closed) and msg.len > 0 and s.writes < MaxWrites and s.isOpen:
# Fast path: Avoid a copy of msg being kept in the closure created by
# `{.async.}` as this drives up memory usage - the conditions are laid out
# in prepareWrite
s.conn.writeMsg(s.id, s.msgCode, msg)
else:
prepareWrite(s, msg)
s.completeWrite(fut, msg.len)
method getWrapped*(s: LPChannel): Connection =
s.conn
proc init*(
L: type LPChannel,
id: uint64,
conn: Connection,
initiator: bool,
name: string = "",
timeout: Duration = DefaultChanTimeout,
): LPChannel =
let chann = L(
id: id,
name: name,
@@ -216,12 +305,17 @@ proc init*(
msgCode: if initiator: MessageType.MsgOut else: MessageType.MsgIn,
closeCode: if initiator: MessageType.CloseOut else: MessageType.CloseIn,
resetCode: if initiator: MessageType.ResetOut else: MessageType.ResetIn,
dir: if initiator: Direction.Out else: Direction.In)
dir: if initiator: Direction.Out else: Direction.In,
)
chann.initStream()
when chronicles.enabledLogLevel == LogLevel.TRACE:
chann.name = if chann.name.len > 0: chann.name else: $chann.oid
chann.name =
if chann.name.len > 0:
chann.name
else:
$chann.oid
trace "Created new lpchannel", s = chann, id, initiator

View File

@@ -1,21 +1,24 @@
## Nim-LibP2P
## Copyright (c) 2019 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-LibP2P
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
{.push raises: [].}
import tables, sequtils, oids
import chronos, chronicles, stew/byteutils, metrics
import ../muxer,
../../stream/connection,
../../stream/bufferstream,
../../utility,
../../peerinfo,
./coder,
./lpchannel
import
../muxer,
../../stream/connection,
../../stream/bufferstream,
../../utility,
../../peerinfo,
./coder,
./lpchannel
export muxer
@@ -24,16 +27,13 @@ logScope:
const MplexCodec* = "/mplex/6.7.0"
const
MaxChannelCount = 200
const MaxChannelCount = 200
when defined(libp2p_expensive_metrics):
declareGauge(libp2p_mplex_channels,
"mplex channels", labels = ["initiator", "peer"])
declareGauge(libp2p_mplex_channels, "mplex channels", labels = ["initiator", "peer"])
type
TooManyChannels* = object of CatchableError
InvalidChannelIdError* = object of CatchableError
InvalidChannelIdError* = object of MuxerError
Mplex* = ref object of Muxer
channels: array[bool, Table[uint64, LPChannel]]
@@ -44,8 +44,11 @@ type
oid*: Oid
maxChannCount: int
func shortLog*(m: MPlex): auto = shortLog(m.connection)
chronicles.formatIt(Mplex): shortLog(it)
func shortLog*(m: Mplex): auto =
shortLog(m.connection)
chronicles.formatIt(Mplex):
shortLog(it)
proc newTooManyChannels(): ref TooManyChannels =
newException(TooManyChannels, "max allowed channel count exceeded")
@@ -53,7 +56,7 @@ proc newTooManyChannels(): ref TooManyChannels =
proc newInvalidChannelIdError(): ref InvalidChannelIdError =
newException(InvalidChannelIdError, "max allowed channel count exceeded")
proc cleanupChann(m: Mplex, chann: LPChannel) {.async, inline.} =
proc cleanupChann(m: Mplex, chann: LPChannel) {.async: (raises: []), inline.} =
## remove the local channel from the internal tables
##
try:
@@ -64,37 +67,37 @@ proc cleanupChann(m: Mplex, chann: LPChannel) {.async, inline.} =
when defined(libp2p_expensive_metrics):
libp2p_mplex_channels.set(
m.channels[chann.initiator].len.int64,
labelValues = [$chann.initiator, $m.connection.peerInfo.peerId])
except CatchableError as exc:
# This is top-level procedure which will work as separate task, so it
# do not need to propogate CancelledError, and no other exceptions should
# happen here
warn "Error cleaning up mplex channel", m, chann, msg = exc.msg
labelValues = [$chann.initiator, $m.connection.peerId],
)
except CancelledError as exc:
warn "Error cleaning up mplex channel", m, chann, description = exc.msg
proc newStreamInternal*(m: Mplex,
initiator: bool = true,
chanId: uint64 = 0,
name: string = "",
timeout: Duration):
LPChannel {.gcsafe.} =
proc newStreamInternal*(
m: Mplex,
initiator: bool = true,
chanId: uint64 = 0,
name: string = "",
timeout: Duration,
): LPChannel {.gcsafe, raises: [InvalidChannelIdError].} =
## create new channel/stream
##
let id = if initiator:
m.currentId.inc(); m.currentId
else: chanId
let id =
if initiator:
m.currentId.inc()
m.currentId
else:
chanId
if id in m.channels[initiator]:
raise newInvalidChannelIdError()
result = LPChannel.init(
id,
m.connection,
initiator,
name,
timeout = timeout)
result = LPChannel.init(id, m.connection, initiator, name, timeout = timeout)
result.peerInfo = m.connection.peerInfo
result.peerId = m.connection.peerId
result.observedAddr = m.connection.observedAddr
result.transportDir = m.connection.transportDir
when defined(libp2p_agents_metrics):
result.shortAgent = m.connection.shortAgent
trace "Creating new channel", m, channel = result, id, initiator, name
@@ -105,23 +108,17 @@ proc newStreamInternal*(m: Mplex,
when defined(libp2p_expensive_metrics):
libp2p_mplex_channels.set(
m.channels[initiator].len.int64,
labelValues = [$initiator, $m.connection.peerInfo.peerId])
m.channels[initiator].len.int64, labelValues = [$initiator, $m.connection.peerId]
)
proc handleStream(m: Mplex, chann: LPChannel) {.async.} =
proc handleStream(m: Mplex, chann: LPChannel) {.async: (raises: []).} =
## call the muxer stream handler for this channel
##
try:
await m.streamHandler(chann)
trace "finished handling stream", m, chann
doAssert(chann.closed, "connection not closed by handler!")
except CatchableError as exc:
# This is top-level procedure which will work as separate task, so it
# do not need to propogate CancelledError.
trace "Exception in mplex stream handler", m, chann, msg = exc.msg
await chann.reset()
await m.streamHandler(chann)
trace "finished handling stream", m, chann
doAssert(chann.closed, "connection not closed by handler!")
method handle*(m: Mplex) {.async, gcsafe.} =
method handle*(m: Mplex) {.async: (raises: []).} =
trace "Starting mplex handler", m
try:
while not m.connection.atEof:
@@ -149,7 +146,7 @@ method handle*(m: Mplex) {.async, gcsafe.} =
else:
if m.channels[false].len > m.maxChannCount - 1:
warn "too many channels created by remote peer",
allowedMax = MaxChannelCount, m
allowedMax = MaxChannelCount, m
raise newTooManyChannels()
let name = string.fromBytes(data)
@@ -157,54 +154,64 @@ method handle*(m: Mplex) {.async, gcsafe.} =
trace "Processing channel message", m, channel, data = data.shortLog
case msgType:
of MessageType.New:
trace "created channel", m, channel
case msgType
of MessageType.New:
trace "created channel", m, channel
if not isNil(m.streamHandler):
# Launch handler task
# All the errors are handled inside `handleStream()` procedure.
asyncSpawn m.handleStream(channel)
if m.streamHandler != nil:
# Launch handler task
# All the errors are handled inside `handleStream()` procedure.
asyncSpawn m.handleStream(channel)
of MessageType.MsgIn, MessageType.MsgOut:
if data.len > MaxMsgSize:
warn "attempting to send a packet larger than allowed",
allowed = MaxMsgSize, channel
raise newLPStreamLimitError()
of MessageType.MsgIn, MessageType.MsgOut:
if data.len > MaxMsgSize:
warn "attempting to send a packet larger than allowed",
allowed = MaxMsgSize, channel
raise newLPStreamLimitError()
trace "pushing data to channel", m, channel, len = data.len
trace "pushing data to channel", m, channel, len = data.len
try:
await channel.pushData(data)
trace "pushed data to channel", m, channel, len = data.len
of MessageType.CloseIn, MessageType.CloseOut:
await channel.pushEof()
of MessageType.ResetIn, MessageType.ResetOut:
await channel.reset()
except LPStreamClosedError as exc:
# Channel is being closed, but `cleanupChann` was not yet triggered.
trace "pushing data to channel failed",
m, channel, len = data.len, description = exc.msg
discard # Ignore message, same as if `cleanupChann` had completed.
of MessageType.CloseIn, MessageType.CloseOut:
await channel.pushEof()
of MessageType.ResetIn, MessageType.ResetOut:
channel.remoteReset = true
await channel.reset()
except CancelledError:
# This procedure is spawned as task and it is not part of public API, so
# there no way for this procedure to be cancelled implicitly.
debug "Unexpected cancellation in mplex handler", m
except LPStreamEOFError as exc:
trace "Stream EOF", m, msg = exc.msg
except CatchableError as exc:
debug "Unexpected exception in mplex read loop", m, msg = exc.msg
trace "Stream EOF", m, description = exc.msg
except LPStreamError as exc:
debug "Unexpected stream exception in mplex read loop", m, description = exc.msg
except MuxerError as exc:
debug "Unexpected muxer exception in mplex read loop", m, description = exc.msg
finally:
await m.close()
trace "Stopped mplex handler", m
proc init*(M: type Mplex,
conn: Connection,
inTimeout, outTimeout: Duration = DefaultChanTimeout,
maxChannCount: int = MaxChannelCount): Mplex =
M(connection: conn,
proc new*(
M: type Mplex,
conn: Connection,
inTimeout: Duration = DefaultChanTimeout,
outTimeout: Duration = DefaultChanTimeout,
maxChannCount: int = MaxChannelCount,
): Mplex =
M(
connection: conn,
inChannTimeout: inTimeout,
outChannTimeout: outTimeout,
oid: genOid(),
maxChannCount: maxChannCount)
maxChannCount: maxChannCount,
)
method newStream*(m: Mplex,
name: string = "",
lazy: bool = false): Future[Connection] {.async, gcsafe.} =
method newStream*(
m: Mplex, name: string = "", lazy: bool = false
): Future[Connection] {.async: (raises: [CancelledError, LPStreamError, MuxerError]).} =
let channel = m.newStreamInternal(timeout = m.inChannTimeout)
if not lazy:
@@ -212,7 +219,7 @@ method newStream*(m: Mplex,
return Connection(channel)
method close*(m: Mplex) {.async, gcsafe.} =
method close*(m: Mplex) {.async: (raises: []).} =
if m.isClosed:
trace "Already closed", m
return
@@ -239,3 +246,9 @@ method close*(m: Mplex) {.async, gcsafe.} =
m.channels[true].clear()
trace "Closed mplex", m
method getStreams*(m: Mplex): seq[Connection] {.gcsafe.} =
for c in m.channels[false].values:
result.add(c)
for c in m.channels[true].values:
result.add(c)

View File

@@ -1,78 +1,71 @@
## Nim-LibP2P
## Copyright (c) 2019 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-LibP2P
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
{.push raises: [].}
import chronos, chronicles
import ../protocols/protocol,
../stream/connection,
../errors
import ../stream/connection, ../errors
logScope:
topics = "libp2p muxer"
const
DefaultChanTimeout* = 5.minutes
const DefaultChanTimeout* = 5.minutes
type
StreamHandler* = proc(conn: Connection): Future[void] {.gcsafe.}
MuxerHandler* = proc(muxer: Muxer): Future[void] {.gcsafe.}
MuxerError* = object of LPError
TooManyChannels* = object of MuxerError
StreamHandler* = proc(conn: Connection): Future[void] {.async: (raises: []).}
MuxerHandler* = proc(muxer: Muxer): Future[void] {.async: (raises: []).}
Muxer* = ref object of RootObj
streamHandler*: StreamHandler
handler*: Future[void].Raising([])
connection*: Connection
# user provider proc that returns a constructed Muxer
MuxerConstructor* = proc(conn: Connection): Muxer {.gcsafe, closure.}
MuxerConstructor* = proc(conn: Connection): Muxer {.gcsafe, closure, raises: [].}
# this wraps a creator proc that knows how to make muxers
MuxerProvider* = ref object of LPProtocol
MuxerProvider* = object
newMuxer*: MuxerConstructor
streamHandler*: StreamHandler # triggered every time there is a new stream, called for any muxer instance
muxerHandler*: MuxerHandler # triggered every time there is a new muxed connection created
codec*: string
func shortLog*(m: Muxer): auto = shortLog(m.connection)
chronicles.formatIt(Muxer): shortLog(it)
func shortLog*(m: Muxer): auto =
if m == nil:
"nil"
else:
shortLog(m.connection)
chronicles.formatIt(Muxer):
shortLog(it)
# muxer interface
method newStream*(m: Muxer, name: string = "", lazy: bool = false):
Future[Connection] {.base, async, gcsafe.} = discard
method close*(m: Muxer) {.base, async, gcsafe.} = discard
method handle*(m: Muxer): Future[void] {.base, async, gcsafe.} = discard
method newStream*(
m: Muxer, name: string = "", lazy: bool = false
): Future[Connection] {.
base, async: (raises: [CancelledError, LPStreamError, MuxerError], raw: true)
.} =
raiseAssert("[Muxer.newStream] abstract method not implemented!")
proc newMuxerProvider*(creator: MuxerConstructor, codec: string): MuxerProvider {.gcsafe.} =
new result
result.newMuxer = creator
result.codec = codec
result.init()
method close*(m: Muxer) {.base, async: (raises: []).} =
if m.connection != nil:
await m.connection.close()
method init(c: MuxerProvider) =
proc handler(conn: Connection, proto: string) {.async, gcsafe, closure.} =
trace "starting muxer handler", proto=proto, conn
try:
let
muxer = c.newMuxer(conn)
method handle*(m: Muxer): Future[void] {.base, async: (raises: []).} =
discard
if not isNil(c.streamHandler):
muxer.streamHandler = c.streamHandler
proc new*(
T: typedesc[MuxerProvider], creator: MuxerConstructor, codec: string
): T {.gcsafe.} =
let muxerProvider = T(newMuxer: creator, codec: codec)
muxerProvider
var futs = newSeq[Future[void]]()
futs &= muxer.handle()
# finally await both the futures
if not isNil(c.muxerHandler):
futs &= c.muxerHandler(muxer)
checkFutures(await allFinished(futs))
except CancelledError as exc:
raise exc
except CatchableError as exc:
trace "exception in muxer handler", exc = exc.msg, conn, proto
finally:
await conn.close()
c.handler = handler
method getStreams*(m: Muxer): seq[Connection] {.base, gcsafe.} =
raiseAssert("[Muxer.getStreams] abstract method not implemented!")

View File

@@ -0,0 +1,670 @@
# Nim-LibP2P
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
{.push raises: [].}
import sequtils, std/[tables]
import chronos, chronicles, metrics, stew/[endians2, byteutils, objects]
import ../muxer, ../../stream/connection
export muxer
logScope:
topics = "libp2p yamux"
const
YamuxCodec* = "/yamux/1.0.0"
YamuxVersion = 0.uint8
YamuxDefaultWindowSize* = 256000
MaxSendQueueSize = 256000
MaxChannelCount = 200
when defined(libp2p_yamux_metrics):
declareGauge libp2p_yamux_channels, "yamux channels", labels = ["initiator", "peer"]
declareHistogram libp2p_yamux_send_queue,
"message send queue length (in byte)",
buckets = [0.0, 100.0, 250.0, 1000.0, 2000.0, 3200.0, 6400.0, 25600.0, 256000.0]
declareHistogram libp2p_yamux_recv_queue,
"message recv queue length (in byte)",
buckets = [0.0, 100.0, 250.0, 1000.0, 2000.0, 3200.0, 6400.0, 25600.0, 256000.0]
type
YamuxError* = object of MuxerError
MsgType = enum
Data = 0x0
WindowUpdate = 0x1
Ping = 0x2
GoAway = 0x3
MsgFlags {.size: 2.} = enum
Syn
Ack
Fin
Rst
GoAwayStatus = enum
NormalTermination = 0x0
ProtocolError = 0x1
InternalError = 0x2
YamuxHeader = object
version: uint8
msgType: MsgType
flags: set[MsgFlags]
streamId: uint32
length: uint32
proc readHeader(
conn: LPStream
): Future[YamuxHeader] {.async: (raises: [CancelledError, LPStreamError, MuxerError]).} =
var buffer: array[12, byte]
await conn.readExactly(addr buffer[0], 12)
result.version = buffer[0]
let flags = fromBytesBE(uint16, buffer[2 .. 3])
if not result.msgType.checkedEnumAssign(buffer[1]) or flags notin 0'u16 .. 15'u16:
raise newException(YamuxError, "Wrong header")
result.flags = cast[set[MsgFlags]](flags)
result.streamId = fromBytesBE(uint32, buffer[4 .. 7])
result.length = fromBytesBE(uint32, buffer[8 .. 11])
return result
proc `$`(header: YamuxHeader): string =
"{" & $header.msgType & ", " & "{" &
header.flags.foldl(
if a != "":
a & ", " & $b
else:
$b,
"",
) & "}, " & "streamId: " & $header.streamId & ", " & "length: " & $header.length &
"}"
proc encode(header: YamuxHeader): array[12, byte] =
result[0] = header.version
result[1] = uint8(header.msgType)
result[2 .. 3] = toBytesBE(uint16(cast[uint8](header.flags)))
# workaround https://github.com/nim-lang/Nim/issues/21789
result[4 .. 7] = toBytesBE(header.streamId)
result[8 .. 11] = toBytesBE(header.length)
proc write(
conn: LPStream, header: YamuxHeader
): Future[void] {.async: (raises: [CancelledError, LPStreamError], raw: true).} =
trace "write directly on stream", h = $header
var buffer = header.encode()
conn.write(@buffer)
proc ping(T: type[YamuxHeader], flag: MsgFlags, pingData: uint32): T =
T(version: YamuxVersion, msgType: MsgType.Ping, flags: {flag}, length: pingData)
proc goAway(T: type[YamuxHeader], status: GoAwayStatus): T =
T(version: YamuxVersion, msgType: MsgType.GoAway, length: uint32(status))
proc data(
T: type[YamuxHeader],
streamId: uint32,
length: uint32 = 0,
flags: set[MsgFlags] = {},
): T =
T(
version: YamuxVersion,
msgType: MsgType.Data,
length: length,
flags: flags,
streamId: streamId,
)
proc windowUpdate(
T: type[YamuxHeader], streamId: uint32, delta: uint32, flags: set[MsgFlags] = {}
): T =
T(
version: YamuxVersion,
msgType: MsgType.WindowUpdate,
length: delta,
flags: flags,
streamId: streamId,
)
type
ToSend =
tuple[
data: seq[byte],
sent: int,
fut: Future[void].Raising([CancelledError, LPStreamError]),
]
YamuxChannel* = ref object of Connection
id: uint32
recvWindow: int
sendWindow: int
maxRecvWindow: int
maxSendQueueSize: int
conn: Connection
isSrc: bool
opened: bool
isSending: bool
sendQueue: seq[ToSend]
recvQueue: seq[byte]
isReset: bool
remoteReset: bool
closedRemotely: AsyncEvent
closedLocally: bool
receivedData: AsyncEvent
proc `$`(channel: YamuxChannel): string =
result = if channel.conn.dir == Out: "=> " else: "<= "
result &= $channel.id
var s: seq[string] = @[]
if channel.closedRemotely.isSet():
s.add("ClosedRemotely")
if channel.closedLocally:
s.add("ClosedLocally")
if channel.isReset:
s.add("Reset")
if s.len > 0:
result &=
" {" &
s.foldl(
if a != "":
a & ", " & b
else:
b,
"",
) & "}"
proc lengthSendQueue(channel: YamuxChannel): int =
## Returns the length of what remains to be sent
##
channel.sendQueue.foldl(a + b.data.len - b.sent, 0)
proc lengthSendQueueWithLimit(channel: YamuxChannel): int =
## Returns the length of what remains to be sent, but limit the size of big messages.
##
# For leniency, limit big messages size to the third of maxSendQueueSize
# This value is arbitrary, it's not in the specs, it permits to store up to
# 3 big messages if the peer is stalling.
channel.sendQueue.foldl(
a + min(b.data.len - b.sent, channel.maxSendQueueSize div 3), 0
)
proc actuallyClose(channel: YamuxChannel) {.async: (raises: []).} =
if channel.closedLocally and channel.sendQueue.len == 0 and
channel.closedRemotely.isSet():
await procCall Connection(channel).closeImpl()
proc remoteClosed(channel: YamuxChannel) {.async: (raises: []).} =
if not channel.closedRemotely.isSet():
channel.closedRemotely.fire()
await channel.actuallyClose()
channel.isClosedRemotely = true
method closeImpl*(channel: YamuxChannel) {.async: (raises: []).} =
if not channel.closedLocally:
trace "Closing yamux channel locally", streamId = channel.id, conn = channel.conn
channel.closedLocally = true
if not channel.isReset and channel.sendQueue.len == 0:
try:
await channel.conn.write(YamuxHeader.data(channel.id, 0, {Fin}))
except CancelledError, LPStreamError:
discard
await channel.actuallyClose()
proc reset(channel: YamuxChannel, isLocal: bool = false) {.async: (raises: []).} =
# If we reset locally, we want to flush up to a maximum of recvWindow
# bytes. It's because the peer we're connected to can send us data before
# it receives the reset.
if channel.isReset:
return
trace "Reset channel"
channel.isReset = true
channel.remoteReset = not isLocal
for (d, s, fut) in channel.sendQueue:
fut.fail(newLPStreamEOFError())
channel.sendQueue = @[]
channel.recvQueue = @[]
channel.sendWindow = 0
if not channel.closedLocally:
if isLocal and not channel.isSending:
try:
await channel.conn.write(YamuxHeader.data(channel.id, 0, {Rst}))
except CancelledError, LPStreamError:
discard
await channel.close()
if not channel.closedRemotely.isSet():
await channel.remoteClosed()
channel.receivedData.fire()
if not isLocal:
# If the reset is remote, there's no reason to flush anything.
channel.recvWindow = 0
proc updateRecvWindow(
channel: YamuxChannel
) {.async: (raises: [CancelledError, LPStreamError]).} =
## Send to the peer a window update when the recvWindow is empty enough
##
# In order to avoid spamming a window update everytime a byte is read,
# we send it everytime half of the maxRecvWindow is read.
let inWindow = channel.recvWindow + channel.recvQueue.len
if inWindow > channel.maxRecvWindow div 2:
return
let delta = channel.maxRecvWindow - inWindow
channel.recvWindow.inc(delta)
await channel.conn.write(YamuxHeader.windowUpdate(channel.id, delta.uint32))
trace "increasing the recvWindow", delta
method readOnce*(
channel: YamuxChannel, pbytes: pointer, nbytes: int
): Future[int] {.async: (raises: [CancelledError, LPStreamError]).} =
## Read from a yamux channel
if channel.isReset:
raise
if channel.remoteReset:
trace "stream is remote reset when readOnce", channel = $channel
newLPStreamResetError()
elif channel.closedLocally:
trace "stream is closed locally when readOnce", channel = $channel
newLPStreamClosedError()
else:
trace "stream is down when readOnce", channel = $channel
newLPStreamConnDownError()
if channel.isEof:
raise newLPStreamRemoteClosedError()
if channel.recvQueue.len == 0:
channel.receivedData.clear()
let
closedRemotelyFut = channel.closedRemotely.wait()
receivedDataFut = channel.receivedData.wait()
defer:
if not closedRemotelyFut.finished():
await closedRemotelyFut.cancelAndWait()
if not receivedDataFut.finished():
await receivedDataFut.cancelAndWait()
await closedRemotelyFut or receivedDataFut
if channel.closedRemotely.isSet() and channel.recvQueue.len == 0:
channel.isEof = true
return
0 # we return 0 to indicate that the channel is closed for reading from now on
let toRead = min(channel.recvQueue.len, nbytes)
var p = cast[ptr UncheckedArray[byte]](pbytes)
toOpenArray(p, 0, nbytes - 1)[0 ..< toRead] =
channel.recvQueue.toOpenArray(0, toRead - 1)
channel.recvQueue = channel.recvQueue[toRead ..^ 1]
# We made some room in the recv buffer let the peer know
await channel.updateRecvWindow()
channel.activity = true
return toRead
proc gotDataFromRemote(
channel: YamuxChannel, b: seq[byte]
) {.async: (raises: [CancelledError, LPStreamError]).} =
channel.recvWindow -= b.len
channel.recvQueue = channel.recvQueue.concat(b)
channel.receivedData.fire()
when defined(libp2p_yamux_metrics):
libp2p_yamux_recv_queue.observe(channel.recvQueue.len.int64)
await channel.updateRecvWindow()
proc setMaxRecvWindow*(channel: YamuxChannel, maxRecvWindow: int) =
channel.maxRecvWindow = maxRecvWindow
proc trySend(
channel: YamuxChannel
) {.async: (raises: [CancelledError, LPStreamError]).} =
if channel.isSending:
return
channel.isSending = true
defer:
channel.isSending = false
while channel.sendQueue.len != 0:
channel.sendQueue.keepItIf(not (it.fut.cancelled() and it.sent == 0))
if channel.sendWindow == 0:
trace "trying to send while the sendWindow is empty"
if channel.lengthSendQueueWithLimit() > channel.maxSendQueueSize:
trace "channel send queue too big, resetting",
maxSendQueueSize = channel.maxSendQueueSize,
currentQueueSize = channel.lengthSendQueueWithLimit()
await channel.reset(isLocal = true)
break
let
bytesAvailable = channel.lengthSendQueue()
toSend = min(channel.sendWindow, bytesAvailable)
var
sendBuffer = newSeqUninitialized[byte](toSend + 12)
header = YamuxHeader.data(channel.id, toSend.uint32)
inBuffer = 0
if toSend >= bytesAvailable and channel.closedLocally:
trace "last buffer we'll sent on this channel", toSend, bytesAvailable
header.flags.incl({Fin})
sendBuffer[0 ..< 12] = header.encode()
var futures: seq[Future[void].Raising([CancelledError, LPStreamError])]
while inBuffer < toSend:
# concatenate the different message we try to send into one buffer
let (data, sent, fut) = channel.sendQueue[0]
let bufferToSend = min(data.len - sent, toSend - inBuffer)
sendBuffer.toOpenArray(12, 12 + toSend - 1)[
inBuffer ..< (inBuffer + bufferToSend)
] = channel.sendQueue[0].data.toOpenArray(sent, sent + bufferToSend - 1)
channel.sendQueue[0].sent.inc(bufferToSend)
if channel.sendQueue[0].sent >= data.len:
# if every byte of the message is in the buffer, add the write future to the
# sequence of futures to be completed (or failed) when the buffer is sent
futures.add(fut)
channel.sendQueue.delete(0)
inBuffer.inc(bufferToSend)
trace "try to send the buffer", h = $header
channel.sendWindow.dec(toSend)
try:
await channel.conn.write(sendBuffer)
except CancelledError:
trace "cancelled sending the buffer"
for fut in futures.items():
fut.cancelSoon()
await channel.reset()
break
except LPStreamError as exc:
trace "failed to send the buffer"
let connDown = newLPStreamConnDownError(exc)
for fut in futures.items():
fut.fail(connDown)
await channel.reset()
break
for fut in futures.items():
fut.complete()
channel.activity = true
method write*(
channel: YamuxChannel, msg: seq[byte]
): Future[void] {.async: (raises: [CancelledError, LPStreamError], raw: true).} =
## Write to yamux channel
##
result = newFuture[void]("Yamux Send")
if channel.remoteReset:
trace "stream is reset when write", channel = $channel
result.fail(newLPStreamResetError())
return result
if channel.closedLocally or channel.isReset:
result.fail(newLPStreamClosedError())
return result
if msg.len == 0:
result.complete()
return result
channel.sendQueue.add((msg, 0, result))
when defined(libp2p_yamux_metrics):
libp2p_yamux_send_queue.observe(channel.lengthSendQueue().int64)
asyncSpawn channel.trySend()
proc open(channel: YamuxChannel) {.async: (raises: [CancelledError, LPStreamError]).} =
## Open a yamux channel by sending a window update with Syn or Ack flag
##
if channel.opened:
trace "Try to open channel twice"
return
channel.opened = true
await channel.conn.write(
YamuxHeader.windowUpdate(
channel.id,
uint32(max(channel.maxRecvWindow - YamuxDefaultWindowSize, 0)),
{if channel.isSrc: Syn else: Ack},
)
)
method getWrapped*(channel: YamuxChannel): Connection =
channel.conn
type Yamux* = ref object of Muxer
channels: Table[uint32, YamuxChannel]
flushed: Table[uint32, int]
currentId: uint32
isClosed: bool
maxChannCount: int
windowSize: int
maxSendQueueSize: int
inTimeout: Duration
outTimeout: Duration
proc lenBySrc(m: Yamux, isSrc: bool): int =
for v in m.channels.values():
if v.isSrc == isSrc:
result += 1
proc cleanupChannel(m: Yamux, channel: YamuxChannel) {.async: (raises: []).} =
try:
await channel.join()
except CancelledError:
discard
m.channels.del(channel.id)
when defined(libp2p_yamux_metrics):
libp2p_yamux_channels.set(
m.lenBySrc(channel.isSrc).int64, [$channel.isSrc, $channel.peerId]
)
if channel.isReset and channel.recvWindow > 0:
m.flushed[channel.id] = channel.recvWindow
proc createStream(
m: Yamux, id: uint32, isSrc: bool, recvWindow: int, maxSendQueueSize: int
): YamuxChannel =
# During initialization, recvWindow can be larger than maxRecvWindow.
# This is because the peer we're connected to will always assume
# that the initial recvWindow is 256k.
# To solve this contradiction, no updateWindow will be sent until
# recvWindow is less than maxRecvWindow
var stream = YamuxChannel(
id: id,
maxRecvWindow: recvWindow,
recvWindow:
if recvWindow > YamuxDefaultWindowSize: recvWindow else: YamuxDefaultWindowSize,
sendWindow: YamuxDefaultWindowSize,
maxSendQueueSize: maxSendQueueSize,
isSrc: isSrc,
conn: m.connection,
receivedData: newAsyncEvent(),
closedRemotely: newAsyncEvent(),
)
stream.objName = "YamuxStream"
if isSrc:
stream.dir = Direction.Out
stream.timeout = m.outTimeout
else:
stream.dir = Direction.In
stream.timeout = m.inTimeout
stream.timeoutHandler = proc(): Future[void] {.async: (raises: [], raw: true).} =
trace "Idle timeout expired, resetting YamuxChannel"
stream.reset(isLocal = true)
stream.initStream()
stream.peerId = m.connection.peerId
stream.observedAddr = m.connection.observedAddr
stream.transportDir = m.connection.transportDir
when defined(libp2p_agents_metrics):
stream.shortAgent = m.connection.shortAgent
m.channels[id] = stream
asyncSpawn m.cleanupChannel(stream)
trace "created channel", id, pid = m.connection.peerId
when defined(libp2p_yamux_metrics):
libp2p_yamux_channels.set(m.lenBySrc(isSrc).int64, [$isSrc, $stream.peerId])
return stream
method close*(m: Yamux) {.async: (raises: []).} =
if m.isClosed == true:
trace "Already closed"
return
m.isClosed = true
trace "Closing yamux"
let channels = toSeq(m.channels.values())
for channel in channels:
await channel.reset(isLocal = true)
try:
await m.connection.write(YamuxHeader.goAway(NormalTermination))
except CancelledError as exc:
trace "cancelled sending goAway", description = exc.msg
except LPStreamError as exc:
trace "failed to send goAway", description = exc.msg
await m.connection.close()
trace "Closed yamux"
proc handleStream(m: Yamux, channel: YamuxChannel) {.async: (raises: []).} =
## Call the muxer stream handler for this channel
##
await m.streamHandler(channel)
trace "finished handling stream"
doAssert(channel.isClosed, "connection not closed by handler!")
method handle*(m: Yamux) {.async: (raises: []).} =
trace "Starting yamux handler", pid = m.connection.peerId
try:
while not m.connection.atEof:
trace "waiting for header"
let header = await m.connection.readHeader()
trace "got message", h = $header
case header.msgType
of Ping:
if MsgFlags.Syn in header.flags:
await m.connection.write(YamuxHeader.ping(MsgFlags.Ack, header.length))
of GoAway:
var status: GoAwayStatus
if status.checkedEnumAssign(header.length):
trace "Received go away", status
else:
trace "Received unexpected error go away"
break
of Data, WindowUpdate:
if MsgFlags.Syn in header.flags:
if header.streamId in m.channels:
debug "Trying to create an existing channel, skipping", id = header.streamId
else:
if header.streamId in m.flushed:
m.flushed.del(header.streamId)
if header.streamId mod 2 == m.currentId mod 2:
debug "Peer used our reserved stream id, skipping",
id = header.streamId,
currentId = m.currentId,
peerId = m.connection.peerId
raise newException(YamuxError, "Peer used our reserved stream id")
let newStream =
m.createStream(header.streamId, false, m.windowSize, m.maxSendQueueSize)
if m.channels.len >= m.maxChannCount:
await newStream.reset()
continue
await newStream.open()
asyncSpawn m.handleStream(newStream)
elif header.streamId notin m.channels:
# Flush the data
m.flushed.withValue(header.streamId, flushed):
if header.msgType == Data:
flushed[].dec(int(header.length))
if flushed[] < 0:
raise
newException(YamuxError, "Peer exhausted the recvWindow after reset")
if header.length > 0:
var buffer = newSeqUninitialized[byte](header.length)
await m.connection.readExactly(addr buffer[0], int(header.length))
do:
raise newException(YamuxError, "Unknown stream ID: " & $header.streamId)
continue
let channel =
try:
m.channels[header.streamId]
except KeyError:
raise newException(
YamuxError,
"Stream was cleaned up before handling data: " & $header.streamId,
)
if header.msgType == WindowUpdate:
channel.sendWindow += int(header.length)
await channel.trySend()
else:
if header.length.int > channel.recvWindow.int:
# check before allocating the buffer
raise newException(YamuxError, "Peer exhausted the recvWindow")
if header.length > 0:
var buffer = newSeqUninitialized[byte](header.length)
await m.connection.readExactly(addr buffer[0], int(header.length))
trace "Msg Rcv", description = shortLog(buffer)
await channel.gotDataFromRemote(buffer)
if MsgFlags.Fin in header.flags:
trace "remote closed channel"
await channel.remoteClosed()
if MsgFlags.Rst in header.flags:
trace "remote reset channel"
await channel.reset()
except CancelledError as exc:
debug "Unexpected cancellation in yamux handler", description = exc.msg
except LPStreamEOFError as exc:
trace "Stream EOF", description = exc.msg
except LPStreamError as exc:
debug "Unexpected stream exception in yamux read loop", description = exc.msg
except YamuxError as exc:
trace "Closing yamux connection", description = exc.msg
try:
await m.connection.write(YamuxHeader.goAway(ProtocolError))
except CancelledError, LPStreamError:
discard
except MuxerError as exc:
debug "Unexpected muxer exception in yamux read loop", description = exc.msg
try:
await m.connection.write(YamuxHeader.goAway(ProtocolError))
except CancelledError, LPStreamError:
discard
finally:
await m.close()
trace "Stopped yamux handler"
method getStreams*(m: Yamux): seq[Connection] {.gcsafe.} =
for c in m.channels.values:
result.add(c)
method newStream*(
m: Yamux, name: string = "", lazy: bool = false
): Future[Connection] {.async: (raises: [CancelledError, LPStreamError, MuxerError]).} =
if m.channels.len > m.maxChannCount - 1:
raise newException(TooManyChannels, "max allowed channel count exceeded")
let stream = m.createStream(m.currentId, true, m.windowSize, m.maxSendQueueSize)
m.currentId += 2
if not lazy:
await stream.open()
return stream
proc new*(
T: type[Yamux],
conn: Connection,
maxChannCount: int = MaxChannelCount,
windowSize: int = YamuxDefaultWindowSize,
maxSendQueueSize: int = MaxSendQueueSize,
inTimeout: Duration = 5.minutes,
outTimeout: Duration = 5.minutes,
): T =
T(
connection: conn,
currentId: if conn.dir == Out: 1 else: 2,
maxChannCount: maxChannCount,
windowSize: windowSize,
maxSendQueueSize: maxSendQueueSize,
inTimeout: inTimeout,
outTimeout: outTimeout,
)

View File

@@ -0,0 +1,203 @@
# Nim-LibP2P
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
{.push raises: [].}
import
std/[streams, sets, sequtils],
chronos,
chronicles,
stew/byteutils,
dnsclientpkg/[protocol, types],
../utility
import nameresolver
logScope:
topics = "libp2p dnsresolver"
type DnsResolver* = ref object of NameResolver
nameServers*: seq[TransportAddress]
proc questionToBuf(address: string, kind: QKind): seq[byte] =
try:
var
header = initHeader()
question = initQuestion(address, kind)
requestStream = header.toStream()
question.toStream(requestStream)
let dataLen = requestStream.getPosition()
requestStream.setPosition(0)
var buf = newSeq[byte](dataLen)
discard requestStream.readData(addr buf[0], dataLen)
buf
except IOError as exc:
info "Failed to created DNS buffer", description = exc.msg
newSeq[byte](0)
except OSError as exc:
info "Failed to created DNS buffer", description = exc.msg
newSeq[byte](0)
except ValueError as exc:
info "Failed to created DNS buffer", description = exc.msg
newSeq[byte](0)
proc getDnsResponse(
dnsServer: TransportAddress, address: string, kind: QKind
): Future[Response] {.
async: (raises: [CancelledError, IOError, OSError, TransportError, ValueError])
.} =
var sendBuf = questionToBuf(address, kind)
if sendBuf.len == 0:
raise newException(ValueError, "Incorrect DNS query")
let receivedDataFuture = Future[void].Raising([CancelledError]).init()
proc datagramDataReceived(
transp: DatagramTransport, raddr: TransportAddress
): Future[void] {.async: (raises: []).} =
receivedDataFuture.complete()
let sock =
if dnsServer.family == AddressFamily.IPv6:
newDatagramTransport6(datagramDataReceived)
else:
newDatagramTransport(datagramDataReceived)
try:
await sock.sendTo(dnsServer, addr sendBuf[0], sendBuf.len)
try:
await receivedDataFuture.wait(5.seconds) #unix default
except AsyncTimeoutError:
raise newException(IOError, "DNS server timeout")
let rawResponse = sock.getMessage()
try:
parseResponse(string.fromBytes(rawResponse))
except IOError as exc:
raise exc
except OSError as exc:
raise exc
except ValueError as exc:
raise exc
except Exception as exc:
# Nim 1.6: parseResponse can has a raises: [Exception, ..] because of
# https://github.com/nim-lang/Nim/commit/035134de429b5d99c5607c5fae912762bebb6008
# it can't actually raise though
raiseAssert exc.msg
finally:
await sock.closeWait()
method resolveIp*(
self: DnsResolver, address: string, port: Port, domain: Domain = Domain.AF_UNSPEC
): Future[seq[TransportAddress]] {.
async: (raises: [CancelledError, TransportAddressError])
.} =
trace "Resolving IP using DNS", address, servers = self.nameServers.mapIt($it), domain
for _ in 0 ..< self.nameServers.len:
let server = self.nameServers[0]
var responseFutures: seq[
Future[Response].Raising(
[CancelledError, IOError, OSError, TransportError, ValueError]
)
]
if domain == Domain.AF_INET or domain == Domain.AF_UNSPEC:
responseFutures.add(getDnsResponse(server, address, A))
if domain == Domain.AF_INET6 or domain == Domain.AF_UNSPEC:
let fut = getDnsResponse(server, address, AAAA)
if server.family == AddressFamily.IPv6:
trace "IPv6 DNS server, puting AAAA records first", server = $server
responseFutures.insert(fut)
else:
responseFutures.add(fut)
var
resolvedAddresses: OrderedSet[string]
resolveFailed = false
template handleFail(e): untyped =
info "Failed to query DNS", address, error = e.msg
resolveFailed = true
break
for fut in responseFutures:
try:
let resp = await fut
for answer in resp.answers:
resolvedAddresses.incl(answer.toString())
except CancelledError as e:
raise e
except ValueError as e:
info "Invalid DNS query", address, error = e.msg
return @[]
except IOError as e:
handleFail(e)
except OSError as e:
handleFail(e)
except TransportError as e:
handleFail(e)
except Exception as e:
# Nim 1.6: answer.toString can has a raises: [Exception, ..] because of
# https://github.com/nim-lang/Nim/commit/035134de429b5d99c5607c5fae912762bebb6008
# it can't actually raise though
raiseAssert e.msg
if resolveFailed:
self.nameServers.add(self.nameServers[0])
self.nameServers.delete(0)
continue
trace "Got IPs from DNS server", resolvedAddresses, server = $server
return resolvedAddresses.toSeq().mapIt(initTAddress(it, port))
debug "Failed to resolve address, returning empty set"
return @[]
method resolveTxt*(
self: DnsResolver, address: string
): Future[seq[string]] {.async: (raises: [CancelledError]).} =
trace "Resolving TXT using DNS", address, servers = self.nameServers.mapIt($it)
for _ in 0 ..< self.nameServers.len:
let server = self.nameServers[0]
template handleFail(e): untyped =
info "Failed to query DNS", address, error = e.msg
self.nameServers.add(self.nameServers[0])
self.nameServers.delete(0)
continue
try:
let response = await getDnsResponse(server, address, TXT)
trace "Got TXT response",
server = $server, answer = response.answers.mapIt(it.toString())
return response.answers.mapIt(it.toString())
except CancelledError as e:
raise e
except IOError as e:
handleFail(e)
except OSError as e:
handleFail(e)
except TransportError as e:
handleFail(e)
except ValueError as e:
handleFail(e)
except Exception as e:
# Nim 1.6: toString can has a raises: [Exception, ..] because of
# https://github.com/nim-lang/Nim/commit/035134de429b5d99c5607c5fae912762bebb6008
# it can't actually raise though
raiseAssert e.msg
debug "Failed to resolve TXT, returning empty set"
return @[]
proc new*(T: typedesc[DnsResolver], nameServers: seq[TransportAddress]): T =
T(nameServers: nameServers)

View File

@@ -0,0 +1,49 @@
# Nim-LibP2P
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
{.push raises: [].}
import std/tables, chronos, chronicles
import nameresolver
export tables
logScope:
topics = "libp2p mockresolver"
type MockResolver* = ref object of NameResolver
txtResponses*: Table[string, seq[string]]
# key: address, isipv6?
ipResponses*: Table[(string, bool), seq[string]]
method resolveIp*(
self: MockResolver, address: string, port: Port, domain: Domain = Domain.AF_UNSPEC
): Future[seq[TransportAddress]] {.
async: (raises: [CancelledError, TransportAddressError])
.} =
var res: seq[TransportAddress]
if domain == Domain.AF_INET or domain == Domain.AF_UNSPEC:
for resp in self.ipResponses.getOrDefault((address, false)):
res.add(initTAddress(resp, port))
if domain == Domain.AF_INET6 or domain == Domain.AF_UNSPEC:
for resp in self.ipResponses.getOrDefault((address, true)):
res.add(initTAddress(resp, port))
res
method resolveTxt*(
self: MockResolver, address: string
): Future[seq[string]] {.async: (raises: [CancelledError]).} =
self.txtResponses.getOrDefault(address)
proc new*(T: typedesc[MockResolver]): T =
T()

Some files were not shown because too many files have changed in this diff Show More