Compare commits

...

330 Commits

Author SHA1 Message Date
Tanguy
15f2b38877 Revert "revert refactos"
This reverts commit 136296fe7e.
2023-07-28 11:06:41 +02:00
Tanguy
f68b32a64c Merge remote-tracking branch 'origin/unstable' into limitfloodpublish 2023-07-28 11:04:45 +02:00
Tanguy
136296fe7e revert refactos 2023-07-28 11:04:41 +02:00
Tanguy
b784167805 GossipSub: IDontWant (#934) 2023-07-28 10:58:05 +02:00
Tanguy
b06e362c71 refacto 2023-07-26 11:46:11 +02:00
Tanguy
440461b24b GS: improve handleIHave (#922) 2023-07-11 12:17:50 +02:00
Jacek Sieka
fab1340020 avoid a few zeroMem (#932) 2023-07-11 12:17:28 +02:00
Tanguy
1721f078c7 Fix crash on empty write (#930) 2023-07-10 13:52:08 +00:00
Ivan Folgueira Bande
74c402ed9d wstransport.nim: avoid re-raising 'TransportOsError' to avoid stopping switch.accept (#929) 2023-07-07 11:32:20 +02:00
diegomrsantos
c45f9705ab Gossipsub scoring improvements (#909) 2023-07-04 00:27:45 +02:00
Diego
d974bc4e9a Fix test 2023-06-29 16:48:13 +02:00
Diego
c10c232424 Fix bandwidth and split publish into smaller procs 2023-06-29 15:58:09 +02:00
Etan Kissling
81b861b34e avoid ProveField warning in crypto.init (#915) 2023-06-29 15:28:25 +02:00
Jacek Sieka
43359dd9d1 standard nimble env vars (#921) 2023-06-28 17:51:51 +02:00
diegomrsantos
f85d0f75ea Handling Opt[PeerId] in logging (#923) 2023-06-28 17:00:33 +02:00
Tanguy
66f9dc9167 Remove all Result.get()s & Option -> Opt (#902)
Co-authored-by: Ludovic Chenut <ludovic@status.im>
Co-authored-by: Diego <diego@status.im>
2023-06-28 16:44:58 +02:00
Tanguy
1c4d0832ce Add GossipSub ping (#912) 2023-06-21 10:40:10 +02:00
Tanguy
224f92e172 Fix #916 regression causing accept loop lockup (#919) 2023-06-20 14:18:49 +00:00
Tanguy
5efa089196 TCP transport: handle getObservedAddr errors (#918) 2023-06-20 10:25:29 +02:00
Tanguy
9d4c4307de Bumper: fix case where target is up to date (#917) 2023-06-15 18:37:01 +02:00
Tanguy
652e7810b6 fix ci 2023-06-15 15:03:12 +02:00
Tanguy
49dfa84c6f Transports: handle TransportAbortedError properly (#916) 2023-06-14 15:55:56 +00:00
Tanguy
a65b7b028f GossipSub: remove peer if we can't establish sendConn (#886) 2023-06-14 17:23:39 +02:00
Tanguy
30e0dae40a fix ci 2023-06-14 10:55:46 +02:00
diegomrsantos
67711478ce Consider dns as public address (#913) 2023-06-13 17:58:41 +02:00
Tanguy
50aaaf4784 Always publish to at least dLow 2023-06-13 15:51:26 +02:00
Tanguy
c28d8bb353 WS Transport: handle 'tls/ws' (#914) 2023-06-12 15:45:53 +00:00
Tanguy
8f78a349ed really fix ci 2023-06-12 14:53:55 +02:00
Tanguy
07f7670b22 even slower for CI 2023-06-12 11:13:39 +02:00
Tanguy
f50cdd4c6d wait longer for slow cI 2023-06-09 10:47:26 +02:00
Tanguy
f2209dc299 Limit flood publishing 2023-06-09 10:26:39 +02:00
Tanguy
eb78292d9c Bump deps (#896) 2023-06-07 17:42:42 +02:00
Vaclav Pavlin
3725f6a95b chore: add basic metrics for rendezvous (#905) 2023-06-07 15:45:06 +02:00
diegomrsantos
3640b4dd89 Autonat and HP changes (#899) 2023-06-07 15:26:58 +02:00
Tanguy
32085ca88a Allow to override TCP connection timeouts (#903) 2023-06-07 14:27:32 +02:00
Tanguy
c76d1e18ef Remove nim 1.2 support (#907) 2023-06-07 11:12:49 +00:00
Tanguy
41649f0999 Version 1.1.0 (#904) 2023-06-06 11:05:49 +00:00
diegomrsantos
67102873ba Fail only if all addresses in PeerRecord are invalid (#898)
Fixes https://github.com/waku-org/nwaku/issues/1768
2023-05-31 08:59:50 +02:00
diegomrsantos
d40d324160 fix missing import (#897) 2023-05-26 10:36:39 +02:00
diegomrsantos
a677b06273 Try a direct connection only if there isn't one already (#891) 2023-05-25 15:48:22 +02:00
diegomrsantos
6050cdef7e Refinement of Hole Punching Service (#892) 2023-05-25 15:47:00 +02:00
Tanguy
fedfa8e817 Fix bumper CI (#894) 2023-05-22 17:37:42 +02:00
diegomrsantos
6887b43777 Improve utility tests (#893) 2023-05-22 11:07:22 +02:00
Tanguy
225accd11b Less warnings (#813)
Co-authored-by: Diego <diego@status.im>
2023-05-18 10:24:17 +02:00
diegomrsantos
7d6bc545e0 Handle dns addrs in HP service (#890) 2023-05-16 14:59:02 +02:00
Tanguy
a1eb53b181 Fix gossipsub dOut handling (#883) 2023-04-26 13:44:45 +02:00
Tanguy
db629dca25 Fix network protocol metrics typo (#874) 2023-04-26 09:52:06 +02:00
diegomrsantos
a5666789b0 Hole Punching (#806)
Co-authored-by: Tanguy <tanguy@status.im>
2023-04-18 12:50:21 +02:00
diegomrsantos
b7726bf68f Dcutr (#824)
Co-authored-by: Tanguy <tanguy@status.im>
2023-04-14 16:23:19 +02:00
diegomrsantos
0221affe98 Invalid MA is ignored (#881) 2023-04-14 12:05:32 +00:00
diegomrsantos
edbd35b16c Fix interop tests (#882) 2023-04-13 19:38:34 +02:00
diegomrsantos
80cca0ecac Does not allow an empty MA (#877) 2023-04-06 14:19:01 +00:00
diegomrsantos
0041ed4cf8 Transport hole punching (#873)
Co-authored-by: Tanguy <tanguy@status.im>
2023-04-06 15:23:35 +02:00
Tanguy
95e98e8c51 Fix traffic metrics (#879) 2023-04-03 12:37:23 +00:00
Tanguy
4aa615c44c GossipSub: TimedEntry & shortAgent fixes (#858) 2023-04-03 11:05:01 +02:00
Tanguy
6b61ce8c91 GossipSub: Better IWANT handling (#875) 2023-04-03 10:56:20 +02:00
Alvaro Revuelta
53b060f8f0 Add getters for conns and streams (#878) 2023-03-31 00:16:39 +02:00
diegomrsantos
af5299f26c Create an ObservedAddrManager and add an AddressMapper in AutonatService and AutoRelayService (#871)
Co-authored-by: Tanguy <tanguy@status.im>
2023-03-24 15:42:49 +00:00
Tanguy
bac754e2ad Various gossipsub fixes (#827) 2023-03-21 17:13:25 +01:00
Tanguy
8d5ea43e2b Upgrade flow refactoring (#807) 2023-03-08 12:30:19 +01:00
Jacek Sieka
e573238705 reexport public types (#872) 2023-03-06 15:36:10 +00:00
Tanguy
c1a3bd8fee Fix pubsub CI logs (#861) 2023-03-01 16:59:44 +01:00
diegomrsantos
ddeb7b3bd4 Handle when peers ask each other at the same time (#865) 2023-02-21 17:49:41 +01:00
Tanguy
382b992e00 Interop tests (#864) 2023-02-20 14:26:53 +01:00
Tanguy
408dcf12bd Fix backward compatibility of #822 (#862) 2023-02-15 17:18:29 +01:00
Ludovic Chenut
0012b639c8 Fix testrelay (#860) 2023-02-15 11:18:42 +01:00
Tanguy
f7f1e89669 TCP Transport: enable NO_DELAY for clients (#822) 2023-02-14 10:35:44 +01:00
Tanguy
f14ada3dcf Move tests flags from 'nimble test' to 'config.nims' (#852) 2023-02-10 11:32:21 +01:00
diegomrsantos
444b837923 Autonat doesn't ask an incoming peer (#857) 2023-02-09 17:40:04 +01:00
diegomrsantos
f89bd0c77c Autonat dials dns addrs (#856) 2023-02-09 16:53:46 +01:00
diegomrsantos
e68186373b Dialing addrs concurrently in autonat (#855) 2023-02-07 18:51:17 +01:00
diegomrsantos
266c7b117a Add anyCompleted proc (#853) 2023-02-07 18:50:42 +01:00
Ludovic Chenut
0e28d3b828 Add the peerId to the transport.dial (#842) 2023-01-31 12:46:10 +01:00
diegomrsantos
4ace70d53b Connect is able to force a new connection (#849) 2023-01-25 11:19:03 +01:00
Tanguy
ca19f8fdbf Autonat service: handle connections limits (#846)
Co-authored-by: diegomrsantos <diego@status.im>
2023-01-24 17:04:42 +01:00
Tanguy
351bda2b56 Add expected connections to connmngr (#845)
Co-authored-by: diegomrsantos <diego@status.im>
2023-01-23 22:28:39 +00:00
Tanguy
7d9c43a5ce Fix CI for nim devel 1.6 (#848) 2023-01-23 15:53:15 +01:00
Tanguy
c11772c94e Happy new year! (#847) 2023-01-20 15:47:40 +01:00
Ludovic Chenut
489c115132 Autorelay service (#819) 2023-01-17 16:18:38 +01:00
Tanguy
166c0d1c87 Fix PubSub subscribe on connection race condition (#809) 2023-01-10 13:33:14 +01:00
Etan Kissling
ba451196e8 Avoid closing connection on channel EOF (#816) 2023-01-06 15:18:16 +01:00
diegomrsantos
9f658c151e Autonat refactoring (#834) 2023-01-06 11:14:38 +01:00
diegomrsantos
e304ad0f7e Remove unnecessary async (#836) 2023-01-05 15:02:52 +01:00
diegomrsantos
5e3323d43f More autonat tests (#833) 2022-12-23 19:10:15 +00:00
diegomrsantos
9532bff983 Ignore unknown answers (#831)
Co-authored-by: Tanguy <tanguy@status.im>
2022-12-23 16:49:25 +01:00
diegomrsantos
676786b00e Add Autonat timeout (#829)
Co-authored-by: Ludovic Chenut <ludovic@status.im>
2022-12-22 19:29:31 +00:00
diegomrsantos
d521c57b82 Handle dial error correctly (#830) 2022-12-22 17:33:59 +01:00
Tanguy
63e1872516 Fix yamux simultaneous closing (#823) 2022-12-16 18:14:40 +01:00
diegomrsantos
67ef25fae0 Autonat service (#814)
Co-authored-by: Tanguy <tanguy@status.im>
2022-12-16 11:32:00 +00:00
Tanguy
fe7a69e389 Fix pinning system on devel (#820) 2022-12-14 12:10:00 +01:00
Ludovic Chenut
a17cad710c Fix devel 32bits (#818) 2022-12-09 17:23:33 +01:00
Tanguy
3863a4cd21 Allow connect with known PeerId in MA (#817) 2022-12-08 16:11:55 +00:00
Tanguy
64cbbe1e0a Limit number of streams per protocol per peer (#811) 2022-12-01 12:20:40 +01:00
Tanguy
31ad4ae205 Fix website (#812) 2022-12-01 12:20:17 +01:00
diegomrsantos
b3d9360dfc Starting switch two times does not crash (#810) 2022-11-29 15:21:51 +00:00
Tanguy
1711c204ea Fix: big messages in pubsub (#808) 2022-11-24 16:43:45 +00:00
Tanguy
c43aacdc81 WS: add timeout to handleRequest (#796) 2022-11-22 10:44:08 +01:00
Tanguy
711609057c Reuseable install_nim action (#803) 2022-11-21 16:56:04 +01:00
Alvaro Revuelta
192cac6254 Store missing pubkey in PeerStore [KeyBook] (#804) 2022-11-21 10:55:25 +01:00
Tanguy
cc3c637c22 Daily CI: run devel with refc (#799) 2022-11-16 16:56:02 +01:00
Tanguy
afbb1b4d3c Fix Codecov (#802)
Also switch from 14 jobs to 1
2022-11-16 14:33:42 +01:00
Dmitriy Ryajov
8c2eca18dc Allow passing max message size (#800)
Co-authored-by: Tanguy <tanguy@status.im>
2022-11-15 14:01:14 +01:00
Tanguy
ce371f3bb4 Update README (#798) 2022-11-10 13:47:41 +01:00
diegomrsantos
23338fceaa Explicit dependency to unittest2 and update other packages (#797) 2022-11-07 22:55:22 +01:00
diegomrsantos
6ab6ab48ef Fix after gcsafe requirements change and other issues on devel (#795) 2022-11-05 02:04:05 +01:00
diegomrsantos
d9305bda84 Add Tor Transport support (#765) 2022-11-04 19:20:23 +01:00
Lorenzo Delgado
f95eda8bf6 Add libp2p label to chronicles log topics (#794) 2022-11-04 11:19:27 +01:00
Tanguy
a3e9d1ed80 Version 1.0.0 (#785) 2022-10-30 13:18:04 +01:00
Tanguy
c9c2f6acdb Tron example (#775)
Co-authored-by: lchenut <ludovic@status.im>
2022-10-30 08:49:02 +01:00
Tanguy
c39c1cbf68 Fix daily CI (#790) 2022-10-29 23:26:44 +02:00
lchenut
b4f96721af Add a tutorial for the discovery manager (#787)
Co-authored-by: Tanguy <tanguy@status.im>
2022-10-29 10:12:12 +00:00
Tanguy
4bce8f38c9 Remove sleepAsync from tests/testswitch (#792) 2022-10-27 23:10:24 +00:00
Tanguy
dc13ff81d3 Revert "Temporarily remove failing test (#788)" & fix tests (#789)
Co-authored-by: Ludovic Chenut <ludovic@status.im>
2022-10-26 11:07:31 +02:00
Tanguy
a69301f392 Merge remote-tracking branch 'origin/unstable' 2022-10-21 19:44:32 +02:00
Tanguy
4b105c6abd GossipSub tutorial (#784) 2022-10-21 17:00:36 +02:00
Tanguy
2e12c7ab73 Temporarily remove failing test (#788) 2022-10-21 16:59:53 +02:00
Tanguy
a086fcba72 Remove shallow copies (#782) 2022-10-20 14:52:02 +02:00
Tanguy
7b103e02f2 Allow public address mapping (#767) 2022-10-20 12:22:28 +02:00
lchenut
32233d36c8 Discovery interface (#783)
Co-authored-by: Tanguy <tanguy@status.im>
2022-10-03 22:00:00 +00:00
lchenut
1c99aca054 RendezVous Protocol (#751) 2022-09-30 10:41:04 +02:00
diegomrsantos
4f18dd30e9 Handle trying to write empty byte seq (#780) 2022-09-29 20:02:10 +02:00
Tanguy
0cd3554ce4 Bump deps (#779) 2022-09-29 10:29:51 +02:00
Tanguy
bcb8f5e3b6 Protobuf tutorial (#778) 2022-09-29 10:28:58 +02:00
Tanguy
eb78660702 Docs rework (#776) 2022-09-28 10:40:53 +02:00
Etan Kissling
103e199bc0 add lodestar to known lib p2p agents documentation (#766) 2022-09-26 11:48:03 +02:00
Tanguy
fa5d102370 Better dnsaddr resolving (#753) 2022-09-26 11:03:24 +02:00
diegomrsantos
a56c3bc296 Make observedAddr optional (#772)
Co-authored-by: Tanguy <tanguy@status.im>
2022-09-22 21:55:59 +02:00
Tanguy
5e7e009445 Move relay & autonat to connectivity folder (#769) 2022-09-15 09:43:40 +02:00
Tanguy
72abe822c0 Fix switch failed start (#770) 2022-09-15 07:06:32 +00:00
Tanguy
a001508490 Add codex & waku to autobump (#768) 2022-09-14 12:05:43 +00:00
lchenut
4d8b50d24c Specify EOF error (#759) 2022-09-14 10:58:41 +02:00
Tanguy
ef594e1e02 Only log multiple missed heartbeats as info (#763) 2022-09-12 17:09:10 +02:00
diegomrsantos
d8a9e93ff7 Add onion3 multiaddr support (#764) 2022-09-08 17:10:11 +02:00
Tanguy
abbeaab684 Keep connection alive when peer doesn't support pubsub (#754) 2022-09-06 13:20:42 +02:00
Jacek Sieka
dfbfbe6eb6 allow connection to a peer with unknown PeerId (#756)
Co-authored-by: Tanguy <tanguy@status.im>
2022-09-05 14:31:14 +02:00
lchenut
1de7508b64 Fix typos (#761) 2022-09-02 18:40:14 +00:00
Tanguy
3ffc03ed16 Gossipsub: Rebalance mesh immediately when peer sub/unsub (#719) 2022-09-02 10:24:54 +02:00
Tanguy
543358b262 Remove unused code 2022-09-02 10:09:12 +02:00
Miran
14d2c3f51e Configuration refactoring (#732)
move -d:nimRawSetjmp to nim.cfg
switch .cfg to .nims
add --skipParentCfg

Co-authored-by: Tanguy <tanguy@status.im>
2022-09-01 14:00:22 +02:00
lchenut
2332813873 Autonat protocol (#739) 2022-08-23 17:49:07 +02:00
Tanguy
124a7a5ffe Add missing stub logger (#752) 2022-08-03 19:04:17 +02:00
Tanguy
2d864633ea ConnManager connection tracking refacto (#749) 2022-08-03 16:48:19 +02:00
Miran
2fbe82bf9d make it more Nim 1.4+ compatible (#742) 2022-08-03 13:33:19 +02:00
Miran
20c02a5f23 ci: test Nim 1.6 branch (#747)
* ci: test Nim 1.6 branch
* drop `devel` from testing

Co-authored-by: Tanguy <tanguy@status.im>
2022-08-02 19:07:08 +02:00
lchenut
a9a7e7eb15 Yamux metrics and limits (#740)
* Add yamux channel gauge
* Add limit to channel
* Add recv/send queue length metrics
* Add yamux stream tracking
* Add timeout to YamuxChannel

Co-authored-by: Tanguy <tanguy@status.im>
2022-08-01 12:52:42 +00:00
lchenut
34c2fb8787 Circuit relay v2 (#717) 2022-08-01 14:31:22 +02:00
tersec
1e598a0239 topicIDs -> topicIds; PubsubPeerEvent -> PubSubPeerEvent; messageIDs -> messageIds (#748) 2022-07-27 17:14:05 +00:00
tersec
4ca1c2d7ed --styleCheck:error (#743)
* --styleCheck:error

* PeerID -> PeerId

* other libp2p styleCheck changes

* MessageID -> messageId; topicID -> topicId
2022-07-27 11:47:50 +00:00
Tanguy
83ad890535 Documentation site (#745) 2022-07-26 12:44:26 +02:00
Tanguy
0b0686ee94 Move auto bump to different CI job (#731) 2022-07-26 12:35:02 +02:00
Tanguy
93ac795aef Bump dependencies (#744) 2022-07-22 21:39:06 +02:00
lchenut
912873f8b3 Replace table by seq for storing muxers (#741)
Replace table by seq for stocking muxers
2022-07-22 12:54:09 +02:00
Ivan Yonchovski
78a65eebcc Add setup files (#738) 2022-07-12 21:03:23 +03:00
Tanguy
533e39ef94 Yamux (#704)
Co-authored-by: Ludovic Chenut <ludovic@status.im>
2022-07-04 15:19:21 +02:00
Tanguy
150fafbee8 Faster CI (#724)
* Only compile once in CI
* Fix codecov fetch
2022-07-01 20:20:42 +02:00
Tanguy
d0523fdc9d Documentation (#716) 2022-07-01 20:19:57 +02:00
lchenut
0ece5eaf12 Start/stop protocols (#730)
Starting/stopping a switch now starts/stops all protocols mounted on that switch
2022-06-30 11:21:33 +02:00
Jacek Sieka
e6440c43c2 lower log level of no-peers message (#735)
the number of peers sent to is returned from the function - the caller
can log this, if need be
2022-06-30 09:57:07 +02:00
Jacek Sieka
597abddba7 don't increase seqno when anonymizing (#734) 2022-06-30 09:56:49 +02:00
Tanguy
5d7024f2e0 Handle more cancellations (#733)
* Mplex: handle write cancellation
* LpChannel: Don't close connection on EOF
2022-06-24 11:11:23 +02:00
Tanguy
a7e335e1bb Remove Br prefix from BearSSL (#728)
ref status-im/nim-bearssl#27
2022-06-16 10:08:52 +02:00
Tanguy
718374d890 Merge pull request #718 from status-im/unstable
Unstable
2022-06-07 10:06:31 +02:00
lchenut
36f3132d9a Fix compilation error on nim 1.4 (#723) 2022-05-27 12:51:56 +02:00
Tanguy
ca3f4e8701 builder: Expose MaxChanCount (#722) 2022-05-25 15:59:42 +00:00
Tanguy
7323ecc9c4 Optimize rebalanceMesh (#708) 2022-05-25 12:59:33 +02:00
Tanguy
60becadcf9 Peer store refacto (#700)
There is now a global PeerStore structure (instead of having one for libp2p, one for waku, etc)

The user can create custom books for new types easily

Also add a pruning system to remove dead peers
2022-05-25 12:12:57 +02:00
Tanguy
1696d0c707 Set activity flag in WS (#721) 2022-05-24 19:41:05 +00:00
Tanguy
d4ff1c88e9 Less warnings (#710) 2022-05-24 15:10:57 +02:00
Tanguy
e536d7cb1b Bump deps (#720) 2022-05-19 17:29:09 +02:00
lchenut
13503f3799 Circuit relay v1 (#670)
Circuit relay v1
2022-05-18 10:19:37 +02:00
Tanguy
991549f391 Gossipsub scoring fixes (#709)
* Use decayInterval as a scoring heartbeat period
* Take mesh delivery window into account
2022-05-11 10:38:43 +02:00
lchenut
32ca1898d9 Gossipsub: Put Peer Exchange behind a flag (#715)
Add a flag to enable Peer Exchange in Gossipsub (disabled by default)
2022-05-10 10:39:43 +02:00
Tanguy
9ba5c069c8 Update SPR when the PeerInfo changes (#711) 2022-04-13 09:35:28 +02:00
Tanguy
c97befb387 Add tests for gossipsub direct peers (#707) 2022-04-12 14:03:31 +00:00
tersec
fc6b8f46f1 encrypt and decrypt empty sequences (#713)
* encrypt and decrypt empty sequences

* use assign in curve25519
2022-04-12 10:41:48 +00:00
Csaba Kiraly
9973b9466d expose more libp2p performance and queuing metrics (#678)
* gossipsub: adding duplicate arrival metrics

Adding counters for received deduplicated messages and for
duplicates recognized by the seen cache. Note that duplicates that
are not recognized (arrive after seenTTL) are not counted as
duplicates here either.

* gossipsub: adding mcache (message cache for responding IWANT) stats

It is generally assumed that IWANT messages arrive when mcache still
has the message. These stats are to verify this assumption.

* libp2p: adding internal TX queuing stats

Messages are queued in TX before getting written on the stream,
but we have no statistics about these queues. This patch adds
some queue length and queuing time related statistics.

* adding Grafana libp2p dashboard

Adding Grafana dashboard with newly exposed metrics.

* enable libp2p_mplex_metrics in nimble test

Signed-off-by: Csaba Kiraly <csaba.kiraly@gmail.com>
2022-04-06 16:00:24 +02:00
Tanguy
868ecab54f Pin go version in CI (#706) 2022-04-05 13:12:55 +02:00
Tanguy
84cbcd8f22 Bump dependencies (#703) 2022-03-29 09:23:24 +02:00
Tanguy
eaa72dcdbe WS Accept timeout (#699)
* Add timeout to WS accept
* Handle more WS errors
2022-03-17 10:16:48 +01:00
Tanguy
c7504d2446 Gossipsub peer exchange (#647)
* Signed envelopes and routing records
* Send signed peer record as part of identify (#649)
* Add SPR from identify to new peer book (#657)
* Send & receive gossipsub PX
* Add Signed Payload

Co-authored-by: Hanno Cornelius <68783915+jm-clius@users.noreply.github.com>
2022-03-14 09:39:30 +01:00
Tanguy
cba3ca3c3e Unstable (#695) 2022-02-25 09:41:32 +01:00
Eric Mastro
44a7260f07 fixes from #688 (#697)
* tests: invert message logic on expect from #688
* fix: export pubsub_errors for backward compatibility
2022-02-24 17:32:20 +01:00
Tanguy
c09d032133 Allow force dial (#696) 2022-02-24 17:31:47 +01:00
Tanguy
f98bf612bd Fix tests of #638 2022-02-21 18:14:43 +01:00
Tanguy
fd59cbc7a9 Fix shuffle of #638 2022-02-21 17:00:18 +01:00
Tanguy
bc318084f4 GS: Publish to fanout when mesh unhealthy (#638)
* Send to fanout when mesh unhealthy

* don't use fanout when floodPublish
2022-02-21 16:22:08 +01:00
Eric Mastro
3b718baa97 feat: allow msgIdProvider to fail (#688)
* feat: allow msgIdProvider to fail

Closes: #642.

Changes the return type of the msgIdProvider to `Result[MessageID, string]` so that message id generation can fail.

String error type was chosen as this `msgIdProvider` mainly because the failed message id generation drops the message and logs the error provided. Because `msgIdProvider` can be externally provided by library consumers, an enum didn’t make sense and a object seemed to be overkill. Exceptions could have been used as well, however, in this case, Result ergonomics were warranted and prevented wrapping quite a large block of code in try/except.

The `defaultMsgIdProvider` function previously allowed message id generation to fail silently for use in the tests: when seqno or source peerid were not valid, the message id generated was based on a hash of the message data and topic ids. The silent failing was moved to the `defaultMsgIdProvider` used only in the tests so that it could not fail silently in applications.

Unit tests were added for the `defaultMsgIdProvider`.

* Change MsgIdProvider error type to ValidationResult
2022-02-21 16:04:17 +01:00
Tanguy
9a7e3bda3c Bump dependencies (#694) 2022-02-10 14:21:12 +01:00
Tanguy
00e1f9342f Fix identify log for json_sink (#690) 2022-02-01 18:35:48 +01:00
Tanguy
07da14a7a7 Fix websocket EOF reading exception (#689) 2022-01-28 18:05:07 +00:00
Tanguy
c18830ad33 Score correctly on mesh peer unsub (#644)
* Score correctly on mesh peer unsub
* remove from mesh before removing from gossipsub
2022-01-15 12:47:41 +01:00
Tanguy
1a97d0a2f5 Validate pubsub subscriptions (#627)
* Check topic before subscribing
* Block subscribe to invalid topics
2022-01-14 12:40:30 -06:00
Ștefan Talpalaru
e72d03bc78 CI: test with multiple Nim versions by default (#684)
* CI: test with 1.2 & devel by default
* Skip buggy windows websocket test
2022-01-10 12:29:52 +01:00
Tanguy
388b92d58f Bump dependencies (#683) 2022-01-07 08:19:22 +01:00
Tanguy
f3dee6865c Chronos strict exception tracking (#652)
* Enable chronos strict exception tracking ( -d:chronosStrictException )
2022-01-05 16:27:33 +01:00
Dmitriy Ryajov
dffe4bed45 mitigate high traffic - drop faulty peers (#460)
Only allow 1 connection per peer

Co-authored-by: Tanguy <tanguy@status.im>
2022-01-04 11:21:24 +00:00
Tanguy
fb0d10b6fd Gossipsub: process messages concurrently (#680)
* Gossip sub: process messages concurrently

* Retries for flaky test
2021-12-27 11:17:00 +01:00
Tanguy
58f383e661 Bump unittest2
Now that we have pinned nimble deps, we get
surprises from the nim version updates
2021-12-17 09:13:17 +01:00
Tanguy
123bf290c3 Merge branch 'unstable' (#674) 2021-12-17 08:31:47 +01:00
Tanguy
1e7e95f5fd Pin dependencies for CI (#677)
Adds a pin-file for CI. When a dependency break something in the libp2p, we don't want for the CI to be
broken on every branch, just on a "bump" branch that will update the pin-file & fix the eventual interop issue.
2021-12-16 18:00:10 +00:00
Tanguy
df566e69db Fixes for style check (#676) 2021-12-16 11:05:20 +01:00
Tanguy
1152012bb0 Update README
Core developers, badges, module list were not up to date
2021-12-15 15:01:28 +01:00
Jacek Sieka
c49932b55a fast path for writes (#659)
avoids several copies of the various message buffers being kept alive
for the lifetime of the future
2021-12-14 10:55:17 +01:00
Tanguy
47a35e26d7 Typo: s/unsubcribeBackoff/unsubscribeBackoff (#675) 2021-12-14 10:50:57 +01:00
Etan Kissling
2373ee0061 harden ecnist byte export against uninitialized key (#671)
Currently, `ecnist`'s `toBytes` and `getBytes` methods operate only on
properly initialized keys. If an un-initialized key is given, an
`IndexError` may be raised if the key's `xlen` / `qlen` property is
larger than the maximum buffer size. This patch hardens those functions
to report a proper error in that case.
Note that the library functions called by `init` and `initRaw` already
reject data that does not have the expected length, so these new checks
should not be reachable in practice.
2021-12-13 18:46:25 +00:00
Etan Kissling
0be9180977 harden ProtoBuffer.finish() assert (#672)
This hardens the length check in `ProtoBuffer.finish` to account for
passed `options`.
2021-12-13 18:31:05 +00:00
Tanguy
8fe44e35c2 Fix flaky tests (#669)
* Fix push identify flaky test
* Fix mplex flaky tests
* Fix switch flaky test
2021-12-13 11:25:33 +01:00
Jacek Sieka
18dc233c9b link to style guide 2021-12-10 10:11:52 +01:00
Eric Mastro
fffa7e8cc2 fix: remove returned Futures from switch.start (#662)
* fix: remove returned Futures from switch.start

The proc `start` returned a seq of futures that was mean to be awaited by the caller. However, the start proc itself awaited each Future before returning it, so the ceremony requiring the caller to await the Future, and returning the Futures themselves was just used to handle errors. But we'll give a better way to handle errors in a future revision

Remove `switch.start` return type (implicit `Future[void]`)

Update tutorials and examples to reflect the change.

* Raise error during failed transport

Replaces logging of error, and adds comment that it should be replaced with a callback in a future PR.
2021-12-03 19:23:12 +01:00
Tanguy
6893bd9dbb Customizable gossipsub backoff on unsubscribe (#665)
* Customizable gossipsub backoff on unsubscribe
* change default to 5s
2021-12-02 14:47:40 +00:00
Eric Mastro
c19b966d23 fix (directchat): auto disconnect self once conn closed (#663)
When closing a connection of clientA(using the command “/disconnect”), the connection is then closed on clientA’s side. However, the connection remained open on clientB and clientB could continue sending message to clientA and clientA would receive those messages.

This PR listens for a closing connection on clientB, then also closes clientB’s connection.
2021-12-02 13:45:55 +01:00
Tanguy
b8c54068a3 Handle websock exceptions in handle (#658) 2021-11-26 13:55:35 +01:00
Tanguy
0dfac6fce7 Signed envelopes and routing records (#656) 2021-11-24 14:03:40 -06:00
Dmitriy Ryajov
73168b6eae Add support for multiple addresses to transports (#598)
* add test for multiple local addresses

* allow transports to listen on multiple addrs

* fix tcp transport accept

* check switch addrs are correct

* switch test to port 0

* close accepted peers on close

* ignore CancelledError in transport accept

* test ci

* only accept in accept loop

* avoid accept greedyness

* close acceptedPeers

* accept doesn't crash on cancelled fut

* add common transport test

* close conn on handling failure

* close accepted peers in two steps

* test for macos

* revert accept greedyness

* fix dialing cancel

* test chronos fix

* add ws

* ws cancellation

* small fix

* remove chronos blocked test

* fix testping

* Fix transport's switch start (like #609)

* bump chronos

* Websocket: handle both ws & wss

Co-authored-by: Tanguy Cizain <tanguycizain@gmail.com>
Co-authored-by: Tanguy <tanguy@status.im>
2021-11-24 14:01:12 -06:00
Tanguy
98184d9dfd Merge pull request #654 from status-im/ci_testslim
CI Multi nim: only test slim
2021-11-19 14:54:51 +01:00
Tanguy
5f18968c3f CI Multi nim: only test slim 2021-11-19 11:52:29 +01:00
Tanguy
32ede6da3c Merge pull request #651 from status-im/unstable
Unstable
2021-11-16 17:36:00 +01:00
Tanguy
6f779c47c8 Gossipsub: don't send to peers seen during validation (#648)
* Gossipsub: don't send to peers seen during validation

* Less error prone code

* add metric

* Fix metric

* remove dangling code test

* address comments

* don't allocate memory
2021-11-14 09:08:05 +01:00
Tanguy
5d1b10f3e7 Short agent fix (#645)
* Copy client type to muxed channel

* Fix incoming shortAgent
2021-11-08 18:33:37 +01:00
Tanguy
7d677f848f Short agent fix (#645)
* Copy client type to muxed channel

* Fix incoming shortAgent
2021-11-08 17:42:56 +01:00
Tanguy
c92125a1a4 Integrate dns resolving (#615)
* integrate dns

* give hostname to transport dial

* add hostname test

* switched to websock master

* Add dnsaddr dial test w multiple transports
2021-11-08 13:02:03 +01:00
Tanguy
e787fc35a6 Add examples to CI (#599)
* add examples to CI

* add markdown runner

* two tutorials
2021-11-08 13:00:44 +01:00
Zahary Karadjov
cf8a1a60a4 Work-around a Nim compilation error 2021-11-04 12:19:17 +02:00
Tanguy
e1d96a0f4d Remove isWire (#640) 2021-10-28 19:11:31 +02:00
Ștefan Talpalaru
b308affb98 CI: test multiple Nim compiler branches (#624)
* CI: test multiple Nim compiler branches

* make it a separate workflow and limit target branches

* make it a daily job
2021-10-26 14:47:30 -06:00
Tanguy
5885e03680 Add maxMessageSize option to pubsub (#634)
* Add maxMessageSize option to pubsub
* Switched default to 1mb
2021-10-25 12:58:38 +02:00
Tanguy
846baf3853 Various cleanups part 1 (#632)
* raise -> raise exc
* replace stdlib random with bearssl
* object init -> new
* Remove deprecated procs
* getMandatoryField
2021-10-25 10:26:32 +02:00
Tanguy
3669b90ceb Fix WS observed address (#631)
* Fix WS observed address

* Unify tcptransport & wstransport
2021-10-14 13:16:34 +02:00
Tanguy
75bfc1b5f7 Fix muxer bandwidth typo (#628) 2021-09-27 14:30:22 +02:00
Tanguy
1b2cdd6aec Merge branch 'master' into unstable 2021-09-09 13:22:45 +02:00
Tanguy
68af8122e9 Fix macos go-daemon CI (#622)
bump libp2p-go-daemon
2021-09-08 16:58:44 +02:00
Menduist
d02735dc46 Remove peer info (#610)
Peer Info is now for local peer data only.
For other peers info, use the peer store.

Previous reference to peer info are replaced with the peerid
2021-09-08 11:07:46 +02:00
Tanguy Cizain
8cddfde837 Rename getKey -> getPublicKey (#621)
* rename getKey to getPublicKey

* use publicKey directly in gossipsub

* update error messages
2021-09-02 12:03:40 +02:00
Tanguy Cizain
8ea90037e5 Fix gossipsub incoming graft backoff (#616)
* Fix gossipsub incoming graft backoff

* Improve debug messages

* clamp to 24h
2021-09-01 08:41:11 +02:00
Tanguy Cizain
62ca6dd9a0 Add missing test to testnative (#618)
* add missing tests

* fix testping
2021-09-01 08:38:24 +02:00
Tanguy Cizain
f274bfe19d DNS Addresses handling (#580)
* add 'dns' multiaddr protocol

* multiaddr: isWire is true for DNS protocols

* resolve dns on connect

* fix typo

* add dns test

* update resolveDns error handling

* handle multiple dns entries

* start of new resolver

* working dns resolver

* use the DnsResolver

* fix json logs

* small overhaul

* fix dns implem in lp2p

* update dnsclient repo

* add dns test to testnative

* dummy dns server for ut

* better mocked

* moved resolving to transport

* moved mockresolver to libp2p

* test resolve in switch test

* try multiple txt & track leaks

* raise e

* catchable error instead of exception

* save failed dns server

* moved resolve back to dialer

* remove nameresolver from dialer
2021-08-18 09:40:12 +02:00
Tanguy Cizain
e58658d822 Fix unstable autobump (#613) 2021-08-03 11:43:00 -06:00
Tanguy Cizain
af3be7966b Websocket Transport (#593)
* start of websocket transport

* more ws tests

* switch to common test

* add close to wsstream

* update ws & chronicles version

* cleanup

* removed multicodec

* clean ws outgoing connections

* renamed to websock

* removed stream from logs

* renamed ws to websock

* add connection closing test to common transport

* close incoming connection on ws stop

* renamed testwebsocket.nim -> testwstransport.nim

* removed raise todo

* split out/in connections

* add wss to tests

* Fix tls (#608)

* change log level

* fixed issue related to stopping

some cosmetic cleanup

* use `allFutures` to stop/close things

Prevent potential race conditions when stopping two or more transports

* misc

* point websock to server-case-object branch

* interop test with go

* removed websock version specification

* add daemon -> native ws test

* fix & test closed read/write

* update readOnce, thanks jangko

Co-authored-by: Dmitriy Ryajov <dryajov@gmail.com>
2021-08-03 15:48:03 +02:00
markspanbroek
8b438142ce Fixes failing peer store test in Nim 1.4.8 (#612)
* Fixes failing test in Nim 1.4.8

This tests failed because the order in which the elements
of a hashset are added to a seq is non-deterministic.

Co-authored-by: Tanguy Cizain <tanguycizain@gmail.com>
2021-07-28 12:06:18 +02:00
Tanguy Cizain
7fc0dfbd55 Muxer bandwidth metrics (#607)
* add stream bytes metrics

* renamed bw metric

* renamed

* new global metric
2021-07-26 16:12:36 +02:00
Tanguy Cizain
cb94baf9c4 Fix transport's switch start (#609)
* fix switch start

* don't fail eagerly on transport start

* handlecancel

* raise exc
2021-07-19 12:51:27 +02:00
Tanguy Cizain
c1b2d45d1b Track ChronosStream in test (#605)
* track chronos stream in tests

* use chronosstreamtrackername
2021-07-13 13:53:08 +02:00
Tanguy Cizain
008dcf7043 Faster CI (#603)
* ci on PR only

* add option to run ci manually

* thanks Stefan
2021-07-13 10:26:49 +02:00
Jacek Sieka
1d4f7c7a43 avoid borrow, breaks logging (#604) 2021-07-11 09:59:42 +02:00
Miran
c12f00c8b6 make isLiteral compatible with ARC and ORC (#602)
(cherry picked from commit 2d4dad0716)
2021-07-01 12:05:50 +02:00
Tanguy Cizain
26e47d7da5 Various transports improvement (#594)
* little transport cleanup

* rename TcpTransport.init -> TcpTransport.new

* moved transport e2e to common file

* remove localAddress

* rename testtransport -> testtcptransport

* add checktrackers to commontransports

* removed multicodec from transports
2021-06-30 10:59:30 +02:00
Dmitriy Ryajov
49f137049f Merge remote-tracking branch 'origin' into unstable 2021-06-29 15:43:11 -06:00
Dmitriy Ryajov
1681197d67 Merge branch 'master' into unstable 2021-06-29 15:42:22 -06:00
Dmitriy Ryajov
398b3a4ce6 Merge branch 'master' into unstable 2021-06-29 11:41:34 -06:00
Ben Tilford
72ccd1d1f5 Correcting paths and file/directory names (#596) 2021-06-27 22:36:57 -06:00
Giovanni Petrantoni
4be0cdcdb4 Docs cleanup (#557)
* remove pointless go_daemon page (made no sense)

* remove confusing go requirement from main readme

* wip directchat fixes

* use matchPartial for connect(wire)

* fix directchat

* revert wire changes

* rewrite directchat partially

* more readme updates

* fix things to follow the update on blog posts
2021-06-27 20:37:00 -06:00
Tanguy Cizain
7e3974f9c8 Fix flaky macos switch test (#592)
* sleep to avoid flaky switch test on macos

* Update tests/testswitch.nim

Co-authored-by: markspanbroek <mark@spanbroek.net>

Co-authored-by: markspanbroek <mark@spanbroek.net>
2021-06-23 09:27:52 -06:00
Tanguy Cizain
bee91538ef builder: add multiple local address support (#591) 2021-06-21 17:14:24 +02:00
Dmitriy Ryajov
b63e064b4a Remove asynccheck (#590)
* Merge master (#555)

* Revisit Floodsub (#543)

Fixes #525

add coverage to unsubscribeAll and testing

* add mounted protos to identify message (#546)

* add stable/unstable auto bumps

* fix auto-bump CI

* merge nbc auto bump with CI in order to bump only on CI success

* put conditional locks on nbc bump (#549)

* Fix minor exception issues (#550)

Makes code compatible with
https://github.com/status-im/nim-chronos/pull/166 without requiring it.

* fix nimbus ref for auto-bump stable's PR

* Split dialer (#542)

* extracting dialing logic to dialer

* exposing upgrade methods on transport

* cleanup

* fixing tests to use new interfaces

* add comments

* add base exception class and fix hierarchy

* fix imports

* `doAssert` is `ValueError` not `AssertionError`?

* revert back to `AssertionError`

Co-authored-by: Giovanni Petrantoni <7008900+sinkingsugar@users.noreply.github.com>
Co-authored-by: Jacek Sieka <jacek@status.im>

* Merge master (#555)

* Revisit Floodsub (#543)

Fixes #525

add coverage to unsubscribeAll and testing

* add mounted protos to identify message (#546)

* add stable/unstable auto bumps

* fix auto-bump CI

* merge nbc auto bump with CI in order to bump only on CI success

* put conditional locks on nbc bump (#549)

* Fix minor exception issues (#550)

Makes code compatible with
https://github.com/status-im/nim-chronos/pull/166 without requiring it.

* fix nimbus ref for auto-bump stable's PR

* Split dialer (#542)

* extracting dialing logic to dialer

* exposing upgrade methods on transport

* cleanup

* fixing tests to use new interfaces

* add comments

* add base exception class and fix hierarchy

* fix imports

* `doAssert` is `ValueError` not `AssertionError`?

* revert back to `AssertionError`

Co-authored-by: Giovanni Petrantoni <7008900+sinkingsugar@users.noreply.github.com>
Co-authored-by: Jacek Sieka <jacek@status.im>

* cleanup

Co-authored-by: Giovanni Petrantoni <7008900+sinkingsugar@users.noreply.github.com>
Co-authored-by: Jacek Sieka <jacek@status.im>
2021-06-14 17:21:44 -06:00
Tanguy Cizain
bed00ec43c Identify Push (#587)
* start of identifypush

* better pushidentify

* push identify test

* fix: make peerid optional
2021-06-14 11:08:47 -06:00
Tanguy Cizain
bd2e9a0462 Fix trackers tests (#588)
* Fix checkTracker typo

* fix testswich tests

* use tracker consts
2021-06-14 10:26:11 +02:00
Dmitriy Ryajov
c7d9770637 Merge master (#555)
* Revisit Floodsub (#543)

Fixes #525

add coverage to unsubscribeAll and testing

* add mounted protos to identify message (#546)

* add stable/unstable auto bumps

* fix auto-bump CI

* merge nbc auto bump with CI in order to bump only on CI success

* put conditional locks on nbc bump (#549)

* Fix minor exception issues (#550)

Makes code compatible with
https://github.com/status-im/nim-chronos/pull/166 without requiring it.

* fix nimbus ref for auto-bump stable's PR

* Split dialer (#542)

* extracting dialing logic to dialer

* exposing upgrade methods on transport

* cleanup

* fixing tests to use new interfaces

* add comments

* add base exception class and fix hierarchy

* fix imports

* `doAssert` is `ValueError` not `AssertionError`?

* revert back to `AssertionError`

Co-authored-by: Giovanni Petrantoni <7008900+sinkingsugar@users.noreply.github.com>
Co-authored-by: Jacek Sieka <jacek@status.im>
2021-06-11 16:34:41 -06:00
Dmitriy Ryajov
9e2b464e14 Cleanup testinterop imports (#589)
* cleanup exports/imports, specially for testinterop

* export pubsub
2021-06-11 16:34:40 -06:00
Dmitriy Ryajov
40e21e3375 Cleanup testinterop imports (#589)
* cleanup exports/imports, specially for testinterop

* export pubsub
2021-06-11 16:33:47 -06:00
Tanguy Cizain
93156447ba Peer Store implement part II (#586)
* Connect & Peer event handlers now receive a peerinfo

* small peerstore refacto

* implement peerstore in switch

* changed PeerStore to final ref object

* revert libp2p/builders.nim
2021-06-08 18:55:24 +02:00
Tanguy Cizain
55a3606ecb add ping protocol (#584)
* add ping protocol

* add ping protocol handler

* ping styling

* more ping tests

* switch ping to bearssl rng

* update ping style

* new cancellation test
2021-06-08 18:53:45 +02:00
Tanguy Cizain
caac8191d2 Change newXXXX procs to XXXX.new (#585)
* newBufferStream -> BufferStream.new

* newMultistream -> MultistreamSelect.new

* newSecio -> Secio.new

* newNoise -> Noise.new

* newPlainText -> PlainText.new

* newPubSubPeer -> PubSubPeer.new

* newIdentify -> Identify.new

* newMuxerProvider -> MuxerProvider.new
2021-06-07 09:32:08 +02:00
Ștefan Talpalaru
fbe888a3aa CI: refactor Nim compiler building (#579) 2021-06-02 12:26:03 -06:00
Dmitriy Ryajov
ac47964377 use pkg namespace 2021-06-02 12:26:02 -06:00
Dmitriy Ryajov
b56ca11b74 use raises defect 2021-06-02 12:26:02 -06:00
Dmitriy Ryajov
ce42674d80 avoid memory safety errors with nim 1.4.x 2021-06-02 12:26:01 -06:00
Dmitriy Ryajov
a6eea0c275 import chronicles otherwise compile breaks 2021-06-02 12:25:37 -06:00
Dmitriy Ryajov
1c3616e3a5 merge latest master 2021-06-02 12:25:36 -06:00
Dmitriy Ryajov
c949f14a99 Merge master to unstable (#570)
* Revisit Floodsub (#543)

Fixes #525

add coverage to unsubscribeAll and testing

* add mounted protos to identify message (#546)

* add stable/unstable auto bumps

* fix auto-bump CI

* merge nbc auto bump with CI in order to bump only on CI success

* put conditional locks on nbc bump (#549)

* Fix minor exception issues (#550)

Makes code compatible with
https://github.com/status-im/nim-chronos/pull/166 without requiring it.

* fix nimbus ref for auto-bump stable's PR

* use a builder pattern to build the switch (#551)

* use a builder pattern to build the switch

* with with

* more refs

* builders (#559)

* More builders (#560)

* address some issues pointed out in review

* re-add to prevent breaking other projects

* mem usage cleanups for pubsub (#564)

In `async` functions, a closure environment is created for variables
that cross an await boundary - this closure environment is kept in
memory for the lifetime of the associated future - this means that
although _some_ variables are no longer used, they still take up memory
for a long time.

In Nimbus, message validation is processed in batches meaning the future
of an incoming gossip message stays around for quite a while - this
leads to memory consumption peaks of 100-200 mb when there are many
attestations in the pipeline.

To avoid excessive memory usage, it's generally better to move non-async
code into proc's such that the variables therein can be released earlier
- this includes the many hidden variables introduced by macro and
template expansion (ie chronicles that does expensive exception
handling)

* move seen table salt to floodsub, use there as well
* shorten seen table salt to size of hash
* avoid unnecessary memory allocations and copies in a few places
* factor out message scoring
* avoid reencoding outgoing message for every peer
* keep checking validators until reject (in case there's both reject and
ignore)
* `readOnce` avoids `readExactly` overhead for single-byte read
* genericAssign -> assign2

* More gossip coverage (#553)

* add floodPublish test

* test delivery via control Iwant/have mechanics

* fix issues in control, and add testing

* fix possible backoff issue with pruned routine overriding it

* fix control messages (#566)

* remove unused control graft check in handleControl

* avoid sending empty Iwant messages

* Split dialer (#542)

* extracting dialing logic to dialer

* exposing upgrade methods on transport

* cleanup

* fixing tests to use new interfaces

* add comments

* add base exception class and fix hierarchy

* fix imports

* Merge master (#555)

* Revisit Floodsub (#543)

Fixes #525

add coverage to unsubscribeAll and testing

* add mounted protos to identify message (#546)

* add stable/unstable auto bumps

* fix auto-bump CI

* merge nbc auto bump with CI in order to bump only on CI success

* put conditional locks on nbc bump (#549)

* Fix minor exception issues (#550)

Makes code compatible with
https://github.com/status-im/nim-chronos/pull/166 without requiring it.

* fix nimbus ref for auto-bump stable's PR

* Split dialer (#542)

* extracting dialing logic to dialer

* exposing upgrade methods on transport

* cleanup

* fixing tests to use new interfaces

* add comments

* add base exception class and fix hierarchy

* fix imports

* `doAssert` is `ValueError` not `AssertionError`?

* revert back to `AssertionError`

Co-authored-by: Giovanni Petrantoni <7008900+sinkingsugar@users.noreply.github.com>
Co-authored-by: Jacek Sieka <jacek@status.im>

* Builders (#558)

* use a builder pattern to build the switch (#551)

* use a builder pattern to build the switch

* with with

* more refs

* Merge master (#555)

* Revisit Floodsub (#543)

Fixes #525

add coverage to unsubscribeAll and testing

* add mounted protos to identify message (#546)

* add stable/unstable auto bumps

* fix auto-bump CI

* merge nbc auto bump with CI in order to bump only on CI success

* put conditional locks on nbc bump (#549)

* Fix minor exception issues (#550)

Makes code compatible with
https://github.com/status-im/nim-chronos/pull/166 without requiring it.

* fix nimbus ref for auto-bump stable's PR

* Split dialer (#542)

* extracting dialing logic to dialer

* exposing upgrade methods on transport

* cleanup

* fixing tests to use new interfaces

* add comments

* add base exception class and fix hierarchy

* fix imports

* `doAssert` is `ValueError` not `AssertionError`?

* revert back to `AssertionError`

Co-authored-by: Giovanni Petrantoni <7008900+sinkingsugar@users.noreply.github.com>
Co-authored-by: Jacek Sieka <jacek@status.im>

* `doAssert` is `ValueError` not `AssertionError`?

* revert back to `AssertionError`

* fix builders

* more builder stuff

* more builders

Co-authored-by: Giovanni Petrantoni <7008900+sinkingsugar@users.noreply.github.com>
Co-authored-by: Jacek Sieka <jacek@status.im>

Co-authored-by: Giovanni Petrantoni <7008900+sinkingsugar@users.noreply.github.com>
Co-authored-by: Jacek Sieka <jacek@status.im>
2021-06-02 12:24:46 -06:00
Dmitriy Ryajov
5ee67f31bf Merge master (#562)
* builders (#559)

* More builders (#560)

* address some issues pointed out in review

* re-add to prevent breaking other projects

* Merge master (#555)

* Revisit Floodsub (#543)

Fixes #525

add coverage to unsubscribeAll and testing

* add mounted protos to identify message (#546)

* add stable/unstable auto bumps

* fix auto-bump CI

* merge nbc auto bump with CI in order to bump only on CI success

* put conditional locks on nbc bump (#549)

* Fix minor exception issues (#550)

Makes code compatible with
https://github.com/status-im/nim-chronos/pull/166 without requiring it.

* fix nimbus ref for auto-bump stable's PR

* Split dialer (#542)

* extracting dialing logic to dialer

* exposing upgrade methods on transport

* cleanup

* fixing tests to use new interfaces

* add comments

* add base exception class and fix hierarchy

* fix imports

* `doAssert` is `ValueError` not `AssertionError`?

* revert back to `AssertionError`

Co-authored-by: Giovanni Petrantoni <7008900+sinkingsugar@users.noreply.github.com>
Co-authored-by: Jacek Sieka <jacek@status.im>

Co-authored-by: Giovanni Petrantoni <7008900+sinkingsugar@users.noreply.github.com>
Co-authored-by: Jacek Sieka <jacek@status.im>
2021-06-02 12:24:46 -06:00
Dmitriy Ryajov
530e589e14 Builders (#558)
* use a builder pattern to build the switch (#551)

* use a builder pattern to build the switch

* with with

* more refs

* Merge master (#555)

* Revisit Floodsub (#543)

Fixes #525

add coverage to unsubscribeAll and testing

* add mounted protos to identify message (#546)

* add stable/unstable auto bumps

* fix auto-bump CI

* merge nbc auto bump with CI in order to bump only on CI success

* put conditional locks on nbc bump (#549)

* Fix minor exception issues (#550)

Makes code compatible with
https://github.com/status-im/nim-chronos/pull/166 without requiring it.

* fix nimbus ref for auto-bump stable's PR

* Split dialer (#542)

* extracting dialing logic to dialer

* exposing upgrade methods on transport

* cleanup

* fixing tests to use new interfaces

* add comments

* add base exception class and fix hierarchy

* fix imports

* `doAssert` is `ValueError` not `AssertionError`?

* revert back to `AssertionError`

Co-authored-by: Giovanni Petrantoni <7008900+sinkingsugar@users.noreply.github.com>
Co-authored-by: Jacek Sieka <jacek@status.im>

* `doAssert` is `ValueError` not `AssertionError`?

* revert back to `AssertionError`

* fix builders

* more builder stuff

* more builders

Co-authored-by: Giovanni Petrantoni <7008900+sinkingsugar@users.noreply.github.com>
Co-authored-by: Jacek Sieka <jacek@status.im>
2021-06-02 12:24:44 -06:00
Dmitriy Ryajov
ee49b76478 Merge master (#555)
* Revisit Floodsub (#543)

Fixes #525

add coverage to unsubscribeAll and testing

* add mounted protos to identify message (#546)

* add stable/unstable auto bumps

* fix auto-bump CI

* merge nbc auto bump with CI in order to bump only on CI success

* put conditional locks on nbc bump (#549)

* Fix minor exception issues (#550)

Makes code compatible with
https://github.com/status-im/nim-chronos/pull/166 without requiring it.

* fix nimbus ref for auto-bump stable's PR

* Split dialer (#542)

* extracting dialing logic to dialer

* exposing upgrade methods on transport

* cleanup

* fixing tests to use new interfaces

* add comments

* add base exception class and fix hierarchy

* fix imports

* `doAssert` is `ValueError` not `AssertionError`?

* revert back to `AssertionError`

Co-authored-by: Giovanni Petrantoni <7008900+sinkingsugar@users.noreply.github.com>
Co-authored-by: Jacek Sieka <jacek@status.im>
2021-06-02 12:24:07 -06:00
Dmitriy Ryajov
eef5dd0042 fix imports 2021-06-02 12:24:06 -06:00
Dmitriy Ryajov
8e3ef540ea add base exception class and fix hierarchy 2021-06-02 12:24:04 -06:00
Dmitriy Ryajov
a3c00af945 Split dialer (#542)
* extracting dialing logic to dialer

* exposing upgrade methods on transport

* cleanup

* fixing tests to use new interfaces

* add comments
2021-06-02 12:23:44 -06:00
Dmitriy Ryajov
3da656687b use LPError more consistently (#582)
* use LPError more consistently

* don't use Exceptino

* annotate with raises

* don't panic on concatenation

* further rework error handling
2021-06-02 15:39:10 +02:00
Dmitriy Ryajov
1b94c3feda fix #581 (#583) 2021-06-01 18:06:08 -06:00
Dmitriy Ryajov
34787728a3 don't use MaResult as default in newStandardSwitch (#578) 2021-05-24 16:48:18 -06:00
Dmitriy Ryajov
24132d7129 More raises cleanup (#575)
* use toException to map errors

* don't initialize address twice

* better error messages

* allow LPError to escape

* allow LPError to escape
2021-05-22 12:27:30 -06:00
Dmitriy Ryajov
ac4e060e1a adding raises defect across the codebase (#572)
* adding raises defect across the codebase

* use unittest2

* add windows deps caching

* update mingw link

* die on failed peerinfo initialization

* use result.expect instead of get

* use expect more consistently and rework inits

* use expect more consistently

* throw on missing public key

* remove unused closure annotation

* merge master
2021-05-21 10:27:01 -06:00
Jacek Sieka
9674a6a6f6 simplify connmanager (#573)
* no need to init orderedset
* array more simple than table
2021-05-19 08:49:55 +02:00
Jacek Sieka
83a20a992a gossipsub: unsubscribe fixes (#569)
* gossipsub: unsubscribe fixes

* fix KeyError when updating metric of unsubscribed topic
* fix unsubscribe message not being sent to all peers causing them to
keep thinking we're still subscribed
* release memory earlier in a few places

* floodsub fix
2021-05-07 00:43:45 +02:00
Giovanni Petrantoni
9f301964ed fix control messages (#566)
* remove unused control graft check in handleControl

* avoid sending empty Iwant messages
2021-04-28 10:03:03 +09:00
Giovanni Petrantoni
f81a085d0b More gossip coverage (#553)
* add floodPublish test

* test delivery via control Iwant/have mechanics

* fix issues in control, and add testing

* fix possible backoff issue with pruned routine overriding it
2021-04-22 18:51:22 +09:00
Jacek Sieka
e285d8bbf4 mem usage cleanups for pubsub (#564)
In `async` functions, a closure environment is created for variables
that cross an await boundary - this closure environment is kept in
memory for the lifetime of the associated future - this means that
although _some_ variables are no longer used, they still take up memory
for a long time.

In Nimbus, message validation is processed in batches meaning the future
of an incoming gossip message stays around for quite a while - this
leads to memory consumption peaks of 100-200 mb when there are many
attestations in the pipeline.

To avoid excessive memory usage, it's generally better to move non-async
code into proc's such that the variables therein can be released earlier
- this includes the many hidden variables introduced by macro and
template expansion (ie chronicles that does expensive exception
handling)

* move seen table salt to floodsub, use there as well
* shorten seen table salt to size of hash
* avoid unnecessary memory allocations and copies in a few places
* factor out message scoring
* avoid reencoding outgoing message for every peer
* keep checking validators until reject (in case there's both reject and
ignore)
* `readOnce` avoids `readExactly` overhead for single-byte read
* genericAssign -> assign2
2021-04-18 10:08:33 +02:00
Dmitriy Ryajov
6b930ae7e6 More builders (#560)
* address some issues pointed out in review

* re-add to prevent breaking other projects
2021-04-06 14:16:23 -06:00
Dmitriy Ryajov
290866dd62 builders (#559) 2021-04-05 16:06:45 -06:00
Giovanni Petrantoni
795a651839 use a builder pattern to build the switch (#551)
* use a builder pattern to build the switch

* with with

* more refs
2021-04-02 10:20:51 +09:00
Giovanni Petrantoni
df497660bc fix nimbus ref for auto-bump stable's PR 2021-03-23 16:46:24 +09:00
Jacek Sieka
54031c9e9b Fix minor exception issues (#550)
Makes code compatible with
https://github.com/status-im/nim-chronos/pull/166 without requiring it.
2021-03-23 07:45:25 +01:00
Giovanni Petrantoni
38333e45ae put conditional locks on nbc bump (#549) 2021-03-22 22:13:00 +09:00
Giovanni Petrantoni
3bf6acef23 merge nbc auto bump with CI in order to bump only on CI success 2021-03-16 09:31:06 +09:00
Giovanni Petrantoni
2d3595708c fix auto-bump CI 2021-03-16 08:45:37 +09:00
Giovanni Petrantoni
9175a9476e add stable/unstable auto bumps 2021-03-16 08:31:39 +09:00
Dmitriy Ryajov
f7a9d83545 add mounted protos to identify message (#546) 2021-03-15 15:29:05 -06:00
Giovanni Petrantoni
03bbdd2261 Revisit Floodsub (#543)
Fixes #525

add coverage to unsubscribeAll and testing
2021-03-15 13:16:03 -06:00
Giovanni Petrantoni
a54e1cc699 add coverage to disabled metrics (#545) 2021-03-15 21:36:57 +09:00
Giovanni Petrantoni
aeb18c4e41 now wild except: 2021-03-15 16:48:11 +09:00
Giovanni Petrantoni
c5b9fd3353 Try fix ci (#544)
* use apt-get

* try bump ubuntu ver
2021-03-15 14:50:36 +09:00
Giovanni Petrantoni
ba0a57ee34 fix reports number 2021-03-15 12:16:01 +09:00
Giovanni Petrantoni
996dc5be9c add testfloodsub coverage for different build options 2021-03-15 12:15:29 +09:00
Giovanni Petrantoni
4760df1e31 fix build with libp2p_agents_metrics switch 2021-03-15 01:42:47 +00:00
Jacek Sieka
70deac9e0d fix peer score accumulation (#541)
* fix accumulating peer score
* fix missing exception handling
* remove unnecessary initHashSet/initTable calls
* simplify peer stats management
* clean up tests a little
* fix some missing raises annotations
2021-03-09 13:22:52 +01:00
Giovanni Petrantoni
269d3df351 Consolidate metrics collection for mesh (#540)
* Consolidate metrics collection for mesh

* more fixes

* wrapping up

* Update libp2p/protocols/pubsub/gossipsub/behavior.nim

Co-authored-by: Jacek Sieka <jacek@status.im>

* Update libp2p/protocols/pubsub/gossipsub/behavior.nim

Co-authored-by: Jacek Sieka <jacek@status.im>

* Update libp2p/protocols/pubsub/gossipsub/behavior.nim

Co-authored-by: Jacek Sieka <jacek@status.im>

Co-authored-by: Jacek Sieka <jacek@status.im>
2021-03-03 22:11:21 +01:00
Giovanni Petrantoni
34c2fbeb16 small gossipsub metrics change 2021-03-03 08:41:21 +00:00
Giovanni Petrantoni
02ad017107 Gossipsub fixes and Initiator flagging fixes (#539)
* properly propagate initiator information for gossipsub

* Fix pubsubpeer lifetime management

* restore old behavior

* tests fixing

* clamp backoff time value received

* fix member name collisions

* internal test fixes

* better names and explaining of the importance of transport direction

* fixes
2021-03-03 08:23:40 +09:00
Giovanni Petrantoni
c1334c6d89 pubsubpeer better address management 2021-02-28 04:53:17 +00:00
Giovanni Petrantoni
7b2727d930 avoid leaking in peersInIP, don't depend on sendConn 2021-02-27 23:49:56 +09:00
Giovanni Petrantoni
67d0926e89 use in any case PeerID for peersInIP to avoid keeping references 2021-02-27 21:31:59 +09:00
Giovanni Petrantoni
fae38e0146 fix PubSubPeer hashing issues 2021-02-26 19:19:15 +09:00
Giovanni Petrantoni
45300c28a9 [SEC] gossipsub - handleIHAVE/handleIWANT recommendations & notes (#535)
Fixes #400
2021-02-26 14:27:42 +09:00
Giovanni Petrantoni
d7469b2286 [SEC] gossipsub - a peer score is not retained up until expiry if abusive peer unsubscribes (#534)
* [SEC] gossipsub - a peer score is not retained up until expiry if abusive peer unsubscribes
Fixes #402

* remove debug logging
2021-02-26 14:15:58 +09:00
Giovanni Petrantoni
c1d8317e3c fix badly merged code in gossipsub.colocationFactor 2021-02-26 12:39:57 +09:00
Giovanni Petrantoni
eac6cd3dbf Debt: cleanup warnings #426 (#536)
* testswitch cleanups

* Debt: cleanup warnings
Fixes #426
2021-02-25 09:24:49 -06:00
Giovanni Petrantoni
2015a878ad fix missing coverage (#533)
* investigate missing coverage

* test different pattern

* wip pattern testing

* try globastar
2021-02-23 12:50:43 +09:00
Giovanni Petrantoni
8236319a91 [SEC] gossipsub - handleGraft/handlePrune recommendations & notes (#530)
Fixes #401
2021-02-22 12:04:20 +09:00
Giovanni Petrantoni
1368bf7ecb test rebalanceMesh with low score peers (#529) 2021-02-22 10:05:25 +09:00
Giovanni Petrantoni
922cd92f94 don't check if peers have sendConn when disconnecting for bad scoring 2021-02-22 10:04:02 +09:00
Giovanni Petrantoni
51d8cd4ade [SEC] gossipsub - rebalanceMesh may prune up to D_lo on oversubscription (#531)
Fixes #403
2021-02-13 13:39:32 +09:00
Giovanni Petrantoni
e124e342b0 n subscription limits (#528)
* subscription high water, cleanups

* subscription limits test

* newline
2021-02-12 12:27:26 +09:00
Dmitriy Ryajov
12adefb4de add multi types to exports (#527)
* add multitypes to exports

* export standard setup
2021-02-10 11:42:46 -06:00
Dmitriy Ryajov
f4145ebbfa More exports cleanup (#522)
* annotate `SecureProtocol.Secio` as deprecated

* dont export varint

* add `errors` to exports - convenient error utils
2021-02-09 15:41:49 -06:00
Giovanni Petrantoni
fff54fa23c add more diagnostics when gossip publish fails 2021-02-09 18:42:59 +09:00
Ștefan Talpalaru
d9563d65ae support compilation with Nim-1.4 HEAD (#521) 2021-02-08 15:21:43 -06:00
Dmitriy Ryajov
2658181df9 Merge unstable (#518)
* Address Book POC implementation (#499)

* Address Book POC implementation

* Feat/peerstore impl (#505)

Co-authored-by: Hanno Cornelius <68783915+jm-clius@users.noreply.github.com>
2021-02-08 15:16:23 -06:00
Dmitriy Ryajov
4dea23c394 Remove secio usage and cleanup exports (#519)
* cleaned up exports

* remove secio use

* added more useful exports

* proper import
2021-02-08 14:33:34 -06:00
Giovanni Petrantoni
646557597d lower some gossipsub logging to debug level 2021-02-08 10:11:41 +09:00
Giovanni Petrantoni
fd73cf9f9d Refactor gossipsub into multiple modules (#515)
* Refactor gossipsub into multiple modules

* splitup further gossipsub

* move more mesh related stuff to behavior

* fix internal tests

* fix PubSubPeer.outbound flag, make it more reliable

* use discard rather then _
2021-02-06 09:13:04 +09:00
Dmitriy Ryajov
3213e5d377 add sane exports to libp2p (#517) 2021-02-05 11:57:32 -06:00
Dmitriy Ryajov
5c234ddd37 add hash proc to support using with containers (#516) 2021-02-05 10:12:44 -06:00
233 changed files with 25855 additions and 9050 deletions

1141
.assets/full-logo.svg Normal file

File diff suppressed because it is too large Load Diff

After

Width:  |  Height:  |  Size: 61 KiB

96
.assets/small-logo.svg Normal file
View File

@@ -0,0 +1,96 @@
<?xml version="1.0" encoding="UTF-8"?>
<svg xmlns="http://www.w3.org/2000/svg" xmlns:v="https://vecta.io/nano" xmlns:xlink="http://www.w3.org/1999/xlink" width="172.071" height="196.414" viewBox="0 0 45.527 51.968">
<g transform="matrix(.2822 0 0 .2822 -212.833275 -150.656248)">
<path d="M835.432 533.821l-12.483 9.783c-6.482-.207-19.197 1.251-26.086 3.769-6.346-4.04-11.923-8.5-11.923-8.5l-7.762 13.071c-4.444 2.375-8.906 5.046-12.883 8.58l-10.162-4.17c6.125 12.414 10.243 24.844 21.445 32.316 17.834-28.299 100.705-25.691 118.907-.16 11.764-6.165 16.339-19.429 20.965-31.674-.507.168-6.802 2.285-10.882 3.849-2.436-2.665-8.179-6.763-11.443-8.741-3.096-5.696-7.602-13.391-7.602-13.391s-5.337 3.988-11.523 8.34c-8.357-1.55-18.465-3.433-26.966-2.967-5.787-4.779-11.603-10.104-11.603-10.104z" fill="#f3d400" />
<g opacity=".9" transform="matrix(.9375 0 0 .9375 765.1166 550.13225)">
<path d="M99.952 106.898l.215-.107 24.755-14.248-24.97-14.535-24.97 14.374z" fill="#cc2a65" />
<use xlink:href="#B" fill="#a21d4c" />
<path d="M124.922 92.542l-24.755 14.248-.215.107v28.89l24.97-14.356z" fill="#b62454" />
<path d="M50.012 106.737l.215-.125 24.755-14.248-24.97-14.517-24.97 14.356z" fill="#c8d92b" />
<path d="M50.012 135.609v-28.872l-24.97-14.535v28.89h.018z" fill="#c2d02f" />
<path d="M74.982 92.381l-24.755 14.23-.215.125v28.872.018h.018l24.952-14.356v-.018z" fill="#b9be33" />
<path d="M74.982 121.253l.215-.107 24.755-14.248-24.97-14.535-24.97 14.374z" fill="#cc2a65" />
<use xlink:href="#B" x="-24.97" y="14.356" fill="#a21d4c" />
<path d="M99.952 106.898l-24.755 14.248-.215.107v28.89H75l24.952-14.356z" fill="#b62454" />
<path d="M124.905 121.415l.215-.125 24.737-14.23-24.952-14.535-24.97 14.356z" fill="#a159a2" />
<path d="M124.905 150.305v-28.89l-24.97-14.535v28.89h.018z" fill="#772a86" />
<path d="M149.875 107.059l-24.755 14.23-.215.125v28.89l24.97-14.356z" fill="#8e3b95" />
<path d="M74.982 92.345l.215-.125 24.737-14.248-24.952-14.517-24.97 14.356z" fill="#bec831" />
<path d="M74.982 121.217V92.345L50.012 77.81v28.89h.018z" fill="#a1a938" />
<path d="M99.952 77.989l-24.755 14.23-.215.125v28.872.018l24.97-14.356v-.018z" fill="#999b37" />
<path d="M75 60.645l.197-.125 24.755-14.23L75 31.755 50.029 46.11l24.952 14.535z" fill="#bec831" />
<path d="M74.982 89.535L75 60.645 50.029 46.11 50.012 75h.018z" fill="#a1a938" />
<path d="M99.97 46.307L75.197 60.52l-.197.125h-.018v28.89l24.97-14.338v-.018z" fill="#999b37" />
<path d="M99.952 75.179l.215-.107 24.755-14.23L99.97 46.306 75 60.644z" fill="#ee539a" />
<path d="M99.952 104.069v-28.89L75 60.644l-.018 28.89H75z" fill="#d01b68" />
<path d="M124.922 60.841l-24.755 14.23-.215.107v28.89l24.97-14.338.018-28.89z" fill="#ec0f68" />
<path d="M124.923 89.731l.215-.125 24.755-14.23-24.952-14.535h-.018l-24.97 14.338z" fill="#a159a2" />
<path d="M124.905 118.622l.018-28.89-24.97-14.535v28.872.018z" fill="#772a86" />
<path d="M149.893 75.376l-24.755 14.23-.215.125-.018 28.89h.018l24.97-14.356z" fill="#8e3b95" />
<path d="M50.03 75l.197-.125 24.755-14.23L50.03 46.109 25.06 60.447l24.952 14.535z" fill="#c8d92b" />
<path d="M50.012 103.872L50.03 75 25.06 60.447l-.018 28.89h.018z" fill="#c2d02f" />
<path d="M75 60.644l-24.773 14.23-.197.125-.018 28.872 24.97-14.338z" fill="#b9be33" />
<path d="M74.982 89.534l.215-.125 24.755-14.23L75 60.644l-24.97 14.338z" fill="#f7af19" />
<path d="M74.982 118.425v-.018.018-28.89L50.029 75l-.018 28.872.018.018z" fill="#f2901f" />
<path d="M99.952 75.179l-24.755 14.23-.215.125v28.89l24.97-14.356.018-28.89z" fill="#f9a120" />
<path d="M99.934 135.769l.215-.125 24.684-14.356-25.042-14.409L74.91 121.36z" fill="#833593" />
<path d="M100.077 164.66l-.143-28.89-25.042-14.409.143 28.89h.018z" fill="#652977" />
<path d="M124.833 121.288l-24.684 14.356-.215.125.143 28.89 24.899-14.481z" fill="#4d1f5b" />
<path d="M99.952 104.069l.215-.107 24.755-14.23L99.97 75.179h-.018l-24.97 14.356z" fill="#a159a2" />
<path d="M99.934 132.959l.018-28.89-24.97-14.535v28.89z" fill="#772a86" />
<path d="M124.922 89.732l-24.755 14.23-.215.107-.018 28.89h.018l24.97-14.338z" fill="#8e3b95" />
<path d="M25.042 121.074l.197-.125 24.755-14.248-24.952-14.517h-.018L.071 106.54l24.952 14.535z" fill="#f6dd03" />
<path d="M25.024 149.947h.018v-28.872L.071 106.54v28.89z" fill="#f9bb1d" />
<path d="M49.994 106.719l-24.755 14.23-.197.125v28.872.018l24.952-14.356h.018v-.018-28.872z" fill="#e9ae20" />
<path d="M25.06 89.338l.197-.125 24.755-14.23L25.06 60.447.089 74.803l24.952 14.535z" fill="#f6dd03" />
<path d="M25.042 118.228l.018-28.89L.089 74.803.072 103.675l.018.018z" fill="#f9bb1d" />
<path d="M50.03 75L25.257 89.212l-.197.125-.018 28.89 24.97-14.356v.018-.018z" fill="#e9ae20" />
<path d="M50.012 135.59l.215-.107 24.737-14.248L50.012 106.7l-24.97 14.374z" fill="#f7af19" />
<path d="M50.012 164.481v-28.89l-24.97-14.517v28.872.018z" fill="#f2901f" />
<path d="M74.964 121.235l-24.755 14.248-.197.107v28.89l24.97-14.356v-28.89z" fill="#f9a120" />
<path d="M50.012 103.872l.215-.107 24.755-14.23L50.03 74.982 25.06 89.338z" fill="#f7af19" />
<path d="M50.012 132.763v-28.89L25.06 89.338l-.018 28.89h.018z" fill="#f2901f" />
<path d="M74.982 89.535l-24.755 14.23-.215.107v28.89l24.97-14.338.018-28.89z" fill="#f9a120" />
<path d="M74.982 150.125l.197-.125 24.755-14.23-24.952-14.535h-.018l-24.952 14.356 24.952 14.535z" fill="#f7af19" />
<path d="M74.964 179.015h.018v-28.89l-24.97-14.517v28.872.018z" fill="#f2901f" />
<path d="M99.934 135.77L75.179 150l-.197.125v28.89l24.97-14.356v-28.89z" fill="#f9a120" />
<path d="M74.982 118.425l.215-.125 24.755-14.23L75 89.535h-.018l-24.97 14.338z" fill="#31838b" />
<path d="M74.964 147.297l.018-28.872-24.97-14.535v-.018 28.89.018z" fill="#22626c" />
<path d="M99.952 104.069L75.197 118.3l-.215.125-.018 28.872v.018h.018l24.97-14.356z" fill="#1b4b56" />
<path d="M74.982 28.962l.215-.125 24.737-14.248L74.982.072l-24.97 14.356 24.97 14.517z" fill="#bec831" />
<path d="M74.982 57.834V28.962l-24.97-14.535v28.89h.018z" fill="#a1a938" />
<path d="M99.952 14.606l-24.755 14.23-.215.125v28.872l24.97-14.356z" fill="#999b37" />
<path d="M74.964 28.944l.215-.125 24.755-14.23L74.982.054h-.018l-24.97 14.338z" fill="#a159a2" />
<path d="M74.946 57.835l.018-28.89-24.97-14.535v28.872.018z" fill="#772a86" />
<path d="M99.934 14.589l-24.755 14.23-.215.125-.018 28.89h.018l24.97-14.356z" fill="#8e3b95" />
<path d="M99.952 43.479l.215-.107 24.755-14.248-24.97-14.535-24.97 14.356z" fill="#ee539a" />
<use xlink:href="#B" y="-63.419" fill="#d01b68" />
<path d="M124.922 29.123l-24.755 14.248-.215.107v28.89l24.97-14.356z" fill="#ec0f68" />
<path d="M50.03 43.317l.215-.125L75 28.961 50.048 14.427h-.018L25.06 28.765z" fill="#31838b" />
<path d="M50.012 72.189l.018-28.872-24.97-14.535v-.018 28.89.018z" fill="#22626c" />
<path d="M75 28.961l-24.755 14.23-.215.125-.018 28.872v.018h.018L75 57.852z" fill="#1b4b56" />
<path d="M124.923 58.013l.215-.125 24.737-14.23-24.952-14.535-24.97 14.356z" fill="#cc2a65" />
<use xlink:href="#B" x="24.971" y="-48.884" fill="#a21d4c" />
<path d="M149.893 43.658l-24.755 14.23-.215.125v28.89l24.97-14.356z" fill="#b62454" />
<path d="M74.982 57.835l.215-.107 24.755-14.248-24.97-14.535L50.012 43.3z" fill="#c8d92b" />
<path d="M74.982 86.725v-28.89l-24.97-14.517V72.19l.018.018z" fill="#c2d02f" />
<path d="M99.952 43.479L75.197 57.727l-.215.107v28.89H75l24.952-14.356z" fill="#b9be33" />
<path d="M99.952 72.369l.215-.125 24.755-14.23-24.97-14.535-24.97 14.356z" fill="#33b4d7" />
<use xlink:href="#B" y="-34.529" fill="#209ac5" />
<path d="M124.922 58.014l-24.755 14.23-.215.125v28.89h.018l24.952-14.356z" fill="#0f8cae" />
<path d="M25.06 57.673l.197-.125L50.012 43.3 25.06 28.783h-.018L.089 43.139l24.952 14.535z" fill="#94d6e3" />
<path d="M25.042 86.546h.018V57.673L.089 43.139v28.89z" fill="#73ccdd" />
<path d="M50.012 43.318l-24.755 14.23-.197.125v28.872.018l24.952-14.356h.018v-.018-28.872z" fill="#3bafbb" />
<path d="M50.03 72.19l.215-.107 24.737-14.248L50.03 43.318 25.06 57.674z" fill="#94d6e3" />
<path d="M50.03 101.08V72.208v-.018L25.06 57.674v28.872.018z" fill="#73ccdd" />
<path d="M74.982 57.835L50.227 72.083l-.197.107v28.89L75 86.725v-28.89z" fill="#3bafbb" />
<path d="M75 86.724l.197-.107 24.755-14.248L75 57.834h-.018L50.029 72.189l24.952 14.535z" fill="#33b4d7" />
<path d="M74.982 115.614H75v-28.89l-24.97-14.517v28.872.018z" fill="#209ac5" />
<path d="M99.952 72.368L75.197 86.617l-.197.107v28.89l24.97-14.356v-28.89z" fill="#0f8cae" />
</g>
<path d="M759.126 567.007s10.273 21.02 16.364 35.698c25.549 33.869 90.792 36.224 119.235.656 9.484-17.619 16.733-36.357 16.733-36.357-7.297 10.862-20.094 18.056-27.408 22.095-5.197 2.861-17.189 4.59-17.189 4.59l-31.482-16.393-31.663 16.065s-11.832-1.91-17.189-4.426c-10.811-5.799-19.735-12.549-27.401-21.928z" fill="#ffe953" />
</g>
<defs>
<path id="B" d="M99.952 135.788v-28.89l-24.97-14.517v28.872l.018.018z" />
</defs>
</svg>

After

Width:  |  Height:  |  Size: 9.4 KiB

131
.github/actions/install_nim/action.yml vendored Normal file
View File

@@ -0,0 +1,131 @@
name: Install Nim
inputs:
os:
description: "Operating system to build for"
required: true
cpu:
description: "CPU to build for"
default: "amd64"
nim_branch:
description: "Nim version"
default: "version-1-6"
shell:
description: "Shell to run commands in"
default: "bash --noprofile --norc -e -o pipefail"
runs:
using: "composite"
steps:
- name: Install build dependencies (Linux i386)
shell: ${{ inputs.shell }}
if: inputs.os == 'Linux' && inputs.cpu == 'i386'
run: |
sudo dpkg --add-architecture i386
sudo apt-get update -qq
sudo DEBIAN_FRONTEND='noninteractive' apt-get install \
--no-install-recommends -yq gcc-multilib g++-multilib \
libssl-dev:i386
mkdir -p external/bin
cat << EOF > external/bin/gcc
#!/bin/bash
exec $(which gcc) -m32 "\$@"
EOF
cat << EOF > external/bin/g++
#!/bin/bash
exec $(which g++) -m32 "\$@"
EOF
chmod 755 external/bin/gcc external/bin/g++
echo '${{ github.workspace }}/external/bin' >> $GITHUB_PATH
- name: MSYS2 (Windows i386)
if: inputs.os == 'Windows' && inputs.cpu == 'i386'
uses: msys2/setup-msys2@v2
with:
path-type: inherit
msystem: MINGW32
install: >-
base-devel
git
mingw-w64-i686-toolchain
- name: MSYS2 (Windows amd64)
if: inputs.os == 'Windows' && inputs.cpu == 'amd64'
uses: msys2/setup-msys2@v2
with:
path-type: inherit
install: >-
base-devel
git
mingw-w64-x86_64-toolchain
- name: Restore Nim DLLs dependencies (Windows) from cache
if: inputs.os == 'Windows'
id: windows-dlls-cache
uses: actions/cache@v3
with:
path: external/dlls
key: 'dlls'
- name: Install DLL dependencies (Windows)
shell: ${{ inputs.shell }}
if: >
steps.windows-dlls-cache.outputs.cache-hit != 'true' &&
inputs.os == 'Windows'
run: |
mkdir external
curl -L "https://nim-lang.org/download/windeps.zip" -o external/windeps.zip
7z x external/windeps.zip -oexternal/dlls
- name: Path to cached dependencies (Windows)
shell: ${{ inputs.shell }}
if: >
inputs.os == 'Windows'
run: |
echo '${{ github.workspace }}'"/external/dlls" >> $GITHUB_PATH
- name: Derive environment variables
shell: ${{ inputs.shell }}
run: |
if [[ '${{ inputs.cpu }}' == 'amd64' ]]; then
PLATFORM=x64
else
PLATFORM=x86
fi
echo "PLATFORM=$PLATFORM" >> $GITHUB_ENV
ncpu=
MAKE_CMD="make"
case '${{ inputs.os }}' in
'Linux')
ncpu=$(nproc)
;;
'macOS')
ncpu=$(sysctl -n hw.ncpu)
;;
'Windows')
ncpu=$NUMBER_OF_PROCESSORS
MAKE_CMD="mingw32-make"
;;
esac
[[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
echo "ncpu=$ncpu" >> $GITHUB_ENV
echo "MAKE_CMD=${MAKE_CMD}" >> $GITHUB_ENV
echo '${{ github.workspace }}/nim/bin' >> $GITHUB_PATH
- name: Restore Nim from cache
id: nim-cache
uses: actions/cache@v3
with:
path: '${{ github.workspace }}/nim'
key: ${{ inputs.os }}-${{ inputs.cpu }}-nim-${{ inputs.nim_branch }}-cache-${{ env.cache_nonce }}
- name: Build Nim and Nimble
shell: ${{ inputs.shell }}
if: ${{ steps.nim-cache.outputs.cache-hit != 'true' }}
run: |
# We don't want partial matches of the cache restored
rm -rf nim
curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh
env MAKE="${MAKE_CMD} -j${ncpu}" ARCH_OVERRIDE=${PLATFORM} NIM_COMMIT=${{ inputs.nim_branch }} \
QUICK_AND_DIRTY_COMPILER=1 QUICK_AND_DIRTY_NIMBLE=1 CC=gcc \
bash build_nim.sh nim csources dist/nimble NimBinaries

45
.github/workflows/bumper.yml vendored Normal file
View File

@@ -0,0 +1,45 @@
name: Bumper
on:
push:
branches:
- unstable
- bumper
workflow_dispatch:
jobs:
bumpProjects:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
target: [
{ repo: status-im/nimbus-eth2, branch: unstable },
{ repo: waku-org/nwaku, branch: master },
{ repo: codex-storage/nim-codex, branch: master }
]
steps:
- name: Clone repo
uses: actions/checkout@v2
with:
repository: ${{ matrix.target.repo }}
ref: ${{ matrix.target.branch }}
path: nbc
fetch-depth: 0
token: ${{ secrets.ACTIONS_GITHUB_TOKEN }}
- name: Checkout this ref
run: |
cd nbc
git submodule update --init vendor/nim-libp2p
cd vendor/nim-libp2p
git checkout $GITHUB_SHA
- name: Commit this bump
run: |
cd nbc
git config --global user.email "${{ github.actor }}@users.noreply.github.com"
git config --global user.name = "${{ github.actor }}"
git commit --allow-empty -a -m "auto-bump nim-libp2p"
git branch -D nim-libp2p-auto-bump-${GITHUB_REF##*/} || true
git switch -c nim-libp2p-auto-bump-${GITHUB_REF##*/}
git push -f origin nim-libp2p-auto-bump-${GITHUB_REF##*/}

View File

@@ -1,207 +1,92 @@
name: nim-libp2p CI
on: [push, pull_request]
name: CI
on:
push:
branches:
- master
- unstable
pull_request:
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
build:
timeout-minutes: 90
strategy:
fail-fast: false
max-parallel: 20
matrix:
branch: [v1.2.6]
target:
# Unit tests
- os: linux
cpu: amd64
TEST_KIND: unit-tests
- os: linux
cpu: i386
TEST_KIND: unit-tests
- os: macos
cpu: amd64
TEST_KIND: unit-tests
- os: windows
cpu: i386
TEST_KIND: unit-tests
- os: windows
cpu: amd64
TEST_KIND: unit-tests
#- os: windows
#cpu: i386
branch: [version-1-6]
include:
- target:
os: linux
builder: ubuntu-18.04
builder: ubuntu-20.04
shell: bash
- target:
os: macos
builder: macos-10.15
builder: macos-12
shell: bash
- target:
os: windows
builder: windows-2019
name: '${{ matrix.target.os }}-${{ matrix.target.cpu }} (${{ matrix.branch }})'
shell: msys2 {0}
defaults:
run:
shell: ${{ matrix.shell }}
name: '${{ matrix.target.os }}-${{ matrix.target.cpu }} (Nim ${{ matrix.branch }})'
runs-on: ${{ matrix.builder }}
continue-on-error: ${{ matrix.branch == 'devel' }}
steps:
- name: Checkout nim-libp2p
- name: Checkout
uses: actions/checkout@v2
with:
path: nim-libp2p
submodules: true
- name: Install build dependencies (Linux i386)
if: runner.os == 'Linux' && matrix.target.cpu == 'i386'
run: |
sudo dpkg --add-architecture i386
sudo apt-fast update -qq
sudo DEBIAN_FRONTEND='noninteractive' apt-fast install \
--no-install-recommends -yq gcc-multilib g++-multilib \
libssl-dev:i386
mkdir -p external/bin
cat << EOF > external/bin/gcc
#!/bin/bash
exec $(which gcc) -m32 "\$@"
EOF
cat << EOF > external/bin/g++
#!/bin/bash
exec $(which g++) -m32 "\$@"
EOF
chmod 755 external/bin/gcc external/bin/g++
echo '${{ github.workspace }}/external/bin' >> $GITHUB_PATH
- name: Install build dependencies (Windows)
if: runner.os == 'Windows'
shell: bash
run: |
mkdir external
if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then
arch=64
else
arch=32
fi
curl -L "https://nim-lang.org/download/mingw$arch-6.3.0.7z" -o "external/mingw$arch.7z"
curl -L "https://nim-lang.org/download/windeps.zip" -o external/windeps.zip
7z x "external/mingw$arch.7z" -oexternal/
7z x external/windeps.zip -oexternal/dlls
echo '${{ github.workspace }}'"/external/mingw$arch/bin" >> $GITHUB_PATH
echo '${{ github.workspace }}'"/external/dlls" >> $GITHUB_PATH
- name: Setup environment
shell: bash
run: echo '${{ github.workspace }}/nim/bin' >> $GITHUB_PATH
- name: Get latest Nim commit hash
id: versions
shell: bash
run: |
getHash() {
git ls-remote "https://github.com/$1" "${2:-HEAD}" | cut -f 1
}
nimHash=$(getHash nim-lang/Nim '${{ matrix.branch }}')
csourcesHash=$(getHash nim-lang/csources)
echo "::set-output name=nim::$nimHash"
echo "::set-output name=csources::$csourcesHash"
- name: Restore prebuilt Nim from cache
id: nim-cache
uses: actions/cache@v1
- name: Setup Nim
uses: "./.github/actions/install_nim"
with:
path: nim
key: "nim-${{ matrix.target.os }}-${{ matrix.target.cpu }}-${{ steps.versions.outputs.nim }}"
- name: Restore prebuilt csources from cache
if: steps.nim-cache.outputs.cache-hit != 'true'
id: csources-cache
uses: actions/cache@v1
with:
path: csources/bin
key: "csources-${{ matrix.target.os }}-${{ matrix.target.cpu }}-${{ steps.versions.outputs.csources }}"
- name: Checkout Nim csources
if: >
steps.csources-cache.outputs.cache-hit != 'true' &&
steps.nim-cache.outputs.cache-hit != 'true'
uses: actions/checkout@v2
with:
repository: nim-lang/csources
path: csources
ref: ${{ steps.versions.outputs.csources }}
- name: Checkout Nim
if: steps.nim-cache.outputs.cache-hit != 'true'
uses: actions/checkout@v2
with:
repository: nim-lang/Nim
path: nim
ref: ${{ steps.versions.outputs.nim }}
- name: Build Nim and associated tools
if: steps.nim-cache.outputs.cache-hit != 'true'
shell: bash
run: |
ncpu=
ext=
case '${{ runner.os }}' in
'Linux')
ncpu=$(nproc)
;;
'macOS')
ncpu=$(sysctl -n hw.ncpu)
;;
'Windows')
ncpu=$NUMBER_OF_PROCESSORS
ext=.exe
;;
esac
[[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
if [[ ! -e csources/bin/nim$ext ]]; then
make -C csources -j $ncpu CC=gcc ucpu='${{ matrix.target.cpu }}'
else
echo 'Using prebuilt csources'
fi
cp -v csources/bin/nim$ext nim/bin
cd nim
nim c koch
./koch boot -d:release
./koch tools -d:release
# clean up to save cache space
rm koch
rm -rf nimcache
rm -rf dist
rm -rf .git
# - name: Install library dependencies (Linux amd64)
# if: runner.os == 'Linux' && matrix.target.cpu == 'amd64'
# run: |
# sudo DEBIAN_FRONTEND='noninteractive' apt-fast install \
# --no-install-recommends -yq libgmp-dev
#
# - name: Install library dependencies (Linux i386)
# if: runner.os == 'Linux' && matrix.target.cpu == 'i386'
# run: |
# sudo DEBIAN_FRONTEND='noninteractive' apt-fast install \
# libgmp-dev:i386
#
# - name: Install library dependencies (macOS)
# if: runner.os == 'macOS'
# run: brew install gmp
#
# - name: Install library dependencies (Windows)
# if: runner.os == 'Windows'
# shell: bash
# run: |
# choco install msys2
# pacman -S mingw-w64-x86_64-gmp
os: ${{ matrix.target.os }}
cpu: ${{ matrix.target.cpu }}
shell: ${{ matrix.shell }}
nim_branch: ${{ matrix.branch }}
- name: Setup Go
uses: actions/setup-go@v2
with:
go-version: '^1.15.5'
go-version: '~1.15.5'
- name: Install p2pd
shell: bash
run: |
cd nim-libp2p
V=1 bash scripts/build_p2pd.sh p2pdCache v0.3.0
V=1 bash scripts/build_p2pd.sh p2pdCache 124530a3
- name: Run nim-libp2p tests
shell: bash
- name: Restore deps from cache
id: deps-cache
uses: actions/cache@v3
with:
path: nimbledeps
key: nimbledeps-${{ hashFiles('.pinned') }}
- name: Install deps
if: ${{ steps.deps-cache.outputs.cache-hit != 'true' }}
run: |
export UCPU="$cpu"
cd nim-libp2p
nimble install -y --depsOnly
nimble install_pinned
- name: Run tests
run: |
nim --version
nimble --version
nimble test

View File

@@ -1,121 +1,65 @@
name: nim-libp2p codecov builds
on: [push, pull_request]
on:
#On push to common branches, this computes the "bases stats" for PRs
push:
branches:
- master
- unstable
pull_request:
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
GossipSub:
Coverage:
runs-on: ubuntu-20.04
strategy:
matrix:
nim-options: [
"",
"-d:libp2p_pubsub_anonymize=true -d:libp2p_pubsub_sign=false -d:libp2p_pubsub_verify=false",
"-d:libp2p_pubsub_sign=true -d:libp2p_pubsub_verify=true"
]
test-program: [
"tests/pubsub/testpubsub",
"tests/pubsub/testgossipinternal"
]
env:
CICOV: YES
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Setup Nim
uses: "./.github/actions/install_nim"
with:
os: linux
cpu: amd64
shell: bash
- name: Restore deps from cache
id: deps-cache
uses: actions/cache@v3
with:
path: nimbledeps
key: nimbledeps-${{ hashFiles('.pinned') }}
- name: Install deps
if: ${{ steps.deps-cache.outputs.cache-hit != 'true' }}
run: |
nimble install_pinned
- name: Run
run: |
sudo apt-get update
sudo apt-get install -y lcov build-essential git curl
mkdir coverage
curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh
env MAKE="make -j${NPROC}" bash build_nim.sh Nim csources dist/nimble NimBinaries
export PATH="$PATH:$PWD/Nim/bin"
nimble install -y --depsOnly
export NIM_OPTIONS="--opt:speed -d:debug --verbosity:0 --hints:off --lineDir:on -d:chronicles_log_level=INFO --warning[CaseTransition]:off --warning[ObservableStores]:off --warning[LockLevel]:off --nimcache:nimcache --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage ${{ matrix.nim-options }}"
nim c $NIM_OPTIONS -r ${{ matrix.test-program }}
cd nimcache; rm *.c; cd ..
export NIMFLAGS="--lineDir:on --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage"
nimble testnative
nimble testpubsub
nimble testfilter
find nimcache -name *.c -delete
lcov --capture --directory nimcache --output-file coverage/coverage.info
lcov --extract coverage/coverage.info `pwd`/libp2p/* `pwd`/libp2p/**/* `pwd`/libp2p/**/**/* --output-file coverage/coverage.f.info
export COV_UUID=`cksum <<< "${{ matrix.test-program }} $NIM_OPTIONS" | cut -f 1 -d ' '`
genhtml coverage/coverage.f.info --output-directory coverage/$COV_UUID-output
echo ${{ matrix.test-program }} > coverage/$COV_UUID-nim_options.txt
echo $NIM_OPTIONS >> coverage/$COV_UUID-nim_options.txt
shopt -s globstar
ls `pwd`/libp2p/{*,**/*}.nim
lcov --extract coverage/coverage.info `pwd`/libp2p/{*,**/*}.nim --output-file coverage/coverage.f.info
genhtml coverage/coverage.f.info --output-directory coverage/output
bash <(curl -s https://codecov.io/bash) -f coverage/coverage.f.info || echo "Codecov did not collect coverage reports"
- uses: actions/upload-artifact@master
with:
name: coverage
path: coverage
Tests:
runs-on: ubuntu-20.04
strategy:
matrix:
nim-options: [
""
]
test-program: [
"tests/testnative",
]
steps:
- uses: actions/checkout@v2
- name: Run
run: |
sudo apt-get update
sudo apt-get install -y lcov build-essential git curl
mkdir coverage
curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh
env MAKE="make -j${NPROC}" bash build_nim.sh Nim csources dist/nimble NimBinaries
export PATH="$PATH:$PWD/Nim/bin"
nimble install -y --depsOnly
export NIM_OPTIONS="--opt:speed -d:debug --verbosity:0 --hints:off --lineDir:on -d:chronicles_log_level=INFO --warning[CaseTransition]:off --warning[ObservableStores]:off --warning[LockLevel]:off --nimcache:nimcache --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage ${{ matrix.nim-options }}"
nim c $NIM_OPTIONS -r ${{ matrix.test-program }}
cd nimcache; rm *.c; cd ..
lcov --capture --directory nimcache --output-file coverage/coverage.info
lcov --extract coverage/coverage.info `pwd`/libp2p/* `pwd`/libp2p/**/* `pwd`/libp2p/**/**/* --output-file coverage/coverage.f.info
export COV_UUID=`cksum <<< "${{ matrix.test-program }} $NIM_OPTIONS" | cut -f 1 -d ' '`
genhtml coverage/coverage.f.info --output-directory coverage/$COV_UUID-output
echo ${{ matrix.test-program }} > coverage/$COV_UUID-nim_options.txt
echo $NIM_OPTIONS >> coverage/$COV_UUID-nim_options.txt
bash <(curl -s https://codecov.io/bash) -f coverage/coverage.f.info || echo "Codecov did not collect coverage reports"
- uses: actions/upload-artifact@master
with:
name: coverage
path: coverage
Filter:
runs-on: ubuntu-20.04
strategy:
matrix:
nim-options: [
"",
"-d:libp2p_pki_schemes=secp256k1",
"-d:libp2p_pki_schemes=secp256k1;ed25519",
"-d:libp2p_pki_schemes=secp256k1;ed25519;ecnist",
]
test-program: [
"tests/testpkifilter",
]
steps:
- uses: actions/checkout@v2
- name: Run
run: |
sudo apt-get update
sudo apt-get install -y lcov build-essential git curl
mkdir coverage
curl -O -L -s -S https://raw.githubusercontent.com/status-im/nimbus-build-system/master/scripts/build_nim.sh
env MAKE="make -j${NPROC}" bash build_nim.sh Nim csources dist/nimble NimBinaries
export PATH="$PATH:$PWD/Nim/bin"
nimble install -y --depsOnly
export NIM_OPTIONS="--opt:speed -d:debug --verbosity:0 --hints:off --lineDir:on -d:chronicles_log_level=INFO --warning[CaseTransition]:off --warning[ObservableStores]:off --warning[LockLevel]:off --nimcache:nimcache --passC:-fprofile-arcs --passC:-ftest-coverage --passL:-fprofile-arcs --passL:-ftest-coverage ${{ matrix.nim-options }}"
nim c $NIM_OPTIONS -r ${{ matrix.test-program }}
cd nimcache; rm *.c; cd ..
lcov --capture --directory nimcache --output-file coverage/coverage.info
lcov --extract coverage/coverage.info `pwd`/libp2p/* `pwd`/libp2p/**/* `pwd`/libp2p/**/**/* --output-file coverage/coverage.f.info
export COV_UUID=`cksum <<< "${{ matrix.test-program }} $NIM_OPTIONS" | cut -f 1 -d ' '`
genhtml coverage/coverage.f.info --output-directory coverage/$COV_UUID-output
echo ${{ matrix.test-program }} > coverage/$COV_UUID-nim_options.txt
echo $NIM_OPTIONS >> coverage/$COV_UUID-nim_options.txt
bash <(curl -s https://codecov.io/bash) -f coverage/coverage.f.info || echo "Codecov did not collect coverage reports"
- uses: actions/upload-artifact@master
with:
name: coverage
path: coverage
#- uses: actions/upload-artifact@master
# with:
# name: coverage
# path: coverage

103
.github/workflows/doc.yml vendored Normal file
View File

@@ -0,0 +1,103 @@
name: Docgen
on:
push:
workflow_dispatch:
jobs:
build:
timeout-minutes: 20
name: 'Generate & upload documentation'
runs-on: 'ubuntu-20.04'
continue-on-error: true
steps:
- name: Checkout
uses: actions/checkout@v2
with:
submodules: true
- uses: jiro4989/setup-nim-action@v1
with:
nim-version: 'stable'
- name: Generate doc
run: |
nim --version
nimble --version
nimble install_pinned
# nim doc can "fail", but the doc is still generated
nim doc --git.url:https://github.com/status-im/nim-libp2p --git.commit:${GITHUB_REF##*/} --outdir:${GITHUB_REF##*/} --project libp2p || true
# check that the folder exists
ls ${GITHUB_REF##*/}
- name: Clone the gh-pages branch
uses: actions/checkout@v2
with:
repository: status-im/nim-libp2p
ref: gh-pages
path: subdoc
submodules: true
fetch-depth: 0
- name: Commit & push
run: |
cd subdoc
# Delete merged branches doc's
for branch in $(git branch -vv | grep ': gone]' | awk '{print $1}'); do rm -rf $branch; done
# Update / create this branch doc
rm -rf ${GITHUB_REF##*/}
mv ../${GITHUB_REF##*/} .
# Remove .idx files
# NOTE: git also uses idx files in his
# internal folder, hence the `*` instead of `.`
find * -name "*.idx" -delete
git add .
git config --global user.email "${{ github.actor }}@users.noreply.github.com"
git config --global user.name = "${{ github.actor }}"
git commit -a -m "update docs for ${GITHUB_REF##*/}"
git push origin gh-pages
update_site:
if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/docs'
name: 'Rebuild website'
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- uses: actions/setup-python@v2
with:
python-version: 3.x
- uses: jiro4989/setup-nim-action@v1
with:
nim-version: 'stable'
- name: Generate website
run: pip install mkdocs-material && nimble website
- name: Clone the gh-pages branch
uses: actions/checkout@v2
with:
repository: status-im/nim-libp2p
ref: gh-pages
path: subdoc
fetch-depth: 0
- name: Commit & push
run: |
cd subdoc
rm -rf docs
mv ../site docs
git add .
git config --global user.email "${{ github.actor }}@users.noreply.github.com"
git config --global user.name = "${{ github.actor }}"
git commit -a -m "update website"
git push origin gh-pages

54
.github/workflows/interop.yml vendored Normal file
View File

@@ -0,0 +1,54 @@
name: Interoperability Testing
on:
pull_request:
push:
branches:
- unstable
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
run-multidim-interop:
name: Run multidimensional interoperability tests
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v3
with:
repository: libp2p/test-plans
submodules: true
fetch-depth: 0
- name: Build image
run: >
cd multidim-interop/impl/nim/v1.0 &&
make commitSha=$GITHUB_SHA image_name=nim-libp2p-head
- name: Create ping-version.json
run: >
(cat << EOF
{
"id": "nim-libp2p-head",
"containerImageID": "nim-libp2p-head",
"transports": [
"tcp",
"ws"
],
"secureChannels": [
"noise"
],
"muxers": [
"mplex",
"yamux"
]
}
EOF
) > ${{ github.workspace }}/test_head.json
- uses: libp2p/test-plans/.github/actions/run-interop-ping-test@master
with:
test-filter: nim-libp2p-head
extra-versions: ${{ github.workspace }}/test_head.json

82
.github/workflows/multi_nim.yml vendored Normal file
View File

@@ -0,0 +1,82 @@
name: Daily
on:
schedule:
- cron: "30 6 * * *"
workflow_dispatch:
jobs:
delete-cache:
runs-on: ubuntu-latest
steps:
- uses: snnaplab/delete-branch-cache-action@v1
build:
needs: delete-cache
timeout-minutes: 120
strategy:
fail-fast: false
matrix:
target:
- os: linux
cpu: amd64
- os: linux
cpu: i386
- os: macos
cpu: amd64
- os: windows
cpu: amd64
#- os: windows
#cpu: i386
branch: [version-1-6, version-2-0, devel]
include:
- target:
os: linux
builder: ubuntu-20.04
shell: bash
- target:
os: macos
builder: macos-12
shell: bash
- target:
os: windows
builder: windows-2019
shell: msys2 {0}
defaults:
run:
shell: ${{ matrix.shell }}
name: '${{ matrix.target.os }}-${{ matrix.target.cpu }} (Nim ${{ matrix.branch }})'
runs-on: ${{ matrix.builder }}
continue-on-error: ${{ matrix.branch == 'devel' || matrix.branch == 'version-2-0' }}
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Setup Nim
uses: "./.github/actions/install_nim"
with:
os: ${{ matrix.target.os }}
shell: ${{ matrix.shell }}
nim_branch: ${{ matrix.branch }}
cpu: ${{ matrix.target.cpu }}
- name: Setup Go
uses: actions/setup-go@v2
with:
go-version: '~1.15.5'
- name: Install p2pd
run: |
V=1 bash scripts/build_p2pd.sh p2pdCache 124530a3
- name: Run tests
run: |
nim --version
nimble --version
nimble install -y --depsOnly
NIMFLAGS="${NIMFLAGS} --gc:refc" nimble test
if [[ "${{ matrix.branch }}" == "devel" ]]; then
echo -e "\nTesting with '--gc:orc':\n"
NIMFLAGS="${NIMFLAGS} --gc:orc" nimble test
fi

View File

@@ -1,46 +0,0 @@
name: NBC Bump PR
on:
push:
branches:
- master
jobs:
bumpNBC:
runs-on: ubuntu-latest
steps:
- uses: status-im/github-app-token@v1
name: Generate token
id: generate-token
with:
app_id: ${{ secrets.BUMP_BOT_APP_ID }}
private_key: ${{ secrets.BUMP_BOT_APP_PRIVATE_KEY }}
- name: Clone NBC
uses: actions/checkout@v2
with:
repository: status-im/nimbus-eth2
ref: unstable
path: nbc
submodules: true
fetch-depth: 0
- name: Checkout this ref
run: |
cd nbc/vendor/nim-libp2p
git checkout $GITHUB_SHA
- name: Commit this bump
run: |
cd nbc
git config --global user.email "${{ github.actor }}@users.noreply.github.com"
git config --global user.name = "${{ github.actor }}"
git commit -a -m "auto-bump nim-libp2p"
- name: Make PR
uses: peter-evans/create-pull-request@v3.5.0
with:
branch: nim-libp2p-auto-bump
path: nbc
token: ${{ steps.generate-token.outputs.token }}
title: nim-libp2p auto bump

4
.gitignore vendored
View File

@@ -1,4 +1,5 @@
nimcache/
nimbledeps/
# Executables shall be put in an ignored build/ directory
# Ignore dynamic, static libs and libtool archive files
@@ -12,3 +13,6 @@ build/
.vscode/
.DS_Store
tests/pubsub/testgossipsub
examples/*.md
nimble.develop
nimble.paths

16
.pinned Normal file
View File

@@ -0,0 +1,16 @@
bearssl;https://github.com/status-im/nim-bearssl@#9372f27a25d0718d3527afad6cc936f6a853f86e
chronicles;https://github.com/status-im/nim-chronicles@#32ac8679680ea699f7dbc046e8e0131cac97d41a
chronos;https://github.com/status-im/nim-chronos@#ba143e029f35fd9b4cd3d89d007cc834d0d5ba3c
dnsclient;https://github.com/ba0f3/dnsclient.nim@#2b3d4b4e35b5e698fbbeafe16a4fa757926a4673
faststreams;https://github.com/status-im/nim-faststreams@#2a771bb91f8aae8520a5553955a2acce5fdd0c87
httputils;https://github.com/status-im/nim-http-utils@#aad684d3758a74c1b327df93da2e956458410b48
json_serialization;https://github.com/status-im/nim-json-serialization@#aa44ee61dd323022d4abe7cbf4e44668aad88454
metrics;https://github.com/status-im/nim-metrics@#abf3acc7f06cee9ee2c287d2f31413dc3df4c04e
nimcrypto;https://github.com/cheatfate/nimcrypto@#4014ef939b51e02053c2e16dd3481d47bc9267dd
secp256k1;https://github.com/status-im/nim-secp256k1@#5fd81357839d57ef38fb17647bd5e31dfa9f55b8
serialization;https://github.com/status-im/nim-serialization@#f0860e1c25acf26ef5e6ea231c7c0537c793b555
stew;https://github.com/status-im/nim-stew@#000eeb14a34832e6c95303e6508e2925db56be7c
testutils;https://github.com/status-im/nim-testutils@#dfc4c1b39f9ded9baf6365014de2b4bfb4dafc34
unittest2;https://github.com/status-im/nim-unittest2@#b178f47527074964f76c395ad0dfc81cf118f379
websock;https://github.com/status-im/nim-websock@#3696e3f3a5b938e478e473a6089bf8de386d2f04
zlib;https://github.com/status-im/nim-zlib@#d65ee2a7611eb9f0ef0e7350caed6e93ccfa9651

221
README.md
View File

@@ -1,13 +1,13 @@
<h1 align="center">
<a href="https://libp2p.io"><img width="250" src="https://github.com/libp2p/libp2p/blob/master/logo/black-bg-2.png?raw=true" alt="libp2p hex logo" /></a>
<a href="https://libp2p.io"><img width="250" src="./.assets/full-logo.svg?raw=true" alt="nim-libp2p logo" /></a>
</h1>
<h3 align="center">The Nim implementation of the libp2p Networking Stack.</h3>
<h3 align="center">The <a href="https://nim-lang.org/">Nim</a> implementation of the <a href="https://libp2p.io/">libp2p</a> Networking Stack.</h3>
<p align="center">
<a href="https://travis-ci.org/status-im/nim-libp2p"><img src="https://travis-ci.org/status-im/nim-libp2p.svg?branch=master" /></a>
<a href="https://ci.appveyor.com/project/nimbus/nim-libp2p/branch/master"><img src="https://ci.appveyor.com/api/projects/status/pqgif5bcie6cp3wi/branch/master?svg=true" /></a>
<a href="https://dev.azure.com/nimbus-dev/nim-libp2p/_build?definitionId=5&branchName=master"><img src="https://img.shields.io/azure-devops/build/nimbus-dev/dc5eed24-3f6c-4c06-8466-3d060abd6c8b/5/master?label=Azure%20%28Linux%2064-bit%2C%20Windows%2032-bit%2F64-bit%2C%20MacOS%2064-bit%29" /></a>
<a href="https://github.com/status-im/nim-libp2p/actions"><img src="https://github.com/status-im/nim-libp2p/actions/workflows/ci.yml/badge.svg" /></a>
<a href="https://codecov.io/gh/status-im/nim-libp2p"><img src="https://codecov.io/gh/status-im/nim-libp2p/branch/master/graph/badge.svg?token=UR5JRQ249W"/></a>
</p>
<p align="center">
@@ -16,168 +16,156 @@
<img src="https://img.shields.io/badge/nim-%3E%3D1.2.0-orange.svg?style=flat-square" />
</p>
## Introduction
An implementation of [libp2p](https://libp2p.io/) in Nim. Also provides a Nim wrapper of the [Libp2p Go daemon](https://github.com/libp2p/go-libp2p).
## Project Status
The current native Nim libp2p implementation support is experimental and shouldn't be relied on for production use. It is under active development and contributions are highly welcomed. :)
Check our [examples folder](/examples) to get started!
# Table of Contents
- [Background](#background)
- [Install](#install)
- [Prerequisite](#prerequisite)
- [Usage](#usage)
- [API](#api)
- [Getting Started](#getting-started)
- [Tutorials and Examples](#tutorials-and-examples)
- [Using the Go Daemon](#using-the-go-daemon)
- [Getting Started](#getting-started)
- [Modules](#modules)
- [Users](#users)
- [Stability](#stability)
- [Development](#development)
- [Tests](#tests)
- [Packages](#packages)
- [Contribute](#contribute)
- [Core Developers](#core-developers)
- [Contribute](#contribute)
- [Contributors](#contributors)
- [Core Maintainers](#core-maintainers)
- [License](#license)
## Background
libp2p is a networking stack and library modularized out of [The IPFS Project](https://github.com/ipfs/ipfs), and bundled separately for other tools to use.
libp2p is a [Peer-to-Peer](https://en.wikipedia.org/wiki/Peer-to-peer) networking stack, with [implementations](https://github.com/libp2p/libp2p#implementations) in multiple languages derived from the same [specifications.](https://github.com/libp2p/specs)
libp2p is the product of a long and arduous quest of understanding; a deep dive into the internet's network stack and the peer-to-peer protocols from the past. Building large scale peer-to-peer systems has been complex and difficult in the last 15 years and libp2p is a way to fix that. It is a "network stack", a suite of networking protocols that cleanly separates concerns and enables sophisticated applications to only use the protocols they absolutely need, without giving up interoperability and upgradeability.
Building large scale peer-to-peer systems has been complex and difficult in the last 15 years and libp2p is a way to fix that. It's striving to be a modular stack, with sane and secure defaults, useful protocols, while remain open and extensible.
This implementation in native Nim, relying on [chronos](https://github.com/status-im/nim-chronos) for async. It's used in production by a few [projects](#users)
libp2p grew out of IPFS, but it is built so that lots of people can use it, for lots of different projects.
- Learn more about libp2p at [**libp2p.io**](https://libp2p.io) and follow our evolving documentation efforts at [**docs.libp2p.io**](https://docs.libp2p.io).
- [Here](https://github.com/libp2p/libp2p#description) is an overview of libp2p and its implementations in other programming languages.
Learn more about libp2p at [**libp2p.io**](https://libp2p.io) and follow libp2p's documentation [**docs.libp2p.io**](https://docs.libp2p.io).
## Install
**Prerequisite**
- [Nim](https://nim-lang.org/install.html)
```
nimble install libp2p
```
### Prerequisite
- [Nim](https://nim-lang.org/install.html)
## Usage
## Getting Started
You'll find the nim-libp2p documentation [here](https://status-im.github.io/nim-libp2p/docs/).
### API
The specification is available in the [docs/api](docs/api) folder.
**Go Daemon:**
Please find the installation and usage intructions in [daemonapi.md](examples/go-daemon/daemonapi.md).
### Getting Started
Please read the [GETTING_STARTED.md](docs/GETTING_STARTED.md) guide.
## Modules
### Tutorials and Examples
Example code can be found in the [examples folder](/examples).
List of packages modules implemented in nim-libp2p:
#### Direct Chat Tutorial
- [Part I](https://our.status.im/nim-libp2p-tutorial-a-peer-to-peer-chat-example-1/): Set up the main function and use multi-thread for processing IO.
- [Part II](https://our.status.im/nim-libp2p-tutorial-a-peer-to-peer-chat-example-2/): Dial remote peer and allow customized user input commands.
- [Part III](https://our.status.im/nim-libp2p-tutorial-a-peer-to-peer-chat-example-3/): Configure and establish a libp2p node.
| Name | Description |
| ---------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------- |
| **Libp2p** | |
| [libp2p](libp2p/switch.nim) | The core of the project |
| [connmanager](libp2p/connmanager.nim) | Connection manager |
| [identify / push identify](libp2p/protocols/identify.nim) | [Identify](https://docs.libp2p.io/concepts/fundamentals/protocols/#identify) protocol |
| [ping](libp2p/protocols/ping.nim) | [Ping](https://docs.libp2p.io/concepts/fundamentals/protocols/#ping) protocol |
| [libp2p-daemon-client](libp2p/daemon/daemonapi.nim) | [go-daemon](https://github.com/libp2p/go-libp2p-daemon) nim wrapper |
| [interop-libp2p](tests/testinterop.nim) | Interop tests |
| **Transports** | |
| [libp2p-tcp](libp2p/transports/tcptransport.nim) | TCP transport |
| [libp2p-ws](libp2p/transports/wstransport.nim) | WebSocket & WebSocket Secure transport |
| [libp2p-tor](libp2p/transports/tortransport.nim) | Tor Transport |
| **Secure Channels** | |
| [libp2p-secio](libp2p/protocols/secure/secio.nim) | Secio secure channel |
| [libp2p-noise](libp2p/protocols/secure/noise.nim) | [Noise](https://docs.libp2p.io/concepts/secure-comm/noise/) secure channel |
| [libp2p-plaintext](libp2p/protocols/secure/plaintext.nim) | Plain Text for development purposes |
| **Stream Multiplexers** | |
| [libp2p-mplex](libp2p/muxers/mplex/mplex.nim) | [MPlex](https://github.com/libp2p/specs/tree/master/mplex) multiplexer |
| [libp2p-yamux](libp2p/muxers/yamux/yamux.nim) | [Yamux](https://docs.libp2p.io/concepts/multiplex/yamux/) multiplexer |
| **Data Types** | |
| [peer-id](libp2p/peerid.nim) | [Cryptographic identifiers](https://docs.libp2p.io/concepts/fundamentals/peers/#peer-id) |
| [peer-store](libp2p/peerstore.nim) | ["Address book" of known peers](https://docs.libp2p.io/concepts/fundamentals/peers/#peer-store) |
| [multiaddress](libp2p/multiaddress.nim) | [Composable network addresses](https://github.com/multiformats/multiaddr) |
| [signed envelope](libp2p/signed_envelope.nim) | [Signed generic data container](https://github.com/libp2p/specs/blob/master/RFC/0002-signed-envelopes.md) |
| [routing record](libp2p/routing_record.nim) | [Signed peer dialing informations](https://github.com/libp2p/specs/blob/master/RFC/0003-routing-records.md) |
| [discovery manager](libp2p/discovery/discoverymngr.nim) | Discovery Manager |
| **Utilities** | |
| [libp2p-crypto](libp2p/crypto) | Cryptographic backend |
| [libp2p-crypto-secp256k1](libp2p/crypto/secp.nim) | |
| **Pubsub** | |
| [libp2p-pubsub](libp2p/protocols/pubsub/pubsub.nim) | Pub-Sub generic interface |
| [libp2p-floodsub](libp2p/protocols/pubsub/floodsub.nim) | FloodSub implementation |
| [libp2p-gossipsub](libp2p/protocols/pubsub/gossipsub.nim) | [GossipSub](https://docs.libp2p.io/concepts/publish-subscribe/) implementation |
## Users
### Using the Go Daemon
Please find the installation and usage intructions in [daemonapi.md](docs/api/libp2p/daemonapi.md).
nim-libp2p is used by:
- [Nimbus](https://github.com/status-im/nimbus-eth2), an Ethereum client
- [nwaku](https://github.com/status-im/nwaku), a decentralized messaging application
- [nim-codex](https://github.com/status-im/nim-codex), a decentralized storage application
- (open a pull request if you want to be included here)
Examples can be found in the [examples/go-daemon folder](https://github.com/status-im/nim-libp2p/tree/readme/examples/go-daemon);
## Stability
nim-libp2p has been used in production for over a year in high-stake scenarios, so its core is considered stable.
Some modules are more recent and less stable.
The versioning follows [semver](https://semver.org/), with some additions:
- Some of libp2p procedures are marked as `.public.`, they will remain compatible during each `MAJOR` version
- The rest of the procedures are considered internal, and can change at any `MINOR` version (but remain compatible for each new `PATCH`)
We aim to be compatible at all time with at least 2 Nim `MINOR` versions, currently `1.6 & 2.0`
## Development
**Clone and Install dependencies:**
Clone and Install dependencies:
```sh
git clone https://github.com/status-im/nim-libp2p
cd nim-libp2p
nimble install
# to use dependencies computed by nimble
nimble install -dy
# OR to install the dependencies versions used in CI
nimble install_pinned
```
### Tests
#### Prerequisite
- [Go 1.12+](https://golang.org/dl/)
#### Run unit tests
Run unit tests:
```sh
# run all the unit tests
nimble test
```
This requires the go daemon to be available. To only run native tests, use `nimble testnative`.
Or use `nimble tasks` to show all available tasks.
### Packages
### Contribute
List of packages currently in existence for nim-libp2p:
The libp2p implementation in Nim is a work in progress. We welcome contributors to help out! Specifically, you can:
- Go through the modules and **check out existing issues**. This would be especially useful for modules in active development. Some knowledge of IPFS/libp2p may be required, as well as the infrastructure behind it.
- **Perform code reviews**. Feel free to let us know if you found anything that can a) speed up the project development b) ensure better quality and c) reduce possible future bugs.
- **Add tests**. Help nim-libp2p to be more robust by adding more tests to the [tests folder](tests/).
#### Libp2p
- [libp2p](https://github.com/status-im/nim-libp2p)
- [libp2p-daemon-client](https://github.com/status-im/nim-libp2p/blob/master/libp2p/daemon/daemonapi.nim)
- [interop-libp2p](https://github.com/status-im/nim-libp2p/blob/master/tests/testinterop.nim)
The code follows the [Status Nim Style Guide](https://status-im.github.io/nim-style-guide/).
#### Transports
- [libp2p-tcp](https://github.com/status-im/nim-libp2p/blob/master/libp2p/transports/tcptransport.nim)
### Contributors
<a href="https://github.com/status-im/nim-libp2p/graphs/contributors"><img src="https://contrib.rocks/image?repo=status-im/nim-libp2p" alt="nim-libp2p contributors"></a>
#### Secure Channels
- [libp2p-secio](https://github.com/status-im/nim-libp2p/blob/master/libp2p/protocols/secure/secio.nim)
### Core Maintainers
<table>
<tbody>
<tr>
<td align="center"><a href="https://github.com/Menduist"><img src="https://avatars.githubusercontent.com/u/13471753?v=4?s=100" width="100px;" alt="Tanguy"/><br /><sub><b>Tanguy (Menduist)</b></sub></a></td>
<td align="center"><a href="https://github.com/lchenut"><img src="https://avatars.githubusercontent.com/u/11214565?v=4?s=100" width="100px;" alt="Ludovic"/><br /><sub><b>Ludovic</b></sub></a></td>
<td align="center"><a href="https://github.com/diegomrsantos"><img src="https://avatars.githubusercontent.com/u/7316595?v=4?s=100" width="100px;" alt="Diego"/><br /><sub><b>Diego</b></sub></a></td>
</tr>
</tbody>
</table>
#### Stream Multiplexers
- [libp2p-mplex](https://github.com/status-im/nim-libp2p/blob/master/libp2p/muxers/mplex/mplex.nim)
#### Utilities
- [libp2p-crypto](https://github.com/status-im/nim-libp2p/tree/master/libp2p/crypto)
- [libp2p-crypto-secp256k1](https://github.com/status-im/nim-libp2p/blob/master/libp2p/crypto/secp.nim)
#### Data Types
- [peer-id](https://github.com/status-im/nim-libp2p/blob/master/libp2p/peer.nim)
- [peer-info](https://github.com/status-im/nim-libp2p/blob/master/libp2p/peerinfo.nim)
#### Pubsub
- [libp2p-pubsub](https://github.com/status-im/nim-libp2p/blob/master/libp2p/protocols/pubsub/pubsub.nim)
- [libp2p-floodsub](https://github.com/status-im/nim-libp2p/blob/master/libp2p/protocols/pubsub/floodsub.nim)
- [libp2p-gossipsub](https://github.com/status-im/nim-libp2p/blob/master/libp2p/protocols/pubsub/gossipsub.nim)
Packages that exist in the original libp2p specs and are under active development:
- libp2p-daemon
- libp2p-webrtc-direct
- libp2p-webrtc-star
- libp2p-websockets
- libp2p-spdy
- libp2p-bootstrap
- libp2p-kad-dht
- libp2p-mdns
- libp2p-webrtc-star
- libp2p-delegated-content-routing
- libp2p-delegated-peer-routing
- libp2p-nat-mgnr
- libp2p-utils
** Note that the current stack reflects the minimal requirements for the upcoming Eth2 implementation.
### Tips and tricks
#### enable expensive metrics:
### Compile time flags
Enable expensive metrics (ie, metrics with per-peer cardinality):
```bash
nim c -d:libp2p_expensive_metrics some_file.nim
```
#### use identify metrics
Set list of known libp2p agents for metrics:
```bash
nim c -d:libp2p_agents_metrics -d:KnownLibP2PAgents=nimbus,lighthouse,prysm,teku some_file.nim
nim c -d:libp2p_agents_metrics -d:KnownLibP2PAgents=nimbus,lighthouse,lodestar,prysm,teku some_file.nim
```
### specify gossipsub specific topics to measure
Specify gossipsub specific topics to measure in the metrics:
```bash
nim c -d:KnownLibP2PTopics=topic1,topic2,topic3 some_file.nim
```
## Contribute
The libp2p implementation in Nim is a work in progress. We welcome contributors to help out! Specifically, you can:
- Go through the modules and **check out existing issues**. This would be especially useful for modules in active development. Some knowledge of IPFS/libp2p may be required, as well as the infrastructure behind it.
- **Perform code reviews**. Feel free to let us know if you found anything that can a) speed up the project development b) ensure better quality and c) reduce possible future bugs.
- **Add tests**. Help nim-libp2p to be more robust by adding more tests to the [tests folder](https://github.com/status-im/nim-libp2p/tree/master/tests).
### Core Developers
[@cheatfate](https://github.com/cheatfate), [Dmitriy Ryajov](https://github.com/dryajov), [Giovanni Petrantoni](https://github.com/sinkingsugar), [Zahary Karadjov](https://github.com/zah)
## License
Licensed and distributed under either of
@@ -189,4 +177,3 @@ or
* Apache License, Version 2.0, ([LICENSE-APACHEv2](LICENSE-APACHEv2) or http://www.apache.org/licenses/LICENSE-2.0)
at your option. These files may not be copied, modified, or distributed except according to those terms.

View File

@@ -1,14 +1,8 @@
codecov:
notify:
require_ci_to_pass: true
# must be the number of coverage report builds
# notice that this number is for PRs;
# like this we disabled notify on pure branches report
# which is fine I guess
after_n_builds: 25
comment:
layout: "reach, diff, flags, files"
after_n_builds: 25 # must be the number of coverage report builds
coverage:
status:
project:
@@ -16,4 +10,4 @@ coverage:
# basic settings
target: auto
threshold: 5%
base: auto
base: auto

23
config.nims Normal file
View File

@@ -0,0 +1,23 @@
# to allow locking
if dirExists("nimbledeps/pkgs"):
switch("NimblePath", "nimbledeps/pkgs")
if dirExists("nimbledeps/pkgs2"):
switch("NimblePath", "nimbledeps/pkgs2")
switch("warning", "CaseTransition:off")
switch("warning", "ObservableStores:off")
switch("warning", "LockLevel:off")
--define:chronosStrictException
--styleCheck:usages
switch("warningAsError", "UseBase:on")
--styleCheck:error
# Avoid some rare stack corruption while using exceptions with a SEH-enabled
# toolchain: https://github.com/status-im/nimbus-eth2/issues/3121
if defined(windows) and not defined(vcc):
--define:nimRawSetjmp
# begin Nimble config (version 1)
when fileExists("nimble.paths"):
include "nimble.paths"
# end Nimble config

View File

@@ -1,3 +0,0 @@
# API
Coming Soon...

View File

@@ -1,98 +0,0 @@
# Getting Started
Welcome to nim-libp2p! This guide will walk you through a peer to peer chat example. <br>
The full code can be found in [directchat.nim](examples/directchat.nim) under the examples folder.
### Direct Chat Example
To run nim-libp2p, add it to your project's nimble file and spawn a node as follows:
```nim
import tables
import chronos
import ../libp2p/[switch,
multistream,
protocols/identify,
connection,
transports/transport,
transports/tcptransport,
multiaddress,
peerinfo,
crypto/crypto,
peerid,
protocols/protocol,
muxers/muxer,
muxers/mplex/mplex,
protocols/secure/secio,
protocols/secure/secure]
const TestCodec = "/test/proto/1.0.0" # custom protocol string
type
TestProto = ref object of LPProtocol # declare a custom protocol
method init(p: TestProto) {.gcsafe.} =
# handle incoming connections in closure
proc handle(conn: Connection, proto: string) {.async, gcsafe.} =
echo "Got from remote - ", cast[string](await conn.readLp(1024))
await conn.writeLp("Hello!")
await conn.close()
p.codec = TestCodec # init proto with the correct string id
p.handler = handle # set proto handler
proc createSwitch(ma: MultiAddress): (Switch, PeerInfo) =
## Helper to create a swith
let seckey = PrivateKey.random(RSA) # use a random key for peer id
var peerInfo = PeerInfo.init(seckey) # create a peer id and assign
peerInfo.addrs.add(ma) # set this peer's multiaddresses (can be any number)
let identify = newIdentify(peerInfo) # create the identify proto
proc createMplex(conn: Connection): Muxer =
# helper proc to create multiplexers,
# use this to perform any custom setup up,
# such as adjusting timeout or anything else
# that the muxer requires
result = newMplex(conn)
let mplexProvider = newMuxerProvider(createMplex, MplexCodec) # create multiplexer
let transports = @[Transport(newTransport(TcpTransport))] # add all transports (tcp only for now, but can be anything in the future)
let muxers = {MplexCodec: mplexProvider}.toTable() # add all muxers
let secureManagers = {SecioCodec: Secure(newSecio(seckey))}.toTable() # setup the secio and any other secure provider
# create the switch
let switch = newSwitch(peerInfo,
transports,
identify,
muxers,
secureManagers)
result = (switch, peerInfo)
proc main() {.async, gcsafe.} =
let ma1: MultiAddress = Multiaddress.init("/ip4/0.0.0.0/tcp/0")
let ma2: MultiAddress = Multiaddress.init("/ip4/0.0.0.0/tcp/0")
var peerInfo1, peerInfo2: PeerInfo
var switch1, switch2: Switch
(switch1, peerInfo1) = createSwitch(ma1) # create node 1
# setup the custom proto
let testProto = new TestProto
testProto.init() # run it's init method to perform any required initialization
switch1.mount(testProto) # mount the proto
var switch1Fut = await switch1.start() # start the node
(switch2, peerInfo2) = createSwitch(ma2) # create node 2
var switch2Fut = await switch2.start() # start second node
let conn = await switch2.dial(switch1.peerInfo, TestCodec) # dial the first node
await conn.writeLp("Hello!") # writeLp send a length prefixed buffer over the wire
# readLp reads length prefixed bytes and returns a buffer without the prefix
echo "Remote responded with - ", cast[string](await conn.readLp(1024))
await allFutures(switch1.stop(), switch2.stop()) # close connections and shutdown all transports
await allFutures(switch1Fut & switch2Fut) # wait for all transports to shutdown
waitFor(main())
```

View File

@@ -1,56 +0,0 @@
# Table of Contents
- [Introduction](#introduction)
- [Installation](#installation)
- [Usage](#usage)
- [Example](#example)
- [Getting Started](#getting-started)
# Introduction
This is a libp2p-backed daemon wrapping the functionalities of go-libp2p for use in Nim. <br>
For more information about the go daemon, check out [this repository](https://github.com/libp2p/go-libp2p-daemon).
# Installation
```sh
# clone and install dependencies
git clone https://github.com/status-im/nim-libp2p
cd nim-libp2p
nimble install
# perform unit tests
nimble test
# update the git submodule to install the go daemon
git submodule update --init --recursive
go version
git clone https://github.com/libp2p/go-libp2p-daemon
cd go-libp2p-daemon
git checkout v0.0.1
go install ./...
cd ..
```
# Usage
## Example
Examples can be found in the [examples folder](https://github.com/status-im/nim-libp2p/tree/readme/examples/go-daemon)
## Getting Started
Try out the chat example. Full code can be found [here](https://github.com/status-im/nim-libp2p/blob/master/examples/chat.nim):
```bash
nim c -r --threads:on examples\chat.nim
```
This will output a peer ID such as `QmbmHfVvouKammmQDJck4hz33WvVktNEe7pasxz2HgseRu` which you can use in another instance to connect to it.
```bash
./example/chat
/connect QmbmHfVvouKammmQDJck4hz33WvVktNEe7pasxz2HgseRu
```
You can now chat between the instances!
![Chat example](https://imgur.com/caYRu8K.gif)

View File

@@ -1,29 +0,0 @@
# Introduction
This folder contains the documentation for each nim-libp2p module and the sample code for the tutorials.
# Table of Contents
### [Getting Started](GETTING_STARTED.md)
### Tutorials
- P2P Chat Example
- [part I](tutorial/directchat/start.nim)
- [part II](tutorial/directchat/second.nim)
### API Specifications
- libp2p
- [libp2p-daemon-client](api/libp2p/daemonapi.md)
- [interop-libp2p](api/libp2p/interop.md)
- transports
- [libp2p-tcp](api/transports/tcptransport.md)
- secure channels
- [libp2p-secio](api/secure_channels/secio.md)
- stream multiplexers
- [libp2p-mplex](api/stream_multiplexers/mplex.md)
- utilities
- [libp2p-crypto](api/utilities/crypto.md)
- [libp2p-crypto-secp256k1](api/utilities/secp256k1.md)
- data types
- [peer-id](api/data_types/peer.md)
- [peer-info](api/data_types/peerinfo.md)
- pubsub
- [libp2p-pubsub](api/pubsub/pubsub.md)
- [libp2p-floodsub](api/pubsub/floodsub.md)
- [libp2p-gossipsub](api/pubsub/gossipsub.md)

View File

@@ -1,149 +0,0 @@
when not(compileOption("threads")):
{.fatal: "Please, compile this program with the --threads:on option!".}
import tables, strformat, strutils
import chronos
import ../libp2p/[switch,
multistream,
crypto/crypto,
protocols/identify,
connection,
transports/transport,
transports/tcptransport,
multiaddress,
peerinfo,
peerid,
protocols/protocol,
protocols/secure/secure,
protocols/secure/secio,
muxers/muxer,
muxers/mplex/mplex]
const ChatCodec = "/nim-libp2p/chat/1.0.0"
const DefaultAddr = "/ip4/127.0.0.1/tcp/55505"
const Help = """
Commands: /[?|hep|connect|disconnect|exit]
help: Prints this help
connect: dials a remote peer
disconnect: ends current session
exit: closes the chat
"""
type ChatProto = ref object of LPProtocol
switch: Switch # a single entry point for dialing and listening to peer
transp: StreamTransport # transport streams between read & write file descriptor
conn: Connection # create and close read & write stream
connected: bool # if the node is connected to another peer
started: bool # if the node has started
# copied from https://github.com/status-im/nimbus-eth2/blob/0ed657e953740a92458f23033d47483ffa17ccb0/beacon_chain/eth2_network.nim#L109-L115
proc initAddress(T: type MultiAddress, str: string): T =
let address = MultiAddress.init(str)
if IPFS.match(address) and matchPartial(multiaddress.TCP, address):
result = address
else:
raise newException(MultiAddressError,
"Invalid bootstrap node multi-address")
proc dialPeer(p: ChatProto, address: string) {.async.} =
let multiAddr = MultiAddress.initAddress(address);
let parts = address.split("/")
let remotePeer = PeerInfo.init(parts[^1],
[multiAddr])
echo &"dialing peer: {multiAddr}"
p.conn = await p.switch.dial(remotePeer, ChatCodec)
p.connected = true
proc readAndPrint(p: ChatProto) {.async.} =
while true:
while p.connected:
echo cast[string](await p.conn.readLp(1024))
await sleepAsync(100.millis)
proc writeAndPrint(p: ChatProto) {.async.} =
while true:
if not p.connected:
echo "type an address or wait for a connection:"
echo "type /[help|?] for help"
let line = await p.transp.readLine()
if line.startsWith("/help") or line.startsWith("/?") or not p.started:
echo Help
continue
if line.startsWith("/disconnect"):
echo "Ending current session"
if p.connected and p.conn.closed.not:
await p.conn.close()
p.connected = false
elif line.startsWith("/connect"):
if p.connected:
var yesno = "N"
echo "a session is already in progress, do you want end it [y/N]?"
yesno = await p.transp.readLine()
if yesno.cmpIgnoreCase("y") == 0:
await p.conn.close()
p.connected = false
elif yesno.cmpIgnoreCase("n") == 0:
continue
else:
echo "unrecognized response"
continue
echo "enter address of remote peer"
let address = await p.transp.readLine()
if address.len > 0:
await p.dialPeer(address)
elif line.startsWith("/exit"):
if p.connected and p.conn.closed.not:
await p.conn.close()
p.connected = false
await p.switch.stop()
echo "quitting..."
quit(0)
else:
if p.connected:
await p.conn.writeLp(line)
else:
try:
if line.startsWith("/") and "ipfs" in line:
await p.dialPeer(line)
except:
echo &"unable to dial remote peer {line}"
echo getCurrentExceptionMsg()
proc readWriteLoop(p: ChatProto) {.async.} =
asyncCheck p.writeAndPrint() # execute the async function but does not block
asyncCheck p.readAndPrint()
proc processInput(rfd: AsyncFD) {.async.} =
let transp = fromPipe(rfd)
while true:
let a = await transp.readLine()
echo "You just entered: " & a
proc readInput(wfd: AsyncFD) {.thread.} =
## This procedure performs reading from `stdin` and sends data over
## pipe to main thread.
let transp = fromPipe(wfd)
while true:
let line = stdin.readLine()
discard waitFor transp.write(line & "\r\n")
proc main() {.async.} =
let (rfd, wfd) = createAsyncPipe()
if rfd == asyncInvalidPipe or wfd == asyncInvalidPipe:
raise newException(ValueError, "Could not initialize pipe!")
var thread: Thread[AsyncFD]
thread.createThread(readInput, wfd)
await processInput(rfd)
when isMainModule: # isMainModule = true when the module is compiled as the main file
waitFor(main())

View File

@@ -1,39 +0,0 @@
when not(compileOption("threads")):
{.fatal: "Please, compile this program with the --threads:on option!".}
import chronos # an efficient library for async
proc processInput(rfd: AsyncFD) {.async.} =
echo "Type something below to see if the multithread IO works:\nType 'exit' to exit."
let transp = fromPipe(rfd)
while true:
let a = await transp.readLine()
if a == "exit":
quit(0);
echo "You just entered: " & a
proc readInput(wfd: AsyncFD) {.thread.} =
## This procedure performs reading from `stdin` and sends data over
## pipe to main thread.
let transp = fromPipe(wfd)
while true:
let line = stdin.readLine()
discard waitFor transp.write(line & "\r\n")
proc main() {.async.} =
let (rfd, wfd) = createAsyncPipe()
if rfd == asyncInvalidPipe or wfd == asyncInvalidPipe:
raise newException(ValueError, "Could not initialize pipe!")
var thread: Thread[AsyncFD]
thread.createThread(readInput, wfd)
await processInput(rfd)
when isMainModule: # isMainModule = true when the module is compiled as the main file
waitFor(main())

View File

@@ -1,149 +0,0 @@
when not(compileOption("threads")):
{.fatal: "Please, compile this program with the --threads:on option!".}
import tables, strformat, strutils
import chronos
import ../libp2p/[switch,
multistream,
crypto/crypto,
protocols/identify,
connection,
transports/transport,
transports/tcptransport,
multiaddress,
peerinfo,
peerid,
protocols/protocol,
protocols/secure/secure,
protocols/secure/secio,
muxers/muxer,
muxers/mplex/mplex]
const ChatCodec = "/nim-libp2p/chat/1.0.0"
const DefaultAddr = "/ip4/127.0.0.1/tcp/55505"
const Help = """
Commands: /[?|hep|connect|disconnect|exit]
help: Prints this help
connect: dials a remote peer
disconnect: ends current session
exit: closes the chat
"""
type ChatProto = ref object of LPProtocol
switch: Switch # a single entry point for dialing and listening to peer
transp: StreamTransport # transport streams between read & write file descriptor
conn: Connection # create and close read & write stream
connected: bool # if the node is connected to another peer
started: bool # if the node has started
# copied from https://github.com/status-im/nimbus-eth2/blob/0ed657e953740a92458f23033d47483ffa17ccb0/beacon_chain/eth2_network.nim#L109-L115
proc initAddress(T: type MultiAddress, str: string): T =
let address = MultiAddress.init(str)
if IPFS.match(address) and matchPartial(multiaddress.TCP, address):
result = address
else:
raise newException(MultiAddressError,
"Invalid bootstrap node multi-address")
proc dialPeer(p: ChatProto, address: string) {.async.} =
let multiAddr = MultiAddress.initAddress(address);
let parts = address.split("/")
let remotePeer = PeerInfo.init(parts[^1],
[multiAddr])
echo &"dialing peer: {multiAddr}"
p.conn = await p.switch.dial(remotePeer, ChatCodec)
p.connected = true
proc readAndPrint(p: ChatProto) {.async.} =
while true:
while p.connected:
echo cast[string](await p.conn.readLp(1024))
await sleepAsync(100.millis)
proc writeAndPrint(p: ChatProto) {.async.} =
while true:
if not p.connected:
echo "type an address or wait for a connection:"
echo "type /[help|?] for help"
let line = await p.transp.readLine()
if line.startsWith("/help") or line.startsWith("/?") or not p.started:
echo Help
continue
if line.startsWith("/disconnect"):
echo "Ending current session"
if p.connected and p.conn.closed.not:
await p.conn.close()
p.connected = false
elif line.startsWith("/connect"):
if p.connected:
var yesno = "N"
echo "a session is already in progress, do you want end it [y/N]?"
yesno = await p.transp.readLine()
if yesno.cmpIgnoreCase("y") == 0:
await p.conn.close()
p.connected = false
elif yesno.cmpIgnoreCase("n") == 0:
continue
else:
echo "unrecognized response"
continue
echo "enter address of remote peer"
let address = await p.transp.readLine()
if address.len > 0:
await p.dialPeer(address)
elif line.startsWith("/exit"):
if p.connected and p.conn.closed.not:
await p.conn.close()
p.connected = false
await p.switch.stop()
echo "quitting..."
quit(0)
else:
if p.connected:
await p.conn.writeLp(line)
else:
try:
if line.startsWith("/") and "ipfs" in line:
await p.dialPeer(line)
except:
echo &"unable to dial remote peer {line}"
echo getCurrentExceptionMsg()
proc readWriteLoop(p: ChatProto) {.async.} =
asyncCheck p.writeAndPrint() # execute the async function but does not block
asyncCheck p.readAndPrint()
proc processInput(rfd: AsyncFD) {.async.} =
let transp = fromPipe(rfd)
while true:
let a = await transp.readLine()
echo "You just entered: " & a
proc readInput(wfd: AsyncFD) {.thread.} =
## This procedure performs reading from `stdin` and sends data over
## pipe to main thread.
let transp = fromPipe(wfd)
while true:
let line = stdin.readLine()
discard waitFor transp.write(line & "\r\n")
proc main() {.async.} =
let (rfd, wfd) = createAsyncPipe()
if rfd == asyncInvalidPipe or wfd == asyncInvalidPipe:
raise newException(ValueError, "Could not initialize pipe!")
var thread: Thread[AsyncFD]
thread.createThread(readInput, wfd)
await processInput(rfd)
when isMainModule: # isMainModule = true when the module is compiled as the main file
waitFor(main())

View File

@@ -1,39 +0,0 @@
when not(compileOption("threads")):
{.fatal: "Please, compile this program with the --threads:on option!".}
import chronos # an efficient library for async
proc processInput(rfd: AsyncFD) {.async.} =
echo "Type something below to see if the multithread IO works:\nType 'exit' to exit."
let transp = fromPipe(rfd)
while true:
let a = await transp.readLine()
if a == "exit":
quit(0);
echo "You just entered: " & a
proc readInput(wfd: AsyncFD) {.thread.} =
## This procedure performs reading from `stdin` and sends data over
## pipe to main thread.
let transp = fromPipe(wfd)
while true:
let line = stdin.readLine()
discard waitFor transp.write(line & "\r\n")
proc main() {.async.} =
let (rfd, wfd) = createAsyncPipe()
if rfd == asyncInvalidPipe or wfd == asyncInvalidPipe:
raise newException(ValueError, "Could not initialize pipe!")
var thread: Thread[AsyncFD]
thread.createThread(readInput, wfd)
await processInput(rfd)
when isMainModule: # isMainModule = true when the module is compiled as the main file
waitFor(main())

5
examples/README.md Normal file
View File

@@ -0,0 +1,5 @@
# nim-libp2p examples
In this folder, you'll find the sources of the [nim-libp2p website](https://status-im.github.io/nim-libp2p/docs/)
We recommand to follow the tutorials on the website, but feel free to grok the sources here!

83
examples/circuitrelay.nim Normal file
View File

@@ -0,0 +1,83 @@
## # Circuit Relay example
##
## Circuit Relay can be used when a node cannot reach another node
## directly, but can reach it through a another node (the Relay).
##
## That may happen because of NAT, Firewalls, or incompatible transports.
##
## More informations [here](https://docs.libp2p.io/concepts/circuit-relay/).
import chronos, stew/byteutils
import libp2p,
libp2p/protocols/connectivity/relay/[relay, client]
# Helper to create a circuit relay node
proc createCircuitRelaySwitch(r: Relay): Switch =
SwitchBuilder.new()
.withRng(newRng())
.withAddresses(@[ MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet() ])
.withTcpTransport()
.withMplex()
.withNoise()
.withCircuitRelay(r)
.build()
proc main() {.async.} =
# Create a custom protocol
let customProtoCodec = "/test"
var proto = new LPProtocol
proto.codec = customProtoCodec
proto.handler = proc(conn: Connection, proto: string) {.async.} =
var msg = string.fromBytes(await conn.readLp(1024))
echo "1 - Dst Received: ", msg
assert "test1" == msg
await conn.writeLp("test2")
msg = string.fromBytes(await conn.readLp(1024))
echo "2 - Dst Received: ", msg
assert "test3" == msg
await conn.writeLp("test4")
let
relay = Relay.new()
clSrc = RelayClient.new()
clDst = RelayClient.new()
# Create three hosts, enable relay client on two of them.
# The third one can relay connections for other peers.
# RelayClient can use a relay, Relay is a relay.
swRel = createCircuitRelaySwitch(relay)
swSrc = createCircuitRelaySwitch(clSrc)
swDst = createCircuitRelaySwitch(clDst)
swDst.mount(proto)
await swRel.start()
await swSrc.start()
await swDst.start()
let
# Create a relay address to swDst using swRel as the relay
addrs = MultiAddress.init($swRel.peerInfo.addrs[0] & "/p2p/" &
$swRel.peerInfo.peerId & "/p2p-circuit").get()
# Connect Dst to the relay
await swDst.connect(swRel.peerInfo.peerId, swRel.peerInfo.addrs)
# Dst reserve a slot on the relay.
let rsvp = await clDst.reserve(swRel.peerInfo.peerId, swRel.peerInfo.addrs)
# Src dial Dst using the relay
let conn = await swSrc.dial(swDst.peerInfo.peerId, @[ addrs ], customProtoCodec)
await conn.writeLp("test1")
var msg = string.fromBytes(await conn.readLp(1024))
echo "1 - Src Received: ", msg
assert "test2" == msg
await conn.writeLp("test3")
msg = string.fromBytes(await conn.readLp(1024))
echo "2 - Src Received: ", msg
assert "test4" == msg
await relay.stop()
await allFutures(swSrc.stop(), swDst.stop(), swRel.stop())
waitFor(main())

View File

@@ -1,146 +1,151 @@
when not(compileOption("threads")):
{.fatal: "Please, compile this program with the --threads:on option!".}
import tables, strformat, strutils, bearssl
import chronos # an efficient library for async
import ../libp2p/[switch, # manage transports, a single entry point for dialing and listening
multistream, # tag stream with short header to identify it
crypto/crypto, # cryptographic functions
errors, # error handling utilities
protocols/identify, # identify the peer info of a peer
stream/connection, # create and close stream read / write connections
transports/transport, # listen and dial to other peers using p2p protocol
transports/tcptransport, # listen and dial to other peers using client-server protocol
multiaddress, # encode different addressing schemes. For example, /ip4/7.7.7.7/tcp/6543 means it is using IPv4 protocol and TCP
peerinfo, # manage the information of a peer, such as peer ID and public / private key
peerid, # Implement how peers interact
protocols/protocol, # define the protocol base type
protocols/secure/secure, # define the protocol of secure connection
protocols/secure/secio, # define the protocol of secure input / output, allows encrypted communication that uses public keys to validate signed messages instead of a certificate authority like in TLS
muxers/muxer, # define an interface for stream multiplexing, allowing peers to offer many protocols over a single connection
muxers/mplex/mplex] # define some contants and message types for stream multiplexing
import
strformat, strutils,
stew/byteutils,
chronos,
libp2p
const ChatCodec = "/nim-libp2p/chat/1.0.0"
const DefaultAddr = "/ip4/127.0.0.1/tcp/55505"
const DefaultAddr = "/ip4/127.0.0.1/tcp/0"
const Help = """
Commands: /[?|hep|connect|disconnect|exit]
Commands: /[?|help|connect|disconnect|exit]
help: Prints this help
connect: dials a remote peer
disconnect: ends current session
exit: closes the chat
"""
type ChatProto = ref object of LPProtocol
switch: Switch # a single entry point for dialing and listening to peer
transp: StreamTransport # transport streams between read & write file descriptor
conn: Connection # create and close read & write stream
connected: bool # if the node is connected to another peer
started: bool # if the node has started
type
Chat = ref object
switch: Switch # a single entry point for dialing and listening to peer
stdinReader: StreamTransport # transport streams between read & write file descriptor
conn: Connection # connection to the other peer
connected: bool # if the node is connected to another peer
##
# Stdout helpers, to write the prompt
##
proc writePrompt(c: Chat) =
if c.connected:
stdout.write '\r' & $c.switch.peerInfo.peerId & ": "
stdout.flushFile()
proc initAddress(T: type MultiAddress, str: string): T =
let address = MultiAddress.init(str).tryGet()
if IPFS.match(address) and matchPartial(multiaddress.TCP, address):
result = address
else:
raise newException(ValueError,
"Invalid bootstrap node multi-address")
proc writeStdout(c: Chat, str: string) =
echo '\r' & str
c.writePrompt()
proc dialPeer(p: ChatProto, address: string) {.async.} =
let multiAddr = MultiAddress.initAddress(address);
let parts = address.split("/")
let remotePeer = PeerInfo.init(parts[^1],
[multiAddr])
##
# Chat Protocol
##
const ChatCodec = "/nim-libp2p/chat/1.0.0"
type
ChatProto = ref object of LPProtocol
proc new(T: typedesc[ChatProto], c: Chat): T =
let chatproto = T()
# create handler for incoming connection
proc handle(stream: Connection, proto: string) {.async.} =
if c.connected and not c.conn.closed:
c.writeStdout "a chat session is already in progress - refusing incoming peer!"
await stream.close()
else:
await c.handlePeer(stream)
await stream.close()
# assign the new handler
chatproto.handler = handle
chatproto.codec = ChatCodec
return chatproto
##
# Chat application
##
proc handlePeer(c: Chat, conn: Connection) {.async.} =
# Handle a peer (incoming or outgoing)
try:
c.conn = conn
c.connected = true
c.writeStdout $conn.peerId & " connected"
# Read loop
while true:
let
strData = await conn.readLp(1024)
str = string.fromBytes(strData)
c.writeStdout $conn.peerId & ": " & $str
except LPStreamEOFError:
defer: c.writeStdout $conn.peerId & " disconnected"
await c.conn.close()
c.connected = false
proc dialPeer(c: Chat, address: string) {.async.} =
# Parse and dial address
let
multiAddr = MultiAddress.init(address).tryGet()
# split the peerId part /p2p/...
peerIdBytes = multiAddr[multiCodec("p2p")]
.tryGet()
.protoAddress()
.tryGet()
remotePeer = PeerId.init(peerIdBytes).tryGet()
# split the wire address
ip4Addr = multiAddr[multiCodec("ip4")].tryGet()
tcpAddr = multiAddr[multiCodec("tcp")].tryGet()
wireAddr = ip4Addr & tcpAddr
echo &"dialing peer: {multiAddr}"
p.conn = await p.switch.dial(remotePeer, ChatCodec)
p.connected = true
asyncSpawn c.handlePeer(await c.switch.dial(remotePeer, @[wireAddr], ChatCodec))
proc readAndPrint(p: ChatProto) {.async.} =
proc readLoop(c: Chat) {.async.} =
while true:
while p.connected:
# TODO: echo &"{p.id} -> "
echo cast[string](await p.conn.readLp(1024))
await sleepAsync(100.millis)
proc writeAndPrint(p: ChatProto) {.async.} =
while true:
if not p.connected:
if not c.connected:
echo "type an address or wait for a connection:"
echo "type /[help|?] for help"
let line = await p.transp.readLine()
if line.startsWith("/help") or line.startsWith("/?") or not p.started:
c.writePrompt()
let line = await c.stdinReader.readLine()
if line.startsWith("/help") or line.startsWith("/?"):
echo Help
continue
if line.startsWith("/disconnect"):
echo "Ending current session"
if p.connected and p.conn.closed.not:
await p.conn.close()
p.connected = false
c.writeStdout "Ending current session"
if c.connected and c.conn.closed.not:
await c.conn.close()
c.connected = false
elif line.startsWith("/connect"):
if p.connected:
var yesno = "N"
echo "a session is already in progress, do you want end it [y/N]?"
yesno = await p.transp.readLine()
if yesno.cmpIgnoreCase("y") == 0:
await p.conn.close()
p.connected = false
elif yesno.cmpIgnoreCase("n") == 0:
continue
else:
echo "unrecognized response"
continue
echo "enter address of remote peer"
let address = await p.transp.readLine()
c.writeStdout "enter address of remote peer"
let address = await c.stdinReader.readLine()
if address.len > 0:
await p.dialPeer(address)
await c.dialPeer(address)
elif line.startsWith("/exit"):
if p.connected and p.conn.closed.not:
await p.conn.close()
p.connected = false
if c.connected and c.conn.closed.not:
await c.conn.close()
c.connected = false
await p.switch.stop()
echo "quitting..."
quit(0)
await c.switch.stop()
c.writeStdout "quitting..."
return
else:
if p.connected:
await p.conn.writeLp(line)
if c.connected:
await c.conn.writeLp(line)
else:
try:
if line.startsWith("/") and "ipfs" in line:
await p.dialPeer(line)
except:
if line.startsWith("/") and "p2p" in line:
await c.dialPeer(line)
except CatchableError as exc:
echo &"unable to dial remote peer {line}"
echo getCurrentExceptionMsg()
proc readWriteLoop(p: ChatProto) {.async.} =
asyncCheck p.writeAndPrint() # execute the async function but does not block
asyncCheck p.readAndPrint()
proc newChatProto(switch: Switch, transp: StreamTransport): ChatProto =
var chatproto = ChatProto(switch: switch, transp: transp, codecs: @[ChatCodec])
# create handler for incoming connection
proc handle(stream: Connection, proto: string) {.async.} =
if chatproto.connected and not chatproto.conn.closed:
echo "a chat session is already in progress - disconnecting!"
await stream.close()
else:
chatproto.conn = stream
chatproto.connected = true
# assign the new handler
chatproto.handler = handle
return chatproto
echo exc.msg
proc readInput(wfd: AsyncFD) {.thread.} =
## This procedure performs reading from `stdin` and sends data over
## This thread performs reading from `stdin` and sends data over
## pipe to main thread.
let transp = fromPipe(wfd)
@@ -148,66 +153,46 @@ proc readInput(wfd: AsyncFD) {.thread.} =
let line = stdin.readLine()
discard waitFor transp.write(line & "\r\n")
proc processInput(rfd: AsyncFD, rng: ref BrHmacDrbgContext) {.async.} =
let transp = fromPipe(rfd)
let seckey = PrivateKey.random(RSA, rng[]).get()
let peerInfo = PeerInfo.init(seckey)
var localAddress = DefaultAddr
while true:
echo &"Type an address to bind to or Enter to use the default {DefaultAddr}"
let a = await transp.readLine()
try:
if a.len > 0:
peerInfo.addrs.add(Multiaddress.init(a).tryGet())
break
peerInfo.addrs.add(Multiaddress.init(localAddress).tryGet())
break
except:
echo "invalid address"
localAddress = DefaultAddr
continue
# a constructor for building different multiplexers under various connections
proc createMplex(conn: Connection): Muxer =
result = Mplex.init(conn)
let mplexProvider = newMuxerProvider(createMplex, MplexCodec)
let transports = @[Transport(TcpTransport.init())]
let muxers = [(MplexCodec, mplexProvider)].toTable()
let identify = newIdentify(peerInfo)
let secureManagers = [Secure(newSecio(rng, seckey))]
let switch = newSwitch(peerInfo,
transports,
identify,
muxers,
secureManagers)
let chatProto = newChatProto(switch, transp)
switch.mount(chatProto)
let libp2pFuts = await switch.start()
chatProto.started = true
let id = $peerInfo.peerId
echo "PeerID: " & id
echo "listening on: "
for a in peerInfo.addrs:
echo &"{a}/ipfs/{id}"
await chatProto.readWriteLoop()
await allFuturesThrowing(libp2pFuts)
proc main() {.async.} =
let rng = newRng() # Singe random number source for the whole application
let (rfd, wfd) = createAsyncPipe()
if rfd == asyncInvalidPipe or wfd == asyncInvalidPipe:
raise newException(ValueError, "Could not initialize pipe!")
let
rng = newRng() # Single random number source for the whole application
# Pipe to read stdin from main thread
(rfd, wfd) = createAsyncPipe()
stdinReader = fromPipe(rfd)
var thread: Thread[AsyncFD]
thread.createThread(readInput, wfd)
try:
thread.createThread(readInput, wfd)
except Exception as exc:
quit("Failed to create thread: " & exc.msg)
await processInput(rfd, rng)
var localAddress = MultiAddress.init(DefaultAddr).tryGet()
when isMainModule: # isMainModule = true when the module is compiled as the main file
waitFor(main())
var switch = SwitchBuilder
.new()
.withRng(rng) # Give the application RNG
.withAddress(localAddress)
.withTcpTransport() # Use TCP as transport
.withMplex() # Use Mplex as muxer
.withNoise() # Use Noise as secure manager
.build()
let chat = Chat(
switch: switch,
stdinReader: stdinReader)
switch.mount(ChatProto.new(chat))
await switch.start()
let id = $switch.peerInfo.peerId
echo "PeerId: " & id
echo "listening on: "
for a in switch.peerInfo.addrs:
echo &"{a}/p2p/{id}"
await chat.readLoop()
quit(0)
waitFor(main())

View File

@@ -35,7 +35,7 @@ proc main() {.async.} =
if item.protoCode() == mcip4 or item.protoCode() == mcip6:
echo $item & "/ipfs/" & id.peer.pretty()
asyncCheck monitor(api)
asyncSpawn monitor(api)
proc pubsubLogger(api: DaemonAPI,
ticket: PubsubTicket,

View File

@@ -19,7 +19,7 @@ proc threadMain(wfd: AsyncFD) {.thread.} =
## This procedure performs reading from `stdin` and sends data over
## pipe to main thread.
var transp = fromPipe(wfd)
while true:
var line = stdin.readLine()
let res = waitFor transp.write(line & "\r\n")
@@ -41,7 +41,7 @@ proc serveThread(udata: CustomData) {.async.} =
if line.startsWith("/connect"):
var parts = line.split(" ")
if len(parts) == 2:
var peerId = PeerID.init(parts[1])
var peerId = PeerId.init(parts[1])
var address = MultiAddress.init(multiCodec("p2p-circuit"))
address &= MultiAddress.init(multiCodec("p2p"), peerId)
echo "= Searching for peer ", peerId.pretty()
@@ -55,11 +55,11 @@ proc serveThread(udata: CustomData) {.async.} =
var stream = await udata.api.openStream(peerId, ServerProtocols)
udata.remotes.add(stream.transp)
echo "= Connected to peer chat ", parts[1]
asyncCheck remoteReader(stream.transp)
asyncSpawn remoteReader(stream.transp)
elif line.startsWith("/search"):
var parts = line.split(" ")
if len(parts) == 2:
var peerId = PeerID.init(parts[1])
var peerId = PeerId.init(parts[1])
echo "= Searching for peer ", peerId.pretty()
var id = await udata.api.dhtFindPeer(peerId)
echo "= Peer " & parts[1] & " found at addresses:"
@@ -68,7 +68,7 @@ proc serveThread(udata: CustomData) {.async.} =
elif line.startsWith("/consearch"):
var parts = line.split(" ")
if len(parts) == 2:
var peerId = PeerID.init(parts[1])
var peerId = PeerId.init(parts[1])
echo "= Searching for peers connected to peer ", parts[1]
var peers = await udata.api.dhtFindPeersConnectedToPeer(peerId)
echo "= Found ", len(peers), " connected to peer ", parts[1]
@@ -127,7 +127,7 @@ proc main() {.async.} =
echo ">> ", line
await data.api.addHandler(ServerProtocols, streamHandler)
echo "= Your PeerID is ", id.peer.pretty()
echo "= Your PeerId is ", id.peer.pretty()
await data.serveFut
when isMainModule:

View File

@@ -38,13 +38,13 @@ Examples can be found in the [examples folder](https://github.com/status-im/nim-
Try out the chat example. Full code can be found [here](https://github.com/status-im/nim-libp2p/blob/master/examples/chat.nim):
```bash
nim c -r --threads:on examples\chat.nim
nim c -r --threads:on examples/directchat.nim
```
This will output a peer ID such as `QmbmHfVvouKammmQDJck4hz33WvVktNEe7pasxz2HgseRu` which you can use in another instance to connect to it.
```bash
./example/chat
./examples/directchat
/connect QmbmHfVvouKammmQDJck4hz33WvVktNEe7pasxz2HgseRu
```

89
examples/helloworld.nim Normal file
View File

@@ -0,0 +1,89 @@
import chronos # an efficient library for async
import stew/byteutils # various utils
import libp2p
##
# Create our custom protocol
##
const TestCodec = "/test/proto/1.0.0" # custom protocol string identifier
type
TestProto = ref object of LPProtocol # declare a custom protocol
proc new(T: typedesc[TestProto]): T =
# every incoming connections will be in handled in this closure
proc handle(conn: Connection, proto: string) {.async, gcsafe.} =
echo "Got from remote - ", string.fromBytes(await conn.readLp(1024))
await conn.writeLp("Roger p2p!")
# We must close the connections ourselves when we're done with it
await conn.close()
return T(codecs: @[TestCodec], handler: handle)
##
# Helper to create a switch/node
##
proc createSwitch(ma: MultiAddress, rng: ref HmacDrbgContext): Switch =
var switch = SwitchBuilder
.new()
.withRng(rng) # Give the application RNG
.withAddress(ma) # Our local address(es)
.withTcpTransport() # Use TCP as transport
.withMplex() # Use Mplex as muxer
.withNoise() # Use Noise as secure manager
.build()
result = switch
##
# The actual application
##
proc main() {.async, gcsafe.} =
let
rng = newRng() # Single random number source for the whole application
# port 0 will take a random available port
# `tryGet` will throw an exception if the MultiAddress failed
# (for instance, if the address is not well formatted)
ma1 = MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet()
ma2 = MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet()
# setup the custom proto
let testProto = TestProto.new()
# setup the two nodes
let
switch1 = createSwitch(ma1, rng) #Create the two switches
switch2 = createSwitch(ma2, rng)
# mount the proto on switch1
# the node will now listen for this proto
# and call the handler everytime a client request it
switch1.mount(testProto)
# Start the nodes. This will start the transports
# and listen on each local addresses
await switch1.start()
await switch2.start()
# the node addrs is populated with it's
# actual port during the start
# use the second node to dial the first node
# using the first node peerid and address
# and specify our custom protocol codec
let conn = await switch2.dial(switch1.peerInfo.peerId, switch1.peerInfo.addrs, TestCodec)
# conn is now a fully setup connection, we talk directly to the node1 custom protocol handler
await conn.writeLp("Hello p2p!") # writeLp send a length prefixed buffer over the wire
# readLp reads length prefixed bytes and returns a buffer without the prefix
echo "Remote responded with - ", string.fromBytes(await conn.readLp(1024))
# We must close the connection ourselves when we're done with it
await conn.close()
await allFutures(switch1.stop(), switch2.stop()) # close connections and shutdown all transports
waitFor(main())

View File

@@ -71,8 +71,8 @@ proc dumpHex*(pbytes: pointer, nbytes: int, items = 1, ascii = true): string =
result = result & asciiText
result = result & "\n"
proc dumpHex*[T](v: openarray[T], items: int = 0, ascii = true): string =
## Return hexadecimal memory dump representation of openarray[T] ``v``.
proc dumpHex*[T](v: openArray[T], items: int = 0, ascii = true): string =
## Return hexadecimal memory dump representation of openArray[T] ``v``.
## ``items`` - number of bytes in group (supported ``items`` count is
## 0, 1, 2, 4, 8). If ``items`` is ``0`` group size will depend on
## ``sizeof(T)``.

6
examples/index.md Normal file
View File

@@ -0,0 +1,6 @@
# nim-libp2p documentation
Welcome to the nim-libp2p documentation!
Here, you'll find [tutorials](tutorial_1_connect.md) to help you get started, as well as
the [full reference](https://status-im.github.io/nim-libp2p/master/libp2p.html).

View File

@@ -0,0 +1,95 @@
## # Simple ping tutorial
##
## Hi all, welcome to the first nim-libp2p tutorial!
##
## !!! tips ""
## This tutorial is for everyone who is interested in building peer-to-peer applications. No Nim programming experience is needed.
##
## To give you a quick overview, **Nim** is the programming language we are using and **nim-libp2p** is the Nim implementation of [libp2p](https://libp2p.io/), a modular library that enables the development of peer-to-peer network applications.
##
## Hope you'll find it helpful in your journey of learning. Happy coding! ;)
##
## ## Before you start
## The only prerequisite here is [Nim](https://nim-lang.org/), the programming language with a Python-like syntax and a performance similar to C. Detailed information can be found [here](https://nim-lang.org/docs/tut1.html).
##
## Install Nim via their [official website](https://nim-lang.org/install.html).
## Check Nim's installation via `nim --version` and its package manager Nimble via `nimble --version`.
##
## You can now install the latest version of `nim-libp2p`:
## ```bash
## nimble install libp2p@#master
## ```
##
## ## A simple ping application
## We'll start by creating a simple application, which is starting two libp2p [switch](https://docs.libp2p.io/concepts/stream-multiplexing/#switch-swarm), and pinging each other using the [Ping](https://docs.libp2p.io/concepts/protocols/#ping) protocol.
##
## !!! tips ""
## You can find the source of this tutorial (and other tutorials) in the [libp2p/examples](https://github.com/status-im/nim-libp2p/tree/master/examples) folder!
##
## Let's create a `part1.nim`, and import our dependencies:
import chronos
import libp2p
import libp2p/protocols/ping
## [chronos](https://github.com/status-im/nim-chronos) the asynchronous framework used by `nim-libp2p`
##
## Next, we'll create an helper procedure to create our switches. A switch needs a bit of configuration, and it will be easier to do this configuration only once:
proc createSwitch(ma: MultiAddress, rng: ref HmacDrbgContext): Switch =
var switch = SwitchBuilder
.new()
.withRng(rng) # Give the application RNG
.withAddress(ma) # Our local address(es)
.withTcpTransport() # Use TCP as transport
.withMplex() # Use Mplex as muxer
.withNoise() # Use Noise as secure manager
.build()
return switch
## This will create a switch using [Mplex](https://docs.libp2p.io/concepts/stream-multiplexing/) as a multiplexer, Noise to secure the communication, and TCP as an underlying transport.
##
## You can of course tweak this, to use a different or multiple transport, or tweak the configuration of Mplex and Noise, but this is some sane defaults that we'll use going forward.
##
##
## Let's now start to create our main procedure:
proc main() {.async, gcsafe.} =
let
rng = newRng()
localAddress = MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet()
pingProtocol = Ping.new(rng=rng)
## We created some variables that we'll need for the rest of the application: the global `rng` instance, our `localAddress`, and an instance of the `Ping` protocol.
## The address is in the [MultiAddress](https://github.com/multiformats/multiaddr) format. The port `0` means "take any port available".
##
## `tryGet` is procedure which is part of [nim-result](https://github.com/arnetheduck/nim-result/), that will throw an exception if the supplied MultiAddress is invalid.
##
## We can now create our two switches:
let
switch1 = createSwitch(localAddress, rng)
switch2 = createSwitch(localAddress, rng)
switch1.mount(pingProtocol)
await switch1.start()
await switch2.start()
## We've **mounted** the `pingProtocol` on our first switch. This means that the first switch will actually listen for any ping requests coming in, and handle them accordingly.
##
## Now that we've started the nodes, they are listening for incoming peers.
## We can find out which port was attributed, and the resulting local addresses, by using `switch1.peerInfo.addrs`.
##
## We'll **dial** the first switch from the second one, by specifying it's **Peer ID**, it's **MultiAddress** and the **`Ping` protocol codec**:
let conn = await switch2.dial(switch1.peerInfo.peerId, switch1.peerInfo.addrs, PingCodec)
## We now have a `Ping` connection setup between the second and the first switch, we can use it to actually ping the node:
# ping the other node and echo the ping duration
echo "ping: ", await pingProtocol.ping(conn)
# We must close the connection ourselves when we're done with it
await conn.close()
## And that's it! Just a little bit of cleanup: shutting down the switches, waiting for them to stop, and we'll call our `main` procedure:
await allFutures(switch1.stop(), switch2.stop()) # close connections and shutdown all transports
waitFor(main())
## You can now run this program using `nim c -r part1.nim`, and you should see the dialing sequence, ending with a ping output.
##
## In the [next tutorial](tutorial_2_customproto.md), we'll look at how to create our own custom protocol.

View File

@@ -0,0 +1,74 @@
## # Custom protocol in libp2p
##
## In the [previous tutorial](tutorial_1_connect.md), we've looked at how to create a simple ping program using the `nim-libp2p`.
##
## We'll now look at how to create a custom protocol inside the libp2p
##
## Let's create a `part2.nim`, and import our dependencies:
import chronos
import stew/byteutils
import libp2p
## This is similar to the first tutorial, except we don't need to import the `Ping` protocol.
##
## Next, we'll declare our custom protocol
const TestCodec = "/test/proto/1.0.0"
type TestProto = ref object of LPProtocol
## We've set a [protocol ID](https://docs.libp2p.io/concepts/protocols/#protocol-ids), and created a custom `LPProtocol`. In a more complex protocol, we could use this structure to store interesting variables.
##
## A protocol generally has two part: and handling/server part, and a dialing/client part.
## Theses two parts can be identical, but in our trivial protocol, the server will wait for a message from the client, and the client will send a message, so we have to handle the two cases separately.
##
## Let's start with the server part:
proc new(T: typedesc[TestProto]): T =
# every incoming connections will in be handled in this closure
proc handle(conn: Connection, proto: string) {.async, gcsafe.} =
# Read up to 1024 bytes from this connection, and transform them into
# a string
echo "Got from remote - ", string.fromBytes(await conn.readLp(1024))
# We must close the connections ourselves when we're done with it
await conn.close()
return T.new(codecs = @[TestCodec], handler = handle)
## This is a constructor for our `TestProto`, that will specify our `codecs` and a `handler`, which will be called for each incoming peer asking for this protocol.
## In our handle, we simply read a message from the connection and `echo` it.
##
## We can now create our client part:
proc hello(p: TestProto, conn: Connection) {.async.} =
await conn.writeLp("Hello p2p!")
## Again, pretty straight-forward, we just send a message on the connection.
##
## We can now create our main procedure:
proc main() {.async, gcsafe.} =
let
rng = newRng()
testProto = TestProto.new()
switch1 = newStandardSwitch(rng=rng)
switch2 = newStandardSwitch(rng=rng)
switch1.mount(testProto)
await switch1.start()
await switch2.start()
let conn = await switch2.dial(switch1.peerInfo.peerId, switch1.peerInfo.addrs, TestCodec)
await testProto.hello(conn)
# We must close the connection ourselves when we're done with it
await conn.close()
await allFutures(switch1.stop(), switch2.stop()) # close connections and shutdown all transports
## This is very similar to the first tutorial's `main`, the only noteworthy difference is that we use `newStandardSwitch`, which is similar to the `createSwitch` of the first tutorial, but is bundled directly in libp2p
##
## We can now wrap our program by calling our main proc:
waitFor(main())
## And that's it!
## In the [next tutorial](tutorial_3_protobuf.md), we'll create a more complex protocol using Protobuf.

View File

@@ -0,0 +1,162 @@
## # Protobuf usage
##
## In the [previous tutorial](tutorial_2_customproto.md), we created a simple "ping" protocol.
## Most real protocol want their messages to be structured and extensible, which is why
## most real protocols use [protobuf](https://developers.google.com/protocol-buffers) to
## define their message structures.
##
## Here, we'll create a slightly more complex protocol, which parses & generate protobuf
## messages. Let's start by importing our dependencies, as usual:
import chronos
import stew/results # for Opt[T]
import libp2p
## ## Protobuf encoding & decoding
## This will be the structure of our messages:
## ```protobuf
## message MetricList {
## message Metric {
## string name = 1;
## float value = 2;
## }
##
## repeated Metric metrics = 2;
## }
## ```
## We'll create our protobuf types, encoders & decoders, according to this format.
## To create the encoders & decoders, we are going to use minprotobuf
## (included in libp2p).
##
## While more modern technics
## (such as [nim-protobuf-serialization](https://github.com/status-im/nim-protobuf-serialization))
## exists, minprotobuf is currently the recommended method to handle protobuf, since it has
## been used in production extensively, and audited.
type
Metric = object
name: string
value: float
MetricList = object
metrics: seq[Metric]
{.push raises: [].}
proc encode(m: Metric): ProtoBuffer =
result = initProtoBuffer()
result.write(1, m.name)
result.write(2, m.value)
result.finish()
proc decode(_: type Metric, buf: seq[byte]): Result[Metric, ProtoError] =
var res: Metric
let pb = initProtoBuffer(buf)
# "getField" will return a Result[bool, ProtoError].
# The Result will hold an error if the protobuf is invalid.
# The Result will hold "false" if the field is missing
#
# We are just checking the error, and ignoring whether the value
# is present or not (default values are valid).
discard ? pb.getField(1, res.name)
discard ? pb.getField(2, res.value)
ok(res)
proc encode(m: MetricList): ProtoBuffer =
result = initProtoBuffer()
for metric in m.metrics:
result.write(1, metric.encode())
result.finish()
proc decode(_: type MetricList, buf: seq[byte]): Result[MetricList, ProtoError] =
var
res: MetricList
metrics: seq[seq[byte]]
let pb = initProtoBuffer(buf)
discard ? pb.getRepeatedField(1, metrics)
for metric in metrics:
res.metrics &= ? Metric.decode(metric)
ok(res)
## ## Results instead of exceptions
## As you can see, this part of the program also uses Results instead of exceptions for error handling.
## We start by `{.push raises: [].}`, which will prevent every non-async function from raising
## exceptions.
##
## Then, we use [nim-result](https://github.com/arnetheduck/nim-result) to convey
## errors to function callers. A `Result[T, E]` will either hold a valid result of type
## T, or an error of type E.
##
## You can check if the call succeeded by using `res.isOk`, and then get the
## value using `res.value` or the error by using `res.error`.
##
## Another useful tool is `?`, which will unpack a Result if it succeeded,
## or if it failed, exit the current procedure returning the error.
##
## nim-result is packed with other functionalities that you'll find in the
## nim-result repository.
##
## Results and exception are generally interchangeable, but have different semantics
## that you may or may not prefer.
##
## ## Creating the protocol
## We'll next create a protocol, like in the last tutorial, to request these metrics from our host
type
MetricCallback = proc: Future[MetricList] {.raises: [], gcsafe.}
MetricProto = ref object of LPProtocol
metricGetter: MetricCallback
proc new(_: typedesc[MetricProto], cb: MetricCallback): MetricProto =
var res: MetricProto
proc handle(conn: Connection, proto: string) {.async, gcsafe.} =
let
metrics = await res.metricGetter()
asProtobuf = metrics.encode()
await conn.writeLp(asProtobuf.buffer)
await conn.close()
res = MetricProto.new(@["/metric-getter/1.0.0"], handle)
res.metricGetter = cb
return res
proc fetch(p: MetricProto, conn: Connection): Future[MetricList] {.async.} =
let protobuf = await conn.readLp(2048)
# tryGet will raise an exception if the Result contains an error.
# It's useful to bridge between exception-world and result-world
return MetricList.decode(protobuf).tryGet()
## We can now create our main procedure:
proc main() {.async, gcsafe.} =
let rng = newRng()
proc randomMetricGenerator: Future[MetricList] {.async.} =
let metricCount = rng[].generate(uint32) mod 16
for i in 0 ..< metricCount + 1:
result.metrics.add(Metric(
name: "metric_" & $i,
value: float(rng[].generate(uint16)) / 1000.0
))
return result
let
metricProto1 = MetricProto.new(randomMetricGenerator)
metricProto2 = MetricProto.new(randomMetricGenerator)
switch1 = newStandardSwitch(rng=rng)
switch2 = newStandardSwitch(rng=rng)
switch1.mount(metricProto1)
await switch1.start()
await switch2.start()
let
conn = await switch2.dial(switch1.peerInfo.peerId, switch1.peerInfo.addrs, metricProto2.codecs)
metrics = await metricProto2.fetch(conn)
await conn.close()
for metric in metrics.metrics:
echo metric.name, " = ", metric.value
await allFutures(switch1.stop(), switch2.stop()) # close connections and shutdown all transports
waitFor(main())
## If you run this program, you should see random metrics being sent from the switch1 to the switch2.

View File

@@ -0,0 +1,163 @@
## # GossipSub
##
## In this tutorial, we'll build a simple GossipSub network
## to broadcast the metrics we built in the previous tutorial.
##
## GossipSub is used to broadcast some messages in a network,
## and allows to balance between latency, bandwidth usage,
## privacy and attack resistance.
##
## You'll find a good explanation on how GossipSub works
## [here.](https://docs.libp2p.io/concepts/publish-subscribe/) There are a lot
## of parameters you can tweak to adjust how GossipSub behaves but here we'll
## use the sane defaults shipped with libp2p.
##
## We'll start by creating our metric structure like previously
import chronos
import stew/results
import libp2p
import libp2p/protocols/pubsub/rpc/messages
type
Metric = object
name: string
value: float
MetricList = object
hostname: string
metrics: seq[Metric]
{.push raises: [].}
proc encode(m: Metric): ProtoBuffer =
result = initProtoBuffer()
result.write(1, m.name)
result.write(2, m.value)
result.finish()
proc decode(_: type Metric, buf: seq[byte]): Result[Metric, ProtoError] =
var res: Metric
let pb = initProtoBuffer(buf)
discard ? pb.getField(1, res.name)
discard ? pb.getField(2, res.value)
ok(res)
proc encode(m: MetricList): ProtoBuffer =
result = initProtoBuffer()
for metric in m.metrics:
result.write(1, metric.encode())
result.write(2, m.hostname)
result.finish()
proc decode(_: type MetricList, buf: seq[byte]): Result[MetricList, ProtoError] =
var
res: MetricList
metrics: seq[seq[byte]]
let pb = initProtoBuffer(buf)
discard ? pb.getRepeatedField(1, metrics)
for metric in metrics:
res.metrics &= ? Metric.decode(metric)
? pb.getRequiredField(2, res.hostname)
ok(res)
## This is exactly like the previous structure, except that we added
## a `hostname` to distinguish where the metric is coming from.
##
## Now we'll create a small GossipSub network to broadcast the metrics,
## and collect them on one of the node.
type Node = tuple[switch: Switch, gossip: GossipSub, hostname: string]
proc oneNode(node: Node, rng: ref HmacDrbgContext) {.async.} =
# This procedure will handle one of the node of the network
node.gossip.addValidator(["metrics"],
proc(topic: string, message: Message): Future[ValidationResult] {.async.} =
let decoded = MetricList.decode(message.data)
if decoded.isErr: return ValidationResult.Reject
return ValidationResult.Accept
)
# This "validator" will attach to the `metrics` topic and make sure
# that every message in this topic is valid. This allows us to stop
# propagation of invalid messages quickly in the network, and punish
# peers sending them.
# `John` will be responsible to log the metrics, the rest of the nodes
# will just forward them in the network
if node.hostname == "John":
node.gossip.subscribe("metrics",
proc (topic: string, data: seq[byte]) {.async.} =
echo MetricList.decode(data).tryGet()
)
else:
node.gossip.subscribe("metrics", nil)
# Create random metrics 10 times and broadcast them
for _ in 0..<10:
await sleepAsync(500.milliseconds)
var metricList = MetricList(hostname: node.hostname)
let metricCount = rng[].generate(uint32) mod 4
for i in 0 ..< metricCount + 1:
metricList.metrics.add(Metric(
name: "metric_" & $i,
value: float(rng[].generate(uint16)) / 1000.0
))
discard await node.gossip.publish("metrics", encode(metricList).buffer)
await node.switch.stop()
## For our main procedure, we'll create a few nodes, and connect them together.
## Note that they are not all interconnected, but GossipSub will take care of
## broadcasting to the full network nonetheless.
proc main {.async.} =
let rng = newRng()
var nodes: seq[Node]
for hostname in ["John", "Walter", "David", "Thuy", "Amy"]:
let
switch = newStandardSwitch(rng=rng)
gossip = GossipSub.init(switch = switch, triggerSelf = true)
switch.mount(gossip)
await switch.start()
nodes.add((switch, gossip, hostname))
for index, node in nodes:
# Connect to a few neighbors
for otherNodeIdx in index - 1 .. index + 2:
if otherNodeIdx notin 0 ..< nodes.len or otherNodeIdx == index: continue
let otherNode = nodes[otherNodeIdx]
await node.switch.connect(
otherNode.switch.peerInfo.peerId,
otherNode.switch.peerInfo.addrs)
var allFuts: seq[Future[void]]
for node in nodes:
allFuts.add(oneNode(node, rng))
await allFutures(allFuts)
waitFor(main())
## If you run this program, you should see something like:
## ```
## (hostname: "John", metrics: @[(name: "metric_0", value: 42.097), (name: "metric_1", value: 50.99), (name: "metric_2", value: 47.86), (name: "metric_3", value: 5.368)])
## (hostname: "Walter", metrics: @[(name: "metric_0", value: 39.452), (name: "metric_1", value: 15.606), (name: "metric_2", value: 14.059), (name: "metric_3", value: 6.68)])
## (hostname: "David", metrics: @[(name: "metric_0", value: 9.82), (name: "metric_1", value: 2.862), (name: "metric_2", value: 15.514)])
## (hostname: "Thuy", metrics: @[(name: "metric_0", value: 59.038)])
## (hostname: "Amy", metrics: @[(name: "metric_0", value: 55.616), (name: "metric_1", value: 23.52), (name: "metric_2", value: 59.081), (name: "metric_3", value: 2.516)])
## ```
##
## This is John receiving & logging everyone's metrics.
##
## ## Going further
## Building efficient & safe GossipSub networks is a tricky subject. By tweaking the [gossip params](https://status-im.github.io/nim-libp2p/master/libp2p/protocols/pubsub/gossipsub/types.html#GossipSubParams)
## and [topic params](https://status-im.github.io/nim-libp2p/master/libp2p/protocols/pubsub/gossipsub/types.html#TopicParams),
## you can achieve very different properties.
##
## Also see reports for [GossipSub v1.1](https://gateway.ipfs.io/ipfs/QmRAFP5DBnvNjdYSbWhEhVRJJDFCLpPyvew5GwCCB4VxM4)
##
## If you are interested in broadcasting for your application, you may want to use [Waku](https://waku.org/), which builds on top of GossipSub,
## and adds features such as history, spam protection, and light node friendliness.

View File

@@ -0,0 +1,132 @@
## # Discovery Manager
##
## In the [previous tutorial](tutorial_4_gossipsub.md), we built a custom protocol using [protobuf](https://developers.google.com/protocol-buffers) and
## spread informations (some metrics) on the network using gossipsub.
## For this tutorial, on the other hand, we'll go back on a simple example
## we'll try to discover a specific peers to greet on the network.
##
## First, as usual, we import the dependencies:
import sequtils
import chronos
import stew/byteutils
import libp2p
import libp2p/protocols/rendezvous
import libp2p/discovery/rendezvousinterface
import libp2p/discovery/discoverymngr
## We'll not use newStandardSwitch this time as we need the discovery protocol
## [RendezVous](https://github.com/libp2p/specs/blob/master/rendezvous/README.md) to be mounted on the switch using withRendezVous.
##
## Note that other discovery methods such as [Kademlia](https://github.com/libp2p/specs/blob/master/kad-dht/README.md) or [discv5](https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md) exist.
proc createSwitch(rdv: RendezVous = RendezVous.new()): Switch =
SwitchBuilder.new()
.withRng(newRng())
.withAddresses(@[ MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet() ])
.withTcpTransport()
.withYamux()
.withNoise()
.withRendezVous(rdv)
.build()
# Create a really simple protocol to log one message received then close the stream
const DumbCodec = "/dumb/proto/1.0.0"
type DumbProto = ref object of LPProtocol
proc new(T: typedesc[DumbProto], nodeNumber: int): T =
proc handle(conn: Connection, proto: string) {.async, gcsafe.} =
echo "Node", nodeNumber, " received: ", string.fromBytes(await conn.readLp(1024))
await conn.close()
return T.new(codecs = @[DumbCodec], handler = handle)
## ## Bootnodes
## The first time a p2p program is ran, he needs to know how to join
## its network. This is generally done by hard-coding a list of stable
## nodes in the binary, called "bootnodes". These bootnodes are a
## critical part of a p2p network, since they are used by every new
## user to onboard the network.
##
## By using libp2p, we can use any node supporting our discovery protocol
## (rendezvous in this case) as a bootnode. For this example, we'll
## create a bootnode, and then every peer will advertise itself on the
## bootnode, and use it to find other peers
proc main() {.async, gcsafe.} =
let bootNode = createSwitch()
await bootNode.start()
# Create 5 nodes in the network
var
switches: seq[Switch] = @[]
discManagers: seq[DiscoveryManager] = @[]
for i in 0..5:
let rdv = RendezVous.new()
# Create a remote future to await at the end of the program
let switch = createSwitch(rdv)
switch.mount(DumbProto.new(i))
switches.add(switch)
# A discovery manager is a simple tool, you can set it up by adding discovery
# interfaces (such as RendezVousInterface) then you can use it to advertise
# something on the network or to request something from it.
let dm = DiscoveryManager()
# A RendezVousInterface is a RendezVous protocol wrapped to be usable by the
# DiscoveryManager.
dm.add(RendezVousInterface.new(rdv))
discManagers.add(dm)
# We can now start the switch and connect to the bootnode
await switch.start()
await switch.connect(bootNode.peerInfo.peerId, bootNode.peerInfo.addrs)
# Each nodes of the network will advertise on some topics (EvenGang or OddClub)
dm.advertise(RdvNamespace(if i mod 2 == 0: "EvenGang" else: "OddClub"))
## We can now create the newcomer. This peer will connect to the boot node, and use
## it to discover peers & greet them.
let
rdv = RendezVous.new()
newcomer = createSwitch(rdv)
dm = DiscoveryManager()
await newcomer.start()
await newcomer.connect(bootNode.peerInfo.peerId, bootNode.peerInfo.addrs)
dm.add(RendezVousInterface.new(rdv, ttr = 250.milliseconds))
# Use the discovery manager to find peers on the OddClub topic to greet them
let queryOddClub = dm.request(RdvNamespace("OddClub"))
for _ in 0..2:
let
# getPeer give you a PeerAttribute containing informations about the peer.
res = await queryOddClub.getPeer()
# Here we will use the PeerId and the MultiAddress to greet him
conn = await newcomer.dial(res[PeerId], res.getAll(MultiAddress), DumbCodec)
await conn.writeLp("Odd Club suuuucks! Even Gang is better!")
# Uh-oh!
await conn.close()
# Wait for the peer to close the stream
await conn.join()
# Queries will run in a loop, so we must stop them when we are done
queryOddClub.stop()
# Maybe it was because he wanted to join the EvenGang
let queryEvenGang = dm.request(RdvNamespace("EvenGang"))
for _ in 0..2:
let
res = await queryEvenGang.getPeer()
conn = await newcomer.dial(res[PeerId], res.getAll(MultiAddress), DumbCodec)
await conn.writeLp("Even Gang is sooo laaame! Odd Club rocks!")
# Or maybe not...
await conn.close()
await conn.join()
queryEvenGang.stop()
# What can I say, some people just want to watch the world burn... Anyway
# Stop all the discovery managers
for d in discManagers:
d.stop()
dm.stop()
# Stop all the switches
await allFutures(switches.mapIt(it.stop()))
await allFutures(bootNode.stop(), newcomer.stop())
waitFor(main())

View File

@@ -0,0 +1,259 @@
## # Tron example
##
## In this tutorial, we will create a video game based on libp2p, using
## all of the features we talked about in the last tutorials.
##
## We will:
## - Discover peers using the Discovery Manager
## - Use GossipSub to find a play mate
## - Create a custom protocol to play with him
##
## While this may look like a daunting project, it's less than 150 lines of code.
##
## The game will be a simple Tron. We will use [nico](https://github.com/ftsf/nico)
## as a game engine. (you need to run `nimble install nico` to have it available)
##
## ![multiplay](https://user-images.githubusercontent.com/13471753/198852714-b55048e3-f233-4723-900d-2193ad259fe1.gif)
##
## We will start by importing our dependencies and creating our types
import os
import nico, chronos, stew/byteutils, stew/endians2
import libp2p
import libp2p/protocols/rendezvous
import libp2p/discovery/rendezvousinterface
import libp2p/discovery/discoverymngr
const
directions = @[(K_UP, 0, -1), (K_LEFT, -1, 0), (K_DOWN, 0, 1), (K_RIGHT, 1, 0)]
mapSize = 32
tickPeriod = 0.2
type
Player = ref object
x, y: int
currentDir, nextDir: int
lost: bool
color: int
Game = ref object
gameMap: array[mapSize * mapSize, int]
tickTime: float
localPlayer, remotePlayer: Player
peerFound: Future[Connection]
hasCandidate: bool
tickFinished: Future[int]
GameProto = ref object of LPProtocol
proc new(_: type[Game]): Game =
# Default state of a game
result = Game(
tickTime: -3.0, # 3 seconds of "warm-up" time
localPlayer: Player(x: 4, y: 16, currentDir: 3, nextDir: 3, color: 8),
remotePlayer: Player(x: 27, y: 16, currentDir: 1, nextDir: 1, color: 12),
peerFound: newFuture[Connection]()
)
for pos in 0 .. result.gameMap.high:
if pos mod mapSize in [0, mapSize - 1] or pos div mapSize in [0, mapSize - 1]:
result.gameMap[pos] = 7
## ## Game Logic
## The networking during the game will work like this:
##
## * Each player will have `tickPeriod` (0.1) seconds to choose
## a direction that he wants to go to (default to current direction)
## * After `tickPeriod`, we will send our choosen direction to the peer,
## and wait for his direction
## * Once we have both direction, we will "tick" the game, and restart the
## loop, as long as both player are alive.
##
## This is a very simplistic scheme, but creating proper networking for
## video games is an [art](https://developer.valvesoftware.com/wiki/Latency_Compensating_Methods_in_Client/Server_In-game_Protocol_Design_and_Optimization)
##
## The main drawback of this scheme is that the more ping you have with
## the peer, the slower the game will run. Or invertedly, the less ping you
## have, the faster it runs!
proc update(g: Game, dt: float32) =
# Will be called at each frame of the game.
#
# Because both Nico and Chronos have a main loop,
# they must share the control of the main thread.
# This is a hacky way to make this happen
waitFor(sleepAsync(1.milliseconds))
# Don't do anything if we are still waiting for an opponent
if not(g.peerFound.finished()) or isNil(g.tickFinished): return
g.tickTime += dt
# Update the wanted direction, making sure we can't go backward
for i in 0 .. directions.high:
if i != (g.localPlayer.currentDir + 2 mod 4) and keyp(directions[i][0]):
g.localPlayer.nextDir = i
if g.tickTime > tickPeriod and not g.tickFinished.finished():
# We choosen our next direction, let the networking know
g.localPlayer.currentDir = g.localPlayer.nextDir
g.tickFinished.complete(g.localPlayer.currentDir)
proc tick(g: Game, p: Player) =
# Move player and check if he lost
p.x += directions[p.currentDir][1]
p.y += directions[p.currentDir][2]
if g.gameMap[p.y * mapSize + p.x] != 0: p.lost = true
g.gameMap[p.y * mapSize + p.x] = p.color
proc mainLoop(g: Game, peer: Connection) {.async.} =
while not (g.localPlayer.lost or g.remotePlayer.lost):
if g.tickTime > 0.0:
g.tickTime = 0
g.tickFinished = newFuture[int]()
# Wait for a choosen direction
let dir = await g.tickFinished
# Send it
await peer.writeLp(toBytes(uint32(dir)))
# Get the one from the peer
g.remotePlayer.currentDir = int uint32.fromBytes(await peer.readLp(8))
# Tick the players & restart
g.tick(g.remotePlayer)
g.tick(g.localPlayer)
## We'll draw the map & put some texts when necessary:
proc draw(g: Game) =
for pos, color in g.gameMap:
setColor(color)
boxFill(pos mod 32 * 4, pos div 32 * 4, 4, 4)
let text = if not(g.peerFound.finished()): "Matchmaking.."
elif g.tickTime < -1.5: "Welcome to Etron"
elif g.tickTime < 0.0: "- " & $(int(abs(g.tickTime) / 0.5) + 1) & " -"
elif g.remotePlayer.lost and g.localPlayer.lost: "DEUCE"
elif g.localPlayer.lost: "YOU LOOSE"
elif g.remotePlayer.lost: "YOU WON"
else: ""
printc(text, screenWidth div 2, screenHeight div 2)
## ## Matchmaking
## To find an opponent, we will broadcast our address on a
## GossipSub topic, and wait for someone to connect to us.
## We will also listen to that topic, and connect to anyone
## broadcasting his address.
##
## If we are looking for a game, we'll send `ok` to let the
## peer know that we are available, check that he is also available,
## and launch the game.
proc new(T: typedesc[GameProto], g: Game): T =
proc handle(conn: Connection, proto: string) {.async, gcsafe.} =
defer: await conn.closeWithEof()
if g.peerFound.finished or g.hasCandidate:
await conn.close()
return
g.hasCandidate = true
await conn.writeLp("ok")
if "ok" != string.fromBytes(await conn.readLp(1024)):
g.hasCandidate = false
return
g.peerFound.complete(conn)
# The handler of a protocol must wait for the stream to
# be finished before returning
await conn.join()
return T.new(codecs = @["/tron/1.0.0"], handler = handle)
proc networking(g: Game) {.async.} =
# Create our switch, similar to the GossipSub example and
# the Discovery examples combined
let
rdv = RendezVous.new()
switch = SwitchBuilder.new()
.withRng(newRng())
.withAddresses(@[ MultiAddress.init("/ip4/0.0.0.0/tcp/0").tryGet() ])
.withTcpTransport()
.withYamux()
.withNoise()
.withRendezVous(rdv)
.build()
dm = DiscoveryManager()
gameProto = GameProto.new(g)
gossip = GossipSub.init(
switch = switch,
triggerSelf = false)
dm.add(RendezVousInterface.new(rdv))
switch.mount(gossip)
switch.mount(gameProto)
gossip.subscribe(
"/tron/matchmaking",
proc (topic: string, data: seq[byte]) {.async.} =
# If we are still looking for an opponent,
# try to match anyone broadcasting it's address
if g.peerFound.finished or g.hasCandidate: return
g.hasCandidate = true
try:
let
(peerId, multiAddress) = parseFullAddress(data).tryGet()
stream = await switch.dial(peerId, @[multiAddress], gameProto.codec)
await stream.writeLp("ok")
if (await stream.readLp(10)) != "ok".toBytes:
g.hasCandidate = false
return
g.peerFound.complete(stream)
# We are "player 2"
swap(g.localPlayer, g.remotePlayer)
except CatchableError as exc:
discard
)
await switch.start()
defer: await switch.stop()
# As explained in the last tutorial, we need a bootnode to be able
# to find peers. We could use any libp2p running rendezvous (or any
# node running tron). We will take it's MultiAddress from the command
# line parameters
if paramCount() > 0:
let (peerId, multiAddress) = paramStr(1).parseFullAddress().tryGet()
await switch.connect(peerId, @[multiAddress])
else:
echo "No bootnode provided, listening on: ", switch.peerInfo.fullAddrs.tryGet()
# Discover peers from the bootnode, and connect to them
dm.advertise(RdvNamespace("tron"))
let discoveryQuery = dm.request(RdvNamespace("tron"))
discoveryQuery.forEach:
try:
await switch.connect(peer[PeerId], peer.getAll(MultiAddress))
except CatchableError as exc:
echo "Failed to dial a peer: ", exc.msg
# We will try to publish our address multiple times, in case
# it takes time to establish connections with other GossipSub peers
var published = false
while not published:
await sleepAsync(500.milliseconds)
for fullAddr in switch.peerInfo.fullAddrs.tryGet():
if (await gossip.publish("/tron/matchmaking", fullAddr.bytes)) == 0:
published = false
break
published = true
discoveryQuery.stop()
# We now wait for someone to connect to us (or for us to connect to someone)
let peerConn = await g.peerFound
defer: await peerConn.closeWithEof()
await g.mainLoop(peerConn)
let
game = Game.new()
netFut = networking(game)
nico.init("Status", "Tron")
nico.createWindow("Tron", mapSize * 4, mapSize * 4, 4, false)
nico.run(proc = discard, proc(dt: float32) = game.update(dt), proc = game.draw())
waitFor(netFut.cancelAndWait())
## And that's it! If you want to run this code locally, the simplest way is to use the
## first node as a boot node for the second one. But you can also use any rendezvous node

View File

@@ -1,17 +1,69 @@
## Nim-LibP2P
## Copyright (c) 2018 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-LibP2P
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
import
libp2p/daemon/[daemonapi, transpool],
libp2p/protobuf/minprotobuf,
libp2p/varint
when defined(nimdoc):
## Welcome to the nim-libp2p reference!
##
## On the left, you'll find a switch that allows you to see private
## procedures. By default, you'll only see the public one (marked with `{.public.}`)
##
## The difference between public and private procedures is that public procedure
## stay backward compatible during the Major version, whereas private ones can
## change at each new Minor version.
##
## If you're new to nim-libp2p, you can find a tutorial `here<https://status-im.github.io/nim-libp2p/docs/tutorial_1_connect/>`_
## that can help you get started.
export
daemonapi, transpool, minprotobuf, varint
# Import stuff for doc
import libp2p/[
protobuf/minprotobuf,
switch,
stream/lpstream,
builders,
transports/tcptransport,
transports/wstransport,
protocols/ping,
protocols/pubsub,
peerid,
peerinfo,
peerstore,
multiaddress]
proc dummyPrivateProc*() =
## A private proc example
discard
else:
import
libp2p/[protobuf/minprotobuf,
muxers/muxer,
muxers/mplex/mplex,
stream/lpstream,
stream/bufferstream,
stream/connection,
transports/transport,
transports/tcptransport,
protocols/secure/noise,
cid,
multihash,
multicodec,
errors,
switch,
peerid,
peerinfo,
multiaddress,
builders,
crypto/crypto,
protocols/pubsub]
export
minprotobuf, switch, peerid, peerinfo,
connection, multiaddress, crypto, lpstream,
bufferstream, muxer, mplex, transport,
tcptransport, noise, errors, cid, multihash,
multicodec, builders, pubsub

View File

@@ -1,41 +1,59 @@
mode = ScriptMode.Verbose
packageName = "libp2p"
version = "0.0.2"
version = "1.1.0"
author = "Status Research & Development GmbH"
description = "LibP2P implementation"
license = "MIT"
skipDirs = @["tests", "examples", "Nim", "tools", "scripts", "docs"]
requires "nim >= 1.2.0",
requires "nim >= 1.6.0",
"nimcrypto >= 0.4.1",
"dnsclient >= 0.3.0 & < 0.4.0",
"bearssl >= 0.1.4",
"chronicles >= 0.7.2",
"chronos >= 2.5.2",
"chronicles >= 0.10.2",
"chronos >= 3.0.6",
"metrics",
"secp256k1",
"stew#head"
"stew#head",
"websock",
"unittest2 >= 0.0.5 & < 0.1.0"
let nimc = getEnv("NIMC", "nim") # Which nim compiler to use
let lang = getEnv("NIMLANG", "c") # Which backend (c/cpp/js)
let flags = getEnv("NIMFLAGS", "") # Extra flags for the compiler
let verbose = getEnv("V", "") notin ["", "0"]
let cfg =
" --styleCheck:usages --styleCheck:error" &
(if verbose: "" else: " --verbosity:0 --hints:off") &
" --skipParentCfg --skipUserCfg -f" &
" --threads:on --opt:speed"
import hashes, strutils
proc runTest(filename: string, verify: bool = true, sign: bool = true,
moreoptions: string = "") =
var excstr = "nim c --opt:speed -d:debug --verbosity:0 --hints:off"
excstr.add(" --warning[CaseTransition]:off --warning[ObservableStores]:off --warning[LockLevel]:off")
var excstr = nimc & " " & lang & " -d:debug " & cfg & " " & flags
excstr.add(" -d:libp2p_pubsub_sign=" & $sign)
excstr.add(" -d:libp2p_pubsub_verify=" & $verify)
excstr.add(" " & moreoptions & " ")
if verify and sign:
# build it with TRACE and JSON logs
exec excstr & " -d:chronicles_log_level=TRACE -d:chronicles_sinks:json" & " tests/" & filename
# build it again, to run it with less verbose logs
exec excstr & " -d:chronicles_log_level=INFO -r" & " tests/" & filename
if getEnv("CICOV").len > 0:
excstr &= " --nimcache:nimcache/" & filename & "-" & $excstr.hash
exec excstr & " -r " & " tests/" & filename
rmFile "tests/" & filename.toExe
proc buildSample(filename: string) =
var excstr = "nim c --opt:speed --threads:on -d:debug --verbosity:0 --hints:off"
excstr.add(" --warning[CaseTransition]:off --warning[ObservableStores]:off --warning[LockLevel]:off")
proc buildSample(filename: string, run = false, extraFlags = "") =
var excstr = nimc & " " & lang & " " & cfg & " " & flags & " -p:. " & extraFlags
excstr.add(" examples/" & filename)
exec excstr
rmFile "examples" & filename.toExe
if run:
exec "./examples/" & filename.toExe
rmFile "examples/" & filename.toExe
proc tutorialToMd(filename: string) =
let markdown = gorge "cat " & filename & " | " & nimc & " " & lang & " -r --verbosity:0 --hints:off tools/markdown_builder.nim "
writeFile(filename.replace(".nim", ".md"), markdown)
task testnative, "Runs libp2p native tests":
runTest("testnative")
@@ -72,12 +90,78 @@ task test, "Runs the test suite":
exec "nimble testdaemon"
exec "nimble testinterop"
exec "nimble testfilter"
exec "nimble examples_build"
task test_slim, "Runs the test suite":
task test_slim, "Runs the (slimmed down) test suite":
exec "nimble testnative"
exec "nimble testpubsub_slim"
exec "nimble testinterop"
exec "nimble testfilter"
exec "nimble examples_build"
task website, "Build the website":
tutorialToMd("examples/tutorial_1_connect.nim")
tutorialToMd("examples/tutorial_2_customproto.nim")
tutorialToMd("examples/tutorial_3_protobuf.nim")
tutorialToMd("examples/tutorial_4_gossipsub.nim")
tutorialToMd("examples/tutorial_5_discovery.nim")
tutorialToMd("examples/tutorial_6_game.nim")
tutorialToMd("examples/circuitrelay.nim")
exec "mkdocs build"
task examples_build, "Build the samples":
buildSample("directchat")
buildSample("helloworld", true)
buildSample("circuitrelay", true)
buildSample("tutorial_1_connect", true)
buildSample("tutorial_2_customproto", true)
buildSample("tutorial_3_protobuf", true)
buildSample("tutorial_4_gossipsub", true)
buildSample("tutorial_5_discovery", true)
exec "nimble install -y nimpng@#HEAD" # this is to fix broken build on 1.7.3, remove it when nimpng version 0.3.2 or later is released
exec "nimble install -y nico"
buildSample("tutorial_6_game", false, "--styleCheck:off")
# pin system
# while nimble lockfile
# isn't available
const PinFile = ".pinned"
task pin, "Create a lockfile":
# pinner.nim was originally here
# but you can't read output from
# a command in a nimscript
exec nimc & " c -r tools/pinner.nim"
import sequtils
import os
task install_pinned, "Reads the lockfile":
let toInstall = readFile(PinFile).splitWhitespace().mapIt((it.split(";", 1)[0], it.split(";", 1)[1]))
# [('packageName', 'packageFullUri')]
rmDir("nimbledeps")
mkDir("nimbledeps")
exec "nimble install -y " & toInstall.mapIt(it[1]).join(" ")
# Remove the automatically installed deps
# (inefficient you say?)
let nimblePkgs =
if system.dirExists("nimbledeps/pkgs"): "nimbledeps/pkgs"
else: "nimbledeps/pkgs2"
for dependency in listDirs(nimblePkgs):
let
fileName = dependency.extractFilename
fileContent = readFile(dependency & "/nimblemeta.json")
packageName = fileName.split('-')[0]
if toInstall.anyIt(
it[0] == packageName and
(
it[1].split('#')[^1] in fileContent or # nimble for nim 2.X
fileName.endsWith(it[1].split('#')[^1]) # nimble for nim 1.X
)
) == false or
fileName.split('-')[^1].len < 20: # safegard for nimble for nim 1.X
rmDir(dependency)
task unpin, "Restore global package use":
rmDir("nimbledeps")

322
libp2p/builders.nim Normal file
View File

@@ -0,0 +1,322 @@
# Nim-Libp2p
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
## This module contains a Switch Building helper.
runnableExamples:
let switch =
SwitchBuilder.new()
.withRng(rng)
.withAddresses(multiaddress)
# etc
.build()
{.push raises: [].}
import
options, tables, chronos, chronicles, sequtils,
switch, peerid, peerinfo, stream/connection, multiaddress,
crypto/crypto, transports/[transport, tcptransport],
muxers/[muxer, mplex/mplex, yamux/yamux],
protocols/[identify, secure/secure, secure/noise, rendezvous],
protocols/connectivity/[autonat/server, relay/relay, relay/client, relay/rtransport],
connmanager, upgrademngrs/muxedupgrade,
nameresolving/nameresolver,
errors, utility
export
switch, peerid, peerinfo, connection, multiaddress, crypto, errors
type
TransportProvider* {.public.} = proc(upgr: Upgrade): Transport {.gcsafe, raises: [].}
SecureProtocol* {.pure.} = enum
Noise,
Secio {.deprecated.}
SwitchBuilder* = ref object
privKey: Option[PrivateKey]
addresses: seq[MultiAddress]
secureManagers: seq[SecureProtocol]
muxers: seq[MuxerProvider]
transports: seq[TransportProvider]
rng: ref HmacDrbgContext
maxConnections: int
maxIn: int
sendSignedPeerRecord: bool
maxOut: int
maxConnsPerPeer: int
protoVersion: string
agentVersion: string
nameResolver: NameResolver
peerStoreCapacity: Opt[int]
autonat: bool
circuitRelay: Relay
rdv: RendezVous
services: seq[Service]
proc new*(T: type[SwitchBuilder]): T {.public.} =
## Creates a SwitchBuilder
let address = MultiAddress
.init("/ip4/127.0.0.1/tcp/0")
.expect("Should initialize to default")
SwitchBuilder(
privKey: none(PrivateKey),
addresses: @[address],
secureManagers: @[],
maxConnections: MaxConnections,
maxIn: -1,
maxOut: -1,
maxConnsPerPeer: MaxConnectionsPerPeer,
protoVersion: ProtoVersion,
agentVersion: AgentVersion)
proc withPrivateKey*(b: SwitchBuilder, privateKey: PrivateKey): SwitchBuilder {.public.} =
## Set the private key of the switch. Will be used to
## generate a PeerId
b.privKey = some(privateKey)
b
proc withAddress*(b: SwitchBuilder, address: MultiAddress): SwitchBuilder {.public.} =
## | Set the listening address of the switch
## | Calling it multiple time will override the value
b.addresses = @[address]
b
proc withAddresses*(b: SwitchBuilder, addresses: seq[MultiAddress]): SwitchBuilder {.public.} =
## | Set the listening addresses of the switch
## | Calling it multiple time will override the value
b.addresses = addresses
b
proc withSignedPeerRecord*(b: SwitchBuilder, sendIt = true): SwitchBuilder {.public.} =
b.sendSignedPeerRecord = sendIt
b
proc withMplex*(
b: SwitchBuilder,
inTimeout = 5.minutes,
outTimeout = 5.minutes,
maxChannCount = 200): SwitchBuilder {.public.} =
## | Uses `Mplex <https://docs.libp2p.io/concepts/stream-multiplexing/#mplex>`_ as a multiplexer
## | `Timeout` is the duration after which a inactive connection will be closed
proc newMuxer(conn: Connection): Muxer =
Mplex.new(
conn,
inTimeout,
outTimeout,
maxChannCount)
assert b.muxers.countIt(it.codec == MplexCodec) == 0, "Mplex build multiple times"
b.muxers.add(MuxerProvider.new(newMuxer, MplexCodec))
b
proc withYamux*(b: SwitchBuilder): SwitchBuilder =
proc newMuxer(conn: Connection): Muxer = Yamux.new(conn)
assert b.muxers.countIt(it.codec == YamuxCodec) == 0, "Yamux build multiple times"
b.muxers.add(MuxerProvider.new(newMuxer, YamuxCodec))
b
proc withNoise*(b: SwitchBuilder): SwitchBuilder {.public.} =
b.secureManagers.add(SecureProtocol.Noise)
b
proc withTransport*(b: SwitchBuilder, prov: TransportProvider): SwitchBuilder {.public.} =
## Use a custom transport
runnableExamples:
let switch =
SwitchBuilder.new()
.withTransport(proc(upgr: Upgrade): Transport = TcpTransport.new(flags, upgr))
.build()
b.transports.add(prov)
b
proc withTcpTransport*(b: SwitchBuilder, flags: set[ServerFlags] = {}): SwitchBuilder {.public.} =
b.withTransport(proc(upgr: Upgrade): Transport = TcpTransport.new(flags, upgr))
proc withRng*(b: SwitchBuilder, rng: ref HmacDrbgContext): SwitchBuilder {.public.} =
b.rng = rng
b
proc withMaxConnections*(b: SwitchBuilder, maxConnections: int): SwitchBuilder {.public.} =
## Maximum concurrent connections of the switch. You should either use this, or
## `withMaxIn <#withMaxIn,SwitchBuilder,int>`_ & `withMaxOut<#withMaxOut,SwitchBuilder,int>`_
b.maxConnections = maxConnections
b
proc withMaxIn*(b: SwitchBuilder, maxIn: int): SwitchBuilder {.public.} =
## Maximum concurrent incoming connections. Should be used with `withMaxOut<#withMaxOut,SwitchBuilder,int>`_
b.maxIn = maxIn
b
proc withMaxOut*(b: SwitchBuilder, maxOut: int): SwitchBuilder {.public.} =
## Maximum concurrent outgoing connections. Should be used with `withMaxIn<#withMaxIn,SwitchBuilder,int>`_
b.maxOut = maxOut
b
proc withMaxConnsPerPeer*(b: SwitchBuilder, maxConnsPerPeer: int): SwitchBuilder {.public.} =
b.maxConnsPerPeer = maxConnsPerPeer
b
proc withPeerStore*(b: SwitchBuilder, capacity: int): SwitchBuilder {.public.} =
b.peerStoreCapacity = Opt.some(capacity)
b
proc withProtoVersion*(b: SwitchBuilder, protoVersion: string): SwitchBuilder {.public.} =
b.protoVersion = protoVersion
b
proc withAgentVersion*(b: SwitchBuilder, agentVersion: string): SwitchBuilder {.public.} =
b.agentVersion = agentVersion
b
proc withNameResolver*(b: SwitchBuilder, nameResolver: NameResolver): SwitchBuilder {.public.} =
b.nameResolver = nameResolver
b
proc withAutonat*(b: SwitchBuilder): SwitchBuilder =
b.autonat = true
b
proc withCircuitRelay*(b: SwitchBuilder, r: Relay = Relay.new()): SwitchBuilder =
b.circuitRelay = r
b
proc withRendezVous*(b: SwitchBuilder, rdv: RendezVous = RendezVous.new()): SwitchBuilder =
b.rdv = rdv
b
proc withServices*(b: SwitchBuilder, services: seq[Service]): SwitchBuilder =
b.services = services
b
proc build*(b: SwitchBuilder): Switch
{.raises: [LPError], public.} =
if b.rng == nil: # newRng could fail
raise newException(Defect, "Cannot initialize RNG")
let pkRes = PrivateKey.random(b.rng[])
let
seckey = b.privKey.get(otherwise = pkRes.expect("Expected default Private Key"))
var
secureManagerInstances: seq[Secure]
if SecureProtocol.Noise in b.secureManagers:
secureManagerInstances.add(Noise.new(b.rng, seckey).Secure)
let
peerInfo = PeerInfo.new(
seckey,
b.addresses,
protoVersion = b.protoVersion,
agentVersion = b.agentVersion)
let
identify = Identify.new(peerInfo, b.sendSignedPeerRecord)
connManager = ConnManager.new(b.maxConnsPerPeer, b.maxConnections, b.maxIn, b.maxOut)
ms = MultistreamSelect.new()
muxedUpgrade = MuxedUpgrade.new(b.muxers, secureManagerInstances, connManager, ms)
let
transports = block:
var transports: seq[Transport]
for tProvider in b.transports:
transports.add(tProvider(muxedUpgrade))
transports
if b.secureManagers.len == 0:
b.secureManagers &= SecureProtocol.Noise
if isNil(b.rng):
b.rng = newRng()
let peerStore = block:
b.peerStoreCapacity.withValue(capacity):
PeerStore.new(identify, capacity)
else:
PeerStore.new(identify)
let switch = newSwitch(
peerInfo = peerInfo,
transports = transports,
secureManagers = secureManagerInstances,
connManager = connManager,
ms = ms,
nameResolver = b.nameResolver,
peerStore = peerStore,
services = b.services)
switch.mount(identify)
if b.autonat:
let autonat = Autonat.new(switch)
switch.mount(autonat)
if not isNil(b.circuitRelay):
if b.circuitRelay of RelayClient:
switch.addTransport(RelayTransport.new(RelayClient(b.circuitRelay), muxedUpgrade))
b.circuitRelay.setup(switch)
switch.mount(b.circuitRelay)
if not isNil(b.rdv):
b.rdv.setup(switch)
switch.mount(b.rdv)
return switch
proc newStandardSwitch*(
privKey = none(PrivateKey),
addrs: MultiAddress | seq[MultiAddress] = MultiAddress.init("/ip4/127.0.0.1/tcp/0").tryGet(),
secureManagers: openArray[SecureProtocol] = [
SecureProtocol.Noise,
],
transportFlags: set[ServerFlags] = {},
rng = newRng(),
inTimeout: Duration = 5.minutes,
outTimeout: Duration = 5.minutes,
maxConnections = MaxConnections,
maxIn = -1,
maxOut = -1,
maxConnsPerPeer = MaxConnectionsPerPeer,
nameResolver: NameResolver = nil,
sendSignedPeerRecord = false,
peerStoreCapacity = 1000): Switch
{.raises: [LPError], public.} =
## Helper for common switch configurations.
{.push warning[Deprecated]:off.}
if SecureProtocol.Secio in secureManagers:
quit("Secio is deprecated!") # use of secio is unsafe
{.pop.}
let addrs = when addrs is MultiAddress: @[addrs] else: addrs
var b = SwitchBuilder
.new()
.withAddresses(addrs)
.withRng(rng)
.withSignedPeerRecord(sendSignedPeerRecord)
.withMaxConnections(maxConnections)
.withMaxIn(maxIn)
.withMaxOut(maxOut)
.withMaxConnsPerPeer(maxConnsPerPeer)
.withPeerStore(capacity=peerStoreCapacity)
.withMplex(inTimeout, outTimeout)
.withTcpTransport(transportFlags)
.withNameResolver(nameResolver)
.withNoise()
privKey.withValue(pkey):
b = b.withPrivateKey(pkey)
b.build()

View File

@@ -1,17 +1,17 @@
## Nim-LibP2P
## Copyright (c) 2018 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-LibP2P
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
## This module implementes CID (Content IDentifier).
{.push raises: [Defect].}
{.push raises: [].}
import tables
import tables, hashes
import multibase, multicodec, multihash, vbuffer, varint
import stew/[base58, results]
@@ -62,7 +62,6 @@ const
]
proc initCidCodeTable(): Table[int, MultiCodec] {.compileTime.} =
result = initTable[int, MultiCodec]()
for item in ContentIdsList:
result[int(item)] = item
@@ -72,7 +71,7 @@ const
template orError*(exp: untyped, err: untyped): untyped =
(exp.mapErr do (_: auto) -> auto: err)
proc decode(data: openarray[byte]): Result[Cid, CidError] =
proc decode(data: openArray[byte]): Result[Cid, CidError] =
if len(data) == 34 and data[0] == 0x12'u8 and data[1] == 0x20'u8:
ok(Cid(
cidver: CIDv0,
@@ -115,7 +114,7 @@ proc decode(data: openarray[byte]): Result[Cid, CidError] =
hpos: offset,
data: vb))
proc decode(data: openarray[char]): Result[Cid, CidError] =
proc decode(data: openArray[char]): Result[Cid, CidError] =
var buffer: seq[byte]
var plen = 0
if len(data) < 2:
@@ -138,7 +137,7 @@ proc decode(data: openarray[char]): Result[Cid, CidError] =
return err(CidError.Incorrect)
decode(buffer)
proc validate*(ctype: typedesc[Cid], data: openarray[byte]): bool =
proc validate*(ctype: typedesc[Cid], data: openArray[byte]): bool =
## Returns ``true`` is data has valid binary CID representation.
var version, codec: uint64
var res: VarintResult[void]
@@ -186,7 +185,7 @@ proc version*(cid: Cid): CidVersion =
## Returns CID version
result = cid.cidver
proc init*[T: char|byte](ctype: typedesc[Cid], data: openarray[T]): Result[Cid, CidError] =
proc init*[T: char|byte](ctype: typedesc[Cid], data: openArray[T]): Result[Cid, CidError] =
## Create new content identifier using array of bytes or string ``data``.
decode(data)
@@ -199,10 +198,10 @@ proc init*(ctype: typedesc[Cid], version: CidVersion, content: MultiCodec,
## Cid.init(CIDv0, multiCodec("dag-pb"), MultiHash.digest("sha2-256", data))
##
## All other encodings and hashes are not supported by CIDv0.
var res: Cid
res.cidver = version
if version == CIDv0:
if content != multiCodec("dag-pb"):
return err(CidError.Unsupported)
@@ -268,15 +267,15 @@ proc encode*(mbtype: typedesc[MultiBase], encoding: string,
## ``encoding``.
result = MultiBase.encode(encoding, cid.data.buffer).tryGet()
proc hash*(cid: Cid): Hash {.inline.} =
hash(cid.data.buffer)
proc `$`*(cid: Cid): string =
## Return official string representation of content identifier ``cid``.
if cid.cidver == CIDv0:
BTCBase58.encode(cid.data.buffer)
elif cid.cidver == CIDv1:
let res = Multibase.encode("base58btc", cid.data.buffer)
if res.isOk():
res.get()
else:
""
let res = MultiBase.encode("base58btc", cid.data.buffer)
res.get("")
else:
""

View File

@@ -1,15 +1,18 @@
## Nim-LibP2P
## Copyright (c) 2020 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-LibP2P
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
import std/[options, tables, sequtils, sets]
import chronos, chronicles, metrics
{.push raises: [].}
import std/[tables, sequtils, sets]
import pkg/[chronos, chronicles, metrics]
import peerinfo,
peerstore,
stream/connection,
muxers/muxer,
utils/semaphore,
@@ -22,12 +25,11 @@ declareGauge(libp2p_peers, "total connected peers")
const
MaxConnections* = 50
MaxConnectionsPerPeer* = 5
MaxConnectionsPerPeer* = 1
type
TooManyConnectionsError* = object of CatchableError
ConnProvider* = proc(): Future[Connection] {.gcsafe, closure.}
TooManyConnectionsError* = object of LPError
AlreadyExpectingConnectionError* = object of LPError
ConnEventKind* {.pure.} = enum
Connected, # A connection was made and securely upgraded - there may be
@@ -45,7 +47,8 @@ type
discard
ConnEventHandler* =
proc(peerId: PeerID, event: ConnEvent): Future[void] {.gcsafe.}
proc(peerId: PeerId, event: ConnEvent): Future[void]
{.gcsafe, raises: [].}
PeerEventKind* {.pure.} = enum
Left,
@@ -59,25 +62,26 @@ type
discard
PeerEventHandler* =
proc(peerId: PeerID, event: PeerEvent): Future[void] {.gcsafe.}
MuxerHolder = object
muxer: Muxer
handle: Future[void]
proc(peerId: PeerId, event: PeerEvent): Future[void] {.gcsafe, raises: [].}
ConnManager* = ref object of RootObj
maxConnsPerPeer: int
inSema*: AsyncSemaphore
outSema*: AsyncSemaphore
conns: Table[PeerID, HashSet[Connection]]
muxed: Table[Connection, MuxerHolder]
connEvents: Table[ConnEventKind, OrderedSet[ConnEventHandler]]
peerEvents: Table[PeerEventKind, OrderedSet[PeerEventHandler]]
muxed: Table[PeerId, seq[Muxer]]
connEvents: array[ConnEventKind, OrderedSet[ConnEventHandler]]
peerEvents: array[PeerEventKind, OrderedSet[PeerEventHandler]]
expectedConnectionsOverLimit*: Table[(PeerId, Direction), Future[Muxer]]
peerStore*: PeerStore
ConnectionSlot* = object
connManager: ConnManager
direction: Direction
proc newTooManyConnectionsError(): ref TooManyConnectionsError {.inline.} =
result = newException(TooManyConnectionsError, "Too many connections")
proc init*(C: type ConnManager,
proc new*(C: type ConnManager,
maxConnsPerPeer = MaxConnectionsPerPeer,
maxConnections = MaxConnections,
maxIn = -1,
@@ -93,36 +97,41 @@ proc init*(C: type ConnManager,
raiseAssert "Invalid connection counts!"
C(maxConnsPerPeer: maxConnsPerPeer,
conns: initTable[PeerID, HashSet[Connection]](),
muxed: initTable[Connection, MuxerHolder](),
inSema: inSema,
outSema: outSema)
proc connCount*(c: ConnManager, peerId: PeerID): int =
c.conns.getOrDefault(peerId).len
proc connCount*(c: ConnManager, peerId: PeerId): int =
c.muxed.getOrDefault(peerId).len
proc connectedPeers*(c: ConnManager, dir: Direction): seq[PeerId] =
var peers = newSeq[PeerId]()
for peerId, mux in c.muxed:
if mux.anyIt(it.connection.dir == dir):
peers.add(peerId)
return peers
proc getConnections*(c: ConnManager): Table[PeerId, seq[Muxer]] =
return c.muxed
proc addConnEventHandler*(c: ConnManager,
handler: ConnEventHandler,
kind: ConnEventKind) =
## Add peer event handler - handlers must not raise exceptions!
##
if isNil(handler): return
c.connEvents.mgetOrPut(kind,
initOrderedSet[ConnEventHandler]()).incl(handler)
c.connEvents[kind].incl(handler)
proc removeConnEventHandler*(c: ConnManager,
handler: ConnEventHandler,
kind: ConnEventKind) =
c.connEvents.withValue(kind, handlers) do:
handlers[].excl(handler)
c.connEvents[kind].excl(handler)
proc triggerConnEvent*(c: ConnManager,
peerId: PeerID,
peerId: PeerId,
event: ConnEvent) {.async, gcsafe.} =
try:
trace "About to trigger connection events", peer = peerId
if event.kind in c.connEvents:
if c.connEvents[event.kind].len() > 0:
trace "triggering connection events", peer = peerId, event = $event.kind
var connEvents: seq[Future[void]]
for h in c.connEvents[event.kind]:
@@ -133,7 +142,7 @@ proc triggerConnEvent*(c: ConnManager,
raise exc
except CatchableError as exc:
warn "Exception in triggerConnEvents",
msg = exc.msg, peerId, event = $event
msg = exc.msg, peer = peerId, event = $event
proc addPeerEventHandler*(c: ConnManager,
handler: PeerEventHandler,
@@ -142,33 +151,23 @@ proc addPeerEventHandler*(c: ConnManager,
##
if isNil(handler): return
c.peerEvents.mgetOrPut(kind,
initOrderedSet[PeerEventHandler]()).incl(handler)
c.peerEvents[kind].incl(handler)
proc removePeerEventHandler*(c: ConnManager,
handler: PeerEventHandler,
kind: PeerEventKind) =
c.peerEvents.withValue(kind, handlers) do:
handlers[].excl(handler)
c.peerEvents[kind].excl(handler)
proc triggerPeerEvents*(c: ConnManager,
peerId: PeerID,
peerId: PeerId,
event: PeerEvent) {.async, gcsafe.} =
trace "About to trigger peer events", peer = peerId
if event.kind notin c.peerEvents:
if c.peerEvents[event.kind].len == 0:
return
try:
let count = c.connCount(peerId)
if event.kind == PeerEventKind.Joined and count != 1:
trace "peer already joined", peerId, event = $event
return
elif event.kind == PeerEventKind.Left and count != 0:
trace "peer still connected or already left", peerId, event = $event
return
trace "triggering peer events", peerId, event = $event
trace "triggering peer events", peer = peerId, event = $event
var peerEvents: seq[Future[void]]
for h in c.peerEvents[event.kind]:
@@ -178,23 +177,24 @@ proc triggerPeerEvents*(c: ConnManager,
except CancelledError as exc:
raise exc
except CatchableError as exc: # handlers should not raise!
warn "Exception in triggerPeerEvents", exc = exc.msg, peerId
warn "Exception in triggerPeerEvents", exc = exc.msg, peer = peerId
proc contains*(c: ConnManager, conn: Connection): bool =
## checks if a connection is being tracked by the
## connection manager
##
proc expectConnection*(c: ConnManager, p: PeerId, dir: Direction): Future[Muxer] {.async.} =
## Wait for a peer to connect to us. This will bypass the `MaxConnectionsPerPeer`
let key = (p, dir)
if key in c.expectedConnectionsOverLimit:
raise newException(AlreadyExpectingConnectionError, "Already expecting an incoming connection from that peer")
if isNil(conn):
return
let future = newFuture[Muxer]()
c.expectedConnectionsOverLimit[key] = future
if isNil(conn.peerInfo):
return
try:
return await future
finally:
c.expectedConnectionsOverLimit.del(key)
return conn in c.conns.getOrDefault(conn.peerInfo.peerId)
proc contains*(c: ConnManager, peerId: PeerID): bool =
peerId in c.conns
proc contains*(c: ConnManager, peerId: PeerId): bool =
peerId in c.muxed
proc contains*(c: ConnManager, muxer: Muxer): bool =
## checks if a muxer is being tracked by the connection
@@ -202,341 +202,214 @@ proc contains*(c: ConnManager, muxer: Muxer): bool =
##
if isNil(muxer):
return
return false
let conn = muxer.connection
if conn notin c:
return
return muxer in c.muxed.getOrDefault(conn.peerId)
if conn notin c.muxed:
return
proc closeMuxer(muxer: Muxer) {.async.} =
trace "Cleaning up muxer", m = muxer
return muxer == c.muxed[conn].muxer
proc closeMuxerHolder(muxerHolder: MuxerHolder) {.async.} =
trace "Cleaning up muxer", m = muxerHolder.muxer
await muxerHolder.muxer.close()
if not(isNil(muxerHolder.handle)):
await muxer.close()
if not(isNil(muxer.handler)):
try:
await muxerHolder.handle # TODO noraises?
await muxer.handler # TODO noraises?
except CatchableError as exc:
trace "Exception in close muxer handler", exc = exc.msg
trace "Cleaned up muxer", m = muxerHolder.muxer
proc delConn(c: ConnManager, conn: Connection) =
let peerId = conn.peerInfo.peerId
if peerId in c.conns:
c.conns[peerId].excl(conn)
if c.conns[peerId].len == 0:
c.conns.del(peerId)
libp2p_peers.set(c.conns.len.int64)
trace "Removed connection", conn
proc cleanupConn(c: ConnManager, conn: Connection) {.async.} =
## clean connection's resources such as muxers and streams
if isNil(conn):
trace "Wont cleanup a nil connection"
return
if isNil(conn.peerInfo):
trace "No peer info for connection"
return
# Remove connection from all tables without async breaks
var muxer = some(MuxerHolder())
if not c.muxed.pop(conn, muxer.get()):
muxer = none(MuxerHolder)
delConn(c, conn)
trace "Cleaned up muxer", m = muxer
proc muxCleanup(c: ConnManager, mux: Muxer) {.async.} =
try:
if muxer.isSome:
await closeMuxerHolder(muxer.get())
finally:
await conn.close()
trace "Triggering disconnect events", mux
let peerId = mux.connection.peerId
trace "Connection cleaned up", conn
let muxers = c.muxed.getOrDefault(peerId).filterIt(it != mux)
if muxers.len > 0:
c.muxed[peerId] = muxers
else:
c.muxed.del(peerId)
libp2p_peers.set(c.muxed.len.int64)
await c.triggerPeerEvents(peerId, PeerEvent(kind: PeerEventKind.Left))
proc onConnUpgraded(c: ConnManager, conn: Connection) {.async.} =
try:
trace "Triggering connect events", conn
conn.upgrade()
if not(c.peerStore.isNil):
c.peerStore.cleanup(peerId)
let peerId = conn.peerInfo.peerId
await c.triggerPeerEvents(
peerId, PeerEvent(kind: PeerEventKind.Joined, initiator: conn.dir == Direction.Out))
await c.triggerConnEvent(
peerId, ConnEvent(kind: ConnEventKind.Connected, incoming: conn.dir == Direction.In))
except CatchableError as exc:
# This is top-level procedure which will work as separate task, so it
# do not need to propagate CancelledError and should handle other errors
warn "Unexpected exception in switch peer connection cleanup",
conn, msg = exc.msg
proc peerCleanup(c: ConnManager, conn: Connection) {.async.} =
try:
trace "Triggering disconnect events", conn
let peerId = conn.peerInfo.peerId
await c.triggerConnEvent(
peerId, ConnEvent(kind: ConnEventKind.Disconnected))
await c.triggerPeerEvents(peerId, PeerEvent(kind: PeerEventKind.Left))
except CatchableError as exc:
# This is top-level procedure which will work as separate task, so it
# do not need to propagate CancelledError and should handle other errors
warn "Unexpected exception peer cleanup handler",
conn, msg = exc.msg
mux, msg = exc.msg
proc onClose(c: ConnManager, conn: Connection) {.async.} =
proc onClose(c: ConnManager, mux: Muxer) {.async.} =
## connection close even handler
##
## triggers the connections resource cleanup
##
try:
await conn.join()
trace "Connection closed, cleaning up", conn
await c.cleanupConn(conn)
except CancelledError:
# This is top-level procedure which will work as separate task, so it
# do not need to propagate CancelledError.
debug "Unexpected cancellation in connection manager's cleanup", conn
await mux.connection.join()
trace "Connection closed, cleaning up", mux
except CatchableError as exc:
debug "Unexpected exception in connection manager's cleanup",
errMsg = exc.msg, conn
errMsg = exc.msg, mux
finally:
trace "Triggering peerCleanup", conn
asyncSpawn c.peerCleanup(conn)
await c.muxCleanup(mux)
proc selectConn*(c: ConnManager,
peerId: PeerID,
dir: Direction): Connection =
proc selectMuxer*(c: ConnManager,
peerId: PeerId,
dir: Direction): Muxer =
## Select a connection for the provided peer and direction
##
let conns = toSeq(
c.conns.getOrDefault(peerId))
.filterIt( it.dir == dir )
c.muxed.getOrDefault(peerId))
.filterIt( it.connection.dir == dir )
if conns.len > 0:
return conns[0]
proc selectConn*(c: ConnManager, peerId: PeerID): Connection =
proc selectMuxer*(c: ConnManager, peerId: PeerId): Muxer =
## Select a connection for the provided giving priority
## to outgoing connections
##
var conn = c.selectConn(peerId, Direction.Out)
if isNil(conn):
conn = c.selectConn(peerId, Direction.In)
if isNil(conn):
var mux = c.selectMuxer(peerId, Direction.Out)
if isNil(mux):
mux = c.selectMuxer(peerId, Direction.In)
if isNil(mux):
trace "connection not found", peerId
return conn
proc selectMuxer*(c: ConnManager, conn: Connection): Muxer =
## select the muxer for the provided connection
##
if isNil(conn):
return
if conn in c.muxed:
return c.muxed[conn].muxer
else:
debug "no muxer for connection", conn
proc storeConn*(c: ConnManager, conn: Connection) =
## store a connection
##
if isNil(conn):
raise newException(CatchableError, "Connection cannot be nil")
if conn.closed or conn.atEof:
raise newException(CatchableError, "Connection closed or EOF")
if isNil(conn.peerInfo):
raise newException(CatchableError, "Empty peer info")
let peerId = conn.peerInfo.peerId
if c.conns.getOrDefault(peerId).len > c.maxConnsPerPeer:
debug "Too many connections for peer",
conn, conns = c.conns.getOrDefault(peerId).len
raise newTooManyConnectionsError()
if peerId notin c.conns:
c.conns[peerId] = initHashSet[Connection]()
c.conns[peerId].incl(conn)
libp2p_peers.set(c.conns.len.int64)
# Launch on close listener
# All the errors are handled inside `onClose()` procedure.
asyncSpawn c.onClose(conn)
trace "Stored connection",
conn, direction = $conn.dir, connections = c.conns.len
proc trackConn(c: ConnManager,
provider: ConnProvider,
sema: AsyncSemaphore):
Future[Connection] {.async.} =
var conn: Connection
try:
conn = await provider()
if isNil(conn):
return
trace "Got connection", conn
proc semaphoreMonitor() {.async.} =
try:
await conn.join()
except CatchableError as exc:
trace "Exception in semaphore monitor, ignoring", exc = exc.msg
sema.release()
asyncSpawn semaphoreMonitor()
except CatchableError as exc:
trace "Exception tracking connection", exc = exc.msg
if not isNil(conn):
await conn.close()
raise exc
return conn
proc trackIncomingConn*(c: ConnManager,
provider: ConnProvider):
Future[Connection] {.async.} =
## await for a connection slot before attempting
## to call the connection provider
##
var conn: Connection
try:
trace "Tracking incoming connection"
await c.inSema.acquire()
conn = await c.trackConn(provider, c.inSema)
if isNil(conn):
trace "Couldn't acquire connection, releasing semaphore slot", dir = $Direction.In
c.inSema.release()
return conn
except CatchableError as exc:
trace "Exception tracking connection", exc = exc.msg
c.inSema.release()
raise exc
proc trackOutgoingConn*(c: ConnManager,
provider: ConnProvider):
Future[Connection] {.async.} =
## try acquiring a connection if all slots
## are already taken, raise TooManyConnectionsError
## exception
##
trace "Tracking outgoing connection", count = c.outSema.count,
max = c.outSema.size
if not c.outSema.tryAcquire():
trace "Too many outgoing connections!", count = c.outSema.count,
max = c.outSema.size
raise newTooManyConnectionsError()
var conn: Connection
try:
conn = await c.trackConn(provider, c.outSema)
if isNil(conn):
trace "Couldn't acquire connection, releasing semaphore slot", dir = $Direction.Out
c.outSema.release()
return conn
except CatchableError as exc:
trace "Exception tracking connection", exc = exc.msg
c.outSema.release()
raise exc
return mux
proc storeMuxer*(c: ConnManager,
muxer: Muxer,
handle: Future[void] = nil) =
muxer: Muxer)
{.raises: [CatchableError].} =
## store the connection and muxer
##
if isNil(muxer):
raise newException(CatchableError, "muxer cannot be nil")
raise newException(LPError, "muxer cannot be nil")
if isNil(muxer.connection):
raise newException(CatchableError, "muxer's connection cannot be nil")
raise newException(LPError, "muxer's connection cannot be nil")
if muxer.connection notin c:
raise newException(CatchableError, "cant add muxer for untracked connection")
if muxer.connection.closed or muxer.connection.atEof:
raise newException(LPError, "Connection closed or EOF")
c.muxed[muxer.connection] = MuxerHolder(
muxer: muxer,
handle: handle)
let
peerId = muxer.connection.peerId
dir = muxer.connection.dir
# we use getOrDefault in the if below instead of [] to avoid the KeyError
if c.muxed.getOrDefault(peerId).len > c.maxConnsPerPeer:
let key = (peerId, dir)
let expectedConn = c.expectedConnectionsOverLimit.getOrDefault(key)
if expectedConn != nil and not expectedConn.finished:
expectedConn.complete(muxer)
else:
debug "Too many connections for peer",
conns = c.muxed.getOrDefault(peerId).len
raise newTooManyConnectionsError()
assert muxer notin c.muxed.getOrDefault(peerId)
let
newPeer = peerId notin c.muxed
assert newPeer or c.muxed[peerId].len > 0
c.muxed.mgetOrPut(peerId, newSeq[Muxer]()).add(muxer)
libp2p_peers.set(c.muxed.len.int64)
asyncSpawn c.triggerConnEvent(
peerId, ConnEvent(kind: ConnEventKind.Connected, incoming: dir == Direction.In))
if newPeer:
asyncSpawn c.triggerPeerEvents(
peerId, PeerEvent(kind: PeerEventKind.Joined, initiator: dir == Direction.Out))
asyncSpawn c.onClose(muxer)
trace "Stored muxer",
muxer, handle = not handle.isNil, connections = c.conns.len
muxer, direction = $muxer.connection.dir, peers = c.muxed.len
asyncSpawn c.onConnUpgraded(muxer.connection)
proc getIncomingSlot*(c: ConnManager): Future[ConnectionSlot] {.async.} =
await c.inSema.acquire()
return ConnectionSlot(connManager: c, direction: In)
proc getOutgoingSlot*(c: ConnManager, forceDial = false): ConnectionSlot {.raises: [TooManyConnectionsError].} =
if forceDial:
c.outSema.forceAcquire()
elif not c.outSema.tryAcquire():
trace "Too many outgoing connections!", count = c.outSema.count,
max = c.outSema.size
raise newTooManyConnectionsError()
return ConnectionSlot(connManager: c, direction: Out)
proc slotsAvailable*(c: ConnManager, dir: Direction): int =
case dir:
of Direction.In:
return c.inSema.count
of Direction.Out:
return c.outSema.count
proc release*(cs: ConnectionSlot) =
if cs.direction == In:
cs.connManager.inSema.release()
else:
cs.connManager.outSema.release()
proc trackConnection*(cs: ConnectionSlot, conn: Connection) =
if isNil(conn):
cs.release()
return
proc semaphoreMonitor() {.async.} =
try:
await conn.join()
except CatchableError as exc:
trace "Exception in semaphore monitor, ignoring", exc = exc.msg
cs.release()
asyncSpawn semaphoreMonitor()
proc trackMuxer*(cs: ConnectionSlot, mux: Muxer) =
if isNil(mux):
cs.release()
return
cs.trackConnection(mux.connection)
proc getStream*(c: ConnManager,
peerId: PeerID,
dir: Direction): Future[Connection] {.async, gcsafe.} =
## get a muxed stream for the provided peer
## with the given direction
muxer: Muxer): Future[Connection] {.async, gcsafe.} =
## get a muxed stream for the passed muxer
##
let muxer = c.selectMuxer(c.selectConn(peerId, dir))
if not(isNil(muxer)):
return await muxer.newStream()
proc getStream*(c: ConnManager,
peerId: PeerID): Future[Connection] {.async, gcsafe.} =
peerId: PeerId): Future[Connection] {.async, gcsafe.} =
## get a muxed stream for the passed peer from any connection
##
let muxer = c.selectMuxer(c.selectConn(peerId))
if not(isNil(muxer)):
return await muxer.newStream()
return await c.getStream(c.selectMuxer(peerId))
proc getStream*(c: ConnManager,
conn: Connection): Future[Connection] {.async, gcsafe.} =
## get a muxed stream for the passed connection
peerId: PeerId,
dir: Direction): Future[Connection] {.async, gcsafe.} =
## get a muxed stream for the passed peer from a connection with `dir`
##
let muxer = c.selectMuxer(conn)
if not(isNil(muxer)):
return await muxer.newStream()
return await c.getStream(c.selectMuxer(peerId, dir))
proc dropPeer*(c: ConnManager, peerId: PeerID) {.async.} =
proc dropPeer*(c: ConnManager, peerId: PeerId) {.async.} =
## drop connections and cleanup resources for peer
##
trace "Dropping peer", peerId
let conns = c.conns.getOrDefault(peerId)
for conn in conns:
trace "Removing connection", conn
delConn(c, conn)
var muxers: seq[MuxerHolder]
for conn in conns:
if conn in c.muxed:
muxers.add c.muxed[conn]
c.muxed.del(conn)
let muxers = c.muxed.getOrDefault(peerId)
for muxer in muxers:
await closeMuxerHolder(muxer)
for conn in conns:
await conn.close()
trace "Dropped peer", peerId
await closeMuxer(muxer)
trace "Peer dropped", peerId
@@ -546,17 +419,18 @@ proc close*(c: ConnManager) {.async.} =
##
trace "Closing ConnManager"
let conns = c.conns
c.conns.clear()
let muxed = c.muxed
c.muxed.clear()
for _, muxer in muxed:
await closeMuxerHolder(muxer)
let expected = c.expectedConnectionsOverLimit
c.expectedConnectionsOverLimit.clear()
for _, conns2 in conns:
for conn in conns2:
await conn.close()
for _, fut in expected:
await fut.cancelAndWait()
for _, muxers in muxed:
for mux in muxers:
await closeMuxer(mux)
trace "Closed ConnManager"

View File

@@ -1,11 +1,11 @@
## Nim-Libp2p
## Copyright (c) 2020 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-Libp2p
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
## This module integrates BearSSL ChaCha20+Poly1305
##
@@ -15,16 +15,11 @@
# RFC @ https://tools.ietf.org/html/rfc7539
{.push raises: [Defect].}
{.push raises: [].}
import bearssl
# have to do this due to a nim bug and raises[] on callbacks
# https://github.com/nim-lang/Nim/issues/13905
proc ourPoly1305CtmulRun*(key: pointer; iv: pointer; data: pointer; len: int;
aad: pointer; aadLen: int; tag: pointer; ichacha: pointer;
encrypt: cint) {.cdecl, importc: "br_poly1305_ctmul_run",
header: "bearssl_block.h".}
import bearssl/blockx
from stew/assign2 import assign
from stew/ptrops import baseAddr
const
ChaChaPolyKeySize = 32
@@ -37,17 +32,17 @@ type
ChaChaPolyNonce* = array[ChaChaPolyNonceSize, byte]
ChaChaPolyTag* = array[ChaChaPolyTagSize, byte]
proc intoChaChaPolyKey*(s: openarray[byte]): ChaChaPolyKey =
proc intoChaChaPolyKey*(s: openArray[byte]): ChaChaPolyKey =
assert s.len == ChaChaPolyKeySize
copyMem(addr result[0], unsafeaddr s[0], ChaChaPolyKeySize)
assign(result, s)
proc intoChaChaPolyNonce*(s: openarray[byte]): ChaChaPolyNonce =
proc intoChaChaPolyNonce*(s: openArray[byte]): ChaChaPolyNonce =
assert s.len == ChaChaPolyNonceSize
copyMem(addr result[0], unsafeaddr s[0], ChaChaPolyNonceSize)
assign(result, s)
proc intoChaChaPolyTag*(s: openarray[byte]): ChaChaPolyTag =
proc intoChaChaPolyTag*(s: openArray[byte]): ChaChaPolyTag =
assert s.len == ChaChaPolyTagSize
copyMem(addr result[0], unsafeaddr s[0], ChaChaPolyTagSize)
assign(result, s)
# bearssl allows us to use optimized versions
# this is reconciled at runtime
@@ -57,44 +52,46 @@ proc encrypt*(_: type[ChaChaPoly],
key: ChaChaPolyKey,
nonce: ChaChaPolyNonce,
tag: var ChaChaPolyTag,
data: var openarray[byte],
aad: openarray[byte]) =
data: var openArray[byte],
aad: openArray[byte]) =
let
ad = if aad.len > 0:
unsafeaddr aad[0]
unsafeAddr aad[0]
else:
nil
ourPoly1305CtmulRun(
unsafeaddr key[0],
unsafeaddr nonce[0],
addr data[0],
data.len,
poly1305CtmulRun(
unsafeAddr key[0],
unsafeAddr nonce[0],
baseAddr(data),
uint(data.len),
ad,
aad.len,
addr tag[0],
chacha20CtRun,
uint(aad.len),
baseAddr(tag),
# cast is required to workaround https://github.com/nim-lang/Nim/issues/13905
cast[Chacha20Run](chacha20CtRun),
#[encrypt]# 1.cint)
proc decrypt*(_: type[ChaChaPoly],
key: ChaChaPolyKey,
nonce: ChaChaPolyNonce,
tag: var ChaChaPolyTag,
data: var openarray[byte],
aad: openarray[byte]) =
data: var openArray[byte],
aad: openArray[byte]) =
let
ad = if aad.len > 0:
unsafeaddr aad[0]
unsafeAddr aad[0]
else:
nil
ourPoly1305CtmulRun(
unsafeaddr key[0],
unsafeaddr nonce[0],
addr data[0],
data.len,
poly1305CtmulRun(
unsafeAddr key[0],
unsafeAddr nonce[0],
baseAddr(data),
uint(data.len),
ad,
aad.len,
addr tag[0],
chacha20CtRun,
uint(aad.len),
baseAddr(tag),
# cast is required to workaround https://github.com/nim-lang/Nim/issues/13905
cast[Chacha20Run](chacha20CtRun),
#[decrypt]# 0.cint)

View File

@@ -1,14 +1,15 @@
## Nim-Libp2p
## Copyright (c) 2018 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-Libp2p
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
## This module implements Public Key and Private Key interface for libp2p.
{.push raises: [Defect].}
{.push raises: [].}
from strutils import split, strip, cmpIgnoreCase
const libp2p_pki_schemes* {.strdefine.} = "rsa,ed25519,secp256k1,ecnist"
@@ -68,19 +69,17 @@ when supported(PKScheme.Secp256k1):
# We are still importing `ecnist` because, it is used for SECIO handshake,
# but it will be impossible to create ECNIST keys or import ECNIST keys.
import ecnist, bearssl
import ecnist, bearssl/rand, bearssl/hash as bhash
import ../protobuf/minprotobuf, ../vbuffer, ../multihash, ../multicodec
import nimcrypto/[rijndael, twofish, sha2, hash, hmac]
# We use `ncrutils` for constant-time hexadecimal encoding/decoding procedures.
import nimcrypto/utils as ncrutils
import ../utility
import stew/results
export results
export results, utility
# This is workaround for Nim's `import` bug
export rijndael, twofish, sha2, hash, hmac, ncrutils
from strutils import split
export rijndael, twofish, sha2, hash, hmac, ncrutils, rand
type
DigestSheme* = enum
@@ -129,8 +128,8 @@ type
skkey*: SkPrivateKey
else:
discard
of PKSCheme.ECDSA:
when supported(PKSCheme.ECDSA):
of PKScheme.ECDSA:
when supported(PKScheme.ECDSA):
eckey*: ecnist.EcPrivateKey
else:
discard
@@ -159,22 +158,37 @@ type
template orError*(exp: untyped, err: untyped): untyped =
(exp.mapErr do (_: auto) -> auto: err)
proc newRng*(): ref BrHmacDrbgContext =
proc newRng*(): ref HmacDrbgContext =
# You should only create one instance of the RNG per application / library
# Ref is used so that it can be shared between components
# TODO consider moving to bearssl
var seeder = brPrngSeederSystem(nil)
var seeder = prngSeederSystem(nil)
if seeder == nil:
return nil
var rng = (ref BrHmacDrbgContext)()
brHmacDrbgInit(addr rng[], addr sha256Vtable, nil, 0)
var rng = (ref HmacDrbgContext)()
hmacDrbgInit(rng[], addr sha256Vtable, nil, 0)
if seeder(addr rng.vtable) == 0:
return nil
rng
proc shuffle*[T](
rng: ref HmacDrbgContext,
x: var openArray[T]) =
if x.len == 0: return
var randValues = newSeqUninitialized[byte](len(x) * 2)
hmacDrbgGenerate(rng[], randValues)
for i in countdown(x.high, 1):
let
rand = randValues[i * 2].int32 or (randValues[i * 2 + 1].int32 shl 8)
y = rand mod i
swap(x[i], x[y])
proc random*(T: typedesc[PrivateKey], scheme: PKScheme,
rng: var BrHmacDrbgContext,
rng: var HmacDrbgContext,
bits = RsaDefaultKeySize): CryptoResult[PrivateKey] =
## Generate random private key for scheme ``scheme``.
##
@@ -206,7 +220,7 @@ proc random*(T: typedesc[PrivateKey], scheme: PKScheme,
else:
err(SchemeError)
proc random*(T: typedesc[PrivateKey], rng: var BrHmacDrbgContext,
proc random*(T: typedesc[PrivateKey], rng: var HmacDrbgContext,
bits = RsaDefaultKeySize): CryptoResult[PrivateKey] =
## Generate random private key using default public-key cryptography scheme.
##
@@ -230,7 +244,7 @@ proc random*(T: typedesc[PrivateKey], rng: var BrHmacDrbgContext,
err(SchemeError)
proc random*(T: typedesc[KeyPair], scheme: PKScheme,
rng: var BrHmacDrbgContext,
rng: var HmacDrbgContext,
bits = RsaDefaultKeySize): CryptoResult[KeyPair] =
## Generate random key pair for scheme ``scheme``.
##
@@ -270,7 +284,7 @@ proc random*(T: typedesc[KeyPair], scheme: PKScheme,
else:
err(SchemeError)
proc random*(T: typedesc[KeyPair], rng: var BrHmacDrbgContext,
proc random*(T: typedesc[KeyPair], rng: var HmacDrbgContext,
bits = RsaDefaultKeySize): CryptoResult[KeyPair] =
## Generate random private pair of keys using default public-key cryptography
## scheme.
@@ -302,36 +316,36 @@ proc random*(T: typedesc[KeyPair], rng: var BrHmacDrbgContext,
else:
err(SchemeError)
proc getKey*(key: PrivateKey): CryptoResult[PublicKey] =
proc getPublicKey*(key: PrivateKey): CryptoResult[PublicKey] =
## Get public key from corresponding private key ``key``.
case key.scheme
of PKScheme.RSA:
when supported(PKScheme.RSA):
let rsakey = key.rsakey.getKey()
let rsakey = key.rsakey.getPublicKey()
ok(PublicKey(scheme: RSA, rsakey: rsakey))
else:
err(SchemeError)
of PKScheme.Ed25519:
when supported(PKScheme.Ed25519):
let edkey = key.edkey.getKey()
let edkey = key.edkey.getPublicKey()
ok(PublicKey(scheme: Ed25519, edkey: edkey))
else:
err(SchemeError)
of PKScheme.ECDSA:
when supported(PKScheme.ECDSA):
let eckey = ? key.eckey.getKey().orError(KeyError)
let eckey = ? key.eckey.getPublicKey().orError(KeyError)
ok(PublicKey(scheme: ECDSA, eckey: eckey))
else:
err(SchemeError)
of PKScheme.Secp256k1:
when supported(PKScheme.Secp256k1):
let skkey = key.skkey.getKey()
let skkey = key.skkey.getPublicKey()
ok(PublicKey(scheme: Secp256k1, skkey: skkey))
else:
err(SchemeError)
proc toRawBytes*(key: PrivateKey | PublicKey,
data: var openarray[byte]): CryptoResult[int] =
data: var openArray[byte]): CryptoResult[int] =
## Serialize private key ``key`` (using scheme's own serialization) and store
## it to ``data``.
##
@@ -383,7 +397,7 @@ proc getRawBytes*(key: PrivateKey | PublicKey): CryptoResult[seq[byte]] =
else:
err(SchemeError)
proc toBytes*(key: PrivateKey, data: var openarray[byte]): CryptoResult[int] =
proc toBytes*(key: PrivateKey, data: var openArray[byte]): CryptoResult[int] =
## Serialize private key ``key`` (using libp2p protobuf scheme) and store
## it to ``data``.
##
@@ -397,7 +411,7 @@ proc toBytes*(key: PrivateKey, data: var openarray[byte]): CryptoResult[int] =
copyMem(addr data[0], addr msg.buffer[0], blen)
ok(blen)
proc toBytes*(key: PublicKey, data: var openarray[byte]): CryptoResult[int] =
proc toBytes*(key: PublicKey, data: var openArray[byte]): CryptoResult[int] =
## Serialize public key ``key`` (using libp2p protobuf scheme) and store
## it to ``data``.
##
@@ -411,7 +425,7 @@ proc toBytes*(key: PublicKey, data: var openarray[byte]): CryptoResult[int] =
copyMem(addr data[0], addr msg.buffer[0], blen)
ok(blen)
proc toBytes*(sig: Signature, data: var openarray[byte]): int =
proc toBytes*(sig: Signature, data: var openArray[byte]): int =
## Serialize signature ``sig`` and store it to ``data``.
##
## Returns number of bytes (octets) needed to store signature ``sig``.
@@ -441,7 +455,8 @@ proc getBytes*(sig: Signature): seq[byte] =
## Return signature ``sig`` in binary form.
result = sig.data
proc init*[T: PrivateKey|PublicKey](key: var T, data: openarray[byte]): bool =
template initImpl[T: PrivateKey|PublicKey](
key: var T, data: openArray[byte]): bool =
## Initialize private key ``key`` from libp2p's protobuf serialized raw
## binary form.
##
@@ -454,7 +469,7 @@ proc init*[T: PrivateKey|PublicKey](key: var T, data: openarray[byte]): bool =
var pb = initProtoBuffer(@data)
let r1 = pb.getField(1, id)
let r2 = pb.getField(2, buffer)
if not(r1.isOk() and r1.get() and r2.isOk() and r2.get()):
if not(r1.get(false) and r2.get(false)):
false
else:
if cast[int8](id) notin SupportedSchemesInt or len(buffer) <= 0:
@@ -503,7 +518,15 @@ proc init*[T: PrivateKey|PublicKey](key: var T, data: openarray[byte]): bool =
else:
false
proc init*(sig: var Signature, data: openarray[byte]): bool =
{.push warning[ProveField]:off.} # https://github.com/nim-lang/Nim/issues/22060
proc init*(key: var PrivateKey, data: openArray[byte]): bool =
initImpl(key, data)
proc init*(key: var PublicKey, data: openArray[byte]): bool =
initImpl(key, data)
{.pop.}
proc init*(sig: var Signature, data: openArray[byte]): bool =
## Initialize signature ``sig`` from raw binary form.
##
## Returns ``true`` on success.
@@ -526,7 +549,7 @@ proc init*(sig: var Signature, data: string): bool =
sig.init(ncrutils.fromHex(data))
proc init*(t: typedesc[PrivateKey],
data: openarray[byte]): CryptoResult[PrivateKey] =
data: openArray[byte]): CryptoResult[PrivateKey] =
## Create new private key from libp2p's protobuf serialized binary form.
var res: t
if not res.init(data):
@@ -535,7 +558,7 @@ proc init*(t: typedesc[PrivateKey],
ok(res)
proc init*(t: typedesc[PublicKey],
data: openarray[byte]): CryptoResult[PublicKey] =
data: openArray[byte]): CryptoResult[PublicKey] =
## Create new public key from libp2p's protobuf serialized binary form.
var res: t
if not res.init(data):
@@ -544,7 +567,7 @@ proc init*(t: typedesc[PublicKey],
ok(res)
proc init*(t: typedesc[Signature],
data: openarray[byte]): CryptoResult[Signature] =
data: openArray[byte]): CryptoResult[Signature] =
## Create new public key from libp2p's protobuf serialized binary form.
var res: t
if not res.init(data):
@@ -699,7 +722,7 @@ proc `$`*(sig: Signature): string =
result = ncrutils.toHex(sig.data)
proc sign*(key: PrivateKey,
data: openarray[byte]): CryptoResult[Signature] {.gcsafe.} =
data: openArray[byte]): CryptoResult[Signature] {.gcsafe.} =
## Sign message ``data`` using private key ``key`` and return generated
## signature in raw binary form.
var res: Signature
@@ -733,7 +756,7 @@ proc sign*(key: PrivateKey,
else:
err(SchemeError)
proc verify*(sig: Signature, message: openarray[byte], key: PublicKey): bool =
proc verify*(sig: Signature, message: openArray[byte], key: PublicKey): bool =
## Verify signature ``sig`` using message ``message`` and public key ``key``.
## Return ``true`` if message signature is valid.
case key.scheme:
@@ -858,7 +881,7 @@ proc mac*(secret: Secret, id: int): seq[byte] {.inline.} =
proc ephemeral*(
scheme: ECDHEScheme,
rng: var BrHmacDrbgContext): CryptoResult[EcKeyPair] =
rng: var HmacDrbgContext): CryptoResult[EcKeyPair] =
## Generate ephemeral keys used to perform ECDHE.
var keypair: EcKeyPair
if scheme == Secp256r1:
@@ -867,10 +890,10 @@ proc ephemeral*(
keypair = ? EcKeyPair.random(Secp384r1, rng).orError(KeyError)
elif scheme == Secp521r1:
keypair = ? EcKeyPair.random(Secp521r1, rng).orError(KeyError)
ok(EcKeyPair(keypair))
ok(keypair)
proc ephemeral*(
scheme: string, rng: var BrHmacDrbgContext): CryptoResult[EcKeyPair] =
scheme: string, rng: var HmacDrbgContext): CryptoResult[EcKeyPair] =
## Generate ephemeral keys used to perform ECDHE using string encoding.
##
## Currently supported encoding strings are P-256, P-384, P-521, if encoding
@@ -884,8 +907,8 @@ proc ephemeral*(
else:
ephemeral(Secp521r1, rng)
proc getOrder*(remotePubkey, localNonce: openarray[byte],
localPubkey, remoteNonce: openarray[byte]): CryptoResult[int] =
proc getOrder*(remotePubkey, localNonce: openArray[byte],
localPubkey, remoteNonce: openArray[byte]): CryptoResult[int] =
## Compare values and calculate `order` parameter.
var ctx: sha256
ctx.init()
@@ -929,7 +952,7 @@ proc selectBest*(order: int, p1, p2: string): string =
if felement == selement:
return felement
proc createProposal*(nonce, pubkey: openarray[byte],
proc createProposal*(nonce, pubkey: openArray[byte],
exchanges, ciphers, hashes: string): seq[byte] =
## Create SecIO proposal message using random ``nonce``, local public key
## ``pubkey``, comma-delimieted list of supported exchange schemes
@@ -959,11 +982,10 @@ proc decodeProposal*(message: seq[byte], nonce, pubkey: var seq[byte],
let r4 = pb.getField(4, ciphers)
let r5 = pb.getField(5, hashes)
r1.isOk() and r1.get() and r2.isOk() and r2.get() and
r3.isOk() and r3.get() and r4.isOk() and r4.get() and
r5.isOk() and r5.get()
r1.get(false) and r2.get(false) and r3.get(false) and
r4.get(false) and r5.get(false)
proc createExchange*(epubkey, signature: openarray[byte]): seq[byte] =
proc createExchange*(epubkey, signature: openArray[byte]): seq[byte] =
## Create SecIO exchange message using ephemeral public key ``epubkey`` and
## signature of proposal blocks ``signature``.
var msg = initProtoBuffer({WithUint32BeLength})
@@ -981,67 +1003,34 @@ proc decodeExchange*(message: seq[byte],
var pb = initProtoBuffer(message)
let r1 = pb.getField(1, pubkey)
let r2 = pb.getField(2, signature)
r1.isOk() and r1.get() and r2.isOk() and r2.get()
r1.get(false) and r2.get(false)
## Serialization/Deserialization helpers
proc write*(vb: var VBuffer, pubkey: PublicKey) {.
inline, raises: [Defect, ResultError[CryptoError]].} =
inline, raises: [ResultError[CryptoError]].} =
## Write PublicKey value ``pubkey`` to buffer ``vb``.
vb.writeSeq(pubkey.getBytes().tryGet())
proc write*(vb: var VBuffer, seckey: PrivateKey) {.
inline, raises: [Defect, ResultError[CryptoError]].} =
inline, raises: [ResultError[CryptoError]].} =
## Write PrivateKey value ``seckey`` to buffer ``vb``.
vb.writeSeq(seckey.getBytes().tryGet())
proc write*(vb: var VBuffer, sig: PrivateKey) {.
inline, raises: [Defect, ResultError[CryptoError]].} =
inline, raises: [ResultError[CryptoError]].} =
## Write Signature value ``sig`` to buffer ``vb``.
vb.writeSeq(sig.getBytes().tryGet())
proc write*[T: PublicKey|PrivateKey](pb: var ProtoBuffer, field: int,
key: T) {.
inline, raises: [Defect, ResultError[CryptoError]].} =
inline, raises: [ResultError[CryptoError]].} =
write(pb, field, key.getBytes().tryGet())
proc write*(pb: var ProtoBuffer, field: int, sig: Signature) {.
inline, raises: [Defect, ResultError[CryptoError]].} =
inline, raises: [].} =
write(pb, field, sig.getBytes())
proc initProtoField*(index: int, key: PublicKey|PrivateKey): ProtoField {.
deprecated, raises: [Defect, ResultError[CryptoError]].} =
## Initialize ProtoField with PublicKey/PrivateKey ``key``.
result = initProtoField(index, key.getBytes().tryGet())
proc initProtoField*(index: int, sig: Signature): ProtoField {.deprecated.} =
## Initialize ProtoField with Signature ``sig``.
result = initProtoField(index, sig.getBytes())
proc getValue*[T: PublicKey|PrivateKey](data: var ProtoBuffer, field: int,
value: var T): int {.deprecated.} =
## Read PublicKey/PrivateKey from ProtoBuf's message and validate it.
var buf: seq[byte]
var key: PublicKey
result = getLengthValue(data, field, buf)
if result > 0:
if not key.init(buf):
result = -1
else:
value = key
proc getValue*(data: var ProtoBuffer, field: int, value: var Signature): int {.
deprecated.} =
## Read ``Signature`` from ProtoBuf's message and validate it.
var buf: seq[byte]
var sig: Signature
result = getLengthValue(data, field, buf)
if result > 0:
if not sig.init(buf):
result = -1
else:
value = sig
proc getField*[T: PublicKey|PrivateKey](pb: ProtoBuffer, field: int,
value: var T): ProtoResult[bool] =
## Deserialize public/private key from protobuf's message ``pb`` using field

View File

@@ -1,11 +1,11 @@
## Nim-Libp2p
## Copyright (c) 2020 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-Libp2p
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
## This module integrates BearSSL Cyrve25519 mul and mulgen
##
@@ -15,10 +15,11 @@
# RFC @ https://tools.ietf.org/html/rfc7748
{.push raises: [Defect].}
{.push raises: [].}
import bearssl
import bearssl/[ec, rand]
import stew/results
from stew/assign2 import assign
export results
const
@@ -27,13 +28,12 @@ const
type
Curve25519* = object
Curve25519Key* = array[Curve25519KeySize, byte]
pcuchar = ptr cuchar
Curve25519Error* = enum
Curver25519GenError
proc intoCurve25519Key*(s: openarray[byte]): Curve25519Key =
proc intoCurve25519Key*(s: openArray[byte]): Curve25519Key =
assert s.len == Curve25519KeySize
copyMem(addr result[0], unsafeaddr s[0], Curve25519KeySize)
assign(result, s)
proc getBytes*(key: Curve25519Key): seq[byte] = @key
@@ -45,7 +45,7 @@ proc byteswap(buf: var Curve25519Key) {.inline.} =
buf[31 - i] = x
proc mul*(_: type[Curve25519], point: var Curve25519Key, multiplier: Curve25519Key) =
let defaultBrEc = brEcGetDefault()
let defaultBrEc = ecGetDefault()
# multiplier needs to be big-endian
var
@@ -53,15 +53,15 @@ proc mul*(_: type[Curve25519], point: var Curve25519Key, multiplier: Curve25519K
multiplierBs.byteswap()
let
res = defaultBrEc.mul(
cast[pcuchar](addr point[0]),
addr point[0],
Curve25519KeySize,
cast[pcuchar](addr multiplierBs[0]),
addr multiplierBs[0],
Curve25519KeySize,
EC_curve25519)
assert res == 1
proc mulgen(_: type[Curve25519], dst: var Curve25519Key, point: Curve25519Key) =
let defaultBrEc = brEcGetDefault()
let defaultBrEc = ecGetDefault()
var
rpoint = point
@@ -69,20 +69,20 @@ proc mulgen(_: type[Curve25519], dst: var Curve25519Key, point: Curve25519Key) =
let
size = defaultBrEc.mulgen(
cast[pcuchar](addr dst[0]),
cast[pcuchar](addr rpoint[0]),
addr dst[0],
addr rpoint[0],
Curve25519KeySize,
EC_curve25519)
assert size == Curve25519KeySize
proc public*(private: Curve25519Key): Curve25519Key =
Curve25519.mulgen(result, private)
proc random*(_: type[Curve25519Key], rng: var BrHmacDrbgContext): Curve25519Key =
proc random*(_: type[Curve25519Key], rng: var HmacDrbgContext): Curve25519Key =
var res: Curve25519Key
let defaultBrEc = brEcGetDefault()
let len = brEcKeygen(
let defaultBrEc = ecGetDefault()
let len = ecKeygen(
addr rng.vtable, defaultBrEc, nil, addr res[0], EC_curve25519)
# Per bearssl documentation, the keygen only fails if the curve is
# unrecognised -

View File

@@ -1,11 +1,11 @@
## Nim-Libp2p
## Copyright (c) 2018 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-Libp2p
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
## This module implements constant-time ECDSA and ECDHE for NIST elliptic
## curves secp256r1, secp384r1 and secp521r1.
@@ -14,14 +14,17 @@
## BearSSL library <https://bearssl.org/>
## Copyright(C) 2018 Thomas Pornin <pornin@bolet.org>.
{.push raises: [Defect].}
{.push raises: [].}
import bearssl
import bearssl/[ec, rand, hash]
# We use `ncrutils` for constant-time hexadecimal encoding/decoding procedures.
import nimcrypto/utils as ncrutils
import minasn1
export minasn1.Asn1Error
import stew/[results, ctops]
import ../utility
export results
const
@@ -40,12 +43,12 @@ const
type
EcPrivateKey* = ref object
buffer*: array[BR_EC_KBUF_PRIV_MAX_SIZE, byte]
key*: BrEcPrivateKey
buffer*: array[EC_KBUF_PRIV_MAX_SIZE, byte]
key*: ec.EcPrivateKey
EcPublicKey* = ref object
buffer*: array[BR_EC_KBUF_PUB_MAX_SIZE, byte]
key*: BrEcPublicKey
buffer*: array[EC_KBUF_PUB_MAX_SIZE, byte]
key*: ec.EcPublicKey
EcKeyPair* = object
seckey*: EcPrivateKey
@@ -55,9 +58,9 @@ type
buffer*: seq[byte]
EcCurveKind* = enum
Secp256r1 = BR_EC_SECP256R1,
Secp384r1 = BR_EC_SECP384R1,
Secp521r1 = BR_EC_SECP521R1
Secp256r1 = EC_secp256r1,
Secp384r1 = EC_secp384r1,
Secp521r1 = EC_secp521r1
EcPKI* = EcPrivateKey | EcPublicKey | EcSignature
@@ -71,7 +74,7 @@ type
EcResult*[T] = Result[T, EcError]
const
EcSupportedCurvesCint* = {cint(Secp256r1), cint(Secp384r1), cint(Secp521r1)}
EcSupportedCurvesCint* = @[cint(Secp256r1), cint(Secp384r1), cint(Secp521r1)]
proc `-`(x: uint32): uint32 {.inline.} =
result = (0xFFFF_FFFF'u32 - x) + 1'u32
@@ -94,37 +97,37 @@ proc NEQ(x, y: uint32): uint32 {.inline.} =
proc LT0(x: int32): uint32 {.inline.} =
result = cast[uint32](x) shr 31
proc checkScalar(scalar: openarray[byte], curve: cint): uint32 =
proc checkScalar(scalar: openArray[byte], curve: cint): uint32 =
## Return ``1`` if all of the following hold:
## - len(``scalar``) <= ``orderlen``
## - ``scalar`` != 0
## - ``scalar`` is lower than the curve ``order``.
##
## Otherwise, return ``0``.
var impl = brEcGetDefault()
var orderlen = 0
var order = cast[ptr UncheckedArray[byte]](impl.order(curve, addr orderlen))
var impl = ecGetDefault()
var orderlen: uint = 0
var order = cast[ptr UncheckedArray[byte]](impl.order(curve, orderlen))
var z = 0'u32
var c = 0'i32
for u in scalar:
z = z or u
if len(scalar) == orderlen:
if len(scalar) == int(orderlen):
for i in 0..<len(scalar):
c = c or (-(cast[int32](EQ0(c))) and CMP(scalar[i], order[i]))
else:
c = -1
result = NEQ(z, 0'u32) and LT0(c)
proc checkPublic(key: openarray[byte], curve: cint): uint32 =
proc checkPublic(key: openArray[byte], curve: cint): uint32 =
## Return ``1`` if public key ``key`` is on curve.
var ckey = @key
var x = [0x00'u8, 0x01'u8]
var impl = brEcGetDefault()
var orderlen = 0
discard impl.order(curve, addr orderlen)
result = impl.mul(cast[ptr cuchar](unsafeAddr ckey[0]), len(ckey),
cast[ptr cuchar](addr x[0]), len(x), curve)
var x = [byte 0x00, 0x01]
var impl = ecGetDefault()
var orderlen: uint = 0
discard impl.order(curve, orderlen)
result = impl.mul(unsafeAddr ckey[0], uint(len(ckey)),
addr x[0], uint(len(x)), curve)
proc getOffset(pubkey: EcPublicKey): int {.inline.} =
let o = cast[uint](pubkey.key.q) - cast[uint](unsafeAddr pubkey.buffer[0])
@@ -174,7 +177,7 @@ proc copy*[T: EcPKI](dst: var T, src: T): bool =
dst.buffer = src.buffer
dst.key.curve = src.key.curve
dst.key.xlen = length
dst.key.x = cast[ptr cuchar](addr dst.buffer[offset])
dst.key.x = addr dst.buffer[offset]
result = true
elif T is EcPublicKey:
let length = src.key.qlen
@@ -184,7 +187,7 @@ proc copy*[T: EcPKI](dst: var T, src: T): bool =
dst.buffer = src.buffer
dst.key.curve = src.key.curve
dst.key.qlen = length
dst.key.q = cast[ptr cuchar](addr dst.buffer[offset])
dst.key.q = addr dst.buffer[offset]
result = true
else:
let length = len(src.buffer)
@@ -230,32 +233,32 @@ proc clear*[T: EcPKI|EcKeyPair](pki: var T) =
proc random*(
T: typedesc[EcPrivateKey], kind: EcCurveKind,
rng: var BrHmacDrbgContext): EcResult[EcPrivateKey] =
rng: var HmacDrbgContext): EcResult[EcPrivateKey] =
## Generate new random EC private key using BearSSL's HMAC-SHA256-DRBG
## algorithm.
##
## ``kind`` elliptic curve kind of your choice (secp256r1, secp384r1 or
## secp521r1).
var ecimp = brEcGetDefault()
var ecimp = ecGetDefault()
var res = new EcPrivateKey
if brEcKeygen(addr rng.vtable, ecimp,
if ecKeygen(addr rng.vtable, ecimp,
addr res.key, addr res.buffer[0],
cast[cint](kind)) == 0:
safeConvert[cint](kind)) == 0:
err(EcKeyGenError)
else:
ok(res)
proc getKey*(seckey: EcPrivateKey): EcResult[EcPublicKey] =
proc getPublicKey*(seckey: EcPrivateKey): EcResult[EcPublicKey] =
## Calculate and return EC public key from private key ``seckey``.
if isNil(seckey):
return err(EcKeyIncorrectError)
var ecimp = brEcGetDefault()
var ecimp = ecGetDefault()
if seckey.key.curve in EcSupportedCurvesCint:
var length = getPublicKeyLength(cast[EcCurveKind](seckey.key.curve))
var res = new EcPublicKey
if brEcComputePublicKey(ecimp, addr res.key,
addr res.buffer[0], unsafeAddr seckey.key) == 0:
assert res.buffer.len > getPublicKeyLength(cast[EcCurveKind](seckey.key.curve))
if ecComputePub(ecimp, addr res.key,
addr res.buffer[0], unsafeAddr seckey.key) == 0:
err(EcKeyIncorrectError)
else:
ok(res)
@@ -264,7 +267,7 @@ proc getKey*(seckey: EcPrivateKey): EcResult[EcPublicKey] =
proc random*(
T: typedesc[EcKeyPair], kind: EcCurveKind,
rng: var BrHmacDrbgContext): EcResult[T] =
rng: var HmacDrbgContext): EcResult[T] =
## Generate new random EC private and public keypair using BearSSL's
## HMAC-SHA256-DRBG algorithm.
##
@@ -272,7 +275,7 @@ proc random*(
## secp521r1).
let
seckey = ? EcPrivateKey.random(kind, rng)
pubkey = ? seckey.getKey()
pubkey = ? seckey.getPublicKey()
key = EcKeyPair(seckey: seckey, pubkey: pubkey)
ok(key)
@@ -315,7 +318,7 @@ proc `$`*(sig: EcSignature): string =
else:
result = ncrutils.toHex(sig.buffer)
proc toRawBytes*(seckey: EcPrivateKey, data: var openarray[byte]): EcResult[int] =
proc toRawBytes*(seckey: EcPrivateKey, data: var openArray[byte]): EcResult[int] =
## Serialize EC private key ``seckey`` to raw binary form and store it
## to ``data``.
##
@@ -331,7 +334,7 @@ proc toRawBytes*(seckey: EcPrivateKey, data: var openarray[byte]): EcResult[int]
else:
err(EcKeyIncorrectError)
proc toRawBytes*(pubkey: EcPublicKey, data: var openarray[byte]): EcResult[int] =
proc toRawBytes*(pubkey: EcPublicKey, data: var openArray[byte]): EcResult[int] =
## Serialize EC public key ``pubkey`` to uncompressed form specified in
## section 4.3.6 of ANSI X9.62.
##
@@ -347,7 +350,7 @@ proc toRawBytes*(pubkey: EcPublicKey, data: var openarray[byte]): EcResult[int]
else:
err(EcKeyIncorrectError)
proc toRawBytes*(sig: EcSignature, data: var openarray[byte]): int =
proc toRawBytes*(sig: EcSignature, data: var openArray[byte]): int =
## Serialize EC signature ``sig`` to raw binary form and store it to ``data``.
##
## Returns number of bytes (octets) needed to store EC signature, or `0`
@@ -358,7 +361,7 @@ proc toRawBytes*(sig: EcSignature, data: var openarray[byte]): int =
if len(sig.buffer) > 0:
copyMem(addr data[0], unsafeAddr sig.buffer[0], len(sig.buffer))
proc toBytes*(seckey: EcPrivateKey, data: var openarray[byte]): EcResult[int] =
proc toBytes*(seckey: EcPrivateKey, data: var openArray[byte]): EcResult[int] =
## Serialize EC private key ``seckey`` to ASN.1 DER binary form and store it
## to ``data``.
##
@@ -368,25 +371,29 @@ proc toBytes*(seckey: EcPrivateKey, data: var openarray[byte]): EcResult[int] =
return err(EcKeyIncorrectError)
if seckey.key.curve in EcSupportedCurvesCint:
var offset, length: int
var pubkey = ? seckey.getKey()
var pubkey = ? seckey.getPublicKey()
var b = Asn1Buffer.init()
var p = Asn1Composite.init(Asn1Tag.Sequence)
var c0 = Asn1Composite.init(0)
var c1 = Asn1Composite.init(1)
if seckey.key.curve == BR_EC_SECP256R1:
if seckey.key.curve == EC_secp256r1:
c0.write(Asn1Tag.Oid, Asn1OidSecp256r1)
elif seckey.key.curve == BR_EC_SECP384R1:
elif seckey.key.curve == EC_secp384r1:
c0.write(Asn1Tag.Oid, Asn1OidSecp384r1)
elif seckey.key.curve == BR_EC_SECP521R1:
elif seckey.key.curve == EC_secp521r1:
c0.write(Asn1Tag.Oid, Asn1OidSecp521r1)
c0.finish()
offset = pubkey.getOffset()
length = pubkey.key.qlen
if offset < 0:
return err(EcKeyIncorrectError)
length = int(pubkey.key.qlen)
c1.write(Asn1Tag.BitString,
pubkey.buffer.toOpenArray(offset, offset + length - 1))
c1.finish()
offset = seckey.getOffset()
length = seckey.key.xlen
if offset < 0:
return err(EcKeyIncorrectError)
length = int(seckey.key.xlen)
p.write(1'u64)
p.write(Asn1Tag.OctetString,
seckey.buffer.toOpenArray(offset, offset + length - 1))
@@ -404,7 +411,7 @@ proc toBytes*(seckey: EcPrivateKey, data: var openarray[byte]): EcResult[int] =
err(EcKeyIncorrectError)
proc toBytes*(pubkey: EcPublicKey, data: var openarray[byte]): EcResult[int] =
proc toBytes*(pubkey: EcPublicKey, data: var openArray[byte]): EcResult[int] =
## Serialize EC public key ``pubkey`` to ASN.1 DER binary form and store it
## to ``data``.
##
@@ -417,16 +424,18 @@ proc toBytes*(pubkey: EcPublicKey, data: var openarray[byte]): EcResult[int] =
var p = Asn1Composite.init(Asn1Tag.Sequence)
var c = Asn1Composite.init(Asn1Tag.Sequence)
c.write(Asn1Tag.Oid, Asn1OidEcPublicKey)
if pubkey.key.curve == BR_EC_SECP256R1:
if pubkey.key.curve == EC_secp256r1:
c.write(Asn1Tag.Oid, Asn1OidSecp256r1)
elif pubkey.key.curve == BR_EC_SECP384R1:
elif pubkey.key.curve == EC_secp384r1:
c.write(Asn1Tag.Oid, Asn1OidSecp384r1)
elif pubkey.key.curve == BR_EC_SECP521R1:
elif pubkey.key.curve == EC_secp521r1:
c.write(Asn1Tag.Oid, Asn1OidSecp521r1)
c.finish()
p.write(c)
let offset = getOffset(pubkey)
let length = pubkey.key.qlen
if offset < 0:
return err(EcKeyIncorrectError)
let length = int(pubkey.key.qlen)
p.write(Asn1Tag.BitString,
pubkey.buffer.toOpenArray(offset, offset + length - 1))
p.finish()
@@ -439,7 +448,7 @@ proc toBytes*(pubkey: EcPublicKey, data: var openarray[byte]): EcResult[int] =
else:
err(EcKeyIncorrectError)
proc toBytes*(sig: EcSignature, data: var openarray[byte]): EcResult[int] =
proc toBytes*(sig: EcSignature, data: var openArray[byte]): EcResult[int] =
## Serialize EC signature ``sig`` to ASN.1 DER binary form and store it
## to ``data``.
##
@@ -586,7 +595,7 @@ proc `==`*(a, b: EcSignature): bool =
else:
CT.isEqual(a.buffer, b.buffer)
proc init*(key: var EcPrivateKey, data: openarray[byte]): Result[void, Asn1Error] =
proc init*(key: var EcPrivateKey, data: openArray[byte]): Result[void, Asn1Error] =
## Initialize EC `private key` or `signature` ``key`` from ASN.1 DER binary
## representation ``data``.
##
@@ -621,25 +630,25 @@ proc init*(key: var EcPrivateKey, data: openarray[byte]): Result[void, Asn1Error
return err(Asn1Error.Incorrect)
if oid == Asn1OidSecp256r1:
curve = cast[cint](Secp256r1)
curve = safeConvert[cint](Secp256r1)
elif oid == Asn1OidSecp384r1:
curve = cast[cint](Secp384r1)
curve = safeConvert[cint](Secp384r1)
elif oid == Asn1OidSecp521r1:
curve = cast[cint](Secp521r1)
curve = safeConvert[cint](Secp521r1)
else:
return err(Asn1Error.Incorrect)
if checkScalar(raw.toOpenArray(), curve) == 1'u32:
key = new EcPrivateKey
copyMem(addr key.buffer[0], addr raw.buffer[raw.offset], raw.length)
key.key.x = cast[ptr cuchar](addr key.buffer[0])
key.key.xlen = raw.length
key.key.x = addr key.buffer[0]
key.key.xlen = uint(raw.length)
key.key.curve = curve
ok()
else:
err(Asn1Error.Incorrect)
proc init*(pubkey: var EcPublicKey, data: openarray[byte]): Result[void, Asn1Error] =
proc init*(pubkey: var EcPublicKey, data: openArray[byte]): Result[void, Asn1Error] =
## Initialize EC public key ``pubkey`` from ASN.1 DER binary representation
## ``data``.
##
@@ -675,11 +684,11 @@ proc init*(pubkey: var EcPublicKey, data: openarray[byte]): Result[void, Asn1Err
return err(Asn1Error.Incorrect)
if oid == Asn1OidSecp256r1:
curve = cast[cint](Secp256r1)
curve = safeConvert[cint](Secp256r1)
elif oid == Asn1OidSecp384r1:
curve = cast[cint](Secp384r1)
curve = safeConvert[cint](Secp384r1)
elif oid == Asn1OidSecp521r1:
curve = cast[cint](Secp521r1)
curve = safeConvert[cint](Secp521r1)
else:
return err(Asn1Error.Incorrect)
@@ -691,14 +700,14 @@ proc init*(pubkey: var EcPublicKey, data: openarray[byte]): Result[void, Asn1Err
if checkPublic(raw.toOpenArray(), curve) != 0:
pubkey = new EcPublicKey
copyMem(addr pubkey.buffer[0], addr raw.buffer[raw.offset], raw.length)
pubkey.key.q = cast[ptr cuchar](addr pubkey.buffer[0])
pubkey.key.qlen = raw.length
pubkey.key.q = addr pubkey.buffer[0]
pubkey.key.qlen = uint(raw.length)
pubkey.key.curve = curve
ok()
else:
err(Asn1Error.Incorrect)
proc init*(sig: var EcSignature, data: openarray[byte]): Result[void, Asn1Error] =
proc init*(sig: var EcSignature, data: openArray[byte]): Result[void, Asn1Error] =
## Initialize EC signature ``sig`` from raw binary representation ``data``.
##
## Procedure returns ``Result[void, Asn1Error]``.
@@ -718,7 +727,7 @@ proc init*[T: EcPKI](sospk: var T,
sospk.init(ncrutils.fromHex(data))
proc init*(t: typedesc[EcPrivateKey],
data: openarray[byte]): EcResult[EcPrivateKey] =
data: openArray[byte]): EcResult[EcPrivateKey] =
## Initialize EC private key from ASN.1 DER binary representation ``data`` and
## return constructed object.
var key: EcPrivateKey
@@ -729,7 +738,7 @@ proc init*(t: typedesc[EcPrivateKey],
ok(key)
proc init*(t: typedesc[EcPublicKey],
data: openarray[byte]): EcResult[EcPublicKey] =
data: openArray[byte]): EcResult[EcPublicKey] =
## Initialize EC public key from ASN.1 DER binary representation ``data`` and
## return constructed object.
var key: EcPublicKey
@@ -740,7 +749,7 @@ proc init*(t: typedesc[EcPublicKey],
ok(key)
proc init*(t: typedesc[EcSignature],
data: openarray[byte]): EcResult[EcSignature] =
data: openArray[byte]): EcResult[EcSignature] =
## Initialize EC signature from raw binary representation ``data`` and
## return constructed object.
var sig: EcSignature
@@ -755,7 +764,7 @@ proc init*[T: EcPKI](t: typedesc[T], data: string): EcResult[T] =
## string representation ``data`` and return constructed object.
t.init(ncrutils.fromHex(data))
proc initRaw*(key: var EcPrivateKey, data: openarray[byte]): bool =
proc initRaw*(key: var EcPrivateKey, data: openArray[byte]): bool =
## Initialize EC `private key` or `scalar` ``key`` from raw binary
## representation ``data``.
##
@@ -765,13 +774,13 @@ proc initRaw*(key: var EcPrivateKey, data: openarray[byte]): bool =
## Procedure returns ``true`` on success, ``false`` otherwise.
var curve: cint
if len(data) == SecKey256Length:
curve = cast[cint](Secp256r1)
curve = safeConvert[cint](Secp256r1)
result = true
elif len(data) == SecKey384Length:
curve = cast[cint](Secp384r1)
curve = safeConvert[cint](Secp384r1)
result = true
elif len(data) == SecKey521Length:
curve = cast[cint](Secp521r1)
curve = safeConvert[cint](Secp521r1)
result = true
if result:
result = false
@@ -779,12 +788,12 @@ proc initRaw*(key: var EcPrivateKey, data: openarray[byte]): bool =
let length = len(data)
key = new EcPrivateKey
copyMem(addr key.buffer[0], unsafeAddr data[0], length)
key.key.x = cast[ptr cuchar](addr key.buffer[0])
key.key.xlen = length
key.key.x = addr key.buffer[0]
key.key.xlen = uint(length)
key.key.curve = curve
result = true
proc initRaw*(pubkey: var EcPublicKey, data: openarray[byte]): bool =
proc initRaw*(pubkey: var EcPublicKey, data: openArray[byte]): bool =
## Initialize EC public key ``pubkey`` from raw binary representation
## ``data``.
##
@@ -796,13 +805,13 @@ proc initRaw*(pubkey: var EcPublicKey, data: openarray[byte]): bool =
if len(data) > 0:
if data[0] == 0x04'u8:
if len(data) == PubKey256Length:
curve = cast[cint](Secp256r1)
curve = safeConvert[cint](Secp256r1)
result = true
elif len(data) == PubKey384Length:
curve = cast[cint](Secp384r1)
curve = safeConvert[cint](Secp384r1)
result = true
elif len(data) == PubKey521Length:
curve = cast[cint](Secp521r1)
curve = safeConvert[cint](Secp521r1)
result = true
if result:
result = false
@@ -810,12 +819,12 @@ proc initRaw*(pubkey: var EcPublicKey, data: openarray[byte]): bool =
let length = len(data)
pubkey = new EcPublicKey
copyMem(addr pubkey.buffer[0], unsafeAddr data[0], length)
pubkey.key.q = cast[ptr cuchar](addr pubkey.buffer[0])
pubkey.key.qlen = length
pubkey.key.q = addr pubkey.buffer[0]
pubkey.key.qlen = uint(length)
pubkey.key.curve = curve
result = true
proc initRaw*(sig: var EcSignature, data: openarray[byte]): bool =
proc initRaw*(sig: var EcSignature, data: openArray[byte]): bool =
## Initialize EC signature ``sig`` from raw binary representation ``data``.
##
## Length of ``data`` array must be ``Sig256Length``, ``Sig384Length``
@@ -838,7 +847,7 @@ proc initRaw*[T: EcPKI](sospk: var T, data: string): bool {.inline.} =
result = sospk.initRaw(ncrutils.fromHex(data))
proc initRaw*(t: typedesc[EcPrivateKey],
data: openarray[byte]): EcResult[EcPrivateKey] =
data: openArray[byte]): EcResult[EcPrivateKey] =
## Initialize EC private key from raw binary representation ``data`` and
## return constructed object.
var res: EcPrivateKey
@@ -848,7 +857,7 @@ proc initRaw*(t: typedesc[EcPrivateKey],
ok(res)
proc initRaw*(t: typedesc[EcPublicKey],
data: openarray[byte]): EcResult[EcPublicKey] =
data: openArray[byte]): EcResult[EcPublicKey] =
## Initialize EC public key from raw binary representation ``data`` and
## return constructed object.
var res: EcPublicKey
@@ -858,7 +867,7 @@ proc initRaw*(t: typedesc[EcPublicKey],
ok(res)
proc initRaw*(t: typedesc[EcSignature],
data: openarray[byte]): EcResult[EcSignature] =
data: openArray[byte]): EcResult[EcSignature] =
## Initialize EC signature from raw binary representation ``data`` and
## return constructed object.
var res: EcSignature
@@ -877,7 +886,7 @@ proc scalarMul*(pub: EcPublicKey, sec: EcPrivateKey): EcPublicKey =
##
## Returns point in curve as ``pub * sec`` or ``nil`` otherwise.
doAssert((not isNil(pub)) and (not isNil(sec)))
var impl = brEcGetDefault()
var impl = ecGetDefault()
if sec.key.curve in EcSupportedCurvesCint:
if pub.key.curve == sec.key.curve:
var key = new EcPublicKey
@@ -885,16 +894,16 @@ proc scalarMul*(pub: EcPublicKey, sec: EcPrivateKey): EcPublicKey =
let poffset = key.getOffset()
let soffset = sec.getOffset()
if poffset >= 0 and soffset >= 0:
let res = impl.mul(cast[ptr cuchar](addr key.buffer[poffset]),
let res = impl.mul(addr key.buffer[poffset],
key.key.qlen,
cast[ptr cuchar](unsafeAddr sec.buffer[soffset]),
unsafeAddr sec.buffer[soffset],
sec.key.xlen,
key.key.curve)
if res != 0:
result = key
proc toSecret*(pubkey: EcPublicKey, seckey: EcPrivateKey,
data: var openarray[byte]): int =
data: var openArray[byte]): int =
## Calculate ECDHE shared secret using Go's elliptic/curve approach, using
## remote public key ``pubkey`` and local private key ``seckey`` and store
## shared secret to ``data``.
@@ -907,11 +916,11 @@ proc toSecret*(pubkey: EcPublicKey, seckey: EcPrivateKey,
doAssert((not isNil(pubkey)) and (not isNil(seckey)))
var mult = scalarMul(pubkey, seckey)
if not isNil(mult):
if seckey.key.curve == BR_EC_SECP256R1:
if seckey.key.curve == EC_secp256r1:
result = Secret256Length
elif seckey.key.curve == BR_EC_SECP384R1:
elif seckey.key.curve == EC_secp384r1:
result = Secret384Length
elif seckey.key.curve == BR_EC_SECP521R1:
elif seckey.key.curve == EC_secp521r1:
result = Secret521Length
if len(data) >= result:
var qplus1 = cast[pointer](cast[uint](mult.key.q) + 1'u)
@@ -931,24 +940,24 @@ proc getSecret*(pubkey: EcPublicKey, seckey: EcPrivateKey): seq[byte] =
copyMem(addr result[0], addr data[0], res)
proc sign*[T: byte|char](seckey: EcPrivateKey,
message: openarray[T]): EcResult[EcSignature] {.gcsafe.} =
message: openArray[T]): EcResult[EcSignature] {.gcsafe.} =
## Get ECDSA signature of data ``message`` using private key ``seckey``.
if isNil(seckey):
return err(EcKeyIncorrectError)
var hc: BrHashCompatContext
var hc: HashCompatContext
var hash: array[32, byte]
var impl = brEcGetDefault()
var impl = ecGetDefault()
if seckey.key.curve in EcSupportedCurvesCint:
var sig = new EcSignature
sig.buffer = newSeq[byte](256)
var kv = addr sha256Vtable
kv.init(addr hc.vtable)
if len(message) > 0:
kv.update(addr hc.vtable, unsafeAddr message[0], len(message))
kv.update(addr hc.vtable, unsafeAddr message[0], uint(len(message)))
else:
kv.update(addr hc.vtable, nil, 0)
kv.output(addr hc.vtable, addr hash[0])
let res = brEcdsaSignAsn1(impl, kv, addr hash[0], addr seckey.key,
kv.out(addr hc.vtable, addr hash[0])
let res = ecdsaI31SignAsn1(impl, kv, addr hash[0], addr seckey.key,
addr sig.buffer[0])
# Clear context with initial value
kv.init(addr hc.vtable)
@@ -960,7 +969,7 @@ proc sign*[T: byte|char](seckey: EcPrivateKey,
else:
err(EcKeyIncorrectError)
proc verify*[T: byte|char](sig: EcSignature, message: openarray[T],
proc verify*[T: byte|char](sig: EcSignature, message: openArray[T],
pubkey: EcPublicKey): bool {.inline.} =
## Verify ECDSA signature ``sig`` using public key ``pubkey`` and data
## ``message``.
@@ -968,20 +977,20 @@ proc verify*[T: byte|char](sig: EcSignature, message: openarray[T],
## Return ``true`` if message verification succeeded, ``false`` if
## verification failed.
doAssert((not isNil(sig)) and (not isNil(pubkey)))
var hc: BrHashCompatContext
var hc: HashCompatContext
var hash: array[32, byte]
var impl = brEcGetDefault()
var impl = ecGetDefault()
if pubkey.key.curve in EcSupportedCurvesCint:
var kv = addr sha256Vtable
kv.init(addr hc.vtable)
if len(message) > 0:
kv.update(addr hc.vtable, unsafeAddr message[0], len(message))
kv.update(addr hc.vtable, unsafeAddr message[0], uint(len(message)))
else:
kv.update(addr hc.vtable, nil, 0)
kv.output(addr hc.vtable, addr hash[0])
let res = brEcdsaVerifyAsn1(impl, addr hash[0], len(hash),
unsafeAddr pubkey.key,
addr sig.buffer[0], len(sig.buffer))
kv.out(addr hc.vtable, addr hash[0])
let res = ecdsaI31VrfyAsn1(impl, addr hash[0], uint(len(hash)),
unsafeAddr pubkey.key,
addr sig.buffer[0], uint(len(sig.buffer)))
# Clear context with initial value
kv.init(addr hc.vtable)
result = (res == 1)

View File

@@ -1,11 +1,11 @@
## Nim-Libp2p
## Copyright (c) 2018 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-Libp2p
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
## This module implements ED25519.
## This is pure nim implementation of ED25519 ref10.

View File

@@ -1,27 +1,31 @@
## Nim-Libp2p
## Copyright (c) 2018 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-Libp2p
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
## This module implements ED25519.
## This code is a port of the public domain, "ref10" implementation of ed25519
## from SUPERCOP.
{.push raises: Defect.}
{.push raises: [].}
import constants, bearssl
import bearssl/rand
import constants
import nimcrypto/[hash, sha2]
# We use `ncrutils` for constant-time hexadecimal encoding/decoding procedures.
import nimcrypto/utils as ncrutils
import stew/[results, ctops]
import ../../utility
export results
# This workaround needed because of some bugs in Nim Static[T].
export hash, sha2
export hash, sha2, rand
const
EdPrivateKeySize* = 64
@@ -165,30 +169,30 @@ proc feCopy(h: var Fe, f: Fe) =
h[8] = f8
h[9] = f9
proc load3(inp: openarray[byte]): uint64 =
result = cast[uint64](inp[0])
result = result or (cast[uint64](inp[1]) shl 8)
result = result or (cast[uint64](inp[2]) shl 16)
proc load_3(inp: openArray[byte]): uint64 =
result = safeConvert[uint64](inp[0])
result = result or (safeConvert[uint64](inp[1]) shl 8)
result = result or (safeConvert[uint64](inp[2]) shl 16)
proc load4(inp: openarray[byte]): uint64 =
result = cast[uint64](inp[0])
result = result or (cast[uint64](inp[1]) shl 8)
result = result or (cast[uint64](inp[2]) shl 16)
result = result or (cast[uint64](inp[3]) shl 24)
proc load_4(inp: openArray[byte]): uint64 =
result = safeConvert[uint64](inp[0])
result = result or (safeConvert[uint64](inp[1]) shl 8)
result = result or (safeConvert[uint64](inp[2]) shl 16)
result = result or (safeConvert[uint64](inp[3]) shl 24)
proc feFromBytes(h: var Fe, s: openarray[byte]) =
proc feFromBytes(h: var Fe, s: openArray[byte]) =
var c0, c1, c2, c3, c4, c5, c6, c7, c8, c9: int64
var h0 = cast[int64](load4(s.toOpenArray(0, 3)))
var h1 = cast[int64](load3(s.toOpenArray(4, 6))) shl 6
var h2 = cast[int64](load3(s.toOpenArray(7, 9))) shl 5
var h3 = cast[int64](load3(s.toOpenArray(10, 12))) shl 3
var h4 = cast[int64](load3(s.toOpenArray(13, 15))) shl 2
var h5 = cast[int64](load4(s.toOpenArray(16, 19)))
var h6 = cast[int64](load3(s.toOpenArray(20, 22))) shl 7
var h7 = cast[int64](load3(s.toOpenArray(23, 25))) shl 5
var h8 = cast[int64](load3(s.toOpenArray(26, 28))) shl 4
var h9 = (cast[int64](load3(s.toOpenArray(29, 31))) and 8388607'i32) shl 2
var h0 = cast[int64](load_4(s.toOpenArray(0, 3)))
var h1 = cast[int64](load_3(s.toOpenArray(4, 6))) shl 6
var h2 = cast[int64](load_3(s.toOpenArray(7, 9))) shl 5
var h3 = cast[int64](load_3(s.toOpenArray(10, 12))) shl 3
var h4 = cast[int64](load_3(s.toOpenArray(13, 15))) shl 2
var h5 = cast[int64](load_4(s.toOpenArray(16, 19)))
var h6 = cast[int64](load_3(s.toOpenArray(20, 22))) shl 7
var h7 = cast[int64](load_3(s.toOpenArray(23, 25))) shl 5
var h8 = cast[int64](load_3(s.toOpenArray(26, 28))) shl 4
var h9 = (cast[int64](load_3(s.toOpenArray(29, 31))) and 8388607'i32) shl 2
c9 = ashr((h9 + (1'i64 shl 24)), 25); h0 = h0 + (c9 * 19); h9 -= (c9 shl 25)
c1 = ashr((h1 + (1'i64 shl 24)), 25); h2 = h2 + c1; h1 -= (c1 shl 25)
@@ -213,7 +217,7 @@ proc feFromBytes(h: var Fe, s: openarray[byte]) =
h[8] = cast[int32](h8)
h[9] = cast[int32](h9)
proc feToBytes(s: var openarray[byte], h: Fe) =
proc feToBytes(s: var openArray[byte], h: Fe) =
var h0 = h[0]; var h1 = h[1]; var h2 = h[2]; var h3 = h[3]; var h4 = h[4]
var h5 = h[5]; var h6 = h[6]; var h7 = h[7]; var h8 = h[8]; var h9 = h[9]
var q, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9: int32
@@ -295,106 +299,106 @@ proc feMul(h: var Fe, f, g: Fe) =
var f5_2 = 2 * f5
var f7_2 = 2 * f7
var f9_2 = 2 * f9
var f0g0 = cast[int64](f0) * cast[int64](g0)
var f0g1 = cast[int64](f0) * cast[int64](g1)
var f0g2 = cast[int64](f0) * cast[int64](g2)
var f0g3 = cast[int64](f0) * cast[int64](g3)
var f0g4 = cast[int64](f0) * cast[int64](g4)
var f0g5 = cast[int64](f0) * cast[int64](g5)
var f0g6 = cast[int64](f0) * cast[int64](g6)
var f0g7 = cast[int64](f0) * cast[int64](g7)
var f0g8 = cast[int64](f0) * cast[int64](g8)
var f0g9 = cast[int64](f0) * cast[int64](g9)
var f1g0 = cast[int64](f1) * cast[int64](g0)
var f1g1_2 = cast[int64](f1_2) * cast[int64](g1)
var f1g2 = cast[int64](f1) * cast[int64](g2)
var f1g3_2 = cast[int64](f1_2) * cast[int64](g3)
var f1g4 = cast[int64](f1) * cast[int64](g4)
var f1g5_2 = cast[int64](f1_2) * cast[int64](g5)
var f1g6 = cast[int64](f1) * cast[int64](g6)
var f1g7_2 = cast[int64](f1_2) * cast[int64](g7)
var f1g8 = cast[int64](f1) * cast[int64](g8)
var f1g9_38 = cast[int64](f1_2) * cast[int64](g9_19)
var f2g0 = cast[int64](f2) * cast[int64](g0)
var f2g1 = cast[int64](f2) * cast[int64](g1)
var f2g2 = cast[int64](f2) * cast[int64](g2)
var f2g3 = cast[int64](f2) * cast[int64](g3)
var f2g4 = cast[int64](f2) * cast[int64](g4)
var f2g5 = cast[int64](f2) * cast[int64](g5)
var f2g6 = cast[int64](f2) * cast[int64](g6)
var f2g7 = cast[int64](f2) * cast[int64](g7)
var f2g8_19 = cast[int64](f2) * cast[int64](g8_19)
var f2g9_19 = cast[int64](f2) * cast[int64](g9_19)
var f3g0 = cast[int64](f3) * cast[int64](g0)
var f3g1_2 = cast[int64](f3_2) * cast[int64](g1)
var f3g2 = cast[int64](f3) * cast[int64](g2)
var f3g3_2 = cast[int64](f3_2) * cast[int64](g3)
var f3g4 = cast[int64](f3) * cast[int64](g4)
var f3g5_2 = cast[int64](f3_2) * cast[int64](g5)
var f3g6 = cast[int64](f3) * cast[int64](g6)
var f3g7_38 = cast[int64](f3_2) * cast[int64](g7_19)
var f3g8_19 = cast[int64](f3) * cast[int64](g8_19)
var f3g9_38 = cast[int64](f3_2) * cast[int64](g9_19)
var f4g0 = cast[int64](f4) * cast[int64](g0)
var f4g1 = cast[int64](f4) * cast[int64](g1)
var f4g2 = cast[int64](f4) * cast[int64](g2)
var f4g3 = cast[int64](f4) * cast[int64](g3)
var f4g4 = cast[int64](f4) * cast[int64](g4)
var f4g5 = cast[int64](f4) * cast[int64](g5)
var f4g6_19 = cast[int64](f4) * cast[int64](g6_19)
var f4g7_19 = cast[int64](f4) * cast[int64](g7_19)
var f4g8_19 = cast[int64](f4) * cast[int64](g8_19)
var f4g9_19 = cast[int64](f4) * cast[int64](g9_19)
var f5g0 = cast[int64](f5) * cast[int64](g0)
var f5g1_2 = cast[int64](f5_2) * cast[int64](g1)
var f5g2 = cast[int64](f5) * cast[int64](g2)
var f5g3_2 = cast[int64](f5_2) * cast[int64](g3)
var f5g4 = cast[int64](f5) * cast[int64](g4)
var f5g5_38 = cast[int64](f5_2) * cast[int64](g5_19)
var f5g6_19 = cast[int64](f5) * cast[int64](g6_19)
var f5g7_38 = cast[int64](f5_2) * cast[int64](g7_19)
var f5g8_19 = cast[int64](f5) * cast[int64](g8_19)
var f5g9_38 = cast[int64](f5_2) * cast[int64](g9_19)
var f6g0 = cast[int64](f6) * cast[int64](g0)
var f6g1 = cast[int64](f6) * cast[int64](g1)
var f6g2 = cast[int64](f6) * cast[int64](g2)
var f6g3 = cast[int64](f6) * cast[int64](g3)
var f6g4_19 = cast[int64](f6) * cast[int64](g4_19)
var f6g5_19 = cast[int64](f6) * cast[int64](g5_19)
var f6g6_19 = cast[int64](f6) * cast[int64](g6_19)
var f6g7_19 = cast[int64](f6) * cast[int64](g7_19)
var f6g8_19 = cast[int64](f6) * cast[int64](g8_19)
var f6g9_19 = cast[int64](f6) * cast[int64](g9_19)
var f7g0 = cast[int64](f7) * cast[int64](g0)
var f7g1_2 = cast[int64](f7_2) * cast[int64](g1)
var f7g2 = cast[int64](f7) * cast[int64](g2)
var f7g3_38 = cast[int64](f7_2) * cast[int64](g3_19)
var f7g4_19 = cast[int64](f7) * cast[int64](g4_19)
var f7g5_38 = cast[int64](f7_2) * cast[int64](g5_19)
var f7g6_19 = cast[int64](f7) * cast[int64](g6_19)
var f7g7_38 = cast[int64](f7_2) * cast[int64](g7_19)
var f7g8_19 = cast[int64](f7) * cast[int64](g8_19)
var f7g9_38 = cast[int64](f7_2) * cast[int64](g9_19)
var f8g0 = cast[int64](f8) * cast[int64](g0)
var f8g1 = cast[int64](f8) * cast[int64](g1)
var f8g2_19 = cast[int64](f8) * cast[int64](g2_19)
var f8g3_19 = cast[int64](f8) * cast[int64](g3_19)
var f8g4_19 = cast[int64](f8) * cast[int64](g4_19)
var f8g5_19 = cast[int64](f8) * cast[int64](g5_19)
var f8g6_19 = cast[int64](f8) * cast[int64](g6_19)
var f8g7_19 = cast[int64](f8) * cast[int64](g7_19)
var f8g8_19 = cast[int64](f8) * cast[int64](g8_19)
var f8g9_19 = cast[int64](f8) * cast[int64](g9_19)
var f9g0 = cast[int64](f9) * cast[int64](g0)
var f9g1_38 = cast[int64](f9_2) * cast[int64](g1_19)
var f9g2_19 = cast[int64](f9) * cast[int64](g2_19)
var f9g3_38 = cast[int64](f9_2) * cast[int64](g3_19)
var f9g4_19 = cast[int64](f9) * cast[int64](g4_19)
var f9g5_38 = cast[int64](f9_2) * cast[int64](g5_19)
var f9g6_19 = cast[int64](f9) * cast[int64](g6_19)
var f9g7_38 = cast[int64](f9_2) * cast[int64](g7_19)
var f9g8_19 = cast[int64](f9) * cast[int64](g8_19)
var f9g9_38 = cast[int64](f9_2) * cast[int64](g9_19)
var f0g0 = safeConvert[int64](f0) * safeConvert[int64](g0)
var f0g1 = safeConvert[int64](f0) * safeConvert[int64](g1)
var f0g2 = safeConvert[int64](f0) * safeConvert[int64](g2)
var f0g3 = safeConvert[int64](f0) * safeConvert[int64](g3)
var f0g4 = safeConvert[int64](f0) * safeConvert[int64](g4)
var f0g5 = safeConvert[int64](f0) * safeConvert[int64](g5)
var f0g6 = safeConvert[int64](f0) * safeConvert[int64](g6)
var f0g7 = safeConvert[int64](f0) * safeConvert[int64](g7)
var f0g8 = safeConvert[int64](f0) * safeConvert[int64](g8)
var f0g9 = safeConvert[int64](f0) * safeConvert[int64](g9)
var f1g0 = safeConvert[int64](f1) * safeConvert[int64](g0)
var f1g1_2 = safeConvert[int64](f1_2) * safeConvert[int64](g1)
var f1g2 = safeConvert[int64](f1) * safeConvert[int64](g2)
var f1g3_2 = safeConvert[int64](f1_2) * safeConvert[int64](g3)
var f1g4 = safeConvert[int64](f1) * safeConvert[int64](g4)
var f1g5_2 = safeConvert[int64](f1_2) * safeConvert[int64](g5)
var f1g6 = safeConvert[int64](f1) * safeConvert[int64](g6)
var f1g7_2 = safeConvert[int64](f1_2) * safeConvert[int64](g7)
var f1g8 = safeConvert[int64](f1) * safeConvert[int64](g8)
var f1g9_38 = safeConvert[int64](f1_2) * safeConvert[int64](g9_19)
var f2g0 = safeConvert[int64](f2) * safeConvert[int64](g0)
var f2g1 = safeConvert[int64](f2) * safeConvert[int64](g1)
var f2g2 = safeConvert[int64](f2) * safeConvert[int64](g2)
var f2g3 = safeConvert[int64](f2) * safeConvert[int64](g3)
var f2g4 = safeConvert[int64](f2) * safeConvert[int64](g4)
var f2g5 = safeConvert[int64](f2) * safeConvert[int64](g5)
var f2g6 = safeConvert[int64](f2) * safeConvert[int64](g6)
var f2g7 = safeConvert[int64](f2) * safeConvert[int64](g7)
var f2g8_19 = safeConvert[int64](f2) * safeConvert[int64](g8_19)
var f2g9_19 = safeConvert[int64](f2) * safeConvert[int64](g9_19)
var f3g0 = safeConvert[int64](f3) * safeConvert[int64](g0)
var f3g1_2 = safeConvert[int64](f3_2) * safeConvert[int64](g1)
var f3g2 = safeConvert[int64](f3) * safeConvert[int64](g2)
var f3g3_2 = safeConvert[int64](f3_2) * safeConvert[int64](g3)
var f3g4 = safeConvert[int64](f3) * safeConvert[int64](g4)
var f3g5_2 = safeConvert[int64](f3_2) * safeConvert[int64](g5)
var f3g6 = safeConvert[int64](f3) * safeConvert[int64](g6)
var f3g7_38 = safeConvert[int64](f3_2) * safeConvert[int64](g7_19)
var f3g8_19 = safeConvert[int64](f3) * safeConvert[int64](g8_19)
var f3g9_38 = safeConvert[int64](f3_2) * safeConvert[int64](g9_19)
var f4g0 = safeConvert[int64](f4) * safeConvert[int64](g0)
var f4g1 = safeConvert[int64](f4) * safeConvert[int64](g1)
var f4g2 = safeConvert[int64](f4) * safeConvert[int64](g2)
var f4g3 = safeConvert[int64](f4) * safeConvert[int64](g3)
var f4g4 = safeConvert[int64](f4) * safeConvert[int64](g4)
var f4g5 = safeConvert[int64](f4) * safeConvert[int64](g5)
var f4g6_19 = safeConvert[int64](f4) * safeConvert[int64](g6_19)
var f4g7_19 = safeConvert[int64](f4) * safeConvert[int64](g7_19)
var f4g8_19 = safeConvert[int64](f4) * safeConvert[int64](g8_19)
var f4g9_19 = safeConvert[int64](f4) * safeConvert[int64](g9_19)
var f5g0 = safeConvert[int64](f5) * safeConvert[int64](g0)
var f5g1_2 = safeConvert[int64](f5_2) * safeConvert[int64](g1)
var f5g2 = safeConvert[int64](f5) * safeConvert[int64](g2)
var f5g3_2 = safeConvert[int64](f5_2) * safeConvert[int64](g3)
var f5g4 = safeConvert[int64](f5) * safeConvert[int64](g4)
var f5g5_38 = safeConvert[int64](f5_2) * safeConvert[int64](g5_19)
var f5g6_19 = safeConvert[int64](f5) * safeConvert[int64](g6_19)
var f5g7_38 = safeConvert[int64](f5_2) * safeConvert[int64](g7_19)
var f5g8_19 = safeConvert[int64](f5) * safeConvert[int64](g8_19)
var f5g9_38 = safeConvert[int64](f5_2) * safeConvert[int64](g9_19)
var f6g0 = safeConvert[int64](f6) * safeConvert[int64](g0)
var f6g1 = safeConvert[int64](f6) * safeConvert[int64](g1)
var f6g2 = safeConvert[int64](f6) * safeConvert[int64](g2)
var f6g3 = safeConvert[int64](f6) * safeConvert[int64](g3)
var f6g4_19 = safeConvert[int64](f6) * safeConvert[int64](g4_19)
var f6g5_19 = safeConvert[int64](f6) * safeConvert[int64](g5_19)
var f6g6_19 = safeConvert[int64](f6) * safeConvert[int64](g6_19)
var f6g7_19 = safeConvert[int64](f6) * safeConvert[int64](g7_19)
var f6g8_19 = safeConvert[int64](f6) * safeConvert[int64](g8_19)
var f6g9_19 = safeConvert[int64](f6) * safeConvert[int64](g9_19)
var f7g0 = safeConvert[int64](f7) * safeConvert[int64](g0)
var f7g1_2 = safeConvert[int64](f7_2) * safeConvert[int64](g1)
var f7g2 = safeConvert[int64](f7) * safeConvert[int64](g2)
var f7g3_38 = safeConvert[int64](f7_2) * safeConvert[int64](g3_19)
var f7g4_19 = safeConvert[int64](f7) * safeConvert[int64](g4_19)
var f7g5_38 = safeConvert[int64](f7_2) * safeConvert[int64](g5_19)
var f7g6_19 = safeConvert[int64](f7) * safeConvert[int64](g6_19)
var f7g7_38 = safeConvert[int64](f7_2) * safeConvert[int64](g7_19)
var f7g8_19 = safeConvert[int64](f7) * safeConvert[int64](g8_19)
var f7g9_38 = safeConvert[int64](f7_2) * safeConvert[int64](g9_19)
var f8g0 = safeConvert[int64](f8) * safeConvert[int64](g0)
var f8g1 = safeConvert[int64](f8) * safeConvert[int64](g1)
var f8g2_19 = safeConvert[int64](f8) * safeConvert[int64](g2_19)
var f8g3_19 = safeConvert[int64](f8) * safeConvert[int64](g3_19)
var f8g4_19 = safeConvert[int64](f8) * safeConvert[int64](g4_19)
var f8g5_19 = safeConvert[int64](f8) * safeConvert[int64](g5_19)
var f8g6_19 = safeConvert[int64](f8) * safeConvert[int64](g6_19)
var f8g7_19 = safeConvert[int64](f8) * safeConvert[int64](g7_19)
var f8g8_19 = safeConvert[int64](f8) * safeConvert[int64](g8_19)
var f8g9_19 = safeConvert[int64](f8) * safeConvert[int64](g9_19)
var f9g0 = safeConvert[int64](f9) * safeConvert[int64](g0)
var f9g1_38 = safeConvert[int64](f9_2) * safeConvert[int64](g1_19)
var f9g2_19 = safeConvert[int64](f9) * safeConvert[int64](g2_19)
var f9g3_38 = safeConvert[int64](f9_2) * safeConvert[int64](g3_19)
var f9g4_19 = safeConvert[int64](f9) * safeConvert[int64](g4_19)
var f9g5_38 = safeConvert[int64](f9_2) * safeConvert[int64](g5_19)
var f9g6_19 = safeConvert[int64](f9) * safeConvert[int64](g6_19)
var f9g7_38 = safeConvert[int64](f9_2) * safeConvert[int64](g7_19)
var f9g8_19 = safeConvert[int64](f9) * safeConvert[int64](g8_19)
var f9g9_38 = safeConvert[int64](f9_2) * safeConvert[int64](g9_19)
var
c0, c1, c2, c3, c4, c5, c6, c7, c8, c9: int64
h0: int64 = f0g0 + f1g9_38 + f2g8_19 + f3g7_38 + f4g6_19 + f5g5_38 +
@@ -450,7 +454,7 @@ proc feNeg(h: var Fe, f: Fe) =
h[0] = h0; h[1] = h1; h[2] = h2; h[3] = h3; h[4] = h4
h[5] = h5; h[6] = h6; h[7] = h7; h[8] = h8; h[9] = h9
proc verify32(x: openarray[byte], y: openarray[byte]): int32 =
proc verify32(x: openArray[byte], y: openArray[byte]): int32 =
var d = 0'u32
d = d or (x[0] xor y[0])
d = d or (x[1] xor y[1])
@@ -489,7 +493,7 @@ proc verify32(x: openarray[byte], y: openarray[byte]): int32 =
proc feIsNegative(f: Fe): int32 =
var s: array[32, byte]
feToBytes(s, f)
result = cast[int32](s[0] and 1'u8)
result = safeConvert[int32](s[0] and 1'u8)
proc feIsNonZero(f: Fe): int32 =
var s: array[32, byte]
@@ -512,61 +516,61 @@ proc feSq(h: var Fe, f: Fe) =
var f7_38: int32 = 38 * f7
var f8_19: int32 = 19 * f8
var f9_38: int32 = 38 * f9
var f0f0: int64 = f0 * cast[int64](f0)
var f0f1_2: int64 = f0_2 * cast[int64](f1)
var f0f2_2: int64 = f0_2 * cast[int64](f2)
var f0f3_2: int64 = f0_2 * cast[int64](f3)
var f0f4_2: int64 = f0_2 * cast[int64](f4)
var f0f5_2: int64 = f0_2 * cast[int64](f5)
var f0f6_2: int64 = f0_2 * cast[int64](f6)
var f0f7_2: int64 = f0_2 * cast[int64](f7)
var f0f8_2: int64 = f0_2 * cast[int64](f8)
var f0f9_2: int64 = f0_2 * cast[int64](f9)
var f1f1_2: int64 = f1_2 * cast[int64](f1)
var f1f2_2: int64 = f1_2 * cast[int64](f2)
var f1f3_4: int64 = f1_2 * cast[int64](f3_2)
var f1f4_2: int64 = f1_2 * cast[int64](f4)
var f1f5_4: int64 = f1_2 * cast[int64](f5_2)
var f1f6_2: int64 = f1_2 * cast[int64](f6)
var f1f7_4: int64 = f1_2 * cast[int64](f7_2)
var f1f8_2: int64 = f1_2 * cast[int64](f8)
var f1f9_76: int64 = f1_2 * cast[int64](f9_38)
var f2f2: int64 = f2 * cast[int64](f2)
var f2f3_2: int64 = f2_2 * cast[int64](f3)
var f2f4_2: int64 = f2_2 * cast[int64](f4)
var f2f5_2: int64 = f2_2 * cast[int64](f5)
var f2f6_2: int64 = f2_2 * cast[int64](f6)
var f2f7_2: int64 = f2_2 * cast[int64](f7)
var f2f8_38: int64 = f2_2 * cast[int64](f8_19)
var f2f9_38: int64 = f2 * cast[int64](f9_38)
var f3f3_2: int64 = f3_2 * cast[int64](f3)
var f3f4_2: int64 = f3_2 * cast[int64](f4)
var f3f5_4: int64 = f3_2 * cast[int64](f5_2)
var f3f6_2: int64 = f3_2 * cast[int64](f6)
var f3f7_76: int64 = f3_2 * cast[int64](f7_38)
var f3f8_38: int64 = f3_2 * cast[int64](f8_19)
var f3f9_76: int64 = f3_2 * cast[int64](f9_38)
var f4f4: int64 = f4 * cast[int64](f4)
var f4f5_2: int64 = f4_2 * cast[int64](f5)
var f4f6_38: int64 = f4_2 * cast[int64](f6_19)
var f4f7_38: int64 = f4 * cast[int64](f7_38)
var f4f8_38: int64 = f4_2 * cast[int64](f8_19)
var f4f9_38: int64 = f4 * cast[int64](f9_38)
var f5f5_38: int64 = f5 * cast[int64](f5_38)
var f5f6_38: int64 = f5_2 * cast[int64](f6_19)
var f5f7_76: int64 = f5_2 * cast[int64](f7_38)
var f5f8_38: int64 = f5_2 * cast[int64](f8_19)
var f5f9_76: int64 = f5_2 * cast[int64](f9_38)
var f6f6_19: int64 = f6 * cast[int64](f6_19)
var f6f7_38: int64 = f6 * cast[int64](f7_38)
var f6f8_38: int64 = f6_2 * cast[int64](f8_19)
var f6f9_38: int64 = f6 * cast[int64](f9_38)
var f7f7_38: int64 = f7 * cast[int64](f7_38)
var f7f8_38: int64 = f7_2 * cast[int64](f8_19)
var f7f9_76: int64 = f7_2 * cast[int64](f9_38)
var f8f8_19: int64 = f8 * cast[int64](f8_19)
var f8f9_38: int64 = f8 * cast[int64](f9_38)
var f9f9_38: int64 = f9 * cast[int64](f9_38)
var f0f0: int64 = f0 * safeConvert[int64](f0)
var f0f1_2: int64 = f0_2 * safeConvert[int64](f1)
var f0f2_2: int64 = f0_2 * safeConvert[int64](f2)
var f0f3_2: int64 = f0_2 * safeConvert[int64](f3)
var f0f4_2: int64 = f0_2 * safeConvert[int64](f4)
var f0f5_2: int64 = f0_2 * safeConvert[int64](f5)
var f0f6_2: int64 = f0_2 * safeConvert[int64](f6)
var f0f7_2: int64 = f0_2 * safeConvert[int64](f7)
var f0f8_2: int64 = f0_2 * safeConvert[int64](f8)
var f0f9_2: int64 = f0_2 * safeConvert[int64](f9)
var f1f1_2: int64 = f1_2 * safeConvert[int64](f1)
var f1f2_2: int64 = f1_2 * safeConvert[int64](f2)
var f1f3_4: int64 = f1_2 * safeConvert[int64](f3_2)
var f1f4_2: int64 = f1_2 * safeConvert[int64](f4)
var f1f5_4: int64 = f1_2 * safeConvert[int64](f5_2)
var f1f6_2: int64 = f1_2 * safeConvert[int64](f6)
var f1f7_4: int64 = f1_2 * safeConvert[int64](f7_2)
var f1f8_2: int64 = f1_2 * safeConvert[int64](f8)
var f1f9_76: int64 = f1_2 * safeConvert[int64](f9_38)
var f2f2: int64 = f2 * safeConvert[int64](f2)
var f2f3_2: int64 = f2_2 * safeConvert[int64](f3)
var f2f4_2: int64 = f2_2 * safeConvert[int64](f4)
var f2f5_2: int64 = f2_2 * safeConvert[int64](f5)
var f2f6_2: int64 = f2_2 * safeConvert[int64](f6)
var f2f7_2: int64 = f2_2 * safeConvert[int64](f7)
var f2f8_38: int64 = f2_2 * safeConvert[int64](f8_19)
var f2f9_38: int64 = f2 * safeConvert[int64](f9_38)
var f3f3_2: int64 = f3_2 * safeConvert[int64](f3)
var f3f4_2: int64 = f3_2 * safeConvert[int64](f4)
var f3f5_4: int64 = f3_2 * safeConvert[int64](f5_2)
var f3f6_2: int64 = f3_2 * safeConvert[int64](f6)
var f3f7_76: int64 = f3_2 * safeConvert[int64](f7_38)
var f3f8_38: int64 = f3_2 * safeConvert[int64](f8_19)
var f3f9_76: int64 = f3_2 * safeConvert[int64](f9_38)
var f4f4: int64 = f4 * safeConvert[int64](f4)
var f4f5_2: int64 = f4_2 * safeConvert[int64](f5)
var f4f6_38: int64 = f4_2 * safeConvert[int64](f6_19)
var f4f7_38: int64 = f4 * safeConvert[int64](f7_38)
var f4f8_38: int64 = f4_2 * safeConvert[int64](f8_19)
var f4f9_38: int64 = f4 * safeConvert[int64](f9_38)
var f5f5_38: int64 = f5 * safeConvert[int64](f5_38)
var f5f6_38: int64 = f5_2 * safeConvert[int64](f6_19)
var f5f7_76: int64 = f5_2 * safeConvert[int64](f7_38)
var f5f8_38: int64 = f5_2 * safeConvert[int64](f8_19)
var f5f9_76: int64 = f5_2 * safeConvert[int64](f9_38)
var f6f6_19: int64 = f6 * safeConvert[int64](f6_19)
var f6f7_38: int64 = f6 * safeConvert[int64](f7_38)
var f6f8_38: int64 = f6_2 * safeConvert[int64](f8_19)
var f6f9_38: int64 = f6 * safeConvert[int64](f9_38)
var f7f7_38: int64 = f7 * safeConvert[int64](f7_38)
var f7f8_38: int64 = f7_2 * safeConvert[int64](f8_19)
var f7f9_76: int64 = f7_2 * safeConvert[int64](f9_38)
var f8f8_19: int64 = f8 * safeConvert[int64](f8_19)
var f8f9_38: int64 = f8 * safeConvert[int64](f9_38)
var f9f9_38: int64 = f9 * safeConvert[int64](f9_38)
var h0: int64 = f0f0 + f1f9_76 + f2f8_38 + f3f7_76 + f4f6_38 + f5f5_38
var h1: int64 = f0f1_2 + f2f9_38 + f3f8_38 + f4f7_38 + f5f6_38
var h2: int64 = f0f2_2 + f1f1_2 + f3f9_76 + f4f8_38 + f5f7_76 + f6f6_19
@@ -619,61 +623,61 @@ proc feSq2(h: var Fe, f: Fe) =
var f7_38 = 38 * f7
var f8_19 = 19 * f8
var f9_38 = 38 * f9
var f0f0 = cast[int64](f0) * cast[int64](f0)
var f0f1_2 = cast[int64](f0_2) * cast[int64](f1)
var f0f2_2 = cast[int64](f0_2) * cast[int64](f2)
var f0f3_2 = cast[int64](f0_2) * cast[int64](f3)
var f0f4_2 = cast[int64](f0_2) * cast[int64](f4)
var f0f5_2 = cast[int64](f0_2) * cast[int64](f5)
var f0f6_2 = cast[int64](f0_2) * cast[int64](f6)
var f0f7_2 = cast[int64](f0_2) * cast[int64](f7)
var f0f8_2 = cast[int64](f0_2) * cast[int64](f8)
var f0f9_2 = cast[int64](f0_2) * cast[int64](f9)
var f1f1_2 = cast[int64](f1_2) * cast[int64](f1)
var f1f2_2 = cast[int64](f1_2) * cast[int64](f2)
var f1f3_4 = cast[int64](f1_2) * cast[int64](f3_2)
var f1f4_2 = cast[int64](f1_2) * cast[int64](f4)
var f1f5_4 = cast[int64](f1_2) * cast[int64](f5_2)
var f1f6_2 = cast[int64](f1_2) * cast[int64](f6)
var f1f7_4 = cast[int64](f1_2) * cast[int64](f7_2)
var f1f8_2 = cast[int64](f1_2) * cast[int64](f8)
var f1f9_76 = cast[int64](f1_2) * cast[int64](f9_38)
var f2f2 = cast[int64](f2) * cast[int64](f2)
var f2f3_2 = cast[int64](f2_2) * cast[int64](f3)
var f2f4_2 = cast[int64](f2_2) * cast[int64](f4)
var f2f5_2 = cast[int64](f2_2) * cast[int64](f5)
var f2f6_2 = cast[int64](f2_2) * cast[int64](f6)
var f2f7_2 = cast[int64](f2_2) * cast[int64](f7)
var f2f8_38 = cast[int64](f2_2) * cast[int64](f8_19)
var f2f9_38 = cast[int64](f2) * cast[int64](f9_38)
var f3f3_2 = cast[int64](f3_2) * cast[int64](f3)
var f3f4_2 = cast[int64](f3_2) * cast[int64](f4)
var f3f5_4 = cast[int64](f3_2) * cast[int64](f5_2)
var f3f6_2 = cast[int64](f3_2) * cast[int64](f6)
var f3f7_76 = cast[int64](f3_2) * cast[int64](f7_38)
var f3f8_38 = cast[int64](f3_2) * cast[int64](f8_19)
var f3f9_76 = cast[int64](f3_2) * cast[int64](f9_38)
var f4f4 = cast[int64](f4) * cast[int64](f4)
var f4f5_2 = cast[int64](f4_2) * cast[int64](f5)
var f4f6_38 = cast[int64](f4_2) * cast[int64](f6_19)
var f4f7_38 = cast[int64](f4) * cast[int64](f7_38)
var f4f8_38 = cast[int64](f4_2) * cast[int64](f8_19)
var f4f9_38 = cast[int64](f4) * cast[int64](f9_38)
var f5f5_38 = cast[int64](f5) * cast[int64](f5_38)
var f5f6_38 = cast[int64](f5_2) * cast[int64](f6_19)
var f5f7_76 = cast[int64](f5_2) * cast[int64](f7_38)
var f5f8_38 = cast[int64](f5_2) * cast[int64](f8_19)
var f5f9_76 = cast[int64](f5_2) * cast[int64](f9_38)
var f6f6_19 = cast[int64](f6) * cast[int64](f6_19)
var f6f7_38 = cast[int64](f6) * cast[int64](f7_38)
var f6f8_38 = cast[int64](f6_2) * cast[int64](f8_19)
var f6f9_38 = cast[int64](f6) * cast[int64](f9_38)
var f7f7_38 = cast[int64](f7) * cast[int64](f7_38)
var f7f8_38 = cast[int64](f7_2) * cast[int64](f8_19)
var f7f9_76 = cast[int64](f7_2) * cast[int64](f9_38)
var f8f8_19 = cast[int64](f8) * cast[int64](f8_19)
var f8f9_38 = cast[int64](f8) * cast[int64](f9_38)
var f9f9_38 = cast[int64](f9) * cast[int64](f9_38)
var f0f0 = safeConvert[int64](f0) * safeConvert[int64](f0)
var f0f1_2 = safeConvert[int64](f0_2) * safeConvert[int64](f1)
var f0f2_2 = safeConvert[int64](f0_2) * safeConvert[int64](f2)
var f0f3_2 = safeConvert[int64](f0_2) * safeConvert[int64](f3)
var f0f4_2 = safeConvert[int64](f0_2) * safeConvert[int64](f4)
var f0f5_2 = safeConvert[int64](f0_2) * safeConvert[int64](f5)
var f0f6_2 = safeConvert[int64](f0_2) * safeConvert[int64](f6)
var f0f7_2 = safeConvert[int64](f0_2) * safeConvert[int64](f7)
var f0f8_2 = safeConvert[int64](f0_2) * safeConvert[int64](f8)
var f0f9_2 = safeConvert[int64](f0_2) * safeConvert[int64](f9)
var f1f1_2 = safeConvert[int64](f1_2) * safeConvert[int64](f1)
var f1f2_2 = safeConvert[int64](f1_2) * safeConvert[int64](f2)
var f1f3_4 = safeConvert[int64](f1_2) * safeConvert[int64](f3_2)
var f1f4_2 = safeConvert[int64](f1_2) * safeConvert[int64](f4)
var f1f5_4 = safeConvert[int64](f1_2) * safeConvert[int64](f5_2)
var f1f6_2 = safeConvert[int64](f1_2) * safeConvert[int64](f6)
var f1f7_4 = safeConvert[int64](f1_2) * safeConvert[int64](f7_2)
var f1f8_2 = safeConvert[int64](f1_2) * safeConvert[int64](f8)
var f1f9_76 = safeConvert[int64](f1_2) * safeConvert[int64](f9_38)
var f2f2 = safeConvert[int64](f2) * safeConvert[int64](f2)
var f2f3_2 = safeConvert[int64](f2_2) * safeConvert[int64](f3)
var f2f4_2 = safeConvert[int64](f2_2) * safeConvert[int64](f4)
var f2f5_2 = safeConvert[int64](f2_2) * safeConvert[int64](f5)
var f2f6_2 = safeConvert[int64](f2_2) * safeConvert[int64](f6)
var f2f7_2 = safeConvert[int64](f2_2) * safeConvert[int64](f7)
var f2f8_38 = safeConvert[int64](f2_2) * safeConvert[int64](f8_19)
var f2f9_38 = safeConvert[int64](f2) * safeConvert[int64](f9_38)
var f3f3_2 = safeConvert[int64](f3_2) * safeConvert[int64](f3)
var f3f4_2 = safeConvert[int64](f3_2) * safeConvert[int64](f4)
var f3f5_4 = safeConvert[int64](f3_2) * safeConvert[int64](f5_2)
var f3f6_2 = safeConvert[int64](f3_2) * safeConvert[int64](f6)
var f3f7_76 = safeConvert[int64](f3_2) * safeConvert[int64](f7_38)
var f3f8_38 = safeConvert[int64](f3_2) * safeConvert[int64](f8_19)
var f3f9_76 = safeConvert[int64](f3_2) * safeConvert[int64](f9_38)
var f4f4 = safeConvert[int64](f4) * safeConvert[int64](f4)
var f4f5_2 = safeConvert[int64](f4_2) * safeConvert[int64](f5)
var f4f6_38 = safeConvert[int64](f4_2) * safeConvert[int64](f6_19)
var f4f7_38 = safeConvert[int64](f4) * safeConvert[int64](f7_38)
var f4f8_38 = safeConvert[int64](f4_2) * safeConvert[int64](f8_19)
var f4f9_38 = safeConvert[int64](f4) * safeConvert[int64](f9_38)
var f5f5_38 = safeConvert[int64](f5) * safeConvert[int64](f5_38)
var f5f6_38 = safeConvert[int64](f5_2) * safeConvert[int64](f6_19)
var f5f7_76 = safeConvert[int64](f5_2) * safeConvert[int64](f7_38)
var f5f8_38 = safeConvert[int64](f5_2) * safeConvert[int64](f8_19)
var f5f9_76 = safeConvert[int64](f5_2) * safeConvert[int64](f9_38)
var f6f6_19 = safeConvert[int64](f6) * safeConvert[int64](f6_19)
var f6f7_38 = safeConvert[int64](f6) * safeConvert[int64](f7_38)
var f6f8_38 = safeConvert[int64](f6_2) * safeConvert[int64](f8_19)
var f6f9_38 = safeConvert[int64](f6) * safeConvert[int64](f9_38)
var f7f7_38 = safeConvert[int64](f7) * safeConvert[int64](f7_38)
var f7f8_38 = safeConvert[int64](f7_2) * safeConvert[int64](f8_19)
var f7f9_76 = safeConvert[int64](f7_2) * safeConvert[int64](f9_38)
var f8f8_19 = safeConvert[int64](f8) * safeConvert[int64](f8_19)
var f8f9_38 = safeConvert[int64](f8) * safeConvert[int64](f9_38)
var f9f9_38 = safeConvert[int64](f9) * safeConvert[int64](f9_38)
var
c0, c1, c2, c3, c4, c5, c6, c7, c8, c9: int64
h0: int64 = f0f0 + f1f9_76 + f2f8_38 + f3f7_76 + f4f6_38 + f5f5_38
@@ -800,7 +804,7 @@ proc geAdd(r: var GeP1P1, p: GeP3, q: GeCached) =
feAdd(r.z, t0, r.t)
feSub(r.t, t0, r.t)
proc geFromBytesNegateVartime(h: var GeP3, s: openarray[byte]): int32 =
proc geFromBytesNegateVartime(h: var GeP3, s: openArray[byte]): int32 =
var u, v, v3, vxx, check: Fe
feFromBytes(h.y, s)
@@ -830,7 +834,7 @@ proc geFromBytesNegateVartime(h: var GeP3, s: openarray[byte]): int32 =
return -1;
feMul(h.x, h.x, SqrTm1)
if feIsNegative(h.x) == cast[int32](s[31] shr 7):
if feIsNegative(h.x) == safeConvert[int32](s[31] shr 7):
feNeg(h.x, h.x)
feMul(h.t, h.x, h.y)
@@ -876,12 +880,12 @@ proc geSub(r: var GeP1P1, p: GeP3, q: GeCached) =
feSub(r.z, t0, r.t)
feAdd(r.t, t0, r.t)
proc geToBytes(s: var openarray[byte], h: GeP2) =
proc geToBytes(s: var openArray[byte], h: GeP2) =
var recip, x, y: Fe
feInvert(recip, h.z)
feMul(x, h.x, recip)
feMul(y, h.y, recip)
feTobytes(s, y)
feToBytes(s, y)
s[31] = s[31] xor cast[byte](feIsNegative(x) shl 7)
proc geP1P1toP2(r: var GeP2, p: GeP1P1) =
@@ -925,10 +929,10 @@ proc geP3toP2(r: var GeP2, p: GeP3) =
proc geP3dbl(r: var GeP1P1, p: GeP3) =
var q: GeP2
geP3ToP2(q, p)
geP3toP2(q, p)
geP2dbl(r, q)
proc geP3ToBytes(s: var openarray[byte], h: GeP3) =
proc geP3ToBytes(s: var openArray[byte], h: GeP3) =
var recip, x, y: Fe
feInvert(recip, h.z);
@@ -952,14 +956,14 @@ proc equal(b, c: int8): byte =
var ub = cast[byte](b)
var uc = cast[byte](c)
var x = ub xor uc
var y = cast[uint32](x)
var y = safeConvert[uint32](x)
y = y - 1
y = y shr 31
result = cast[byte](y)
proc negative(b: int8): byte =
var x = cast[uint64](b)
x = x shr 63
var x = cast[uint8](b)
x = x shr 7
result = cast[byte](x)
proc cmov(t: var GePrecomp, u: GePrecomp, b: byte) =
@@ -985,7 +989,7 @@ proc select(t: var GePrecomp, pos: int, b: int8) =
feNeg(minust.xy2d, t.xy2d)
cmov(t, minust, bnegative)
proc geScalarMultBase(h: var GeP3, a: openarray[byte]) =
proc geScalarMultBase(h: var GeP3, a: openArray[byte]) =
var e: array[64, int8]
var carry: int8
var r: GeP1P1
@@ -1010,8 +1014,8 @@ proc geScalarMultBase(h: var GeP3, a: openarray[byte]) =
geMadd(r, h, t)
geP1P1toP3(h, r)
geP3dbl(r, h); geP1P1ToP2(s, r)
geP2dbl(r, s); geP1P1ToP2(s, r)
geP3dbl(r, h); geP1P1toP2(s, r)
geP2dbl(r, s); geP1P1toP2(s, r)
geP2dbl(r, s); geP1P1toP2(s, r)
geP2dbl(r, s); geP1P1toP3(h, r)
@@ -1020,7 +1024,7 @@ proc geScalarMultBase(h: var GeP3, a: openarray[byte]) =
geMadd(r, h, t)
geP1P1toP3(h, r)
proc scMulAdd(s: var openarray[byte], a, b, c: openarray[byte]) =
proc scMulAdd(s: var openArray[byte], a, b, c: openArray[byte]) =
var a0 = 2097151'i64 and cast[int64](load_3(a.toOpenArray(0, 2)))
var a1 = 2097151'i64 and cast[int64](load_4(a.toOpenArray(2, 5)) shr 5)
var a2 = 2097151'i64 and cast[int64](load_3(a.toOpenArray(5, 7)) shr 2)
@@ -1320,7 +1324,7 @@ proc scMulAdd(s: var openarray[byte], a, b, c: openarray[byte]) =
s[30] = cast[uint8](ashr(s11, 9))
s[31] = cast[uint8](ashr(s11, 17))
proc scReduce(s: var openarray[byte]) =
proc scReduce(s: var openArray[byte]) =
var s0 = 2097151'i64 and cast[int64](load_3(s.toOpenArray(0, 2)));
var s1 = 2097151'i64 and cast[int64](load_4(s.toOpenArray(2, 5)) shr 5)
var s2 = 2097151'i64 and cast[int64](load_3(s.toOpenArray(5, 7)) shr 2)
@@ -1546,7 +1550,7 @@ proc scReduce(s: var openarray[byte]) =
s[30] = cast[byte](ashr(s11, 9))
s[31] = cast[byte](ashr(s11, 17))
proc slide(r: var openarray[int8], a: openarray[byte]) =
proc slide(r: var openArray[int8], a: openArray[byte]) =
for i in 0..<256:
r[i] = cast[int8](1'u8 and (a[i shr 3] shr (i and 7)))
for i in 0..<256:
@@ -1567,8 +1571,8 @@ proc slide(r: var openarray[int8], a: openarray[byte]) =
break
inc(b)
proc geDoubleScalarMultVartime(r: var GeP2, a: openarray[byte], A: GeP3,
b: openarray[byte]) =
proc geDoubleScalarMultVartime(r: var GeP2, a: openArray[byte], A: GeP3,
b: openArray[byte]) =
var
aslide: array[256, int8]
bslide: array[256, int8]
@@ -1632,7 +1636,7 @@ proc NEQ(x, y: uint32): uint32 {.inline.} =
proc LT0(x: int32): uint32 {.inline.} =
result = cast[uint32](x) shr 31
proc checkScalar*(scalar: openarray[byte]): uint32 =
proc checkScalar*(scalar: openArray[byte]): uint32 =
var z = 0'u32
var c = 0'i32
for u in scalar:
@@ -1644,14 +1648,14 @@ proc checkScalar*(scalar: openarray[byte]): uint32 =
c = -1
result = NEQ(z, 0'u32) and LT0(c)
proc random*(t: typedesc[EdPrivateKey], rng: var BrHmacDrbgContext): EdPrivateKey =
proc random*(t: typedesc[EdPrivateKey], rng: var HmacDrbgContext): EdPrivateKey =
## Generate new random ED25519 private key using the given random number generator
var
point: GeP3
pk: array[EdPublicKeySize, byte]
res: EdPrivateKey
brHmacDrbgGenerate(addr rng, addr res.data[0], 32)
hmacDrbgGenerate(rng, res.data.toOpenArray(0, 31))
var hh = sha512.digest(res.data.toOpenArray(0, 31))
hh.data[0] = hh.data[0] and 0xF8'u8
@@ -1663,14 +1667,14 @@ proc random*(t: typedesc[EdPrivateKey], rng: var BrHmacDrbgContext): EdPrivateKe
res
proc random*(t: typedesc[EdKeyPair], rng: var BrHmacDrbgContext): EdKeyPair =
proc random*(t: typedesc[EdKeyPair], rng: var HmacDrbgContext): EdKeyPair =
## Generate new random ED25519 private and public keypair using OS specific
## CSPRNG.
var
point: GeP3
res: EdKeyPair
brHmacDrbgGenerate(addr rng, addr res.seckey.data[0], 32)
hmacDrbgGenerate(rng, res.seckey.data.toOpenArray(0, 31))
var hh = sha512.digest(res.seckey.data.toOpenArray(0, 31))
hh.data[0] = hh.data[0] and 0xF8'u8
@@ -1682,11 +1686,11 @@ proc random*(t: typedesc[EdKeyPair], rng: var BrHmacDrbgContext): EdKeyPair =
res
proc getKey*(key: EdPrivateKey): EdPublicKey =
proc getPublicKey*(key: EdPrivateKey): EdPublicKey =
## Calculate and return ED25519 public key from private key ``key``.
copyMem(addr result.data[0], unsafeAddr key.data[32], 32)
proc toBytes*(key: EdPrivateKey, data: var openarray[byte]): int =
proc toBytes*(key: EdPrivateKey, data: var openArray[byte]): int =
## Serialize ED25519 `private key` ``key`` to raw binary form and store it
## to ``data``.
##
@@ -1696,7 +1700,7 @@ proc toBytes*(key: EdPrivateKey, data: var openarray[byte]): int =
if len(data) >= result:
copyMem(addr data[0], unsafeAddr key.data[0], len(key.data))
proc toBytes*(key: EdPublicKey, data: var openarray[byte]): int =
proc toBytes*(key: EdPublicKey, data: var openArray[byte]): int =
## Serialize ED25519 `public key` ``key`` to raw binary form and store it
## to ``data``.
##
@@ -1706,7 +1710,7 @@ proc toBytes*(key: EdPublicKey, data: var openarray[byte]): int =
if len(data) >= result:
copyMem(addr data[0], unsafeAddr key.data[0], len(key.data))
proc toBytes*(sig: EdSignature, data: var openarray[byte]): int =
proc toBytes*(sig: EdSignature, data: var openArray[byte]): int =
## Serialize ED25519 `signature` ``sig`` to raw binary form and store it
## to ``data``.
##
@@ -1749,7 +1753,7 @@ proc `$`*(sig: EdSignature): string =
## Return string representation of ED25519 `signature`.
ncrutils.toHex(sig.data)
proc init*(key: var EdPrivateKey, data: openarray[byte]): bool =
proc init*(key: var EdPrivateKey, data: openArray[byte]): bool =
## Initialize ED25519 `private key` ``key`` from raw binary
## representation ``data``.
##
@@ -1759,7 +1763,7 @@ proc init*(key: var EdPrivateKey, data: openarray[byte]): bool =
copyMem(addr key.data[0], unsafeAddr data[0], length)
result = true
proc init*(key: var EdPublicKey, data: openarray[byte]): bool =
proc init*(key: var EdPublicKey, data: openArray[byte]): bool =
## Initialize ED25519 `public key` ``key`` from raw binary
## representation ``data``.
##
@@ -1769,7 +1773,7 @@ proc init*(key: var EdPublicKey, data: openarray[byte]): bool =
copyMem(addr key.data[0], unsafeAddr data[0], length)
result = true
proc init*(sig: var EdSignature, data: openarray[byte]): bool =
proc init*(sig: var EdSignature, data: openArray[byte]): bool =
## Initialize ED25519 `signature` ``sig`` from raw binary
## representation ``data``.
##
@@ -1801,7 +1805,7 @@ proc init*(sig: var EdSignature, data: string): bool =
init(sig, ncrutils.fromHex(data))
proc init*(t: typedesc[EdPrivateKey],
data: openarray[byte]): Result[EdPrivateKey, EdError] =
data: openArray[byte]): Result[EdPrivateKey, EdError] =
## Initialize ED25519 `private key` from raw binary representation ``data``
## and return constructed object.
var res: t
@@ -1811,7 +1815,7 @@ proc init*(t: typedesc[EdPrivateKey],
ok(res)
proc init*(t: typedesc[EdPublicKey],
data: openarray[byte]): Result[EdPublicKey, EdError] =
data: openArray[byte]): Result[EdPublicKey, EdError] =
## Initialize ED25519 `public key` from raw binary representation ``data``
## and return constructed object.
var res: t
@@ -1821,7 +1825,7 @@ proc init*(t: typedesc[EdPublicKey],
ok(res)
proc init*(t: typedesc[EdSignature],
data: openarray[byte]): Result[EdSignature, EdError] =
data: openArray[byte]): Result[EdSignature, EdError] =
## Initialize ED25519 `signature` from raw binary representation ``data``
## and return constructed object.
var res: t
@@ -1878,7 +1882,7 @@ proc clear*(pair: var EdKeyPair) =
burnMem(pair.pubkey.data)
proc sign*[T: byte|char](key: EdPrivateKey,
message: openarray[T]): EdSignature {.gcsafe, noinit.} =
message: openArray[T]): EdSignature {.gcsafe, noinit.} =
## Create ED25519 signature of data ``message`` using private key ``key``.
var ctx: sha512
var r: GeP3
@@ -1911,7 +1915,7 @@ proc sign*[T: byte|char](key: EdPrivateKey,
scMulAdd(result.data.toOpenArray(32, 63), hram.data.toOpenArray(0, 31),
hash.data.toOpenArray(0, 31), nonce.data.toOpenArray(0, 31))
proc verify*[T: byte|char](sig: EdSignature, message: openarray[T],
proc verify*[T: byte|char](sig: EdSignature, message: openArray[T],
key: EdPublicKey): bool =
## Verify ED25519 signature ``sig`` using public key ``key`` and data
## ``message``.

View File

@@ -1,40 +1,33 @@
## Nim-LibP2P
## Copyright (c) 2020 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-LibP2P
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
# https://tools.ietf.org/html/rfc5869
{.push raises: [Defect].}
{.push raises: [].}
import nimcrypto
import bearssl
import bearssl/[kdf, hash]
type
BearHKDFContext {.importc: "br_hkdf_context", header: "bearssl_kdf.h".} = object
HKDFResult*[len: static int] = array[len, byte]
type HkdfResult*[len: static int] = array[len, byte]
proc br_hkdf_init(ctx: ptr BearHKDFContext; hashClass: ptr HashClass; salt: pointer; len: csize_t) {.importc: "br_hkdf_init", header: "bearssl_kdf.h", raises: [].}
proc br_hkdf_inject(ctx: ptr BearHKDFContext; ikm: pointer; len: csize_t) {.importc: "br_hkdf_inject", header: "bearssl_kdf.h", raises: [].}
proc br_hkdf_flip(ctx: ptr BearHKDFContext) {.importc: "br_hkdf_flip", header: "bearssl_kdf.h", raises: [].}
proc br_hkdf_produce(ctx: ptr BearHKDFContext; info: pointer; infoLen: csize_t; output: pointer; outputLen: csize_t) {.importc: "br_hkdf_produce", header: "bearssl_kdf.h", raises: [].}
proc hkdf*[T: sha256; len: static int](_: type[T]; salt, ikm, info: openarray[byte]; outputs: var openarray[HKDFResult[len]]) =
proc hkdf*[T: sha256; len: static int](_: type[T]; salt, ikm, info: openArray[byte]; outputs: var openArray[HkdfResult[len]]) =
var
ctx: BearHKDFContext
br_hkdf_init(
addr ctx, addr sha256Vtable,
if salt.len > 0: unsafeaddr salt[0] else: nil, csize_t(salt.len))
br_hkdf_inject(
addr ctx, if ikm.len > 0: unsafeaddr ikm[0] else: nil, csize_t(ikm.len))
br_hkdf_flip(addr ctx)
ctx: HkdfContext
hkdfInit(
ctx, addr sha256Vtable,
if salt.len > 0: unsafeAddr salt[0] else: nil, csize_t(salt.len))
hkdfInject(
ctx, if ikm.len > 0: unsafeAddr ikm[0] else: nil, csize_t(ikm.len))
hkdfFlip(ctx)
for i in 0..outputs.high:
br_hkdf_produce(
addr ctx,
if info.len > 0: unsafeaddr info[0]
discard hkdfProduce(
ctx,
if info.len > 0: unsafeAddr info[0]
else: nil, csize_t(info.len),
addr outputs[i][0], csize_t(outputs[i].len))

View File

@@ -1,20 +1,21 @@
## Nim-Libp2p
## Copyright (c) 2018 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-Libp2p
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
## This module implements minimal ASN.1 encoding/decoding primitives.
{.push raises: [Defect].}
{.push raises: [].}
import stew/[endians2, results, ctops]
export results
# We use `ncrutils` for constant-time hexadecimal encoding/decoding procedures.
import nimcrypto/utils as ncrutils
import ../utility
type
Asn1Error* {.pure.} = enum
@@ -116,7 +117,7 @@ template toOpenArray*(af: Asn1Field): untyped =
template isEmpty*(ab: Asn1Buffer): bool =
ab.offset >= len(ab.buffer)
template isEnough*(ab: Asn1Buffer, length: int): bool =
template isEnough*(ab: Asn1Buffer, length: int64): bool =
len(ab.buffer) >= ab.offset + length
proc len*[T: Asn1Buffer|Asn1Composite](abc: T): int {.inline.} =
@@ -154,7 +155,7 @@ proc code*(tag: Asn1Tag): byte {.inline.} =
of Asn1Tag.Context:
0xA0'u8
proc asn1EncodeLength*(dest: var openarray[byte], length: uint64): int =
proc asn1EncodeLength*(dest: var openArray[byte], length: uint64): int =
## Encode ASN.1 DER length part of TLV triple and return number of bytes
## (octets) used.
##
@@ -181,8 +182,8 @@ proc asn1EncodeLength*(dest: var openarray[byte], length: uint64): int =
# then 9, so it is safe to convert it to `int`.
int(res)
proc asn1EncodeInteger*(dest: var openarray[byte],
value: openarray[byte]): int =
proc asn1EncodeInteger*(dest: var openArray[byte],
value: openArray[byte]): int =
## Encode big-endian binary representation of integer as ASN.1 DER `INTEGER`
## and return number of bytes (octets) used.
##
@@ -228,7 +229,7 @@ proc asn1EncodeInteger*(dest: var openarray[byte],
len(value) - offset)
destlen
proc asn1EncodeInteger*[T: SomeUnsignedInt](dest: var openarray[byte],
proc asn1EncodeInteger*[T: SomeUnsignedInt](dest: var openArray[byte],
value: T): int =
## Encode Nim's unsigned integer as ASN.1 DER `INTEGER` and return number of
## bytes (octets) used.
@@ -238,7 +239,7 @@ proc asn1EncodeInteger*[T: SomeUnsignedInt](dest: var openarray[byte],
## but number of bytes (octets) required will be returned.
dest.asn1EncodeInteger(value.toBytesBE())
proc asn1EncodeBoolean*(dest: var openarray[byte], value: bool): int =
proc asn1EncodeBoolean*(dest: var openArray[byte], value: bool): int =
## Encode Nim's boolean as ASN.1 DER `BOOLEAN` and return number of bytes
## (octets) used.
##
@@ -252,7 +253,7 @@ proc asn1EncodeBoolean*(dest: var openarray[byte], value: bool): int =
dest[2] = if value: 0xFF'u8 else: 0x00'u8
res
proc asn1EncodeNull*(dest: var openarray[byte]): int =
proc asn1EncodeNull*(dest: var openArray[byte]): int =
## Encode ASN.1 DER `NULL` and return number of bytes (octets) used.
##
## If length of ``dest`` is less then number of required bytes to encode
@@ -264,8 +265,8 @@ proc asn1EncodeNull*(dest: var openarray[byte]): int =
dest[1] = 0x00'u8
res
proc asn1EncodeOctetString*(dest: var openarray[byte],
value: openarray[byte]): int =
proc asn1EncodeOctetString*(dest: var openArray[byte],
value: openArray[byte]): int =
## Encode array of bytes as ASN.1 DER `OCTET STRING` and return number of
## bytes (octets) used.
##
@@ -282,8 +283,8 @@ proc asn1EncodeOctetString*(dest: var openarray[byte],
copyMem(addr dest[1 + lenlen], unsafeAddr value[0], len(value))
res
proc asn1EncodeBitString*(dest: var openarray[byte],
value: openarray[byte], bits = 0): int =
proc asn1EncodeBitString*(dest: var openArray[byte],
value: openArray[byte], bits = 0): int =
## Encode array of bytes as ASN.1 DER `BIT STRING` and return number of bytes
## (octets) used.
##
@@ -318,7 +319,7 @@ proc asn1EncodeBitString*(dest: var openarray[byte],
dest[2 + lenlen + bytelen - 1] = lastbyte and mask
res
proc asn1EncodeTag[T: SomeUnsignedInt](dest: var openarray[byte],
proc asn1EncodeTag[T: SomeUnsignedInt](dest: var openArray[byte],
value: T): int =
var v = value
if value <= cast[T](0x7F):
@@ -341,33 +342,7 @@ proc asn1EncodeTag[T: SomeUnsignedInt](dest: var openarray[byte],
dest[k - 1] = dest[k - 1] and 0x7F'u8
res
proc asn1EncodeOid*(dest: var openarray[byte], value: openarray[int]): int =
## Encode array of integers ``value`` as ASN.1 DER `OBJECT IDENTIFIER` and
## return number of bytes (octets) used.
##
## If length of ``dest`` is less then number of required bytes to encode
## ``value``, then result of encoding will not be stored in ``dest``
## but number of bytes (octets) required will be returned.
var buffer: array[16, byte]
var res = 1
var oidlen = 1
for i in 2..<len(value):
oidlen += asn1EncodeTag(buffer, cast[uint64](value[i]))
res += asn1EncodeLength(buffer, uint64(oidlen))
res += oidlen
if len(dest) >= res:
let last = dest.high
var offset = 1
dest[0] = Asn1Tag.Oid.code()
offset += asn1EncodeLength(dest.toOpenArray(offset, last), uint64(oidlen))
dest[offset] = cast[byte](value[0] * 40 + value[1])
offset += 1
for i in 2..<len(value):
offset += asn1EncodeTag(dest.toOpenArray(offset, last),
cast[uint64](value[i]))
res
proc asn1EncodeOid*(dest: var openarray[byte], value: openarray[byte]): int =
proc asn1EncodeOid*(dest: var openArray[byte], value: openArray[byte]): int =
## Encode array of bytes ``value`` as ASN.1 DER `OBJECT IDENTIFIER` and return
## number of bytes (octets) used.
##
@@ -386,8 +361,8 @@ proc asn1EncodeOid*(dest: var openarray[byte], value: openarray[byte]): int =
copyMem(addr dest[1 + lenlen], unsafeAddr value[0], len(value))
res
proc asn1EncodeSequence*(dest: var openarray[byte],
value: openarray[byte]): int =
proc asn1EncodeSequence*(dest: var openArray[byte],
value: openArray[byte]): int =
## Encode ``value`` as ASN.1 DER `SEQUENCE` and return number of bytes
## (octets) used.
##
@@ -403,7 +378,7 @@ proc asn1EncodeSequence*(dest: var openarray[byte],
copyMem(addr dest[1 + lenlen], unsafeAddr value[0], len(value))
res
proc asn1EncodeComposite*(dest: var openarray[byte],
proc asn1EncodeComposite*(dest: var openArray[byte],
value: Asn1Composite): int =
## Encode composite value and return number of bytes (octets) used.
##
@@ -420,7 +395,7 @@ proc asn1EncodeComposite*(dest: var openarray[byte],
len(value.buffer))
res
proc asn1EncodeContextTag*(dest: var openarray[byte], value: openarray[byte],
proc asn1EncodeContextTag*(dest: var openArray[byte], value: openArray[byte],
tag: int): int =
## Encode ASN.1 DER `CONTEXT SPECIFIC TAG` ``tag`` for value ``value`` and
## return number of bytes (octets) used.
@@ -440,26 +415,29 @@ proc asn1EncodeContextTag*(dest: var openarray[byte], value: openarray[byte],
copyMem(addr dest[1 + lenlen], unsafeAddr value[0], len(value))
res
proc getLength(ab: var Asn1Buffer): Asn1Result[uint64] =
proc getLength(ab: var Asn1Buffer): Asn1Result[int] =
## Decode length part of ASN.1 TLV triplet.
if not ab.isEmpty():
let b = ab.buffer[ab.offset]
if (b and 0x80'u8) == 0x00'u8:
let length = cast[uint64](b)
let length = safeConvert[int](b)
ab.offset += 1
return ok(length)
if b == 0x80'u8:
return err(Asn1Error.Indefinite)
if b == 0xFF'u8:
return err(Asn1Error.Incorrect)
let octets = cast[uint64](b and 0x7F'u8)
if octets > 8'u64:
let octets = safeConvert[int](b and 0x7F'u8)
if octets > 8:
return err(Asn1Error.Overflow)
if ab.isEnough(int(octets)):
var length: uint64 = 0
for i in 0..<int(octets):
length = (length shl 8) or cast[uint64](ab.buffer[ab.offset + i + 1])
ab.offset = ab.offset + int(octets) + 1
if ab.isEnough(octets):
var lengthU: uint64 = 0
for i in 0..<octets:
lengthU = (lengthU shl 8) or safeConvert[uint64](ab.buffer[ab.offset + i + 1])
if lengthU > uint64(int64.high):
return err(Asn1Error.Overflow)
let length = int(lengthU)
ab.offset = ab.offset + octets + 1
return ok(length)
else:
return err(Asn1Error.Incomplete)
@@ -471,8 +449,8 @@ proc getTag(ab: var Asn1Buffer, tag: var int): Asn1Result[Asn1Class] =
if not ab.isEmpty():
let
b = ab.buffer[ab.offset]
c = int((b and 0xC0'u8) shr 6)
tag = int(b and 0x3F)
c = safeConvert[int]((b and 0xC0'u8) shr 6)
tag = safeConvert[int](b and 0x3F)
ab.offset += 1
if c >= 0 and c < 4:
ok(cast[Asn1Class](c))
@@ -486,7 +464,7 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] =
var
field: Asn1Field
tag, ttag, offset: int
length, tlength: uint64
length, tlength: int
aclass: Asn1Class
inclass: bool
@@ -516,7 +494,7 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] =
if length != 1:
return err(Asn1Error.Incorrect)
if not ab.isEnough(int(length)):
if not ab.isEnough(length):
return err(Asn1Error.Incomplete)
let b = ab.buffer[ab.offset]
@@ -524,9 +502,8 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] =
return err(Asn1Error.Incorrect)
field = Asn1Field(kind: Asn1Tag.Boolean, klass: aclass,
index: ttag, offset: int(ab.offset),
length: 1)
shallowCopy(field.buffer, ab.buffer)
index: ttag, offset: ab.offset,
length: 1, buffer: ab.buffer)
field.vbool = (b == 0xFF'u8)
ab.offset += 1
return ok(field)
@@ -536,12 +513,12 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] =
if length == 0:
return err(Asn1Error.Incorrect)
if not ab.isEnough(int(length)):
if not ab.isEnough(length):
return err(Asn1Error.Incomplete)
# Count number of leading zeroes
var zc = 0
while (zc < int(length)) and (ab.buffer[ab.offset + zc] == 0x00'u8):
while (zc < length) and (ab.buffer[ab.offset + zc] == 0x00'u8):
inc(zc)
if zc > 1:
@@ -550,47 +527,45 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] =
if zc == 0:
# Negative or Positive integer
field = Asn1Field(kind: Asn1Tag.Integer, klass: aclass,
index: ttag, offset: int(ab.offset),
length: int(length))
shallowCopy(field.buffer, ab.buffer)
index: ttag, offset: ab.offset,
length: length, buffer: ab.buffer)
if (ab.buffer[ab.offset] and 0x80'u8) == 0x80'u8:
# Negative integer
if length <= 8:
# We need this transformation because our field.vint is uint64.
for i in 0 ..< 8:
if i < 8 - int(length):
if i < 8 - length:
field.vint = (field.vint shl 8) or 0xFF'u64
else:
let offset = ab.offset + i - (8 - int(length))
field.vint = (field.vint shl 8) or uint64(ab.buffer[offset])
let offset = ab.offset + i - (8 - length)
field.vint = (field.vint shl 8) or safeConvert[uint64](ab.buffer[offset])
else:
# Positive integer
if length <= 8:
for i in 0 ..< int(length):
for i in 0 ..< length:
field.vint = (field.vint shl 8) or
uint64(ab.buffer[ab.offset + i])
ab.offset += int(length)
safeConvert[uint64](ab.buffer[ab.offset + i])
ab.offset += length
return ok(field)
else:
if length == 1:
# Zero value integer
field = Asn1Field(kind: Asn1Tag.Integer, klass: aclass,
index: ttag, offset: int(ab.offset),
length: int(length), vint: 0'u64)
shallowCopy(field.buffer, ab.buffer)
ab.offset += int(length)
index: ttag, offset: ab.offset,
length: length, vint: 0'u64,
buffer: ab.buffer)
ab.offset += length
return ok(field)
else:
# Positive integer with leading zero
field = Asn1Field(kind: Asn1Tag.Integer, klass: aclass,
index: ttag, offset: int(ab.offset) + 1,
length: int(length) - 1)
shallowCopy(field.buffer, ab.buffer)
index: ttag, offset: ab.offset + 1,
length: length - 1, buffer: ab.buffer)
if length <= 9:
for i in 1 ..< int(length):
for i in 1 ..< length:
field.vint = (field.vint shl 8) or
uint64(ab.buffer[ab.offset + i])
ab.offset += int(length)
safeConvert[uint64](ab.buffer[ab.offset + i])
ab.offset += length
return ok(field)
of Asn1Tag.BitString.code():
@@ -606,14 +581,13 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] =
else:
# Zero-length BIT STRING.
field = Asn1Field(kind: Asn1Tag.BitString, klass: aclass,
index: ttag, offset: int(ab.offset + 1),
length: 0, ubits: 0)
shallowCopy(field.buffer, ab.buffer)
ab.offset += int(length)
index: ttag, offset: ab.offset + 1,
length: 0, ubits: 0, buffer: ab.buffer)
ab.offset += length
return ok(field)
else:
if not ab.isEnough(int(length)):
if not ab.isEnough(length):
return err(Asn1Error.Incomplete)
let unused = ab.buffer[ab.offset]
@@ -621,28 +595,27 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] =
# Number of unused bits should not be bigger then `7`.
return err(Asn1Error.Incorrect)
let mask = (1'u8 shl int(unused)) - 1'u8
if (ab.buffer[ab.offset + int(length) - 1] and mask) != 0x00'u8:
let mask = (1'u8 shl safeConvert[int](unused)) - 1'u8
if (ab.buffer[ab.offset + length - 1] and mask) != 0x00'u8:
## All unused bits should be set to `0`.
return err(Asn1Error.Incorrect)
field = Asn1Field(kind: Asn1Tag.BitString, klass: aclass,
index: ttag, offset: int(ab.offset + 1),
length: int(length - 1), ubits: int(unused))
shallowCopy(field.buffer, ab.buffer)
ab.offset += int(length)
index: ttag, offset: ab.offset + 1,
length: length - 1, ubits: safeConvert[int](unused),
buffer: ab.buffer)
ab.offset += length
return ok(field)
of Asn1Tag.OctetString.code():
# OCTET STRING
if not ab.isEnough(int(length)):
if not ab.isEnough(length):
return err(Asn1Error.Incomplete)
field = Asn1Field(kind: Asn1Tag.OctetString, klass: aclass,
index: ttag, offset: int(ab.offset),
length: int(length))
shallowCopy(field.buffer, ab.buffer)
ab.offset += int(length)
index: ttag, offset: ab.offset,
length: length, buffer: ab.buffer)
ab.offset += length
return ok(field)
of Asn1Tag.Null.code():
@@ -651,33 +624,30 @@ proc read*(ab: var Asn1Buffer): Asn1Result[Asn1Field] =
return err(Asn1Error.Incorrect)
field = Asn1Field(kind: Asn1Tag.Null, klass: aclass, index: ttag,
offset: int(ab.offset), length: 0)
shallowCopy(field.buffer, ab.buffer)
ab.offset += int(length)
offset: ab.offset, length: 0, buffer: ab.buffer)
ab.offset += length
return ok(field)
of Asn1Tag.Oid.code():
# OID
if not ab.isEnough(int(length)):
if not ab.isEnough(length):
return err(Asn1Error.Incomplete)
field = Asn1Field(kind: Asn1Tag.Oid, klass: aclass,
index: ttag, offset: int(ab.offset),
length: int(length))
shallowCopy(field.buffer, ab.buffer)
ab.offset += int(length)
index: ttag, offset: ab.offset,
length: length, buffer: ab.buffer)
ab.offset += length
return ok(field)
of Asn1Tag.Sequence.code():
# SEQUENCE
if not ab.isEnough(int(length)):
if not ab.isEnough(length):
return err(Asn1Error.Incomplete)
field = Asn1Field(kind: Asn1Tag.Sequence, klass: aclass,
index: ttag, offset: int(ab.offset),
length: int(length))
shallowCopy(field.buffer, ab.buffer)
ab.offset += int(length)
index: ttag, offset: ab.offset,
length: length, buffer: ab.buffer)
ab.offset += length
return ok(field)
else:
@@ -692,7 +662,7 @@ proc getBuffer*(field: Asn1Field): Asn1Buffer {.inline.} =
## Return ``field`` as Asn1Buffer to enter composite types.
Asn1Buffer(buffer: field.buffer, offset: field.offset, length: field.length)
proc `==`*(field: Asn1Field, data: openarray[byte]): bool =
proc `==`*(field: Asn1Field, data: openArray[byte]): bool =
## Compares field ``field`` data with ``data`` and returns ``true`` if both
## buffers are equal.
let length = len(field.buffer)
@@ -710,7 +680,7 @@ proc `==`*(field: Asn1Field, data: openarray[byte]): bool =
else:
false
proc init*(t: typedesc[Asn1Buffer], data: openarray[byte]): Asn1Buffer =
proc init*(t: typedesc[Asn1Buffer], data: openArray[byte]): Asn1Buffer =
## Initialize ``Asn1Buffer`` from array of bytes ``data``.
Asn1Buffer(buffer: @data)
@@ -825,7 +795,7 @@ proc write*[T: Asn1Buffer|Asn1Composite](abc: var T, value: bool) =
abc.offset += length
proc write*[T: Asn1Buffer|Asn1Composite](abc: var T, tag: Asn1Tag,
value: openarray[byte], bits = 0) =
value: openArray[byte], bits = 0) =
## Write array ``value`` using ``tag``.
##
## This procedure is used to write ASN.1 `INTEGER`, `OCTET STRING`,

View File

@@ -1,11 +1,11 @@
## Nim-Libp2p
## Copyright (c) 2018 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-Libp2p
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
## This module implements constant-time RSA PKCS#1.5 DSA.
##
@@ -13,8 +13,9 @@
## BearSSL library <https://bearssl.org/>
## Copyright(C) 2018 Thomas Pornin <pornin@bolet.org>.
{.push raises: Defect.}
import bearssl
{.push raises: [].}
import bearssl/[rsa, rand, hash]
import minasn1
import stew/[results, ctops]
# We use `ncrutils` for constant-time hexadecimal encoding/decoding procedures.
@@ -33,41 +34,41 @@ const
## Default RSA key size in bits.
RsaOidSha1* = [
0x05'u8, 0x2B'u8, 0x0E'u8, 0x03'u8, 0x02'u8, 0x1A'u8
byte 0x05, 0x2B, 0x0E, 0x03, 0x02, 0x1A
]
## RSA PKCS#1.5 SHA-1 hash object identifier.
RsaOidSha224* = [
0x09'u8, 0x60'u8, 0x86'u8, 0x48'u8, 0x01'u8, 0x65'u8, 0x03'u8, 0x04'u8,
0x02'u8, 0x04'u8
byte 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04,
0x02, 0x04
]
## RSA PKCS#1.5 SHA-224 hash object identifier.
RsaOidSha256* = [
0x09'u8, 0x60'u8, 0x86'u8, 0x48'u8, 0x01'u8, 0x65'u8, 0x03'u8, 0x04'u8,
0x02'u8, 0x01'u8
byte 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04,
0x02, 0x01
]
## RSA PKCS#1.5 SHA-256 hash object identifier.
RsaOidSha384* = [
0x09'u8, 0x60'u8, 0x86'u8, 0x48'u8, 0x01'u8, 0x65'u8, 0x03'u8, 0x04'u8,
0x02'u8, 0x02'u8
byte 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04,
0x02, 0x02
]
## RSA PKCS#1.5 SHA-384 hash object identifier.
RsaOidSha512* = [
0x09'u8, 0x60'u8, 0x86'u8, 0x48'u8, 0x01'u8, 0x65'u8, 0x03'u8, 0x04'u8,
0x02'u8, 0x03'u8
byte 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04,
0x02, 0x03
]
## RSA PKCS#1.5 SHA-512 hash object identifier.
type
RsaPrivateKey* = ref object
buffer*: seq[byte]
seck*: BrRsaPrivateKey
pubk*: BrRsaPublicKey
pexp*: ptr cuchar
pexplen*: int
seck*: rsa.RsaPrivateKey
pubk*: rsa.RsaPublicKey
pexp*: ptr byte
pexplen*: uint
RsaPublicKey* = ref object
buffer*: seq[byte]
key*: BrRsaPublicKey
key*: rsa.RsaPublicKey
RsaKeyPair* = RsaPrivateKey
@@ -99,8 +100,8 @@ template getFinish(bs, os, ls: untyped): untyped =
var eo = -1
if p >= s:
let so = cast[int](p - s)
if so + ls <= len(bs):
eo = so + ls - 1
if so + int(ls) <= len(bs):
eo = so + int(ls) - 1
eo
template getArray*(bs, os, ls: untyped): untyped =
@@ -109,12 +110,12 @@ template getArray*(bs, os, ls: untyped): untyped =
template trimZeroes(b: seq[byte], pt, ptlen: untyped) =
var length = ptlen
for i in 0..<length:
if pt[] != cast[cuchar](0x00'u8):
if pt[] != byte(0x00):
break
pt = cast[ptr cuchar](cast[uint](pt) + 1)
pt = cast[ptr byte](cast[uint](pt) + 1)
ptlen -= 1
proc random*[T: RsaKP](t: typedesc[T], rng: var BrHmacDrbgContext,
proc random*[T: RsaKP](t: typedesc[T], rng: var HmacDrbgContext,
bits = DefaultKeySize,
pubexp = DefaultPublicExponent): RsaResult[T] =
## Generate new random RSA private key using BearSSL's HMAC-SHA256-DRBG
@@ -129,14 +130,14 @@ proc random*[T: RsaKP](t: typedesc[T], rng: var BrHmacDrbgContext,
let
sko = 0
pko = brRsaPrivateKeyBufferSize(bits)
eko = pko + brRsaPublicKeyBufferSize(bits)
pko = rsaKbufPrivSize(bits)
eko = pko + rsaKbufPubSize(bits)
length = eko + ((bits + 7) shr 3)
let res = new T
res.buffer = newSeq[byte](length)
var keygen = brRsaKeygenGetDefault()
var keygen = rsaKeygenGetDefault()
if keygen(addr rng.vtable,
addr res.seck, addr res.buffer[sko],
@@ -145,12 +146,12 @@ proc random*[T: RsaKP](t: typedesc[T], rng: var BrHmacDrbgContext,
return err(RsaGenError)
let
compute = brRsaComputePrivexpGetDefault()
compute = rsaComputePrivexpGetDefault()
computed = compute(addr res.buffer[eko], addr res.seck, pubexp)
if computed == 0:
return err(RsaGenError)
res.pexp = cast[ptr cuchar](addr res.buffer[eko])
res.pexp = addr res.buffer[eko]
res.pexplen = computed
trimZeroes(res.buffer, res.seck.p, res.seck.plen)
@@ -169,12 +170,12 @@ proc copy*[T: RsaPKI](key: T): T =
doAssert(not isNil(key))
when T is RsaPrivateKey:
if len(key.buffer) > 0:
let length = key.seck.plen + key.seck.qlen + key.seck.dplen +
key.seck.dqlen + key.seck.iqlen + key.pubk.nlen +
key.pubk.elen + key.pexplen
let length = key.seck.plen.uint + key.seck.qlen.uint + key.seck.dplen.uint +
key.seck.dqlen.uint + key.seck.iqlen.uint + key.pubk.nlen.uint +
key.pubk.elen.uint + key.pexplen.uint
result = new RsaPrivateKey
result.buffer = newSeq[byte](length)
let po = 0
let po: uint = 0
let qo = po + key.seck.plen
let dpo = qo + key.seck.qlen
let dqo = dpo + key.seck.dplen
@@ -190,14 +191,14 @@ proc copy*[T: RsaPKI](key: T): T =
copyMem(addr result.buffer[no], key.pubk.n, key.pubk.nlen)
copyMem(addr result.buffer[eo], key.pubk.e, key.pubk.elen)
copyMem(addr result.buffer[peo], key.pexp, key.pexplen)
result.seck.p = cast[ptr cuchar](addr result.buffer[po])
result.seck.q = cast[ptr cuchar](addr result.buffer[qo])
result.seck.dp = cast[ptr cuchar](addr result.buffer[dpo])
result.seck.dq = cast[ptr cuchar](addr result.buffer[dqo])
result.seck.iq = cast[ptr cuchar](addr result.buffer[iqo])
result.pubk.n = cast[ptr cuchar](addr result.buffer[no])
result.pubk.e = cast[ptr cuchar](addr result.buffer[eo])
result.pexp = cast[ptr cuchar](addr result.buffer[peo])
result.seck.p = addr result.buffer[po]
result.seck.q = addr result.buffer[qo]
result.seck.dp = addr result.buffer[dpo]
result.seck.dq = addr result.buffer[dqo]
result.seck.iq = addr result.buffer[iqo]
result.pubk.n = addr result.buffer[no]
result.pubk.e = addr result.buffer[eo]
result.pexp = addr result.buffer[peo]
result.seck.plen = key.seck.plen
result.seck.qlen = key.seck.qlen
result.seck.dplen = key.seck.dplen
@@ -216,8 +217,8 @@ proc copy*[T: RsaPKI](key: T): T =
let eo = no + key.key.nlen
copyMem(addr result.buffer[no], key.key.n, key.key.nlen)
copyMem(addr result.buffer[eo], key.key.e, key.key.elen)
result.key.n = cast[ptr cuchar](addr result.buffer[no])
result.key.e = cast[ptr cuchar](addr result.buffer[eo])
result.key.n = cast[ptr char](addr result.buffer[no])
result.key.e = cast[ptr char](addr result.buffer[eo])
result.key.nlen = key.key.nlen
result.key.elen = key.key.elen
elif T is RsaSignature:
@@ -225,14 +226,14 @@ proc copy*[T: RsaPKI](key: T): T =
result = new RsaSignature
result.buffer = key.buffer
proc getKey*(key: RsaPrivateKey): RsaPublicKey =
proc getPublicKey*(key: RsaPrivateKey): RsaPublicKey =
## Get RSA public key from RSA private key.
doAssert(not isNil(key))
let length = key.pubk.nlen + key.pubk.elen
result = new RsaPublicKey
result.buffer = newSeq[byte](length)
result.key.n = cast[ptr cuchar](addr result.buffer[0])
result.key.e = cast[ptr cuchar](addr result.buffer[key.pubk.nlen])
result.key.n = addr result.buffer[0]
result.key.e = addr result.buffer[key.pubk.nlen]
copyMem(addr result.buffer[0], cast[pointer](key.pubk.n), key.pubk.nlen)
copyMem(addr result.buffer[key.pubk.nlen], cast[pointer](key.pubk.e),
key.pubk.elen)
@@ -245,7 +246,7 @@ proc seckey*(pair: RsaKeyPair): RsaPrivateKey {.inline.} =
proc pubkey*(pair: RsaKeyPair): RsaPublicKey {.inline.} =
## Get RSA public key from pair ``pair``.
result = cast[RsaPrivateKey](pair).getKey()
result = cast[RsaPrivateKey](pair).getPublicKey()
proc clear*[T: RsaPKI|RsaKeyPair](pki: var T) =
## Wipe and clear EC private key, public key or scalar object.
@@ -279,7 +280,7 @@ proc clear*[T: RsaPKI|RsaKeyPair](pki: var T) =
burnMem(pki.buffer)
pki.buffer.setLen(0)
proc toBytes*(key: RsaPrivateKey, data: var openarray[byte]): RsaResult[int] =
proc toBytes*(key: RsaPrivateKey, data: var openArray[byte]): RsaResult[int] =
## Serialize RSA private key ``key`` to ASN.1 DER binary form and store it
## to ``data``.
##
@@ -316,7 +317,7 @@ proc toBytes*(key: RsaPrivateKey, data: var openarray[byte]): RsaResult[int] =
else:
err(RsaKeyIncorrectError)
proc toBytes*(key: RsaPublicKey, data: var openarray[byte]): RsaResult[int] =
proc toBytes*(key: RsaPublicKey, data: var openArray[byte]): RsaResult[int] =
## Serialize RSA public key ``key`` to ASN.1 DER binary form and store it
## to ``data``.
##
@@ -350,7 +351,7 @@ proc toBytes*(key: RsaPublicKey, data: var openarray[byte]): RsaResult[int] =
else:
err(RsaKeyIncorrectError)
proc toBytes*(sig: RsaSignature, data: var openarray[byte]): RSaResult[int] =
proc toBytes*(sig: RsaSignature, data: var openArray[byte]): RsaResult[int] =
## Serialize RSA signature ``sig`` to raw binary form and store it
## to ``data``.
##
@@ -402,7 +403,7 @@ proc getBytes*(sig: RsaSignature): RsaResult[seq[byte]] =
else:
err(RsaSignatureError)
proc init*(key: var RsaPrivateKey, data: openarray[byte]): Result[void, Asn1Error] =
proc init*(key: var RsaPrivateKey, data: openArray[byte]): Result[void, Asn1Error] =
## Initialize RSA private key ``key`` from ASN.1 DER binary representation
## ``data``.
##
@@ -472,28 +473,28 @@ proc init*(key: var RsaPrivateKey, data: openarray[byte]): Result[void, Asn1Erro
len(rawdp) > 0 and len(rawdq) > 0 and len(rawiq) > 0:
key = new RsaPrivateKey
key.buffer = @data
key.pubk.n = cast[ptr cuchar](addr key.buffer[rawn.offset])
key.pubk.e = cast[ptr cuchar](addr key.buffer[rawpube.offset])
key.seck.p = cast[ptr cuchar](addr key.buffer[rawp.offset])
key.seck.q = cast[ptr cuchar](addr key.buffer[rawq.offset])
key.seck.dp = cast[ptr cuchar](addr key.buffer[rawdp.offset])
key.seck.dq = cast[ptr cuchar](addr key.buffer[rawdq.offset])
key.seck.iq = cast[ptr cuchar](addr key.buffer[rawiq.offset])
key.pexp = cast[ptr cuchar](addr key.buffer[rawprie.offset])
key.pubk.nlen = len(rawn)
key.pubk.elen = len(rawpube)
key.seck.plen = len(rawp)
key.seck.qlen = len(rawq)
key.seck.dplen = len(rawdp)
key.seck.dqlen = len(rawdq)
key.seck.iqlen = len(rawiq)
key.pexplen = len(rawprie)
key.pubk.n = addr key.buffer[rawn.offset]
key.pubk.e = addr key.buffer[rawpube.offset]
key.seck.p = addr key.buffer[rawp.offset]
key.seck.q = addr key.buffer[rawq.offset]
key.seck.dp = addr key.buffer[rawdp.offset]
key.seck.dq = addr key.buffer[rawdq.offset]
key.seck.iq = addr key.buffer[rawiq.offset]
key.pexp = addr key.buffer[rawprie.offset]
key.pubk.nlen = uint(len(rawn))
key.pubk.elen = uint(len(rawpube))
key.seck.plen = uint(len(rawp))
key.seck.qlen = uint(len(rawq))
key.seck.dplen = uint(len(rawdp))
key.seck.dqlen = uint(len(rawdq))
key.seck.iqlen = uint(len(rawiq))
key.pexplen = uint(len(rawprie))
key.seck.nBitlen = cast[uint32](len(rawn) shl 3)
ok()
else:
err(Asn1Error.Incorrect)
proc init*(key: var RsaPublicKey, data: openarray[byte]): Result[void, Asn1Error] =
proc init*(key: var RsaPublicKey, data: openArray[byte]): Result[void, Asn1Error] =
## Initialize RSA public key ``key`` from ASN.1 DER binary representation
## ``data``.
##
@@ -554,15 +555,15 @@ proc init*(key: var RsaPublicKey, data: openarray[byte]): Result[void, Asn1Error
if len(rawn) >= (MinKeySize shr 3) and len(rawe) > 0:
key = new RsaPublicKey
key.buffer = @data
key.key.n = cast[ptr cuchar](addr key.buffer[rawn.offset])
key.key.e = cast[ptr cuchar](addr key.buffer[rawe.offset])
key.key.nlen = len(rawn)
key.key.elen = len(rawe)
key.key.n = addr key.buffer[rawn.offset]
key.key.e = addr key.buffer[rawe.offset]
key.key.nlen = uint(len(rawn))
key.key.elen = uint(len(rawe))
ok()
else:
err(Asn1Error.Incorrect)
proc init*(sig: var RsaSignature, data: openarray[byte]): Result[void, Asn1Error] =
proc init*(sig: var RsaSignature, data: openArray[byte]): Result[void, Asn1Error] =
## Initialize RSA signature ``sig`` from ASN.1 DER binary representation
## ``data``.
##
@@ -583,7 +584,7 @@ proc init*[T: RsaPKI](sospk: var T,
sospk.init(ncrutils.fromHex(data))
proc init*(t: typedesc[RsaPrivateKey],
data: openarray[byte]): RsaResult[RsaPrivateKey] =
data: openArray[byte]): RsaResult[RsaPrivateKey] =
## Initialize RSA private key from ASN.1 DER binary representation ``data``
## and return constructed object.
var res: RsaPrivateKey
@@ -593,7 +594,7 @@ proc init*(t: typedesc[RsaPrivateKey],
ok(res)
proc init*(t: typedesc[RsaPublicKey],
data: openarray[byte]): RsaResult[RsaPublicKey] =
data: openArray[byte]): RsaResult[RsaPublicKey] =
## Initialize RSA public key from ASN.1 DER binary representation ``data``
## and return constructed object.
var res: RsaPublicKey
@@ -603,7 +604,7 @@ proc init*(t: typedesc[RsaPublicKey],
ok(res)
proc init*(t: typedesc[RsaSignature],
data: openarray[byte]): RsaResult[RsaSignature] =
data: openArray[byte]): RsaResult[RsaSignature] =
## Initialize RSA signature from raw binary representation ``data`` and
## return constructed object.
var res: RsaSignature
@@ -682,7 +683,7 @@ proc `==`*(a, b: RsaPrivateKey): bool =
false
else:
if a.seck.nBitlen == b.seck.nBitlen:
if cast[int](a.seck.nBitlen) > 0:
if a.seck.nBitlen > 0'u:
let r1 = CT.isEqual(getArray(a.buffer, a.seck.p, a.seck.plen),
getArray(b.buffer, b.seck.p, b.seck.plen))
let r2 = CT.isEqual(getArray(a.buffer, a.seck.q, a.seck.qlen),
@@ -743,34 +744,34 @@ proc `==`*(a, b: RsaPublicKey): bool =
(r1 and r2)
proc sign*[T: byte|char](key: RsaPrivateKey,
message: openarray[T]): RsaResult[RsaSignature] {.gcsafe.} =
message: openArray[T]): RsaResult[RsaSignature] {.gcsafe.} =
## Get RSA PKCS1.5 signature of data ``message`` using SHA256 and private
## key ``key``.
if isNil(key):
return err(RsaKeyIncorrectError)
var hc: BrHashCompatContext
var hc: HashCompatContext
var hash: array[32, byte]
let impl = BrRsaPkcs1SignGetDefault()
let impl = rsaPkcs1SignGetDefault()
var res = new RsaSignature
res.buffer = newSeq[byte]((key.seck.nBitlen + 7) shr 3)
var kv = addr sha256Vtable
kv.init(addr hc.vtable)
if len(message) > 0:
kv.update(addr hc.vtable, unsafeAddr message[0], len(message))
kv.update(addr hc.vtable, unsafeAddr message[0], uint(len(message)))
else:
kv.update(addr hc.vtable, nil, 0)
kv.output(addr hc.vtable, addr hash[0])
kv.out(addr hc.vtable, addr hash[0])
var oid = RsaOidSha256
let implRes = impl(cast[ptr cuchar](addr oid[0]),
cast[ptr cuchar](addr hash[0]), len(hash),
addr key.seck, cast[ptr cuchar](addr res.buffer[0]))
let implRes = impl(addr oid[0],
addr hash[0], uint(len(hash)),
addr key.seck, addr res.buffer[0])
if implRes == 0:
err(RsaSignatureError)
else:
ok(res)
proc verify*[T: byte|char](sig: RsaSignature, message: openarray[T],
proc verify*[T: byte|char](sig: RsaSignature, message: openArray[T],
pubkey: RsaPublicKey): bool {.inline.} =
## Verify RSA signature ``sig`` using public key ``pubkey`` and data
## ``message``.
@@ -779,20 +780,20 @@ proc verify*[T: byte|char](sig: RsaSignature, message: openarray[T],
## verification failed.
doAssert((not isNil(sig)) and (not isNil(pubkey)))
if len(sig.buffer) > 0:
var hc: BrHashCompatContext
var hc: HashCompatContext
var hash: array[32, byte]
var check: array[32, byte]
var impl = BrRsaPkcs1VrfyGetDefault()
var impl = rsaPkcs1VrfyGetDefault()
var kv = addr sha256Vtable
kv.init(addr hc.vtable)
if len(message) > 0:
kv.update(addr hc.vtable, unsafeAddr message[0], len(message))
kv.update(addr hc.vtable, unsafeAddr message[0], uint(len(message)))
else:
kv.update(addr hc.vtable, nil, 0)
kv.output(addr hc.vtable, addr hash[0])
kv.out(addr hc.vtable, addr hash[0])
var oid = RsaOidSha256
let res = impl(cast[ptr cuchar](addr sig.buffer[0]), len(sig.buffer),
cast[ptr cuchar](addr oid[0]),
len(check), addr pubkey.key, cast[ptr cuchar](addr check[0]))
let res = impl(addr sig.buffer[0], uint(len(sig.buffer)),
addr oid[0],
uint(len(check)), addr pubkey.key, addr check[0])
if res == 1:
result = equalMem(addr check[0], addr hash[0], len(hash))

View File

@@ -1,20 +1,21 @@
## Nim-Libp2p
## Copyright (c) 2018 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-Libp2p
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
{.push raises: [Defect].}
{.push raises: [].}
import bearssl/rand
import
secp256k1, bearssl,
secp256k1,
stew/[byteutils, results],
nimcrypto/[hash, sha2]
export sha2, results
export sha2, results, rand
const
SkRawPrivateKeySize* = 256 div 8
@@ -31,20 +32,18 @@ type
SkSignature* = distinct secp256k1.SkSignature
SkKeyPair* = distinct secp256k1.SkKeyPair
template pubkey*(v: SkKeyPair): SkPublicKey = SkPublicKey(secp256k1.SkKeyPair(v).pubkey)
template seckey*(v: SkKeyPair): SkPrivateKey = SkPrivateKey(secp256k1.SkKeyPair(v).seckey)
proc random*(t: typedesc[SkPrivateKey], rng: var BrHmacDrbgContext): SkPrivateKey =
let rngPtr = unsafeAddr rng # doesn't escape
proc random*(t: typedesc[SkPrivateKey], rng: var HmacDrbgContext): SkPrivateKey =
#TODO is there a better way?
var rngPtr = addr rng
proc callRng(data: var openArray[byte]) =
brHmacDrbgGenerate(rngPtr[], data)
hmacDrbgGenerate(rngPtr[], data)
SkPrivateKey(SkSecretKey.random(callRng))
proc random*(t: typedesc[SkKeyPair], rng: var BrHmacDrbgContext): SkKeyPair =
let rngPtr = unsafeAddr rng # doesn't escape
proc random*(t: typedesc[SkKeyPair], rng: var HmacDrbgContext): SkKeyPair =
let rngPtr = addr rng
proc callRng(data: var openArray[byte]) =
brHmacDrbgGenerate(rngPtr[], data)
hmacDrbgGenerate(rngPtr[], data)
SkKeyPair(secp256k1.SkKeyPair.random(callRng))
@@ -54,7 +53,7 @@ template seckey*(v: SkKeyPair): SkPrivateKey =
template pubkey*(v: SkKeyPair): SkPublicKey =
SkPublicKey(secp256k1.SkKeyPair(v).pubkey)
proc init*(key: var SkPrivateKey, data: openarray[byte]): SkResult[void] =
proc init*(key: var SkPrivateKey, data: openArray[byte]): SkResult[void] =
## Initialize Secp256k1 `private key` ``key`` from raw binary
## representation ``data``.
key = SkPrivateKey(? secp256k1.SkSecretKey.fromRaw(data))
@@ -66,7 +65,7 @@ proc init*(key: var SkPrivateKey, data: string): SkResult[void] =
key = SkPrivateKey(? secp256k1.SkSecretKey.fromHex(data))
ok()
proc init*(key: var SkPublicKey, data: openarray[byte]): SkResult[void] =
proc init*(key: var SkPublicKey, data: openArray[byte]): SkResult[void] =
## Initialize Secp256k1 `public key` ``key`` from raw binary
## representation ``data``.
key = SkPublicKey(? secp256k1.SkPublicKey.fromRaw(data))
@@ -78,7 +77,7 @@ proc init*(key: var SkPublicKey, data: string): SkResult[void] =
key = SkPublicKey(? secp256k1.SkPublicKey.fromHex(data))
ok()
proc init*(sig: var SkSignature, data: openarray[byte]): SkResult[void] =
proc init*(sig: var SkSignature, data: openArray[byte]): SkResult[void] =
## Initialize Secp256k1 `signature` ``sig`` from raw binary
## representation ``data``.
sig = SkSignature(? secp256k1.SkSignature.fromDer(data))
@@ -95,7 +94,7 @@ proc init*(sig: var SkSignature, data: string): SkResult[void] =
return err("secp: Hex to bytes failed")
init(sig, buffer)
proc init*(t: typedesc[SkPrivateKey], data: openarray[byte]): SkResult[SkPrivateKey] =
proc init*(t: typedesc[SkPrivateKey], data: openArray[byte]): SkResult[SkPrivateKey] =
## Initialize Secp256k1 `private key` from raw binary
## representation ``data``.
##
@@ -109,7 +108,7 @@ proc init*(t: typedesc[SkPrivateKey], data: string): SkResult[SkPrivateKey] =
## Procedure returns `private key` on success.
SkSecretKey.fromHex(data).mapConvert(SkPrivateKey)
proc init*(t: typedesc[SkPublicKey], data: openarray[byte]): SkResult[SkPublicKey] =
proc init*(t: typedesc[SkPublicKey], data: openArray[byte]): SkResult[SkPublicKey] =
## Initialize Secp256k1 `public key` from raw binary
## representation ``data``.
##
@@ -125,7 +124,7 @@ proc init*(t: typedesc[SkPublicKey], data: string): SkResult[SkPublicKey] =
var key: SkPublicKey
key.init(data) and ok(key)
proc init*(t: typedesc[SkSignature], data: openarray[byte]): SkResult[SkSignature] =
proc init*(t: typedesc[SkSignature], data: openArray[byte]): SkResult[SkSignature] =
## Initialize Secp256k1 `signature` from raw binary
## representation ``data``.
##
@@ -141,11 +140,11 @@ proc init*(t: typedesc[SkSignature], data: string): SkResult[SkSignature] =
var sig: SkSignature
sig.init(data) and ok(sig)
proc getKey*(key: SkPrivateKey): SkPublicKey =
proc getPublicKey*(key: SkPrivateKey): SkPublicKey =
## Calculate and return Secp256k1 `public key` from `private key` ``key``.
SkPublicKey(SkSecretKey(key).toPublicKey())
proc toBytes*(key: SkPrivateKey, data: var openarray[byte]): SkResult[int] =
proc toBytes*(key: SkPrivateKey, data: var openArray[byte]): SkResult[int] =
## Serialize Secp256k1 `private key` ``key`` to raw binary form and store it
## to ``data``.
##
@@ -157,7 +156,7 @@ proc toBytes*(key: SkPrivateKey, data: var openarray[byte]): SkResult[int] =
else:
err("secp: Not enough bytes")
proc toBytes*(key: SkPublicKey, data: var openarray[byte]): SkResult[int] =
proc toBytes*(key: SkPublicKey, data: var openArray[byte]): SkResult[int] =
## Serialize Secp256k1 `public key` ``key`` to raw binary form and store it
## to ``data``.
##
@@ -169,7 +168,7 @@ proc toBytes*(key: SkPublicKey, data: var openarray[byte]): SkResult[int] =
else:
err("secp: Not enough bytes")
proc toBytes*(sig: SkSignature, data: var openarray[byte]): int =
proc toBytes*(sig: SkSignature, data: var openArray[byte]): int =
## Serialize Secp256k1 `signature` ``sig`` to raw binary form and store it
## to ``data``.
##
@@ -191,24 +190,28 @@ proc getBytes*(sig: SkSignature): seq[byte] {.inline.} =
let length = toBytes(sig, result)
result.setLen(length)
proc sign*[T: byte|char](key: SkPrivateKey, msg: openarray[T]): SkSignature =
proc sign*[T: byte|char](key: SkPrivateKey, msg: openArray[T]): SkSignature =
## Sign message `msg` using private key `key` and return signature object.
let h = sha256.digest(msg)
SkSignature(sign(SkSecretKey(key), SkMessage(h.data)))
proc verify*[T: byte|char](sig: SkSignature, msg: openarray[T],
proc verify*[T: byte|char](sig: SkSignature, msg: openArray[T],
key: SkPublicKey): bool =
let h = sha256.digest(msg)
verify(secp256k1.SkSignature(sig), SkMessage(h.data), secp256k1.SkPublicKey(key))
func clear*(key: var SkPrivateKey) {.borrow.}
func clear*(key: var SkPrivateKey) = clear(secp256k1.SkSecretKey(key))
proc `$`*(key: SkPrivateKey): string {.borrow.}
proc `$`*(key: SkPublicKey): string {.borrow.}
proc `$`*(key: SkSignature): string {.borrow.}
proc `$`*(key: SkKeyPair): string {.borrow.}
func `$`*(key: SkPrivateKey): string = $secp256k1.SkSecretKey(key)
func `$`*(key: SkPublicKey): string = $secp256k1.SkPublicKey(key)
func `$`*(key: SkSignature): string = $secp256k1.SkSignature(key)
func `$`*(key: SkKeyPair): string = $secp256k1.SkKeyPair(key)
proc `==`*(a, b: SkPrivateKey): bool {.borrow.}
proc `==`*(a, b: SkPublicKey): bool {.borrow.}
proc `==`*(a, b: SkSignature): bool {.borrow.}
proc `==`*(a, b: SkKeyPair): bool {.borrow.}
func `==`*(a, b: SkPrivateKey): bool =
secp256k1.SkSecretKey(a) == secp256k1.SkSecretKey(b)
func `==`*(a, b: SkPublicKey): bool =
secp256k1.SkPublicKey(a) == secp256k1.SkPublicKey(b)
func `==`*(a, b: SkSignature): bool =
secp256k1.SkSignature(a) == secp256k1.SkSignature(b)
func `==`*(a, b: SkKeyPair): bool =
secp256k1.SkKeyPair(a) == secp256k1.SkKeyPair(b)

File diff suppressed because it is too large Load Diff

View File

@@ -1,11 +1,13 @@
## Nim-Libp2p
## Copyright (c) 2018 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-Libp2p
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
{.push raises: [].}
## This module implements Pool of StreamTransport.
import chronos

View File

@@ -1,19 +1,19 @@
## Nim-LibP2P
## Copyright (c) 2018 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-LibP2P
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
## To enable dump of all incoming and outgoing unencrypted messages you need
## to compile project with ``-d:libp2p_dump`` compile-time option. When this
## option enabled ``nim-libp2p`` will create dumps of unencrypted messages for
## every peer libp2p communicates.
##
## Every file is created with name "<PeerID>.pbcap". One file represents
## all the communication with peer which identified by ``PeerID``.
## Every file is created with name "<PeerId>.pbcap". One file represents
## all the communication with peer which identified by ``PeerId``.
##
## File can have multiple protobuf encoded messages of this format:
##
@@ -25,7 +25,7 @@
## 5. LocalAddress: optional bytes
## 6. RemoteAddress: optional bytes
## 7. Message: required bytes
import os, options
import os
import nimcrypto/utils, stew/endians2
import protobuf/minprotobuf, stream/connection, protocols/secure/secure,
multiaddress, peerid, varint, muxers/mplex/coder
@@ -33,7 +33,7 @@ import protobuf/minprotobuf, stream/connection, protocols/secure/secure,
from times import getTime, toUnix, fromUnix, nanosecond, format, Time,
NanosecondRange, initTime
from strutils import toHex, repeat
export peerid, options, multiaddress
export peerid, multiaddress
type
FlowDirection* = enum
@@ -43,10 +43,10 @@ type
timestamp*: uint64
direction*: FlowDirection
message*: seq[byte]
seqID*: Option[uint64]
mtype*: Option[uint64]
local*: Option[MultiAddress]
remote*: Option[MultiAddress]
seqID*: Opt[uint64]
mtype*: Opt[uint64]
local*: Opt[MultiAddress]
remote*: Opt[MultiAddress]
const
libp2p_dump_dir* {.strdefine.} = "nim-libp2p"
@@ -72,8 +72,8 @@ proc dumpMessage*(conn: SecureConn, direction: FlowDirection,
var pb = initProtoBuffer(options = {WithVarintLength})
pb.write(2, getTimestamp())
pb.write(4, uint64(direction))
if len(conn.peerInfo.addrs) > 0:
pb.write(6, conn.peerInfo.addrs[0])
conn.observedAddr.withValue(oaddr):
pb.write(6, oaddr)
pb.write(7, data)
pb.finish()
@@ -83,7 +83,7 @@ proc dumpMessage*(conn: SecureConn, direction: FlowDirection,
else:
getHomeDir() / libp2p_dump_dir
let fileName = $(conn.peerInfo.peerId) & ".pbcap"
let fileName = $(conn.peerId) & ".pbcap"
# This is debugging procedure so it should not generate any exceptions,
# and we going to return at every possible OS error.
@@ -101,7 +101,7 @@ proc dumpMessage*(conn: SecureConn, direction: FlowDirection,
finally:
close(handle)
proc decodeDumpMessage*(data: openArray[byte]): Option[ProtoMessage] =
proc decodeDumpMessage*(data: openArray[byte]): Opt[ProtoMessage] =
## Decode protobuf's message ProtoMessage from array of bytes ``data``.
var
pb = initProtoBuffer(data)
@@ -109,13 +109,12 @@ proc decodeDumpMessage*(data: openArray[byte]): Option[ProtoMessage] =
ma1, ma2: MultiAddress
pmsg: ProtoMessage
let res2 = pb.getField(2, pmsg.timestamp)
if res2.isErr() or not(res2.get()):
return none[ProtoMessage]()
let res4 = pb.getField(4, value)
if res4.isErr() or not(res4.get()):
return none[ProtoMessage]()
let
r2 = pb.getField(2, pmsg.timestamp)
r4 = pb.getField(4, value)
r7 = pb.getField(7, pmsg.message)
if not r2.get(false) or not r4.get(false) or not r7.get(false):
return Opt.none(ProtoMessage)
# `case` statement could not work here with an error "selector must be of an
# ordinal type, float or string"
@@ -125,30 +124,27 @@ proc decodeDumpMessage*(data: openArray[byte]): Option[ProtoMessage] =
elif value == uint64(Incoming):
Incoming
else:
return none[ProtoMessage]()
return Opt.none(ProtoMessage)
let res7 = pb.getField(7, pmsg.message)
if res7.isErr() or not(res7.get()):
return none[ProtoMessage]()
let r1 = pb.getField(1, value)
if r1.get(false):
pmsg.seqID = Opt.some(value)
value = 0'u64
let res1 = pb.getField(1, value)
if res1.isOk() and res1.get():
pmsg.seqID = some(value)
value = 0'u64
let res3 = pb.getField(3, value)
if res3.isOk() and res3.get():
pmsg.mtype = some(value)
let res5 = pb.getField(5, ma1)
if res5.isOk() and res5.get():
pmsg.local = some(ma1)
let res6 = pb.getField(6, ma2)
if res6.isOk() and res6.get():
pmsg.remote = some(ma2)
let r3 = pb.getField(3, value)
if r3.get(false):
pmsg.mtype = Opt.some(value)
some(pmsg)
let
r5 = pb.getField(5, ma1)
r6 = pb.getField(6, ma2)
if r5.get(false):
pmsg.local = Opt.some(ma1)
if r6.get(false):
pmsg.remote = Opt.some(ma2)
iterator messages*(data: seq[byte]): Option[ProtoMessage] =
Opt.some(pmsg)
iterator messages*(data: seq[byte]): Opt[ProtoMessage] =
## Iterate over sequence of bytes and decode all the ``ProtoMessage``
## messages we found.
var value: uint64
@@ -171,7 +167,7 @@ iterator messages*(data: seq[byte]): Option[ProtoMessage] =
else:
break
proc dumpHex*(pbytes: openarray[byte], groupBy = 1, ascii = true): string =
proc dumpHex*(pbytes: openArray[byte], groupBy = 1, ascii = true): string =
## Get hexadecimal dump of memory for array ``pbytes``.
var res = ""
var offset = 0
@@ -243,27 +239,19 @@ proc toString*(msg: ProtoMessage, dump = true): string =
" >> "
let address =
block:
let local =
if msg.local.isSome():
"[" & $(msg.local.get()) & "]"
else:
"[LOCAL]"
let remote =
if msg.remote.isSome():
"[" & $(msg.remote.get()) & "]"
else:
"[REMOTE]"
let local = block:
msg.local.withValue(loc): "[" & $loc & "]"
else: "[LOCAL]"
let remote = block:
msg.remote.withValue(rem): "[" & $rem & "]"
else: "[REMOTE]"
local & direction & remote
let seqid =
if msg.seqID.isSome():
"seqID = " & $(msg.seqID.get()) & " "
else:
""
let mtype =
if msg.mtype.isSome():
"type = " & $(msg.mtype.get()) & " "
else:
""
let seqid = block:
msg.seqID.wihValue(seqid): "seqID = " & $seqid & " "
else: ""
let mtype = block:
msg.mtype.withValue(typ): "type = " & $typ & " "
else: ""
res.add(" ")
res.add(address)
res.add(" ")

76
libp2p/dial.nim Normal file
View File

@@ -0,0 +1,76 @@
# Nim-LibP2P
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
{.push raises: [].}
import chronos
import stew/results
import peerid,
stream/connection,
transports/transport
export results
type
Dial* = ref object of RootObj
method connect*(
self: Dial,
peerId: PeerId,
addrs: seq[MultiAddress],
forceDial = false,
reuseConnection = true,
upgradeDir = Direction.Out) {.async, base.} =
## connect remote peer without negotiating
## a protocol
##
doAssert(false, "Not implemented!")
method connect*(
self: Dial,
address: MultiAddress,
allowUnknownPeerId = false): Future[PeerId] {.async, base.} =
## Connects to a peer and retrieve its PeerId
doAssert(false, "Not implemented!")
method dial*(
self: Dial,
peerId: PeerId,
protos: seq[string],
): Future[Connection] {.async, base.} =
## create a protocol stream over an
## existing connection
##
doAssert(false, "Not implemented!")
method dial*(
self: Dial,
peerId: PeerId,
addrs: seq[MultiAddress],
protos: seq[string],
forceDial = false): Future[Connection] {.async, base.} =
## create a protocol stream and establish
## a connection if one doesn't exist already
##
doAssert(false, "Not implemented!")
method addTransport*(
self: Dial,
transport: Transport) {.base.} =
doAssert(false, "Not implemented!")
method tryDial*(
self: Dial,
peerId: PeerId,
addrs: seq[MultiAddress]): Future[Opt[MultiAddress]] {.async, base.} =
doAssert(false, "Not implemented!")

346
libp2p/dialer.nim Normal file
View File

@@ -0,0 +1,346 @@
# Nim-LibP2P
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
import std/tables
import stew/results
import pkg/[chronos,
chronicles,
metrics]
import dial,
peerid,
peerinfo,
peerstore,
multicodec,
muxers/muxer,
multistream,
connmanager,
stream/connection,
transports/transport,
nameresolving/nameresolver,
upgrademngrs/upgrade,
errors
export dial, errors, results
logScope:
topics = "libp2p dialer"
declareCounter(libp2p_total_dial_attempts, "total attempted dials")
declareCounter(libp2p_successful_dials, "dialed successful peers")
declareCounter(libp2p_failed_dials, "failed dials")
type
DialFailedError* = object of LPError
Dialer* = ref object of Dial
localPeerId*: PeerId
connManager: ConnManager
dialLock: Table[PeerId, AsyncLock]
transports: seq[Transport]
peerStore: PeerStore
nameResolver: NameResolver
proc dialAndUpgrade(
self: Dialer,
peerId: Opt[PeerId],
hostname: string,
address: MultiAddress,
upgradeDir = Direction.Out):
Future[Muxer] {.async.} =
for transport in self.transports: # for each transport
if transport.handles(address): # check if it can dial it
trace "Dialing address", address, peerId = peerId.get(default(PeerId)), hostname
let dialed =
try:
libp2p_total_dial_attempts.inc()
await transport.dial(hostname, address, peerId)
except CancelledError as exc:
debug "Dialing canceled", err = exc.msg, peerId = peerId.get(default(PeerId))
raise exc
except CatchableError as exc:
debug "Dialing failed", err = exc.msg, peerId = peerId.get(default(PeerId))
libp2p_failed_dials.inc()
return nil # Try the next address
libp2p_successful_dials.inc()
let mux =
try:
dialed.transportDir = upgradeDir
await transport.upgrade(dialed, upgradeDir, peerId)
except CatchableError as exc:
# If we failed to establish the connection through one transport,
# we won't succeeded through another - no use in trying again
await dialed.close()
debug "Upgrade failed", err = exc.msg, peerId = peerId.get(default(PeerId))
if exc isnot CancelledError:
if upgradeDir == Direction.Out:
libp2p_failed_upgrades_outgoing.inc()
else:
libp2p_failed_upgrades_incoming.inc()
# Try other address
return nil
doAssert not isNil(mux), "connection died after upgrade " & $upgradeDir
debug "Dial successful", peerId = mux.connection.peerId
return mux
return nil
proc expandDnsAddr(
self: Dialer,
peerId: Opt[PeerId],
address: MultiAddress): Future[seq[(MultiAddress, Opt[PeerId])]] {.async.} =
if not DNSADDR.matchPartial(address): return @[(address, peerId)]
if isNil(self.nameResolver):
info "Can't resolve DNSADDR without NameResolver", ma=address
return @[]
let
toResolve =
if peerId.isSome:
address & MultiAddress.init(multiCodec("p2p"), peerId.tryGet()).tryGet()
else:
address
resolved = await self.nameResolver.resolveDnsAddr(toResolve)
for resolvedAddress in resolved:
let lastPart = resolvedAddress[^1].tryGet()
if lastPart.protoCode == Result[MultiCodec, string].ok(multiCodec("p2p")):
let
peerIdBytes = lastPart.protoArgument().tryGet()
addrPeerId = PeerId.init(peerIdBytes).tryGet()
result.add((resolvedAddress[0..^2].tryGet(), Opt.some(addrPeerId)))
else:
result.add((resolvedAddress, peerId))
proc dialAndUpgrade(
self: Dialer,
peerId: Opt[PeerId],
addrs: seq[MultiAddress],
upgradeDir = Direction.Out):
Future[Muxer] {.async.} =
debug "Dialing peer", peerId = peerId.get(default(PeerId))
for rawAddress in addrs:
# resolve potential dnsaddr
let addresses = await self.expandDnsAddr(peerId, rawAddress)
for (expandedAddress, addrPeerId) in addresses:
# DNS resolution
let
hostname = expandedAddress.getHostname()
resolvedAddresses =
if isNil(self.nameResolver): @[expandedAddress]
else: await self.nameResolver.resolveMAddress(expandedAddress)
for resolvedAddress in resolvedAddresses:
result = await self.dialAndUpgrade(addrPeerId, hostname, resolvedAddress, upgradeDir)
if not isNil(result):
return result
proc tryReusingConnection(self: Dialer, peerId: PeerId): Opt[Muxer] =
let muxer = self.connManager.selectMuxer(peerId)
if muxer == nil:
return Opt.none(Muxer)
trace "Reusing existing connection", muxer, direction = $muxer.connection.dir
return Opt.some(muxer)
proc internalConnect(
self: Dialer,
peerId: Opt[PeerId],
addrs: seq[MultiAddress],
forceDial: bool,
reuseConnection = true,
upgradeDir = Direction.Out):
Future[Muxer] {.async.} =
if Opt.some(self.localPeerId) == peerId:
raise newException(CatchableError, "can't dial self!")
# Ensure there's only one in-flight attempt per peer
let lock = self.dialLock.mgetOrPut(peerId.get(default(PeerId)), newAsyncLock())
try:
await lock.acquire()
if reuseConnection:
peerId.withValue(peerId):
self.tryReusingConnection(peerId).withValue(mux):
return mux
let slot = self.connManager.getOutgoingSlot(forceDial)
let muxed =
try:
await self.dialAndUpgrade(peerId, addrs, upgradeDir)
except CatchableError as exc:
slot.release()
raise exc
slot.trackMuxer(muxed)
if isNil(muxed): # None of the addresses connected
raise newException(DialFailedError, "Unable to establish outgoing link")
try:
self.connManager.storeMuxer(muxed)
await self.peerStore.identify(muxed)
except CatchableError as exc:
trace "Failed to finish outgoung upgrade", err=exc.msg
await muxed.close()
raise exc
return muxed
finally:
if lock.locked():
lock.release()
method connect*(
self: Dialer,
peerId: PeerId,
addrs: seq[MultiAddress],
forceDial = false,
reuseConnection = true,
upgradeDir = Direction.Out) {.async.} =
## connect remote peer without negotiating
## a protocol
##
if self.connManager.connCount(peerId) > 0 and reuseConnection:
return
discard await self.internalConnect(Opt.some(peerId), addrs, forceDial, reuseConnection, upgradeDir)
method connect*(
self: Dialer,
address: MultiAddress,
allowUnknownPeerId = false): Future[PeerId] {.async.} =
## Connects to a peer and retrieve its PeerId
parseFullAddress(address).toOpt().withValue(fullAddress):
return (await self.internalConnect(
Opt.some(fullAddress[0]),
@[fullAddress[1]],
false)).connection.peerId
if allowUnknownPeerId == false:
raise newException(DialFailedError, "Address without PeerID and unknown peer id disabled!")
return (await self.internalConnect(
Opt.none(PeerId),
@[address],
false)).connection.peerId
proc negotiateStream(
self: Dialer,
conn: Connection,
protos: seq[string]): Future[Connection] {.async.} =
trace "Negotiating stream", conn, protos
let selected = await MultistreamSelect.select(conn, protos)
if not protos.contains(selected):
await conn.closeWithEOF()
raise newException(DialFailedError, "Unable to select sub-protocol " & $protos)
return conn
method tryDial*(
self: Dialer,
peerId: PeerId,
addrs: seq[MultiAddress]): Future[Opt[MultiAddress]] {.async.} =
## Create a protocol stream in order to check
## if a connection is possible.
## Doesn't use the Connection Manager to save it.
##
trace "Check if it can dial", peerId, addrs
try:
let mux = await self.dialAndUpgrade(Opt.some(peerId), addrs)
if mux.isNil():
raise newException(DialFailedError, "No valid multiaddress")
await mux.close()
return mux.connection.observedAddr
except CancelledError as exc:
raise exc
except CatchableError as exc:
raise newException(DialFailedError, exc.msg)
method dial*(
self: Dialer,
peerId: PeerId,
protos: seq[string]): Future[Connection] {.async.} =
## create a protocol stream over an
## existing connection
##
trace "Dialing (existing)", peerId, protos
let stream = await self.connManager.getStream(peerId)
if stream.isNil:
raise newException(DialFailedError, "Couldn't get muxed stream")
return await self.negotiateStream(stream, protos)
method dial*(
self: Dialer,
peerId: PeerId,
addrs: seq[MultiAddress],
protos: seq[string],
forceDial = false): Future[Connection] {.async.} =
## create a protocol stream and establish
## a connection if one doesn't exist already
##
var
conn: Muxer
stream: Connection
proc cleanup() {.async.} =
if not(isNil(stream)):
await stream.closeWithEOF()
if not(isNil(conn)):
await conn.close()
try:
trace "Dialing (new)", peerId, protos
conn = await self.internalConnect(Opt.some(peerId), addrs, forceDial)
trace "Opening stream", conn
stream = await self.connManager.getStream(conn)
if isNil(stream):
raise newException(DialFailedError,
"Couldn't get muxed stream")
return await self.negotiateStream(stream, protos)
except CancelledError as exc:
trace "Dial canceled", conn
await cleanup()
raise exc
except CatchableError as exc:
debug "Error dialing", conn, err = exc.msg
await cleanup()
raise exc
method addTransport*(self: Dialer, t: Transport) =
self.transports &= t
proc new*(
T: type Dialer,
localPeerId: PeerId,
connManager: ConnManager,
peerStore: PeerStore,
transports: seq[Transport],
nameResolver: NameResolver = nil): Dialer =
T(localPeerId: localPeerId,
connManager: connManager,
transports: transports,
peerStore: peerStore,
nameResolver: nameResolver)

View File

@@ -0,0 +1,179 @@
# Nim-LibP2P
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
{.push raises: [].}
import std/sequtils
import chronos, chronicles, stew/results
import ../errors
type
BaseAttr = ref object of RootObj
comparator: proc(f, c: BaseAttr): bool {.gcsafe, raises: [].}
Attribute[T] = ref object of BaseAttr
value: T
PeerAttributes* = object
attributes: seq[BaseAttr]
DiscoveryService* = distinct string
proc `==`*(a, b: DiscoveryService): bool {.borrow.}
proc ofType*[T](f: BaseAttr, _: type[T]): bool =
return f of Attribute[T]
proc to*[T](f: BaseAttr, _: type[T]): T =
Attribute[T](f).value
proc add*[T](pa: var PeerAttributes,
value: T) =
pa.attributes.add(Attribute[T](
value: value,
comparator: proc(f: BaseAttr, c: BaseAttr): bool =
f.ofType(T) and c.ofType(T) and f.to(T) == c.to(T)
)
)
iterator items*(pa: PeerAttributes): BaseAttr =
for f in pa.attributes:
yield f
proc getAll*[T](pa: PeerAttributes, t: typedesc[T]): seq[T] =
for f in pa.attributes:
if f.ofType(T):
result.add(f.to(T))
proc `{}`*[T](pa: PeerAttributes, t: typedesc[T]): Opt[T] =
for f in pa.attributes:
if f.ofType(T):
return Opt.some(f.to(T))
Opt.none(T)
proc `[]`*[T](pa: PeerAttributes, t: typedesc[T]): T {.raises: [KeyError].} =
pa{T}.valueOr: raise newException(KeyError, "Attritute not found")
proc match*(pa, candidate: PeerAttributes): bool =
for f in pa.attributes:
block oneAttribute:
for field in candidate.attributes:
if field.comparator(field, f):
break oneAttribute
return false
return true
type
PeerFoundCallback* = proc(pa: PeerAttributes) {.raises: [], gcsafe.}
DiscoveryInterface* = ref object of RootObj
onPeerFound*: PeerFoundCallback
toAdvertise*: PeerAttributes
advertisementUpdated*: AsyncEvent
advertiseLoop*: Future[void]
method request*(self: DiscoveryInterface, pa: PeerAttributes) {.async, base.} =
doAssert(false, "Not implemented!")
method advertise*(self: DiscoveryInterface) {.async, base.} =
doAssert(false, "Not implemented!")
type
DiscoveryError* = object of LPError
DiscoveryFinished* = object of LPError
DiscoveryQuery* = ref object
attr: PeerAttributes
peers: AsyncQueue[PeerAttributes]
finished: bool
futs: seq[Future[void]]
DiscoveryManager* = ref object
interfaces: seq[DiscoveryInterface]
queries: seq[DiscoveryQuery]
proc add*(dm: DiscoveryManager, di: DiscoveryInterface) =
dm.interfaces &= di
di.onPeerFound = proc (pa: PeerAttributes) =
for query in dm.queries:
if query.attr.match(pa):
try:
query.peers.putNoWait(pa)
except AsyncQueueFullError as exc:
debug "Cannot push discovered peer to queue"
proc request*(dm: DiscoveryManager, pa: PeerAttributes): DiscoveryQuery =
var query = DiscoveryQuery(attr: pa, peers: newAsyncQueue[PeerAttributes]())
for i in dm.interfaces:
query.futs.add(i.request(pa))
dm.queries.add(query)
dm.queries.keepItIf(it.futs.anyIt(not it.finished()))
return query
proc request*[T](dm: DiscoveryManager, value: T): DiscoveryQuery =
var pa: PeerAttributes
pa.add(value)
return dm.request(pa)
proc advertise*(dm: DiscoveryManager, pa: PeerAttributes) =
for i in dm.interfaces:
i.toAdvertise = pa
if i.advertiseLoop.isNil:
i.advertisementUpdated = newAsyncEvent()
i.advertiseLoop = i.advertise()
else:
i.advertisementUpdated.fire()
proc advertise*[T](dm: DiscoveryManager, value: T) =
var pa: PeerAttributes
pa.add(value)
dm.advertise(pa)
template forEach*(query: DiscoveryQuery, code: untyped) =
## Will execute `code` for each discovered peer. The
## peer attritubtes are available through the variable
## `peer`
proc forEachInternal(q: DiscoveryQuery) {.async.} =
while true:
let peer {.inject.} =
try: await q.getPeer()
except DiscoveryFinished: return
code
asyncSpawn forEachInternal(query)
proc stop*(query: DiscoveryQuery) =
query.finished = true
for r in query.futs:
if not r.finished(): r.cancel()
proc stop*(dm: DiscoveryManager) =
for q in dm.queries:
q.stop()
for i in dm.interfaces:
if isNil(i.advertiseLoop): continue
i.advertiseLoop.cancel()
proc getPeer*(query: DiscoveryQuery): Future[PeerAttributes] {.async.} =
let getter = query.peers.popFirst()
try:
await getter or allFinished(query.futs)
except CancelledError as exc:
getter.cancel()
raise exc
if not finished(getter):
if query.finished:
raise newException(DiscoveryFinished, "Discovery query stopped")
# discovery loops only finish when they don't handle the query
raise newException(DiscoveryError, "Unable to find any peer matching this request")
return await getter

View File

@@ -0,0 +1,73 @@
# Nim-LibP2P
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
{.push raises: [].}
import chronos
import ./discoverymngr,
../protocols/rendezvous,
../peerid
type
RendezVousInterface* = ref object of DiscoveryInterface
rdv*: RendezVous
timeToRequest: Duration
timeToAdvertise: Duration
RdvNamespace* = distinct string
proc `==`*(a, b: RdvNamespace): bool {.borrow.}
method request*(self: RendezVousInterface, pa: PeerAttributes) {.async.} =
var namespace = ""
for attr in pa:
if attr.ofType(RdvNamespace):
namespace = string attr.to(RdvNamespace)
elif attr.ofType(DiscoveryService):
namespace = string attr.to(DiscoveryService)
elif attr.ofType(PeerId):
namespace = $attr.to(PeerId)
else:
# unhandled type
return
while true:
for pr in await self.rdv.request(namespace):
var peer: PeerAttributes
peer.add(pr.peerId)
for address in pr.addresses:
peer.add(address.address)
peer.add(DiscoveryService(namespace))
peer.add(RdvNamespace(namespace))
self.onPeerFound(peer)
await sleepAsync(self.timeToRequest)
method advertise*(self: RendezVousInterface) {.async.} =
while true:
var toAdvertise: seq[string]
for attr in self.toAdvertise:
if attr.ofType(RdvNamespace):
toAdvertise.add string attr.to(RdvNamespace)
elif attr.ofType(DiscoveryService):
toAdvertise.add string attr.to(DiscoveryService)
elif attr.ofType(PeerId):
toAdvertise.add $attr.to(PeerId)
self.advertisementUpdated.clear()
for toAdv in toAdvertise:
await self.rdv.advertise(toAdv, self.timeToAdvertise)
await sleepAsync(self.timeToAdvertise) or self.advertisementUpdated.wait()
proc new*(T: typedesc[RendezVousInterface],
rdv: RendezVous,
ttr: Duration = 1.minutes,
tta: Duration = MinimumDuration): RendezVousInterface =
T(rdv: rdv, timeToRequest: ttr, timeToAdvertise: tta)

View File

@@ -5,11 +5,20 @@ import chronos
import chronicles
import macros
# could not figure how to make it with a simple template
# sadly nim needs more love for hygenic templates
type
# Base exception type for libp2p
LPError* = object of CatchableError
func toException*(e: cstring): ref LPError =
(ref LPError)(msg: $e)
func toException*(e: string): ref LPError =
(ref LPError)(msg: e)
# TODO: could not figure how to make it with a simple template
# sadly nim needs more love for hygienic templates
# so here goes the macro, its based on the proc/template version
# and uses quote do so it's quite readable
macro checkFutures*[T](futs: seq[Future[T]], exclude: untyped = []): untyped =
let nexclude = exclude.len
case nexclude
@@ -40,7 +49,7 @@ proc allFuturesThrowing*[T](args: varargs[Future[T]]): Future[void] =
for fut in args:
futs &= fut
proc call() {.async.} =
var first: ref Exception = nil
var first: ref CatchableError = nil
futs = await allFinished(futs)
for fut in futs:
if fut.failed:

View File

@@ -1,23 +1,27 @@
## Nim-Libp2p
## Copyright (c) 2018 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-Libp2p
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
## This module implements MultiAddress.
{.push raises: [Defect].}
{.push raises: [].}
{.push public.}
import nativesockets, hashes
import tables, strutils, stew/shims/net
import chronos
import pkg/chronos, chronicles
import std/[nativesockets, hashes]
import tables, strutils, sets, stew/shims/net
import multicodec, multihash, multibase, transcoder, vbuffer, peerid,
protobuf/minprotobuf
protobuf/minprotobuf, errors, utility
import stew/[base58, base32, endians2, results]
export results, minprotobuf, vbuffer
export results, minprotobuf, vbuffer, utility
logScope:
topics = "libp2p multiaddress"
type
MAKind* = enum
@@ -30,7 +34,7 @@ type
coder*: Transcoder
MultiAddress* = object
data*: VBuffer
data: VBuffer
MaPatternOp* = enum
Eq, Or, And
@@ -46,7 +50,7 @@ type
MaResult*[T] = Result[T, string]
MaError* = object of CatchableError
MaError* = object of LPError
MaInvalidAddress* = object of MaError
IpTransportProtocol* = enum
@@ -59,6 +63,10 @@ const
IPPROTO_TCP = Protocol.IPPROTO_TCP
IPPROTO_UDP = Protocol.IPPROTO_UDP
proc data*(ma: MultiAddress): VBuffer =
## Returns the data buffer of the MultiAddress.
return ma.data
proc hash*(a: MultiAddress): Hash =
var h: Hash = 0
h = h !& hash(a.data.buffer)
@@ -72,7 +80,7 @@ proc ip4StB(s: string, vb: var VBuffer): bool =
if a.family == IpAddressFamily.IPv4:
vb.writeArray(a.address_v4)
result = true
except:
except CatchableError:
discard
proc ip4BtS(vb: var VBuffer, s: var string): bool =
@@ -95,7 +103,7 @@ proc ip6StB(s: string, vb: var VBuffer): bool =
if a.family == IpAddressFamily.IPv6:
vb.writeArray(a.address_v6)
result = true
except:
except CatchableError:
discard
proc ip6BtS(vb: var VBuffer, s: var string): bool =
@@ -139,14 +147,14 @@ proc portStB(s: string, vb: var VBuffer): bool =
port[1] = cast[byte](nport and 0xFF)
vb.writeArray(port)
result = true
except:
except CatchableError:
discard
proc portBtS(vb: var VBuffer, s: var string): bool =
## Port number bufferToString() implementation.
var port: array[2, byte]
if vb.readArray(port) == 2:
var nport = (cast[uint16](port[0]) shl 8) or cast[uint16](port[1])
var nport = (safeConvert[uint16](port[0]) shl 8) or safeConvert[uint16](port[1])
s = $nport
result = true
@@ -164,7 +172,7 @@ proc p2pStB(s: string, vb: var VBuffer): bool =
if MultiHash.decode(data, mh).isOk:
vb.writeSeq(data)
result = true
except:
except CatchableError:
discard
proc p2pBtS(vb: var VBuffer, s: var string): bool =
@@ -199,14 +207,14 @@ proc onionStB(s: string, vb: var VBuffer): bool =
address[11] = cast[byte](nport and 0xFF)
vb.writeArray(address)
result = true
except:
except CatchableError:
discard
proc onionBtS(vb: var VBuffer, s: var string): bool =
## ONION address bufferToString() implementation.
var buf: array[12, byte]
if vb.readArray(buf) == 12:
var nport = (cast[uint16](buf[10]) shl 8) or cast[uint16](buf[11])
var nport = (safeConvert[uint16](buf[10]) shl 8) or safeConvert[uint16](buf[11])
s = Base32Lower.encode(buf.toOpenArray(0, 9))
s.add(":")
s.add($nport)
@@ -218,6 +226,40 @@ proc onionVB(vb: var VBuffer): bool =
if vb.readArray(buf) == 12:
result = true
proc onion3StB(s: string, vb: var VBuffer): bool =
try:
var parts = s.split(':')
if len(parts) != 2:
return false
if len(parts[0]) != 56:
return false
var address = Base32Lower.decode(parts[0].toLowerAscii())
var nport = parseInt(parts[1])
if (nport > 0 and nport < 65536) and len(address) == 35:
address.setLen(37)
address[35] = cast[byte]((nport shr 8) and 0xFF)
address[36] = cast[byte](nport and 0xFF)
vb.writeArray(address)
result = true
except CatchableError:
discard
proc onion3BtS(vb: var VBuffer, s: var string): bool =
## ONION address bufferToString() implementation.
var buf: array[37, byte]
if vb.readArray(buf) == 37:
var nport = (safeConvert[uint16](buf[35]) shl 8) or safeConvert[uint16](buf[36])
s = Base32Lower.encode(buf.toOpenArray(0, 34))
s.add(":")
s.add($nport)
result = true
proc onion3VB(vb: var VBuffer): bool =
## ONION address validateBuffer() implementation.
var buf: array[37, byte]
if vb.readArray(buf) == 37:
result = true
proc unixStB(s: string, vb: var VBuffer): bool =
## Unix socket name stringToBuffer() implementation.
if len(s) > 0:
@@ -306,6 +348,11 @@ const
bufferToString: onionBtS,
validateBuffer: onionVB
)
TranscoderOnion3* = Transcoder(
stringToBuffer: onion3StB,
bufferToString: onion3BtS,
validateBuffer: onion3VB
)
TranscoderDNS* = Transcoder(
stringToBuffer: dnsStB,
bufferToString: dnsBtS,
@@ -359,9 +406,19 @@ const
mcodec: multiCodec("onion"), kind: Fixed, size: 10,
coder: TranscoderOnion
),
MAProtocol(
mcodec: multiCodec("onion3"), kind: Fixed, size: 37,
coder: TranscoderOnion3
),
MAProtocol(
mcodec: multiCodec("ws"), kind: Marker, size: 0
),
MAProtocol(
mcodec: multiCodec("wss"), kind: Marker, size: 0
),
MAProtocol(
mcodec: multiCodec("tls"), kind: Marker, size: 0
),
MAProtocol(
mcodec: multiCodec("ipfs"), kind: Length, size: 0,
coder: TranscoderP2P
@@ -374,6 +431,10 @@ const
mcodec: multiCodec("unix"), kind: Path, size: 0,
coder: TranscoderUnix
),
MAProtocol(
mcodec: multiCodec("dns"), kind: Length, size: 0,
coder: TranscoderDNS
),
MAProtocol(
mcodec: multiCodec("dns4"), kind: Length, size: 0,
coder: TranscoderDNS
@@ -400,23 +461,44 @@ const
)
]
DNSANY* = mapEq("dns")
DNS4* = mapEq("dns4")
DNS6* = mapEq("dns6")
DNSADDR* = mapEq("dnsaddr")
IP4* = mapEq("ip4")
IP6* = mapEq("ip6")
DNS* = mapOr(mapEq("dnsaddr"), DNS4, DNS6)
DNS* = mapOr(DNSANY, DNS4, DNS6, DNSADDR)
IP* = mapOr(IP4, IP6)
TCP* = mapOr(mapAnd(DNS, mapEq("tcp")), mapAnd(IP, mapEq("tcp")))
UDP* = mapOr(mapAnd(DNS, mapEq("udp")), mapAnd(IP, mapEq("udp")))
DNS_OR_IP* = mapOr(DNS, IP)
TCP_DNS* = mapAnd(DNS, mapEq("tcp"))
TCP_IP* = mapAnd(IP, mapEq("tcp"))
TCP* = mapOr(TCP_DNS, TCP_IP)
UDP_DNS* = mapAnd(DNS, mapEq("udp"))
UDP_IP* = mapAnd(IP, mapEq("udp"))
UDP* = mapOr(UDP_DNS, UDP_IP)
UTP* = mapAnd(UDP, mapEq("utp"))
QUIC* = mapAnd(UDP, mapEq("quic"))
UNIX* = mapEq("unix")
WS_DNS* = mapAnd(TCP_DNS, mapEq("ws"))
WS_IP* = mapAnd(TCP_IP, mapEq("ws"))
WS* = mapAnd(TCP, mapEq("ws"))
TLS_WS* = mapOr(mapEq("wss"), mapAnd(mapEq("tls"), mapEq("ws")))
WSS_DNS* = mapAnd(TCP_DNS, TLS_WS)
WSS_IP* = mapAnd(TCP_IP, TLS_WS)
WSS* = mapAnd(TCP, TLS_WS)
WebSockets_DNS* = mapOr(WS_DNS, WSS_DNS)
WebSockets_IP* = mapOr(WS_IP, WSS_IP)
WebSockets* = mapOr(WS, WSS)
Onion3* = mapEq("onion3")
TcpOnion3* = mapAnd(TCP, Onion3)
Unreliable* = mapOr(UDP)
Reliable* = mapOr(TCP, UTP, QUIC)
Reliable* = mapOr(TCP, UTP, QUIC, WebSockets)
IPFS* = mapAnd(Reliable, mapEq("p2p"))
P2PPattern* = mapEq("p2p")
IPFS* = mapAnd(Reliable, P2PPattern)
HTTP* = mapOr(
mapAnd(TCP, mapEq("http")),
@@ -435,9 +517,10 @@ const
mapAnd(HTTPS, mapEq("p2p-webrtc-direct"))
)
CircuitRelay* = mapEq("p2p-circuit")
proc initMultiAddressCodeTable(): Table[MultiCodec,
MAProtocol] {.compileTime.} =
result = initTable[MultiCodec, MAProtocol]()
for item in ProtocolsList:
result[item.mcodec] = item
@@ -454,15 +537,10 @@ proc trimRight(s: string, ch: char): string =
break
result = s[0..(s.high - m)]
proc shcopy*(m1: var MultiAddress, m2: MultiAddress) =
shallowCopy(m1.data.buffer, m2.data.buffer)
m1.data.offset = m2.data.offset
proc protoCode*(ma: MultiAddress): MaResult[MultiCodec] =
## Returns MultiAddress ``ma`` protocol code.
var header: uint64
var vb: MultiAddress
shcopy(vb, ma)
var vb = ma
if vb.data.readVarint(header) == -1:
err("multiaddress: Malformed binary address!")
else:
@@ -475,8 +553,7 @@ proc protoCode*(ma: MultiAddress): MaResult[MultiCodec] =
proc protoName*(ma: MultiAddress): MaResult[string] =
## Returns MultiAddress ``ma`` protocol name.
var header: uint64
var vb: MultiAddress
shcopy(vb, ma)
var vb = ma
if vb.data.readVarint(header) == -1:
err("multiaddress: Malformed binary address!")
else:
@@ -487,15 +564,14 @@ proc protoName*(ma: MultiAddress): MaResult[string] =
ok($(proto.mcodec))
proc protoArgument*(ma: MultiAddress,
value: var openarray[byte]): MaResult[int] =
value: var openArray[byte]): MaResult[int] =
## Returns MultiAddress ``ma`` protocol argument value.
##
## If current MultiAddress do not have argument value, then result will be
## ``0``.
var header: uint64
var vb: MultiAddress
var vb = ma
var buffer: seq[byte]
shcopy(vb, ma)
if vb.data.readVarint(header) == -1:
err("multiaddress: Malformed binary address!")
else:
@@ -511,7 +587,7 @@ proc protoArgument*(ma: MultiAddress,
err("multiaddress: Decoding protocol error")
else:
ok(res)
elif proto.kind in {Length, Path}:
elif proto.kind in {MAKind.Length, Path}:
if vb.data.readSeq(buffer) == -1:
err("multiaddress: Decoding protocol error")
else:
@@ -532,6 +608,13 @@ proc protoAddress*(ma: MultiAddress): MaResult[seq[byte]] =
buffer.setLen(res)
ok(buffer)
proc protoArgument*(ma: MultiAddress): MaResult[seq[byte]] =
## Returns MultiAddress ``ma`` protocol address binary blob.
##
## If current MultiAddress do not have argument value, then result array will
## be empty.
ma.protoAddress()
proc getPart(ma: MultiAddress, index: int): MaResult[MultiAddress] =
var header: uint64
var data = newSeq[byte]()
@@ -539,6 +622,9 @@ proc getPart(ma: MultiAddress, index: int): MaResult[MultiAddress] =
var vb = ma
var res: MultiAddress
res.data = initVBuffer()
if index < 0: return err("multiaddress: negative index gived to getPart")
while offset <= index:
if vb.data.readVarint(header) == -1:
return err("multiaddress: Malformed binary address!")
@@ -556,7 +642,7 @@ proc getPart(ma: MultiAddress, index: int): MaResult[MultiAddress] =
res.data.writeVarint(header)
res.data.writeArray(data)
res.data.finish()
elif proto.kind in {Length, Path}:
elif proto.kind in {MAKind.Length, Path}:
if vb.data.readSeq(data) == -1:
return err("multiaddress: Decoding protocol error")
@@ -571,9 +657,31 @@ proc getPart(ma: MultiAddress, index: int): MaResult[MultiAddress] =
inc(offset)
ok(res)
proc `[]`*(ma: MultiAddress, i: int): MaResult[MultiAddress] {.inline.} =
proc getParts[U, V](ma: MultiAddress, slice: HSlice[U, V]): MaResult[MultiAddress] =
when slice.a is BackwardsIndex or slice.b is BackwardsIndex:
let maLength = ? len(ma)
template normalizeIndex(index): int =
when index is BackwardsIndex: maLength - int(index)
else: int(index)
let
indexStart = normalizeIndex(slice.a)
indexEnd = normalizeIndex(slice.b)
var res: MultiAddress
for i in indexStart..indexEnd:
? res.append(? ma[i])
ok(res)
proc `[]`*(ma: MultiAddress, i: int | BackwardsIndex): MaResult[MultiAddress] {.inline.} =
## Returns part with index ``i`` of MultiAddress ``ma``.
ma.getPart(i)
when i is BackwardsIndex:
let maLength = ? len(ma)
ma.getPart(maLength - int(i))
else:
ma.getPart(i)
proc `[]`*(ma: MultiAddress, slice: HSlice): MaResult[MultiAddress] {.inline.} =
## Returns parts with slice ``slice`` of MultiAddress ``ma``.
ma.getParts(slice)
iterator items*(ma: MultiAddress): MaResult[MultiAddress] =
## Iterates over all addresses inside of MultiAddress ``ma``.
@@ -600,7 +708,7 @@ iterator items*(ma: MultiAddress): MaResult[MultiAddress] =
res.data.writeVarint(header)
res.data.writeArray(data)
elif proto.kind in {Length, Path}:
elif proto.kind in {MAKind.Length, Path}:
if vb.data.readSeq(data) == -1:
yield err(MaResult[MultiAddress], "Decoding protocol error")
@@ -611,6 +719,13 @@ iterator items*(ma: MultiAddress): MaResult[MultiAddress] =
res.data.finish()
yield ok(MaResult[MultiAddress], res)
proc len*(ma: MultiAddress): MaResult[int] =
var counter: int
for part in ma:
if part.isErr: return err(part.error)
counter.inc()
ok(counter)
proc contains*(ma: MultiAddress, codec: MultiCodec): MaResult[bool] {.inline.} =
## Returns ``true``, if address with MultiCodec ``codec`` present in
## MultiAddress ``ma``.
@@ -661,7 +776,7 @@ proc toString*(value: MultiAddress): MaResult[string] =
res = "/" & parts.join("/")
ok(res)
proc `$`*(value: MultiAddress): string {.raises: [Defect].} =
proc `$`*(value: MultiAddress): string =
## Return string representation of MultiAddress ``value``.
let s = value.toString()
if s.isErr: s.error
@@ -691,8 +806,7 @@ proc encode*(mbtype: typedesc[MultiBase], encoding: string,
proc validate*(ma: MultiAddress): bool =
## Returns ``true`` if MultiAddress ``ma`` is valid.
var header: uint64
var vb: MultiAddress
shcopy(vb, ma)
var vb = ma
while true:
if vb.data.isEmpty():
break
@@ -712,7 +826,7 @@ proc validate*(ma: MultiAddress): bool =
proc init*(
mtype: typedesc[MultiAddress], protocol: MultiCodec,
value: openarray[byte] = []): MaResult[MultiAddress] =
value: openArray[byte] = []): MaResult[MultiAddress] =
## Initialize MultiAddress object from protocol id ``protocol`` and array
## of bytes ``value``.
let proto = CodeAddresses.getOrDefault(protocol)
@@ -743,7 +857,7 @@ proc init*(
raiseAssert "None checked above"
proc init*(mtype: typedesc[MultiAddress], protocol: MultiCodec,
value: PeerID): MaResult[MultiAddress] {.inline.} =
value: PeerId): MaResult[MultiAddress] {.inline.} =
## Initialize MultiAddress object from protocol id ``protocol`` and peer id
## ``value``.
init(mtype, protocol, value.data)
@@ -777,6 +891,8 @@ proc getProtocol(name: string): MAProtocol {.inline.} =
proc init*(mtype: typedesc[MultiAddress],
value: string): MaResult[MultiAddress] =
## Initialize MultiAddress object from string representation ``value``.
if len(value) == 0 or value == "/":
return err("multiaddress: Address must not be empty!")
var parts = value.trimRight('/').split('/')
if len(parts[0]) != 0:
err("multiaddress: Invalid MultiAddress, must start with `/`")
@@ -821,10 +937,10 @@ proc init*(mtype: typedesc[MultiAddress],
ok(res)
proc init*(mtype: typedesc[MultiAddress],
data: openarray[byte]): MaResult[MultiAddress] =
data: openArray[byte]): MaResult[MultiAddress] =
## Initialize MultiAddress with array of bytes ``data``.
if len(data) == 0:
err("multiaddress: Address could not be empty!")
err("multiaddress: Address must not be empty!")
else:
var res: MultiAddress
res.data = initVBuffer()
@@ -910,77 +1026,41 @@ proc append*(m1: var MultiAddress, m2: MultiAddress): MaResult[void] =
ok()
proc `&`*(m1, m2: MultiAddress): MultiAddress {.
raises: [Defect, ResultError[string]].} =
raises: [LPError].} =
## Concatenates two addresses ``m1`` and ``m2``, and returns result.
##
## This procedure performs validation of concatenated result and can raise
## exception on error.
##
concat(m1, m2).tryGet()
proc `&=`*(m1: var MultiAddress, m2: MultiAddress) {.
raises: [Defect, ResultError[string]].} =
raises: [LPError].} =
## Concatenates two addresses ``m1`` and ``m2``.
##
## This procedure performs validation of concatenated result and can raise
## exception on error.
##
m1.append(m2).tryGet()
proc isWire*(ma: MultiAddress): bool =
## Returns ``true`` if MultiAddress ``ma`` is one of:
## - {IP4}/{TCP, UDP}
## - {IP6}/{TCP, UDP}
## - {UNIX}/{PATH}
var
state = 0
try:
for rpart in ma.items():
if rpart.isErr():
return false
let part = rpart.get()
if state == 0:
let rcode = part.protoCode()
if rcode.isErr():
return false
let code = rcode.get()
if code == multiCodec("ip4") or code == multiCodec("ip6"):
inc(state)
continue
elif code == multiCodec("unix"):
result = true
break
else:
result = false
break
elif state == 1:
let rcode = part.protoCode()
if rcode.isErr():
return false
let code = rcode.get()
if code == multiCodec("tcp") or code == multiCodec("udp"):
inc(state)
result = true
else:
result = false
break
else:
result = false
break
except:
result = false
proc `==`*(m1: var MultiAddress, m2: MultiAddress): bool =
## Check of two MultiAddress are equal
m1.data == m2.data
proc matchPart(pat: MaPattern, protos: seq[MultiCodec]): MaPatResult =
var empty: seq[MultiCodec]
var pcs = protos
if pat.operator == Or:
result = MaPatResult(flag: false, rem: empty)
for a in pat.args:
let res = a.matchPart(pcs)
if res.flag:
return MaPatResult(flag: true, rem: res.rem)
result = MaPatResult(flag: false, rem: empty)
#Greedy Or
if result.flag == false or
result.rem.len > res.rem.len:
result = res
elif pat.operator == And:
if len(pcs) < len(pat.args):
return MaPatResult(flag: false, rem: empty)
@@ -1000,19 +1080,15 @@ proc matchPart(pat: MaPattern, protos: seq[MultiCodec]): MaPatResult =
proc match*(pat: MaPattern, address: MultiAddress): bool =
## Match full ``address`` using pattern ``pat`` and return ``true`` if
## ``address`` satisfies pattern.
let protos = address.protocols()
if protos.isErr():
return false
let res = matchPart(pat, protos.get())
let protos = address.protocols().valueOr: return false
let res = matchPart(pat, protos)
res.flag and (len(res.rem) == 0)
proc matchPartial*(pat: MaPattern, address: MultiAddress): bool =
## Match prefix part of ``address`` using pattern ``pat`` and return
## ``true`` if ``address`` starts with pattern.
let protos = address.protocols()
if protos.isErr():
return false
let res = matchPart(pat, protos.get())
let protos = address.protocols().valueOr: return false
let res = matchPart(pat, protos)
res.flag
proc `$`*(pat: MaPattern): string =
@@ -1027,10 +1103,13 @@ proc `$`*(pat: MaPattern): string =
elif pat.operator == Eq:
result = $pat.value
proc bytes*(value: MultiAddress): seq[byte] =
value.data.buffer
proc write*(pb: var ProtoBuffer, field: int, value: MultiAddress) {.inline.} =
write(pb, field, value.data.buffer)
proc getField*(pb: var ProtoBuffer, field: int,
proc getField*(pb: ProtoBuffer, field: int,
value: var MultiAddress): ProtoResult[bool] {.
inline.} =
var buffer: seq[byte]
@@ -1038,16 +1117,16 @@ proc getField*(pb: var ProtoBuffer, field: int,
if not(res):
ok(false)
else:
let ma = MultiAddress.init(buffer)
if ma.isOk():
value = ma.get()
ok(true)
else:
err(ProtoError.IncorrectBlob)
value = MultiAddress.init(buffer).valueOr: return err(ProtoError.IncorrectBlob)
ok(true)
proc getRepeatedField*(pb: var ProtoBuffer, field: int,
proc getRepeatedField*(pb: ProtoBuffer, field: int,
value: var seq[MultiAddress]): ProtoResult[bool] {.
inline.} =
## Read repeated field from protobuf message. ``field`` is field number. If the message is malformed, an error is returned.
## If field is not present in message, then ``ok(false)`` is returned and value is empty. If field is present,
## but no items could be parsed, then ``err(ProtoError.IncorrectBlob)`` is returned and value is empty.
## If field is present and some item could be parsed, then ``true`` is returned and value contains the parsed values.
var items: seq[seq[byte]]
value.setLen(0)
let res = ? pb.getRepeatedField(field, items)
@@ -1055,10 +1134,12 @@ proc getRepeatedField*(pb: var ProtoBuffer, field: int,
ok(false)
else:
for item in items:
let ma = MultiAddress.init(item)
if ma.isOk():
value.add(ma.get())
else:
value.setLen(0)
return err(ProtoError.IncorrectBlob)
ok(true)
let ma = MultiAddress.init(item).valueOr:
debug "Unsupported MultiAddress in blob", ma = item
continue
value.add(ma)
if value.len == 0:
err(ProtoError.IncorrectBlob)
else:
ok(true)

View File

@@ -1,11 +1,11 @@
## Nim-Libp2p
## Copyright (c) 2018 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-Libp2p
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
## This module implements MultiBase.
##
@@ -13,185 +13,185 @@
## 1. base32z
##
{.push raises: [Defect].}
{.push raises: [].}
import tables
import stew/[base32, base58, base64, results]
type
MultibaseStatus* {.pure.} = enum
MultiBaseStatus* {.pure.} = enum
Error, Success, Overrun, Incorrect, BadCodec, NotSupported
MultiBase* = object
MBCodeSize = proc(length: int): int {.nimcall, gcsafe, noSideEffect, raises: [Defect].}
MBCodeSize = proc(length: int): int {.nimcall, gcsafe, noSideEffect, raises: [].}
MBCodec = object
code: char
name: string
encr: proc(inbytes: openarray[byte],
outbytes: var openarray[char],
outlen: var int): MultibaseStatus {.nimcall, gcsafe, noSideEffect, raises: [Defect].}
decr: proc(inbytes: openarray[char],
outbytes: var openarray[byte],
outlen: var int): MultibaseStatus {.nimcall, gcsafe, noSideEffect, raises: [Defect].}
encr: proc(inbytes: openArray[byte],
outbytes: var openArray[char],
outlen: var int): MultiBaseStatus {.nimcall, gcsafe, noSideEffect, raises: [].}
decr: proc(inbytes: openArray[char],
outbytes: var openArray[byte],
outlen: var int): MultiBaseStatus {.nimcall, gcsafe, noSideEffect, raises: [].}
encl: MBCodeSize
decl: MBCodeSize
proc idd(inbytes: openarray[char], outbytes: var openarray[byte],
outlen: var int): MultibaseStatus =
proc idd(inbytes: openArray[char], outbytes: var openArray[byte],
outlen: var int): MultiBaseStatus =
let length = len(inbytes)
if length > len(outbytes):
outlen = length
result = MultibaseStatus.Overrun
result = MultiBaseStatus.Overrun
else:
copyMem(addr outbytes[0], unsafeAddr inbytes[0], length)
outlen = length
result = MultibaseStatus.Success
result = MultiBaseStatus.Success
proc ide(inbytes: openarray[byte],
outbytes: var openarray[char],
outlen: var int): MultibaseStatus =
proc ide(inbytes: openArray[byte],
outbytes: var openArray[char],
outlen: var int): MultiBaseStatus =
let length = len(inbytes)
if length > len(outbytes):
outlen = length
result = MultibaseStatus.Overrun
result = MultiBaseStatus.Overrun
else:
copyMem(addr outbytes[0], unsafeAddr inbytes[0], length)
outlen = length
result = MultibaseStatus.Success
result = MultiBaseStatus.Success
proc idel(length: int): int = length
proc iddl(length: int): int = length
proc b16d(inbytes: openarray[char],
outbytes: var openarray[byte],
outlen: var int): MultibaseStatus =
proc b16d(inbytes: openArray[char],
outbytes: var openArray[byte],
outlen: var int): MultiBaseStatus =
discard
proc b16e(inbytes: openarray[byte],
outbytes: var openarray[char],
outlen: var int): MultibaseStatus =
proc b16e(inbytes: openArray[byte],
outbytes: var openArray[char],
outlen: var int): MultiBaseStatus =
discard
proc b16ud(inbytes: openarray[char],
outbytes: var openarray[byte],
outlen: var int): MultibaseStatus =
proc b16ud(inbytes: openArray[char],
outbytes: var openArray[byte],
outlen: var int): MultiBaseStatus =
discard
proc b16ue(inbytes: openarray[byte],
outbytes: var openarray[char],
outlen: var int): MultibaseStatus =
proc b16ue(inbytes: openArray[byte],
outbytes: var openArray[char],
outlen: var int): MultiBaseStatus =
discard
proc b16el(length: int): int = length shl 1
proc b16dl(length: int): int = (length + 1) div 2
proc b32ce(r: Base32Status): MultibaseStatus {.inline.} =
result = MultibaseStatus.Error
proc b32ce(r: Base32Status): MultiBaseStatus {.inline.} =
result = MultiBaseStatus.Error
if r == Base32Status.Incorrect:
result = MultibaseStatus.Incorrect
result = MultiBaseStatus.Incorrect
elif r == Base32Status.Overrun:
result = MultibaseStatus.Overrun
result = MultiBaseStatus.Overrun
elif r == Base32Status.Success:
result = MultibaseStatus.Success
result = MultiBaseStatus.Success
proc b58ce(r: Base58Status): MultibaseStatus {.inline.} =
result = MultibaseStatus.Error
proc b58ce(r: Base58Status): MultiBaseStatus {.inline.} =
result = MultiBaseStatus.Error
if r == Base58Status.Incorrect:
result = MultibaseStatus.Incorrect
result = MultiBaseStatus.Incorrect
elif r == Base58Status.Overrun:
result = MultibaseStatus.Overrun
result = MultiBaseStatus.Overrun
elif r == Base58Status.Success:
result = MultibaseStatus.Success
result = MultiBaseStatus.Success
proc b64ce(r: Base64Status): MultibaseStatus {.inline.} =
proc b64ce(r: Base64Status): MultiBaseStatus {.inline.} =
result = MultiBaseStatus.Error
if r == Base64Status.Incorrect:
result = MultibaseStatus.Incorrect
result = MultiBaseStatus.Incorrect
elif r == Base64Status.Overrun:
result = MultiBaseStatus.Overrun
elif r == Base64Status.Success:
result = MultibaseStatus.Success
result = MultiBaseStatus.Success
proc b32hd(inbytes: openarray[char],
outbytes: var openarray[byte],
outlen: var int): MultibaseStatus =
proc b32hd(inbytes: openArray[char],
outbytes: var openArray[byte],
outlen: var int): MultiBaseStatus =
result = b32ce(HexBase32Lower.decode(inbytes, outbytes, outlen))
proc b32he(inbytes: openarray[byte],
outbytes: var openarray[char],
outlen: var int): MultibaseStatus =
proc b32he(inbytes: openArray[byte],
outbytes: var openArray[char],
outlen: var int): MultiBaseStatus =
result = b32ce(HexBase32Lower.encode(inbytes, outbytes, outlen))
proc b32hud(inbytes: openarray[char],
outbytes: var openarray[byte],
outlen: var int): MultibaseStatus =
proc b32hud(inbytes: openArray[char],
outbytes: var openArray[byte],
outlen: var int): MultiBaseStatus =
result = b32ce(HexBase32Upper.decode(inbytes, outbytes, outlen))
proc b32hue(inbytes: openarray[byte],
outbytes: var openarray[char],
outlen: var int): MultibaseStatus =
proc b32hue(inbytes: openArray[byte],
outbytes: var openArray[char],
outlen: var int): MultiBaseStatus =
result = b32ce(HexBase32Upper.encode(inbytes, outbytes, outlen))
proc b32hpd(inbytes: openarray[char],
outbytes: var openarray[byte],
outlen: var int): MultibaseStatus =
proc b32hpd(inbytes: openArray[char],
outbytes: var openArray[byte],
outlen: var int): MultiBaseStatus =
result = b32ce(HexBase32LowerPad.decode(inbytes, outbytes, outlen))
proc b32hpe(inbytes: openarray[byte],
outbytes: var openarray[char],
outlen: var int): MultibaseStatus =
proc b32hpe(inbytes: openArray[byte],
outbytes: var openArray[char],
outlen: var int): MultiBaseStatus =
result = b32ce(HexBase32LowerPad.encode(inbytes, outbytes, outlen))
proc b32hpud(inbytes: openarray[char],
outbytes: var openarray[byte],
outlen: var int): MultibaseStatus =
proc b32hpud(inbytes: openArray[char],
outbytes: var openArray[byte],
outlen: var int): MultiBaseStatus =
result = b32ce(HexBase32UpperPad.decode(inbytes, outbytes, outlen))
proc b32hpue(inbytes: openarray[byte],
outbytes: var openarray[char],
outlen: var int): MultibaseStatus =
proc b32hpue(inbytes: openArray[byte],
outbytes: var openArray[char],
outlen: var int): MultiBaseStatus =
result = b32ce(HexBase32UpperPad.encode(inbytes, outbytes, outlen))
proc b32d(inbytes: openarray[char],
outbytes: var openarray[byte],
outlen: var int): MultibaseStatus =
proc b32d(inbytes: openArray[char],
outbytes: var openArray[byte],
outlen: var int): MultiBaseStatus =
result = b32ce(Base32Lower.decode(inbytes, outbytes, outlen))
proc b32e(inbytes: openarray[byte],
outbytes: var openarray[char],
outlen: var int): MultibaseStatus =
proc b32e(inbytes: openArray[byte],
outbytes: var openArray[char],
outlen: var int): MultiBaseStatus =
result = b32ce(Base32Lower.encode(inbytes, outbytes, outlen))
proc b32ud(inbytes: openarray[char],
outbytes: var openarray[byte],
outlen: var int): MultibaseStatus =
proc b32ud(inbytes: openArray[char],
outbytes: var openArray[byte],
outlen: var int): MultiBaseStatus =
result = b32ce(Base32Upper.decode(inbytes, outbytes, outlen))
proc b32ue(inbytes: openarray[byte],
outbytes: var openarray[char],
outlen: var int): MultibaseStatus =
proc b32ue(inbytes: openArray[byte],
outbytes: var openArray[char],
outlen: var int): MultiBaseStatus =
result = b32ce(Base32Upper.encode(inbytes, outbytes, outlen))
proc b32pd(inbytes: openarray[char],
outbytes: var openarray[byte],
outlen: var int): MultibaseStatus =
proc b32pd(inbytes: openArray[char],
outbytes: var openArray[byte],
outlen: var int): MultiBaseStatus =
result = b32ce(Base32LowerPad.decode(inbytes, outbytes, outlen))
proc b32pe(inbytes: openarray[byte],
outbytes: var openarray[char],
outlen: var int): MultibaseStatus =
proc b32pe(inbytes: openArray[byte],
outbytes: var openArray[char],
outlen: var int): MultiBaseStatus =
result = b32ce(Base32LowerPad.encode(inbytes, outbytes, outlen))
proc b32pud(inbytes: openarray[char],
outbytes: var openarray[byte],
outlen: var int): MultibaseStatus =
proc b32pud(inbytes: openArray[char],
outbytes: var openArray[byte],
outlen: var int): MultiBaseStatus =
result = b32ce(Base32UpperPad.decode(inbytes, outbytes, outlen))
proc b32pue(inbytes: openarray[byte],
outbytes: var openarray[char],
outlen: var int): MultibaseStatus =
proc b32pue(inbytes: openArray[byte],
outbytes: var openArray[char],
outlen: var int): MultiBaseStatus =
result = b32ce(Base32UpperPad.encode(inbytes, outbytes, outlen))
proc b32el(length: int): int = Base32Lower.encodedLength(length)
@@ -199,24 +199,24 @@ proc b32dl(length: int): int = Base32Lower.decodedLength(length)
proc b32pel(length: int): int = Base32LowerPad.encodedLength(length)
proc b32pdl(length: int): int = Base32LowerPad.decodedLength(length)
proc b58fd(inbytes: openarray[char],
outbytes: var openarray[byte],
outlen: var int): MultibaseStatus =
proc b58fd(inbytes: openArray[char],
outbytes: var openArray[byte],
outlen: var int): MultiBaseStatus =
result = b58ce(FLCBase58.decode(inbytes, outbytes, outlen))
proc b58fe(inbytes: openarray[byte],
outbytes: var openarray[char],
outlen: var int): MultibaseStatus =
proc b58fe(inbytes: openArray[byte],
outbytes: var openArray[char],
outlen: var int): MultiBaseStatus =
result = b58ce(FLCBase58.encode(inbytes, outbytes, outlen))
proc b58bd(inbytes: openarray[char],
outbytes: var openarray[byte],
outlen: var int): MultibaseStatus =
proc b58bd(inbytes: openArray[char],
outbytes: var openArray[byte],
outlen: var int): MultiBaseStatus =
result = b58ce(BTCBase58.decode(inbytes, outbytes, outlen))
proc b58be(inbytes: openarray[byte],
outbytes: var openarray[char],
outlen: var int): MultibaseStatus =
proc b58be(inbytes: openArray[byte],
outbytes: var openArray[char],
outlen: var int): MultiBaseStatus =
result = b58ce(BTCBase58.encode(inbytes, outbytes, outlen))
proc b58el(length: int): int = Base58.encodedLength(length)
@@ -227,48 +227,48 @@ proc b64dl(length: int): int = Base64.decodedLength(length)
proc b64pel(length: int): int = Base64Pad.encodedLength(length)
proc b64pdl(length: int): int = Base64Pad.decodedLength(length)
proc b64e(inbytes: openarray[byte],
outbytes: var openarray[char],
outlen: var int): MultibaseStatus =
proc b64e(inbytes: openArray[byte],
outbytes: var openArray[char],
outlen: var int): MultiBaseStatus =
result = b64ce(Base64.encode(inbytes, outbytes, outlen))
proc b64d(inbytes: openarray[char],
outbytes: var openarray[byte],
outlen: var int): MultibaseStatus =
proc b64d(inbytes: openArray[char],
outbytes: var openArray[byte],
outlen: var int): MultiBaseStatus =
result = b64ce(Base64.decode(inbytes, outbytes, outlen))
proc b64pe(inbytes: openarray[byte],
outbytes: var openarray[char],
outlen: var int): MultibaseStatus =
proc b64pe(inbytes: openArray[byte],
outbytes: var openArray[char],
outlen: var int): MultiBaseStatus =
result = b64ce(Base64Pad.encode(inbytes, outbytes, outlen))
proc b64pd(inbytes: openarray[char],
outbytes: var openarray[byte],
outlen: var int): MultibaseStatus =
proc b64pd(inbytes: openArray[char],
outbytes: var openArray[byte],
outlen: var int): MultiBaseStatus =
result = b64ce(Base64Pad.decode(inbytes, outbytes, outlen))
proc b64ue(inbytes: openarray[byte],
outbytes: var openarray[char],
outlen: var int): MultibaseStatus =
proc b64ue(inbytes: openArray[byte],
outbytes: var openArray[char],
outlen: var int): MultiBaseStatus =
result = b64ce(Base64Url.encode(inbytes, outbytes, outlen))
proc b64ud(inbytes: openarray[char],
outbytes: var openarray[byte],
outlen: var int): MultibaseStatus =
proc b64ud(inbytes: openArray[char],
outbytes: var openArray[byte],
outlen: var int): MultiBaseStatus =
result = b64ce(Base64Url.decode(inbytes, outbytes, outlen))
proc b64upe(inbytes: openarray[byte],
outbytes: var openarray[char],
outlen: var int): MultibaseStatus =
proc b64upe(inbytes: openArray[byte],
outbytes: var openArray[char],
outlen: var int): MultiBaseStatus =
result = b64ce(Base64UrlPad.encode(inbytes, outbytes, outlen))
proc b64upd(inbytes: openarray[char],
outbytes: var openarray[byte],
outlen: var int): MultibaseStatus =
proc b64upd(inbytes: openArray[char],
outbytes: var openArray[byte],
outlen: var int): MultiBaseStatus =
result = b64ce(Base64UrlPad.decode(inbytes, outbytes, outlen))
const
MultibaseCodecs = [
MultiBaseCodecs = [
MBCodec(name: "identity", code: chr(0x00),
decr: idd, encr: ide, decl: iddl, encl: idel
),
@@ -328,18 +328,16 @@ const
]
proc initMultiBaseCodeTable(): Table[char, MBCodec] {.compileTime.} =
result = initTable[char, MBCodec]()
for item in MultibaseCodecs:
for item in MultiBaseCodecs:
result[item.code] = item
proc initMultiBaseNameTable(): Table[string, MBCodec] {.compileTime.} =
result = initTable[string, MBCodec]()
for item in MultibaseCodecs:
for item in MultiBaseCodecs:
result[item.name] = item
const
CodeMultibases = initMultiBaseCodeTable()
NameMultibases = initMultiBaseNameTable()
CodeMultiBases = initMultiBaseCodeTable()
NameMultiBases = initMultiBaseNameTable()
proc encodedLength*(mbtype: typedesc[MultiBase], encoding: string,
length: int): int =
@@ -348,7 +346,7 @@ proc encodedLength*(mbtype: typedesc[MultiBase], encoding: string,
##
## Procedure returns ``-1`` if ``encoding`` scheme is not supported or
## not present.
let mb = NameMultibases.getOrDefault(encoding)
let mb = NameMultiBases.getOrDefault(encoding)
if len(mb.name) == 0 or isNil(mb.encl):
result = -1
else:
@@ -361,7 +359,7 @@ proc decodedLength*(mbtype: typedesc[MultiBase], encoding: char,
length: int): int =
## Return estimated size of buffer to store MultiBase decoded value with
## encoding character ``encoding`` of length ``length``.
let mb = CodeMultibases.getOrDefault(encoding)
let mb = CodeMultiBases.getOrDefault(encoding)
if len(mb.name) == 0 or isNil(mb.decl) or length == 0:
result = -1
else:
@@ -371,8 +369,8 @@ proc decodedLength*(mbtype: typedesc[MultiBase], encoding: char,
result = mb.decl(length - 1)
proc encode*(mbtype: typedesc[MultiBase], encoding: string,
inbytes: openarray[byte], outbytes: var openarray[char],
outlen: var int): MultibaseStatus =
inbytes: openArray[byte], outbytes: var openArray[char],
outlen: var int): MultiBaseStatus =
## Encode array ``inbytes`` using MultiBase encoding scheme ``encoding`` and
## store encoded value to ``outbytes``.
##
@@ -388,11 +386,11 @@ proc encode*(mbtype: typedesc[MultiBase], encoding: string,
##
## On successfull encoding ``MultiBaseStatus.Success`` will be returned and
## ``outlen`` will be set to number of encoded octets (bytes).
let mb = NameMultibases.getOrDefault(encoding)
let mb = NameMultiBases.getOrDefault(encoding)
if len(mb.name) == 0:
return MultibaseStatus.BadCodec
return MultiBaseStatus.BadCodec
if isNil(mb.encr) or isNil(mb.encl):
return MultibaseStatus.NotSupported
return MultiBaseStatus.NotSupported
if len(outbytes) > 1:
result = mb.encr(inbytes, outbytes.toOpenArray(1, outbytes.high),
outlen)
@@ -410,8 +408,8 @@ proc encode*(mbtype: typedesc[MultiBase], encoding: string,
result = MultiBaseStatus.Overrun
outlen = mb.encl(len(inbytes)) + 1
proc decode*(mbtype: typedesc[MultiBase], inbytes: openarray[char],
outbytes: var openarray[byte], outlen: var int): MultibaseStatus =
proc decode*(mbtype: typedesc[MultiBase], inbytes: openArray[char],
outbytes: var openArray[byte], outlen: var int): MultiBaseStatus =
## Decode array ``inbytes`` using MultiBase encoding and store decoded value
## to ``outbytes``.
##
@@ -428,24 +426,24 @@ proc decode*(mbtype: typedesc[MultiBase], inbytes: openarray[char],
## ``outlen`` will be set to number of encoded octets (bytes).
let length = len(inbytes)
if length == 0:
return MultibaseStatus.Incorrect
let mb = CodeMultibases.getOrDefault(inbytes[0])
return MultiBaseStatus.Incorrect
let mb = CodeMultiBases.getOrDefault(inbytes[0])
if len(mb.name) == 0:
return MultibaseStatus.BadCodec
return MultiBaseStatus.BadCodec
if isNil(mb.decr) or isNil(mb.decl):
return MultibaseStatus.NotSupported
return MultiBaseStatus.NotSupported
if length == 1:
outlen = 0
result = MultibaseStatus.Success
result = MultiBaseStatus.Success
else:
result = mb.decr(inbytes.toOpenArray(1, length - 1), outbytes, outlen)
proc encode*(mbtype: typedesc[MultiBase], encoding: string,
inbytes: openarray[byte]): Result[string, string] =
inbytes: openArray[byte]): Result[string, string] =
## Encode array ``inbytes`` using MultiBase encoding scheme ``encoding`` and
## return encoded string.
let length = len(inbytes)
let mb = NameMultibases.getOrDefault(encoding)
let mb = NameMultiBases.getOrDefault(encoding)
if len(mb.name) == 0:
return err("multibase: Encoding scheme is incorrect!")
if isNil(mb.encr) or isNil(mb.encl):
@@ -464,13 +462,13 @@ proc encode*(mbtype: typedesc[MultiBase], encoding: string,
buffer[0] = mb.code
ok(buffer)
proc decode*(mbtype: typedesc[MultiBase], inbytes: openarray[char]): Result[seq[byte], string] =
proc decode*(mbtype: typedesc[MultiBase], inbytes: openArray[char]): Result[seq[byte], string] =
## Decode MultiBase encoded array ``inbytes`` and return decoded sequence of
## bytes.
let length = len(inbytes)
if length == 0:
return err("multibase: Could not decode zero-length string")
let mb = CodeMultibases.getOrDefault(inbytes[0])
let mb = CodeMultiBases.getOrDefault(inbytes[0])
if len(mb.name) == 0:
return err("multibase: MultiBase scheme is incorrect!")
if isNil(mb.decr) or isNil(mb.decl):

View File

@@ -1,23 +1,21 @@
## Nim-Libp2p
## Copyright (c) 2018 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not BE copied, modified, or distributed except according to
## those terms.
# Nim-Libp2p
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not BE copied, modified, or distributed except according to
# those terms.
## This module implements MultiCodec.
{.push raises: [Defect].}
{.push raises: [].}
import tables, hashes
import varint, vbuffer
import stew/results
export results
{.deadCodeElim: on.}
## List of officially supported codecs can BE found here
## https://github.com/multiformats/multicodec/blob/master/table.csv
const MultiCodecList = [
@@ -193,14 +191,18 @@ const MultiCodecList = [
("p2p", 0x01A5),
("http", 0x01E0),
("https", 0x01BB),
("tls", 0x01C0),
("quic", 0x01CC),
("ws", 0x01DD),
("wss", 0x01DE), # not in multicodec list
("wss", 0x01DE),
("p2p-websocket-star", 0x01DF), # not in multicodec list
("p2p-webrtc-star", 0x0113), # not in multicodec list
("p2p-webrtc-direct", 0x0114), # not in multicodec list
("onion", 0x01BC),
("onion3", 0x01BD),
("p2p-circuit", 0x0122),
("libp2p-peer-record", 0x0301),
("dns", 0x35),
("dns4", 0x36),
("dns6", 0x37),
("dnsaddr", 0x38),
@@ -242,12 +244,10 @@ const
InvalidMultiCodec* = MultiCodec(-1)
proc initMultiCodecNameTable(): Table[string, int] {.compileTime.} =
result = initTable[string, int]()
for item in MultiCodecList:
result[item[0]] = item[1]
proc initMultiCodecCodeTable(): Table[int, string] {.compileTime.} =
result = initTable[int, string]()
for item in MultiCodecList:
result[item[1]] = item[0]
@@ -271,7 +271,7 @@ proc `$`*(mc: MultiCodec): string =
## Returns string representation of MultiCodec ``mc``.
let name = CodeCodecs.getOrDefault(int(mc), "")
doAssert(name != "")
name
name
proc `==`*(mc: MultiCodec, name: string): bool {.inline.} =
## Compares MultiCodec ``mc`` with string ``name``.

View File

@@ -1,11 +1,11 @@
## Nim-Libp2p
## Copyright (c) 2018 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-Libp2p
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
## This module implements MultiHash.
## Supported hashes are:
@@ -21,7 +21,7 @@
## 1. SKEIN
## 2. MURMUR
{.push raises: [Defect].}
{.push raises: [].}
import tables
import nimcrypto/[sha, sha2, keccak, blake2, hash, utils]
@@ -41,8 +41,8 @@ const
ErrParseError = "Parse error fromHex"
type
MHashCoderProc* = proc(data: openarray[byte],
output: var openarray[byte]) {.nimcall, gcsafe, noSideEffect, raises: [Defect].}
MHashCoderProc* = proc(data: openArray[byte],
output: var openArray[byte]) {.nimcall, gcsafe, noSideEffect, raises: [].}
MHash* = object
mcodec*: MultiCodec
size*: int
@@ -56,20 +56,20 @@ type
MhResult*[T] = Result[T, cstring]
proc identhash(data: openarray[byte], output: var openarray[byte]) =
proc identhash(data: openArray[byte], output: var openArray[byte]) =
if len(output) > 0:
var length = if len(data) > len(output): len(output)
else: len(data)
copyMem(addr output[0], unsafeAddr data[0], length)
proc sha1hash(data: openarray[byte], output: var openarray[byte]) =
proc sha1hash(data: openArray[byte], output: var openArray[byte]) =
if len(output) > 0:
var digest = sha1.digest(data)
var length = if sha1.sizeDigest > len(output): len(output)
else: sha1.sizeDigest
copyMem(addr output[0], addr digest.data[0], length)
proc dblsha2_256hash(data: openarray[byte], output: var openarray[byte]) =
proc dblsha2_256hash(data: openArray[byte], output: var openArray[byte]) =
if len(output) > 0:
var digest1 = sha256.digest(data)
var digest2 = sha256.digest(digest1.data)
@@ -77,91 +77,91 @@ proc dblsha2_256hash(data: openarray[byte], output: var openarray[byte]) =
else: sha256.sizeDigest
copyMem(addr output[0], addr digest2.data[0], length)
proc blake2Bhash(data: openarray[byte], output: var openarray[byte]) =
proc blake2Bhash(data: openArray[byte], output: var openArray[byte]) =
if len(output) > 0:
var digest = blake2_512.digest(data)
var length = if blake2_512.sizeDigest > len(output): len(output)
else: blake2_512.sizeDigest
copyMem(addr output[0], addr digest.data[0], length)
proc blake2Shash(data: openarray[byte], output: var openarray[byte]) =
proc blake2Shash(data: openArray[byte], output: var openArray[byte]) =
if len(output) > 0:
var digest = blake2_256.digest(data)
var length = if blake2_256.sizeDigest > len(output): len(output)
else: blake2_256.sizeDigest
copyMem(addr output[0], addr digest.data[0], length)
proc sha2_256hash(data: openarray[byte], output: var openarray[byte]) =
proc sha2_256hash(data: openArray[byte], output: var openArray[byte]) =
if len(output) > 0:
var digest = sha256.digest(data)
var length = if sha256.sizeDigest > len(output): len(output)
else: sha256.sizeDigest
copyMem(addr output[0], addr digest.data[0], length)
proc sha2_512hash(data: openarray[byte], output: var openarray[byte]) =
proc sha2_512hash(data: openArray[byte], output: var openArray[byte]) =
if len(output) > 0:
var digest = sha512.digest(data)
var length = if sha512.sizeDigest > len(output): len(output)
else: sha512.sizeDigest
copyMem(addr output[0], addr digest.data[0], length)
proc sha3_224hash(data: openarray[byte], output: var openarray[byte]) =
proc sha3_224hash(data: openArray[byte], output: var openArray[byte]) =
if len(output) > 0:
var digest = sha3_224.digest(data)
var length = if sha3_224.sizeDigest > len(output): len(output)
else: sha3_224.sizeDigest
copyMem(addr output[0], addr digest.data[0], length)
proc sha3_256hash(data: openarray[byte], output: var openarray[byte]) =
proc sha3_256hash(data: openArray[byte], output: var openArray[byte]) =
if len(output) > 0:
var digest = sha3_256.digest(data)
var length = if sha3_256.sizeDigest > len(output): len(output)
else: sha3_256.sizeDigest
copyMem(addr output[0], addr digest.data[0], length)
proc sha3_384hash(data: openarray[byte], output: var openarray[byte]) =
proc sha3_384hash(data: openArray[byte], output: var openArray[byte]) =
if len(output) > 0:
var digest = sha3_384.digest(data)
var length = if sha3_384.sizeDigest > len(output): len(output)
else: sha3_384.sizeDigest
copyMem(addr output[0], addr digest.data[0], length)
proc sha3_512hash(data: openarray[byte], output: var openarray[byte]) =
proc sha3_512hash(data: openArray[byte], output: var openArray[byte]) =
if len(output) > 0:
var digest = sha3_512.digest(data)
var length = if sha3_512.sizeDigest > len(output): len(output)
else: sha3_512.sizeDigest
copyMem(addr output[0], addr digest.data[0], length)
proc keccak_224hash(data: openarray[byte], output: var openarray[byte]) =
proc keccak_224hash(data: openArray[byte], output: var openArray[byte]) =
if len(output) > 0:
var digest = keccak224.digest(data)
var length = if keccak224.sizeDigest > len(output): len(output)
else: keccak224.sizeDigest
copyMem(addr output[0], addr digest.data[0], length)
proc keccak_256hash(data: openarray[byte], output: var openarray[byte]) =
proc keccak_256hash(data: openArray[byte], output: var openArray[byte]) =
if len(output) > 0:
var digest = keccak256.digest(data)
var length = if keccak256.sizeDigest > len(output): len(output)
else: keccak256.sizeDigest
copyMem(addr output[0], addr digest.data[0], length)
proc keccak_384hash(data: openarray[byte], output: var openarray[byte]) =
proc keccak_384hash(data: openArray[byte], output: var openArray[byte]) =
if len(output) > 0:
var digest = keccak384.digest(data)
var length = if keccak384.sizeDigest > len(output): len(output)
else: keccak384.sizeDigest
copyMem(addr output[0], addr digest.data[0], length)
proc keccak_512hash(data: openarray[byte], output: var openarray[byte]) =
proc keccak_512hash(data: openArray[byte], output: var openArray[byte]) =
if len(output) > 0:
var digest = keccak512.digest(data)
var length = if keccak512.sizeDigest > len(output): len(output)
else: keccak512.sizeDigest
copyMem(addr output[0], addr digest.data[0], length)
proc shake_128hash(data: openarray[byte], output: var openarray[byte]) =
proc shake_128hash(data: openArray[byte], output: var openArray[byte]) =
var sctx: shake128
if len(output) > 0:
sctx.init()
@@ -170,7 +170,7 @@ proc shake_128hash(data: openarray[byte], output: var openarray[byte]) =
discard sctx.output(addr output[0], uint(len(output)))
sctx.clear()
proc shake_256hash(data: openarray[byte], output: var openarray[byte]) =
proc shake_256hash(data: openArray[byte], output: var openArray[byte]) =
var sctx: shake256
if len(output) > 0:
sctx.init()
@@ -208,16 +208,16 @@ const
),
MHash(mcodec: multiCodec("shake-128"), size: 32, coder: shake_128hash),
MHash(mcodec: multiCodec("shake-256"), size: 64, coder: shake_256hash),
MHash(mcodec: multiCodec("keccak-224"), size: keccak_224.sizeDigest,
MHash(mcodec: multiCodec("keccak-224"), size: keccak224.sizeDigest,
coder: keccak_224hash
),
MHash(mcodec: multiCodec("keccak-256"), size: keccak_256.sizeDigest,
MHash(mcodec: multiCodec("keccak-256"), size: keccak256.sizeDigest,
coder: keccak_256hash
),
MHash(mcodec: multiCodec("keccak-384"), size: keccak_384.sizeDigest,
MHash(mcodec: multiCodec("keccak-384"), size: keccak384.sizeDigest,
coder: keccak_384hash
),
MHash(mcodec: multiCodec("keccak-512"), size: keccak_512.sizeDigest,
MHash(mcodec: multiCodec("keccak-512"), size: keccak512.sizeDigest,
coder: keccak_512hash
),
MHash(mcodec: multiCodec("blake2b-8"), size: 1, coder: blake2Bhash),
@@ -319,14 +319,13 @@ const
]
proc initMultiHashCodeTable(): Table[MultiCodec, MHash] {.compileTime.} =
result = initTable[MultiCodec, MHash]()
for item in HashesList:
result[item.mcodec] = item
const
CodeHashes = initMultiHashCodeTable()
proc digestImplWithHash(hash: MHash, data: openarray[byte]): MultiHash =
proc digestImplWithHash(hash: MHash, data: openArray[byte]): MultiHash =
var buffer: array[MaxHashSize, byte]
result.data = initVBuffer()
result.mcodec = hash.mcodec
@@ -344,7 +343,7 @@ proc digestImplWithHash(hash: MHash, data: openarray[byte]): MultiHash =
result.size = hash.size
result.data.finish()
proc digestImplWithoutHash(hash: MHash, data: openarray[byte]): MultiHash =
proc digestImplWithoutHash(hash: MHash, data: openArray[byte]): MultiHash =
result.data = initVBuffer()
result.mcodec = hash.mcodec
result.size = len(data)
@@ -355,7 +354,7 @@ proc digestImplWithoutHash(hash: MHash, data: openarray[byte]): MultiHash =
result.data.finish()
proc digest*(mhtype: typedesc[MultiHash], hashname: string,
data: openarray[byte]): MhResult[MultiHash] {.inline.} =
data: openArray[byte]): MhResult[MultiHash] {.inline.} =
## Perform digest calculation using hash algorithm with name ``hashname`` on
## data array ``data``.
let mc = MultiCodec.codec(hashname)
@@ -369,7 +368,7 @@ proc digest*(mhtype: typedesc[MultiHash], hashname: string,
ok(digestImplWithHash(hash, data))
proc digest*(mhtype: typedesc[MultiHash], hashcode: int,
data: openarray[byte]): MhResult[MultiHash] {.inline.} =
data: openArray[byte]): MhResult[MultiHash] {.inline.} =
## Perform digest calculation using hash algorithm with code ``hashcode`` on
## data array ``data``.
let hash = CodeHashes.getOrDefault(hashcode)
@@ -407,7 +406,7 @@ proc init*[T](mhtype: typedesc[MultiHash], hashcode: MultiCodec,
ok(digestImplWithoutHash(hash, mdigest.data))
proc init*(mhtype: typedesc[MultiHash], hashname: string,
bdigest: openarray[byte]): MhResult[MultiHash] {.inline.} =
bdigest: openArray[byte]): MhResult[MultiHash] {.inline.} =
## Create MultiHash from array of bytes ``bdigest`` and hash algorithm code
## ``hashcode``.
let mc = MultiCodec.codec(hashname)
@@ -423,7 +422,7 @@ proc init*(mhtype: typedesc[MultiHash], hashname: string,
ok(digestImplWithoutHash(hash, bdigest))
proc init*(mhtype: typedesc[MultiHash], hashcode: MultiCodec,
bdigest: openarray[byte]): MhResult[MultiHash] {.inline.} =
bdigest: openArray[byte]): MhResult[MultiHash] {.inline.} =
## Create MultiHash from array of bytes ``bdigest`` and hash algorithm code
## ``hashcode``.
let hash = CodeHashes.getOrDefault(hashcode)
@@ -434,7 +433,7 @@ proc init*(mhtype: typedesc[MultiHash], hashcode: MultiCodec,
else:
ok(digestImplWithoutHash(hash, bdigest))
proc decode*(mhtype: typedesc[MultiHash], data: openarray[byte],
proc decode*(mhtype: typedesc[MultiHash], data: openArray[byte],
mhash: var MultiHash): MhResult[int] =
## Decode MultiHash value from array of bytes ``data``.
##
@@ -479,7 +478,7 @@ proc decode*(mhtype: typedesc[MultiHash], data: openarray[byte],
vb.offset + int(size) - 1))
ok(vb.offset + int(size))
proc validate*(mhtype: typedesc[MultiHash], data: openarray[byte]): bool =
proc validate*(mhtype: typedesc[MultiHash], data: openArray[byte]): bool =
## Returns ``true`` if array of bytes ``data`` has correct MultiHash inside.
var code, size: uint64
var res: VarintResult[void]
@@ -510,7 +509,7 @@ proc validate*(mhtype: typedesc[MultiHash], data: openarray[byte]): bool =
result = true
proc init*(mhtype: typedesc[MultiHash],
data: openarray[byte]): MhResult[MultiHash] {.inline.} =
data: openArray[byte]): MhResult[MultiHash] {.inline.} =
## Create MultiHash from bytes array ``data``.
var hash: MultiHash
discard ? MultiHash.decode(data, hash)
@@ -531,7 +530,7 @@ proc init58*(mhtype: typedesc[MultiHash],
if MultiHash.decode(Base58.decode(data), result) == -1:
raise newException(MultihashError, "Incorrect MultiHash binary format")
proc cmp(a: openarray[byte], b: openarray[byte]): bool {.inline.} =
proc cmp(a: openArray[byte], b: openArray[byte]): bool {.inline.} =
if len(a) != len(b):
return false
var n = len(a)

View File

@@ -1,58 +1,62 @@
## Nim-LibP2P
## Copyright (c) 2019 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-LibP2P
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
import std/[strutils]
{.push raises: [].}
import std/[strutils, sequtils, tables]
import chronos, chronicles, stew/byteutils
import stream/connection,
vbuffer,
protocols/protocol
logScope:
topics = "libp2p multistream"
const
MsgSize* = 64*1024
Codec* = "/multistream/1.0.0"
MsgSize = 1024
Codec = "/multistream/1.0.0"
MSCodec* = "\x13" & Codec & "\n"
Na* = "\x03na\n"
Ls* = "\x03ls\n"
Na = "na\n"
Ls = "ls\n"
type
Matcher* = proc (proto: string): bool {.gcsafe.}
Matcher* = proc (proto: string): bool {.gcsafe, raises: [].}
HandlerHolder* = object
MultiStreamError* = object of LPError
HandlerHolder* = ref object
protos*: seq[string]
protocol*: LPProtocol
match*: Matcher
openedStreams: CountTable[PeerId]
MultistreamSelect* = ref object of RootObj
handlers*: seq[HandlerHolder]
codec*: string
proc newMultistream*(): MultistreamSelect =
new result
result.codec = MSCodec
proc new*(T: typedesc[MultistreamSelect]): T =
T(
codec: Codec,
)
template validateSuffix(str: string): untyped =
if str.endsWith("\n"):
str.removeSuffix("\n")
else:
raise newException(CatchableError, "MultistreamSelect failed, malformed message")
raise newException(MultiStreamError, "MultistreamSelect failed, malformed message")
proc select*(m: MultistreamSelect,
proc select*(_: MultistreamSelect | type MultistreamSelect,
conn: Connection,
proto: seq[string]):
Future[string] {.async.} =
trace "initiating handshake", conn, codec = m.codec
trace "initiating handshake", conn, codec = Codec
## select a remote protocol
await conn.write(m.codec) # write handshake
await conn.writeLp(Codec & "\n") # write handshake
if proto.len() > 0:
trace "selecting proto", conn, proto = proto[0]
await conn.writeLp((proto[0] & "\n")) # select proto
@@ -62,7 +66,7 @@ proc select*(m: MultistreamSelect,
if s != Codec:
notice "handshake failed", conn, codec = s
raise newException(CatchableError, "MultistreamSelect handshake failed")
raise newException(MultiStreamError, "MultistreamSelect handshake failed")
else:
trace "multistream handshake success", conn
@@ -74,6 +78,7 @@ proc select*(m: MultistreamSelect,
trace "reading first requested proto", conn
if s == proto[0]:
trace "successfully selected ", conn, proto = proto[0]
conn.protocol = proto[0]
return proto[0]
elif proto.len > 1:
# Try to negotiate alternatives
@@ -86,19 +91,20 @@ proc select*(m: MultistreamSelect,
validateSuffix(s)
if s == p:
trace "selected protocol", conn, protocol = s
conn.protocol = s
return s
return ""
else:
# No alternatives, fail
return ""
proc select*(m: MultistreamSelect,
proc select*(_: MultistreamSelect | type MultistreamSelect,
conn: Connection,
proto: string): Future[bool] {.async.} =
if proto.len > 0:
return (await m.select(conn, @[proto])) == proto
return (await MultistreamSelect.select(conn, @[proto])) == proto
else:
return (await m.select(conn, @[])) == Codec
return (await MultistreamSelect.select(conn, @[])) == Codec
proc select*(m: MultistreamSelect, conn: Connection): Future[bool] =
m.select(conn, "")
@@ -109,7 +115,7 @@ proc list*(m: MultistreamSelect,
if not await m.select(conn):
return
await conn.write(Ls) # send ls
await conn.writeLp(Ls) # send ls
var list = newSeq[string]()
let ms = string.fromBytes(await conn.readLp(MsgSize))
@@ -119,54 +125,86 @@ proc list*(m: MultistreamSelect,
result = list
proc handle*(
_: type MultistreamSelect,
conn: Connection,
protos: seq[string],
matchers = newSeq[Matcher](),
active: bool = false,
): Future[string] {.async, gcsafe.} =
trace "Starting multistream negotiation", conn, handshaked = active
var handshaked = active
while not conn.atEof:
var ms = string.fromBytes(await conn.readLp(MsgSize))
validateSuffix(ms)
if not handshaked and ms != Codec:
debug "expected handshake message", conn, instead=ms
raise newException(CatchableError,
"MultistreamSelect handling failed, invalid first message")
trace "handle: got request", conn, ms
if ms.len() <= 0:
trace "handle: invalid proto", conn
await conn.writeLp(Na)
case ms:
of "ls":
trace "handle: listing protos", conn
#TODO this doens't seem to follow spec, each protocol
# should be length prefixed. Not very important
# since LS is getting deprecated
await conn.writeLp(protos.join("\n") & "\n")
of Codec:
if not handshaked:
await conn.writeLp(Codec & "\n")
handshaked = true
else:
trace "handle: sending `na` for duplicate handshake while handshaked",
conn
await conn.writeLp(Na)
elif ms in protos or matchers.anyIt(it(ms)):
trace "found handler", conn, protocol = ms
await conn.writeLp(ms & "\n")
conn.protocol = ms
return ms
else:
trace "no handlers", conn, protocol = ms
await conn.writeLp(Na)
proc handle*(m: MultistreamSelect, conn: Connection, active: bool = false) {.async, gcsafe.} =
trace "Starting multistream handler", conn, handshaked = active
var handshaked = active
var
handshaked = active
protos: seq[string]
matchers: seq[Matcher]
for h in m.handlers:
if not isNil(h.match):
matchers.add(h.match)
for proto in h.protos:
protos.add(proto)
try:
while not conn.atEof:
var ms = string.fromBytes(await conn.readLp(MsgSize))
validateSuffix(ms)
let ms = await MultistreamSelect.handle(conn, protos, matchers, active)
for h in m.handlers:
if (not isNil(h.match) and h.match(ms)) or h.protos.contains(ms):
trace "found handler", conn, protocol = ms
if not handshaked and ms != Codec:
notice "expected handshake message", conn, instead=ms
raise newException(CatchableError,
"MultistreamSelect handling failed, invalid first message")
trace "handle: got request", conn, ms
if ms.len() <= 0:
trace "handle: invalid proto", conn
await conn.write(Na)
if m.handlers.len() == 0:
trace "handle: sending `na` for protocol", conn, protocol = ms
await conn.write(Na)
continue
case ms:
of "ls":
trace "handle: listing protos", conn
var protos = ""
for h in m.handlers:
for proto in h.protos:
protos &= (proto & "\n")
await conn.writeLp(protos)
of Codec:
if not handshaked:
await conn.write(m.codec)
handshaked = true
else:
trace "handle: sending `na` for duplicate handshake while handshaked",
conn
await conn.write(Na)
else:
for h in m.handlers:
if (not isNil(h.match) and h.match(ms)) or h.protos.contains(ms):
trace "found handler", conn, protocol = ms
await conn.writeLp(ms & "\n")
await h.protocol.handler(conn, ms)
return
debug "no handlers", conn, protocol = ms
await conn.write(Na)
var protocolHolder = h
let maxIncomingStreams = protocolHolder.protocol.maxIncomingStreams
if protocolHolder.openedStreams.getOrDefault(conn.peerId) >= maxIncomingStreams:
debug "Max streams for protocol reached, blocking new stream",
conn, protocol = ms, maxIncomingStreams
return
protocolHolder.openedStreams.inc(conn.peerId)
try:
await protocolHolder.protocol.handler(conn, ms)
finally:
protocolHolder.openedStreams.inc(conn.peerId, -1)
if protocolHolder.openedStreams[conn.peerId] == 0:
protocolHolder.openedStreams.del(conn.peerId)
return
debug "no handlers", conn, ms
except CancelledError as exc:
raise exc
except CatchableError as exc:
@@ -204,3 +242,9 @@ proc addHandler*(m: MultistreamSelect,
m.handlers.add(HandlerHolder(protos: @[codec],
protocol: protocol,
match: matcher))
proc start*(m: MultistreamSelect) {.async.} =
await allFutures(m.handlers.mapIt(it.protocol.start()))
proc stop*(m: MultistreamSelect) {.async.} =
await allFutures(m.handlers.mapIt(it.protocol.stop()))

View File

@@ -1,18 +1,20 @@
## Nim-LibP2P
## Copyright (c) 2019 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-LibP2P
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
import chronos
import nimcrypto/utils, chronicles, stew/byteutils
{.push raises: [].}
import pkg/[chronos, chronicles, stew/byteutils]
import ../../stream/connection,
../../utility,
../../varint,
../../vbuffer
../../vbuffer,
../muxer
logScope:
topics = "libp2p mplexcoder"
@@ -32,7 +34,7 @@ type
msgType: MessageType
data: seq[byte]
InvalidMplexMsgType = object of CatchableError
InvalidMplexMsgType* = object of MuxerError
# https://github.com/libp2p/specs/tree/master/mplex#writing-to-a-stream
const MaxMsgSize* = 1 shl 20 # 1mb

View File

@@ -1,17 +1,18 @@
## Nim-LibP2P
## Copyright (c) 2019 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-LibP2P
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
{.push raises: [].}
import std/[oids, strformat]
import chronos, chronicles, metrics
import pkg/[chronos, chronicles, metrics]
import ./coder,
../muxer,
nimcrypto/utils,
../../stream/[bufferstream, connection, streamseq],
../../peerinfo
@@ -20,6 +21,15 @@ export connection
logScope:
topics = "libp2p mplexchannel"
when defined(libp2p_mplex_metrics):
declareHistogram libp2p_mplex_qlen, "message queue length",
buckets = [0.0, 1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0]
declareCounter libp2p_mplex_qlenclose, "closed because of max queuelen"
declareHistogram libp2p_mplex_qtime, "message queuing time"
when defined(libp2p_network_protocols_metrics):
declareCounter libp2p_protocols_bytes, "total sent or received bytes", ["protocol", "direction"]
## Channel half-closed states
##
## | State | Closed local | Closed remote
@@ -45,17 +55,22 @@ type
initiator*: bool # initiated remotely or locally flag
isOpen*: bool # has channel been opened
closedLocal*: bool # has channel been closed locally
remoteReset*: bool # has channel been remotely reset
localReset*: bool # has channel been reset locally
msgCode*: MessageType # cached in/out message code
closeCode*: MessageType # cached in/out close code
resetCode*: MessageType # cached in/out reset code
writes*: int # In-flight writes
func shortLog*(s: LPChannel): auto =
if s.isNil: "LPChannel(nil)"
elif s.conn.peerInfo.isNil: $s.oid
elif s.name != $s.oid and s.name.len > 0:
&"{shortLog(s.conn.peerInfo.peerId)}:{s.oid}:{s.name}"
else: &"{shortLog(s.conn.peerInfo.peerId)}:{s.oid}"
try:
if s.isNil: "LPChannel(nil)"
elif s.name != $s.oid and s.name.len > 0:
&"{shortLog(s.conn.peerId)}:{s.oid}:{s.name}"
else: &"{shortLog(s.conn.peerId)}:{s.oid}"
except ValueError as exc:
raise newException(Defect, exc.msg)
chronicles.formatIt(LPChannel): shortLog(it)
proc open*(s: LPChannel) {.async, gcsafe.} =
@@ -65,6 +80,8 @@ proc open*(s: LPChannel) {.async, gcsafe.} =
try:
await s.conn.writeMsg(s.id, MessageType.New, s.name)
s.isOpen = true
except CancelledError as exc:
raise exc
except CatchableError as exc:
await s.conn.close()
raise exc
@@ -85,6 +102,7 @@ proc reset*(s: LPChannel) {.async, gcsafe.} =
s.isClosed = true
s.closedLocal = true
s.localReset = not s.remoteReset
trace "Resetting channel", s, len = s.len
@@ -134,7 +152,7 @@ method close*(s: LPChannel) {.async, gcsafe.} =
method initStream*(s: LPChannel) =
if s.objName.len == 0:
s.objName = "LPChannel"
s.objName = LPChannelTrackerName
s.timeoutHandler = proc(): Future[void] {.gcsafe.} =
trace "Idle timeout expired, resetting LPChannel", s
@@ -150,8 +168,20 @@ method readOnce*(s: LPChannel,
## channels are blocked - in particular, this means that reading from one
## channel must not be done from within a callback / read handler of another
## or the reads will lock each other.
if s.remoteReset:
raise newLPStreamResetError()
if s.localReset:
raise newLPStreamClosedError()
if s.atEof():
raise newLPStreamRemoteClosedError()
if s.conn.closed:
raise newLPStreamConnDownError()
try:
let bytes = await procCall BufferStream(s).readOnce(pbytes, nbytes)
when defined(libp2p_network_protocols_metrics):
if s.protocol.len > 0:
libp2p_protocols_bytes.inc(bytes.int64, labelValues=[s.protocol, "in"])
trace "readOnce", s, bytes
if bytes == 0:
await s.closeUnderlying()
@@ -162,13 +192,17 @@ method readOnce*(s: LPChannel,
# data has been lost in s.readBuf and there's no way to gracefully recover /
# use the channel any more
await s.reset()
raise exc
raise newLPStreamConnDownError(exc)
method write*(s: LPChannel, msg: seq[byte]): Future[void] {.async.} =
## Write to mplex channel - there may be up to MaxWrite concurrent writes
## pending after which the peer is disconnected
if s.closedLocal or s.conn.closed:
proc prepareWrite(s: LPChannel, msg: seq[byte]): Future[void] {.async.} =
# prepareWrite is the slow path of writing a message - see conditions in
# write
if s.remoteReset:
raise newLPStreamResetError()
if s.closedLocal:
raise newLPStreamClosedError()
if s.conn.closed:
raise newLPStreamConnDownError()
if msg.len == 0:
return
@@ -176,28 +210,71 @@ method write*(s: LPChannel, msg: seq[byte]): Future[void] {.async.} =
if s.writes >= MaxWrites:
debug "Closing connection, too many in-flight writes on channel",
s, conn = s.conn, writes = s.writes
when defined(libp2p_mplex_metrics):
libp2p_mplex_qlenclose.inc()
await s.reset()
await s.conn.close()
return
s.writes += 1
if not s.isOpen:
await s.open()
await s.conn.writeMsg(s.id, s.msgCode, msg)
proc completeWrite(
s: LPChannel, fut: Future[void], msgLen: int): Future[void] {.async.} =
try:
if not s.isOpen:
await s.open()
s.writes += 1
# writes should happen in sequence
trace "write msg", s, conn = s.conn, len = msg.len
when defined(libp2p_mplex_metrics):
libp2p_mplex_qlen.observe(s.writes.int64 - 1)
libp2p_mplex_qtime.time:
await fut
else:
await fut
when defined(libp2p_network_protocols_metrics):
if s.protocol.len > 0:
libp2p_protocols_bytes.inc(msgLen.int64, labelValues=[s.protocol, "out"])
await s.conn.writeMsg(s.id, s.msgCode, msg)
s.activity = true
except CancelledError as exc:
# Chronos may still send the data
raise exc
except LPStreamConnDownError as exc:
await s.reset()
await s.conn.close()
raise exc
except LPStreamEOFError as exc:
raise exc
except CatchableError as exc:
trace "exception in lpchannel write handler", s, msg = exc.msg
await s.reset()
await s.conn.close()
raise exc
raise newLPStreamConnDownError(exc)
finally:
s.writes -= 1
method write*(s: LPChannel, msg: seq[byte]): Future[void] =
## Write to mplex channel - there may be up to MaxWrite concurrent writes
## pending after which the peer is disconnected
let
closed = s.closedLocal or s.conn.closed
let fut =
if (not closed) and msg.len > 0 and s.writes < MaxWrites and s.isOpen:
# Fast path: Avoid a copy of msg being kept in the closure created by
# `{.async.}` as this drives up memory usage - the conditions are laid out
# in prepareWrite
s.conn.writeMsg(s.id, s.msgCode, msg)
else:
prepareWrite(s, msg)
s.completeWrite(fut, msg.len)
method getWrapped*(s: LPChannel): Connection = s.conn
proc init*(
L: type LPChannel,
id: uint64,

View File

@@ -1,11 +1,13 @@
## Nim-LibP2P
## Copyright (c) 2019 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-LibP2P
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
{.push raises: [].}
import tables, sequtils, oids
import chronos, chronicles, stew/byteutils, metrics
@@ -32,8 +34,7 @@ when defined(libp2p_expensive_metrics):
"mplex channels", labels = ["initiator", "peer"])
type
TooManyChannels* = object of CatchableError
InvalidChannelIdError* = object of CatchableError
InvalidChannelIdError* = object of MuxerError
Mplex* = ref object of Muxer
channels: array[bool, Table[uint64, LPChannel]]
@@ -44,7 +45,9 @@ type
oid*: Oid
maxChannCount: int
func shortLog*(m: MPlex): auto = shortLog(m.connection)
func shortLog*(m: Mplex): auto =
shortLog(m.connection)
chronicles.formatIt(Mplex): shortLog(it)
proc newTooManyChannels(): ref TooManyChannels =
@@ -64,19 +67,16 @@ proc cleanupChann(m: Mplex, chann: LPChannel) {.async, inline.} =
when defined(libp2p_expensive_metrics):
libp2p_mplex_channels.set(
m.channels[chann.initiator].len.int64,
labelValues = [$chann.initiator, $m.connection.peerInfo.peerId])
labelValues = [$chann.initiator, $m.connection.peerId])
except CatchableError as exc:
# This is top-level procedure which will work as separate task, so it
# do not need to propogate CancelledError, and no other exceptions should
# happen here
warn "Error cleaning up mplex channel", m, chann, msg = exc.msg
proc newStreamInternal*(m: Mplex,
initiator: bool = true,
chanId: uint64 = 0,
name: string = "",
timeout: Duration):
LPChannel {.gcsafe.} =
timeout: Duration): LPChannel
{.gcsafe, raises: [InvalidChannelIdError].} =
## create new channel/stream
##
let id = if initiator:
@@ -93,8 +93,11 @@ proc newStreamInternal*(m: Mplex,
name,
timeout = timeout)
result.peerInfo = m.connection.peerInfo
result.peerId = m.connection.peerId
result.observedAddr = m.connection.observedAddr
result.transportDir = m.connection.transportDir
when defined(libp2p_agents_metrics):
result.shortAgent = m.connection.shortAgent
trace "Creating new channel", m, channel = result, id, initiator, name
@@ -106,7 +109,7 @@ proc newStreamInternal*(m: Mplex,
when defined(libp2p_expensive_metrics):
libp2p_mplex_channels.set(
m.channels[initiator].len.int64,
labelValues = [$initiator, $m.connection.peerInfo.peerId])
labelValues = [$initiator, $m.connection.peerId])
proc handleStream(m: Mplex, chann: LPChannel) {.async.} =
## call the muxer stream handler for this channel
@@ -116,8 +119,6 @@ proc handleStream(m: Mplex, chann: LPChannel) {.async.} =
trace "finished handling stream", m, chann
doAssert(chann.closed, "connection not closed by handler!")
except CatchableError as exc:
# This is top-level procedure which will work as separate task, so it
# do not need to propogate CancelledError.
trace "Exception in mplex stream handler", m, chann, msg = exc.msg
await chann.reset()
@@ -173,16 +174,21 @@ method handle*(m: Mplex) {.async, gcsafe.} =
raise newLPStreamLimitError()
trace "pushing data to channel", m, channel, len = data.len
await channel.pushData(data)
trace "pushed data to channel", m, channel, len = data.len
try:
await channel.pushData(data)
trace "pushed data to channel", m, channel, len = data.len
except LPStreamClosedError as exc:
# Channel is being closed, but `cleanupChann` was not yet triggered.
trace "pushing data to channel failed", m, channel, len = data.len,
msg = exc.msg
discard # Ignore message, same as if `cleanupChann` had completed.
of MessageType.CloseIn, MessageType.CloseOut:
await channel.pushEof()
of MessageType.ResetIn, MessageType.ResetOut:
channel.remoteReset = true
await channel.reset()
except CancelledError:
# This procedure is spawned as task and it is not part of public API, so
# there no way for this procedure to be cancelled implicitly.
debug "Unexpected cancellation in mplex handler", m
except LPStreamEOFError as exc:
trace "Stream EOF", m, msg = exc.msg
@@ -192,9 +198,10 @@ method handle*(m: Mplex) {.async, gcsafe.} =
await m.close()
trace "Stopped mplex handler", m
proc init*(M: type Mplex,
proc new*(M: type Mplex,
conn: Connection,
inTimeout, outTimeout: Duration = DefaultChanTimeout,
inTimeout: Duration = DefaultChanTimeout,
outTimeout: Duration = DefaultChanTimeout,
maxChannCount: int = MaxChannelCount): Mplex =
M(connection: conn,
inChannTimeout: inTimeout,
@@ -239,3 +246,7 @@ method close*(m: Mplex) {.async, gcsafe.} =
m.channels[true].clear()
trace "Closed mplex", m
method getStreams*(m: Mplex): seq[Connection] =
for c in m.channels[false].values: result.add(c)
for c in m.channels[true].values: result.add(c)

View File

@@ -1,15 +1,16 @@
## Nim-LibP2P
## Copyright (c) 2019 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-LibP2P
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
{.push raises: [].}
import chronos, chronicles
import ../protocols/protocol,
../stream/connection,
import ../stream/connection,
../errors
logScope:
@@ -19,60 +20,44 @@ const
DefaultChanTimeout* = 5.minutes
type
StreamHandler* = proc(conn: Connection): Future[void] {.gcsafe.}
MuxerHandler* = proc(muxer: Muxer): Future[void] {.gcsafe.}
MuxerError* = object of LPError
TooManyChannels* = object of MuxerError
StreamHandler* = proc(conn: Connection): Future[void] {.gcsafe, raises: [].}
MuxerHandler* = proc(muxer: Muxer): Future[void] {.gcsafe, raises: [].}
Muxer* = ref object of RootObj
streamHandler*: StreamHandler
handler*: Future[void]
connection*: Connection
# user provider proc that returns a constructed Muxer
MuxerConstructor* = proc(conn: Connection): Muxer {.gcsafe, closure.}
MuxerConstructor* = proc(conn: Connection): Muxer {.gcsafe, closure, raises: [].}
# this wraps a creator proc that knows how to make muxers
MuxerProvider* = ref object of LPProtocol
MuxerProvider* = object
newMuxer*: MuxerConstructor
streamHandler*: StreamHandler # triggered every time there is a new stream, called for any muxer instance
muxerHandler*: MuxerHandler # triggered every time there is a new muxed connection created
codec*: string
func shortLog*(m: Muxer): auto = shortLog(m.connection)
func shortLog*(m: Muxer): auto =
if isNil(m): "nil"
else: shortLog(m.connection)
chronicles.formatIt(Muxer): shortLog(it)
# muxer interface
method newStream*(m: Muxer, name: string = "", lazy: bool = false):
Future[Connection] {.base, async, gcsafe.} = discard
method close*(m: Muxer) {.base, async, gcsafe.} = discard
method close*(m: Muxer) {.base, async, gcsafe.} =
if not isNil(m.connection):
await m.connection.close()
method handle*(m: Muxer): Future[void] {.base, async, gcsafe.} = discard
proc newMuxerProvider*(creator: MuxerConstructor, codec: string): MuxerProvider {.gcsafe.} =
new result
result.newMuxer = creator
result.codec = codec
result.init()
proc new*(
T: typedesc[MuxerProvider],
creator: MuxerConstructor,
codec: string): T {.gcsafe.} =
method init(c: MuxerProvider) =
proc handler(conn: Connection, proto: string) {.async, gcsafe, closure.} =
trace "starting muxer handler", proto=proto, conn
try:
let
muxer = c.newMuxer(conn)
let muxerProvider = T(newMuxer: creator, codec: codec)
muxerProvider
if not isNil(c.streamHandler):
muxer.streamHandler = c.streamHandler
var futs = newSeq[Future[void]]()
futs &= muxer.handle()
# finally await both the futures
if not isNil(c.muxerHandler):
futs &= c.muxerHandler(muxer)
checkFutures(await allFinished(futs))
except CancelledError as exc:
raise exc
except CatchableError as exc:
trace "exception in muxer handler", exc = exc.msg, conn, proto
finally:
await conn.close()
c.handler = handler
method getStreams*(m: Muxer): seq[Connection] {.base.} = doAssert false, "not implemented"

View File

@@ -0,0 +1,529 @@
# Nim-LibP2P
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
{.push raises: [].}
import sequtils, std/[tables]
import chronos, chronicles, metrics, stew/[endians2, byteutils, objects]
import ../muxer,
../../stream/connection
export muxer
logScope:
topics = "libp2p yamux"
const
YamuxCodec* = "/yamux/1.0.0"
YamuxVersion = 0.uint8
DefaultWindowSize = 256000
MaxChannelCount = 200
when defined(libp2p_yamux_metrics):
declareGauge(libp2p_yamux_channels, "yamux channels", labels = ["initiator", "peer"])
declareHistogram libp2p_yamux_send_queue, "message send queue length (in byte)",
buckets = [0.0, 100.0, 250.0, 1000.0, 2000.0, 1600.0, 6400.0, 25600.0, 256000.0]
declareHistogram libp2p_yamux_recv_queue, "message recv queue length (in byte)",
buckets = [0.0, 100.0, 250.0, 1000.0, 2000.0, 1600.0, 6400.0, 25600.0, 256000.0]
type
YamuxError* = object of CatchableError
MsgType = enum
Data = 0x0
WindowUpdate = 0x1
Ping = 0x2
GoAway = 0x3
MsgFlags {.size: 2.} = enum
Syn
Ack
Fin
Rst
GoAwayStatus = enum
NormalTermination = 0x0,
ProtocolError = 0x1,
InternalError = 0x2,
YamuxHeader = object
version: uint8
msgType: MsgType
flags: set[MsgFlags]
streamId: uint32
length: uint32
proc readHeader(conn: LPStream): Future[YamuxHeader] {.async, gcsafe.} =
var buffer: array[12, byte]
await conn.readExactly(addr buffer[0], 12)
result.version = buffer[0]
let flags = fromBytesBE(uint16, buffer[2..3])
if not result.msgType.checkedEnumAssign(buffer[1]) or flags notin 0'u16..15'u16:
raise newException(YamuxError, "Wrong header")
result.flags = cast[set[MsgFlags]](flags)
result.streamId = fromBytesBE(uint32, buffer[4..7])
result.length = fromBytesBE(uint32, buffer[8..11])
return result
proc `$`(header: YamuxHeader): string =
result = "{" & $header.msgType & ", "
result &= "{" & header.flags.foldl(if a != "": a & ", " & $b else: $b, "") & "}, "
result &= "streamId: " & $header.streamId & ", "
result &= "length: " & $header.length & "}"
proc encode(header: YamuxHeader): array[12, byte] =
result[0] = header.version
result[1] = uint8(header.msgType)
result[2..3] = toBytesBE(uint16(cast[uint8](header.flags))) # workaround https://github.com/nim-lang/Nim/issues/21789
result[4..7] = toBytesBE(header.streamId)
result[8..11] = toBytesBE(header.length)
proc write(conn: LPStream, header: YamuxHeader): Future[void] {.gcsafe.} =
trace "write directly on stream", h = $header
var buffer = header.encode()
return conn.write(@buffer)
proc ping(T: type[YamuxHeader], flag: MsgFlags, pingData: uint32): T =
T(
version: YamuxVersion,
msgType: MsgType.Ping,
flags: {flag},
length: pingData
)
proc goAway(T: type[YamuxHeader], status: GoAwayStatus): T =
T(
version: YamuxVersion,
msgType: MsgType.GoAway,
length: uint32(status)
)
proc data(
T: type[YamuxHeader],
streamId: uint32,
length: uint32 = 0,
flags: set[MsgFlags] = {},
): T =
T(
version: YamuxVersion,
msgType: MsgType.Data,
length: length,
flags: flags,
streamId: streamId
)
proc windowUpdate(
T: type[YamuxHeader],
streamId: uint32,
delta: uint32,
flags: set[MsgFlags] = {},
): T =
T(
version: YamuxVersion,
msgType: MsgType.WindowUpdate,
length: delta,
flags: flags,
streamId: streamId
)
type
ToSend = tuple
data: seq[byte]
sent: int
fut: Future[void]
YamuxChannel* = ref object of Connection
id: uint32
recvWindow: int
sendWindow: int
maxRecvWindow: int
conn: Connection
isSrc: bool
opened: bool
isSending: bool
sendQueue: seq[ToSend]
recvQueue: seq[byte]
isReset: bool
remoteReset: bool
closedRemotely: Future[void]
closedLocally: bool
receivedData: AsyncEvent
returnedEof: bool
proc `$`(channel: YamuxChannel): string =
result = if channel.conn.dir == Out: "=> " else: "<= "
result &= $channel.id
var s: seq[string] = @[]
if channel.closedRemotely.done():
s.add("ClosedRemotely")
if channel.closedLocally:
s.add("ClosedLocally")
if channel.isReset:
s.add("Reset")
if s.len > 0:
result &= " {" & s.foldl(if a != "": a & ", " & b else: b, "") & "}"
proc sendQueueBytes(channel: YamuxChannel, limit: bool = false): int =
for (elem, sent, _) in channel.sendQueue:
result.inc(min(elem.len - sent, if limit: channel.maxRecvWindow div 3 else: elem.len - sent))
proc actuallyClose(channel: YamuxChannel) {.async.} =
if channel.closedLocally and channel.sendQueue.len == 0 and
channel.closedRemotely.done():
await procCall Connection(channel).closeImpl()
proc remoteClosed(channel: YamuxChannel) {.async.} =
if not channel.closedRemotely.done():
channel.closedRemotely.complete()
await channel.actuallyClose()
method closeImpl*(channel: YamuxChannel) {.async, gcsafe.} =
if not channel.closedLocally:
channel.closedLocally = true
if channel.isReset == false and channel.sendQueue.len == 0:
await channel.conn.write(YamuxHeader.data(channel.id, 0, {Fin}))
await channel.actuallyClose()
proc reset(channel: YamuxChannel, isLocal: bool = false) {.async.} =
if channel.isReset:
return
trace "Reset channel"
channel.isReset = true
channel.remoteReset = not isLocal
for (d, s, fut) in channel.sendQueue:
fut.fail(newLPStreamEOFError())
channel.sendQueue = @[]
channel.recvQueue = @[]
channel.sendWindow = 0
if not channel.closedLocally:
if isLocal:
try: await channel.conn.write(YamuxHeader.data(channel.id, 0, {Rst}))
except LPStreamEOFError as exc: discard
except LPStreamClosedError as exc: discard
await channel.close()
if not channel.closedRemotely.done():
await channel.remoteClosed()
channel.receivedData.fire()
if not isLocal:
# If we reset locally, we want to flush up to a maximum of recvWindow
# bytes. We use the recvWindow in the proc cleanupChann.
channel.recvWindow = 0
proc updateRecvWindow(channel: YamuxChannel) {.async.} =
let inWindow = channel.recvWindow + channel.recvQueue.len
if inWindow > channel.maxRecvWindow div 2:
return
let delta = channel.maxRecvWindow - inWindow
channel.recvWindow.inc(delta)
await channel.conn.write(YamuxHeader.windowUpdate(
channel.id,
delta.uint32
))
trace "increasing the recvWindow", delta
method readOnce*(
channel: YamuxChannel,
pbytes: pointer,
nbytes: int):
Future[int] {.async.} =
if channel.isReset:
raise if channel.remoteReset:
newLPStreamResetError()
elif channel.closedLocally:
newLPStreamClosedError()
else:
newLPStreamConnDownError()
if channel.returnedEof:
raise newLPStreamRemoteClosedError()
if channel.recvQueue.len == 0:
channel.receivedData.clear()
await channel.closedRemotely or channel.receivedData.wait()
if channel.closedRemotely.done() and channel.recvQueue.len == 0:
channel.returnedEof = true
return 0
let toRead = min(channel.recvQueue.len, nbytes)
var p = cast[ptr UncheckedArray[byte]](pbytes)
toOpenArray(p, 0, nbytes - 1)[0..<toRead] = channel.recvQueue.toOpenArray(0, toRead - 1)
channel.recvQueue = channel.recvQueue[toRead..^1]
# We made some room in the recv buffer let the peer know
await channel.updateRecvWindow()
channel.activity = true
return toRead
proc gotDataFromRemote(channel: YamuxChannel, b: seq[byte]) {.async.} =
channel.recvWindow -= b.len
channel.recvQueue = channel.recvQueue.concat(b)
channel.receivedData.fire()
when defined(libp2p_yamux_metrics):
libp2p_yamux_recv_queue.observe(channel.recvQueue.len.int64)
await channel.updateRecvWindow()
proc setMaxRecvWindow*(channel: YamuxChannel, maxRecvWindow: int) =
channel.maxRecvWindow = maxRecvWindow
proc trySend(channel: YamuxChannel) {.async.} =
if channel.isSending:
return
channel.isSending = true
defer: channel.isSending = false
while channel.sendQueue.len != 0:
channel.sendQueue.keepItIf(not (it.fut.cancelled() and it.sent == 0))
if channel.sendWindow == 0:
trace "send window empty"
if channel.sendQueueBytes(true) > channel.maxRecvWindow:
debug "channel send queue too big, resetting", maxSendWindow=channel.maxRecvWindow,
currentQueueSize = channel.sendQueueBytes(true)
try:
await channel.reset(true)
except CatchableError as exc:
debug "failed to reset", msg=exc.msg
break
let
bytesAvailable = channel.sendQueueBytes()
toSend = min(channel.sendWindow, bytesAvailable)
var
sendBuffer = newSeqUninitialized[byte](toSend + 12)
header = YamuxHeader.data(channel.id, toSend.uint32)
inBuffer = 0
if toSend >= bytesAvailable and channel.closedLocally:
trace "last buffer we'll sent on this channel", toSend, bytesAvailable
header.flags.incl({Fin})
sendBuffer[0..<12] = header.encode()
var futures: seq[Future[void]]
while inBuffer < toSend:
let (data, sent, fut) = channel.sendQueue[0]
let bufferToSend = min(data.len - sent, toSend - inBuffer)
sendBuffer.toOpenArray(12, 12 + toSend - 1)[inBuffer..<(inBuffer+bufferToSend)] =
channel.sendQueue[0].data.toOpenArray(sent, sent + bufferToSend - 1)
channel.sendQueue[0].sent.inc(bufferToSend)
if channel.sendQueue[0].sent >= data.len:
futures.add(fut)
channel.sendQueue.delete(0)
inBuffer.inc(bufferToSend)
trace "build send buffer", h = $header, msg=string.fromBytes(sendBuffer[12..^1])
channel.sendWindow.dec(toSend)
try: await channel.conn.write(sendBuffer)
except CatchableError as exc:
let connDown = newLPStreamConnDownError(exc)
for fut in futures.items():
fut.fail(connDown)
await channel.reset()
break
for fut in futures.items():
fut.complete()
channel.activity = true
method write*(channel: YamuxChannel, msg: seq[byte]): Future[void] =
result = newFuture[void]("Yamux Send")
if channel.remoteReset:
result.fail(newLPStreamResetError())
return result
if channel.closedLocally or channel.isReset:
result.fail(newLPStreamClosedError())
return result
if msg.len == 0:
result.complete()
return result
channel.sendQueue.add((msg, 0, result))
when defined(libp2p_yamux_metrics):
libp2p_yamux_recv_queue.observe(channel.sendQueueBytes().int64)
asyncSpawn channel.trySend()
proc open*(channel: YamuxChannel) {.async, gcsafe.} =
if channel.opened:
trace "Try to open channel twice"
return
channel.opened = true
await channel.conn.write(YamuxHeader.data(channel.id, 0, {if channel.isSrc: Syn else: Ack}))
method getWrapped*(channel: YamuxChannel): Connection = channel.conn
type
Yamux* = ref object of Muxer
channels: Table[uint32, YamuxChannel]
flushed: Table[uint32, int]
currentId: uint32
isClosed: bool
maxChannCount: int
proc lenBySrc(m: Yamux, isSrc: bool): int =
for v in m.channels.values():
if v.isSrc == isSrc: result += 1
proc cleanupChann(m: Yamux, channel: YamuxChannel) {.async.} =
await channel.join()
m.channels.del(channel.id)
when defined(libp2p_yamux_metrics):
libp2p_yamux_channels.set(m.lenBySrc(channel.isSrc).int64, [$channel.isSrc, $channel.peerId])
if channel.isReset and channel.recvWindow > 0:
m.flushed[channel.id] = channel.recvWindow
proc createStream(m: Yamux, id: uint32, isSrc: bool): YamuxChannel =
result = YamuxChannel(
id: id,
maxRecvWindow: DefaultWindowSize,
recvWindow: DefaultWindowSize,
sendWindow: DefaultWindowSize,
isSrc: isSrc,
conn: m.connection,
receivedData: newAsyncEvent(),
closedRemotely: newFuture[void]()
)
result.objName = "YamuxStream"
result.dir = if isSrc: Direction.Out else: Direction.In
result.timeoutHandler = proc(): Future[void] {.gcsafe.} =
trace "Idle timeout expired, resetting YamuxChannel"
result.reset()
result.initStream()
result.peerId = m.connection.peerId
result.observedAddr = m.connection.observedAddr
result.transportDir = m.connection.transportDir
when defined(libp2p_agents_metrics):
result.shortAgent = m.connection.shortAgent
m.channels[id] = result
asyncSpawn m.cleanupChann(result)
trace "created channel", id, pid=m.connection.peerId
when defined(libp2p_yamux_metrics):
libp2p_yamux_channels.set(m.lenBySrc(isSrc).int64, [$isSrc, $result.peerId])
method close*(m: Yamux) {.async.} =
if m.isClosed == true:
trace "Already closed"
return
m.isClosed = true
trace "Closing yamux"
let channels = toSeq(m.channels.values())
for channel in channels:
await channel.reset(true)
try: await m.connection.write(YamuxHeader.goAway(NormalTermination))
except CatchableError as exc: trace "failed to send goAway", msg=exc.msg
await m.connection.close()
trace "Closed yamux"
proc handleStream(m: Yamux, channel: YamuxChannel) {.async.} =
## call the muxer stream handler for this channel
##
try:
await m.streamHandler(channel)
trace "finished handling stream"
doAssert(channel.isClosed, "connection not closed by handler!")
except CatchableError as exc:
trace "Exception in yamux stream handler", msg = exc.msg
await channel.reset()
method handle*(m: Yamux) {.async, gcsafe.} =
trace "Starting yamux handler", pid=m.connection.peerId
try:
while not m.connection.atEof:
trace "waiting for header"
let header = await m.connection.readHeader()
trace "got message", h = $header
case header.msgType:
of Ping:
if MsgFlags.Syn in header.flags:
await m.connection.write(YamuxHeader.ping(MsgFlags.Ack, header.length))
of GoAway:
var status: GoAwayStatus
if status.checkedEnumAssign(header.length): trace "Received go away", status
else: trace "Received unexpected error go away"
break
of Data, WindowUpdate:
if MsgFlags.Syn in header.flags:
if header.streamId in m.channels:
debug "Trying to create an existing channel, skipping", id=header.streamId
else:
if header.streamId in m.flushed:
m.flushed.del(header.streamId)
if header.streamId mod 2 == m.currentId mod 2:
raise newException(YamuxError, "Peer used our reserved stream id")
let newStream = m.createStream(header.streamId, false)
if m.channels.len >= m.maxChannCount:
await newStream.reset()
continue
await newStream.open()
asyncSpawn m.handleStream(newStream)
elif header.streamId notin m.channels:
if header.streamId notin m.flushed:
raise newException(YamuxError, "Unknown stream ID: " & $header.streamId)
elif header.msgType == Data:
# Flush the data
m.flushed[header.streamId].dec(int(header.length))
if m.flushed[header.streamId] < 0:
raise newException(YamuxError, "Peer exhausted the recvWindow after reset")
if header.length > 0:
var buffer = newSeqUninitialized[byte](header.length)
await m.connection.readExactly(addr buffer[0], int(header.length))
continue
let channel = m.channels[header.streamId]
if header.msgType == WindowUpdate:
channel.sendWindow += int(header.length)
await channel.trySend()
else:
if header.length.int > channel.recvWindow.int:
# check before allocating the buffer
raise newException(YamuxError, "Peer exhausted the recvWindow")
if header.length > 0:
var buffer = newSeqUninitialized[byte](header.length)
await m.connection.readExactly(addr buffer[0], int(header.length))
trace "Msg Rcv", msg=string.fromBytes(buffer)
await channel.gotDataFromRemote(buffer)
if MsgFlags.Fin in header.flags:
trace "remote closed channel"
await channel.remoteClosed()
if MsgFlags.Rst in header.flags:
trace "remote reset channel"
await channel.reset()
except LPStreamEOFError as exc:
trace "Stream EOF", msg = exc.msg
except YamuxError as exc:
trace "Closing yamux connection", error=exc.msg
await m.connection.write(YamuxHeader.goAway(ProtocolError))
finally:
await m.close()
trace "Stopped yamux handler"
method getStreams*(m: Yamux): seq[Connection] =
for c in m.channels.values: result.add(c)
method newStream*(
m: Yamux,
name: string = "",
lazy: bool = false): Future[Connection] {.async, gcsafe.} =
if m.channels.len > m.maxChannCount - 1:
raise newException(TooManyChannels, "max allowed channel count exceeded")
let stream = m.createStream(m.currentId, true)
m.currentId += 2
if not lazy:
await stream.open()
return stream
proc new*(T: type[Yamux], conn: Connection, maxChannCount: int = MaxChannelCount): T =
T(
connection: conn,
currentId: if conn.dir == Out: 1 else: 2,
maxChannCount: maxChannCount
)

View File

@@ -0,0 +1,169 @@
# Nim-LibP2P
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
{.push raises: [].}
import
std/[streams, strutils, sets, sequtils],
chronos, chronicles, stew/byteutils,
dnsclientpkg/[protocol, types],
../utility
import
nameresolver
logScope:
topics = "libp2p dnsresolver"
type
DnsResolver* = ref object of NameResolver
nameServers*: seq[TransportAddress]
proc questionToBuf(address: string, kind: QKind): seq[byte] =
try:
var
header = initHeader()
question = initQuestion(address, kind)
requestStream = header.toStream()
question.toStream(requestStream)
let dataLen = requestStream.getPosition()
requestStream.setPosition(0)
var buf = newSeq[byte](dataLen)
discard requestStream.readData(addr buf[0], dataLen)
return buf
except CatchableError as exc:
info "Failed to created DNS buffer", msg = exc.msg
return newSeq[byte](0)
proc getDnsResponse(
dnsServer: TransportAddress,
address: string,
kind: QKind): Future[Response] {.async.} =
var sendBuf = questionToBuf(address, kind)
if sendBuf.len == 0:
raise newException(ValueError, "Incorrect DNS query")
let receivedDataFuture = newFuture[void]()
proc datagramDataReceived(transp: DatagramTransport,
raddr: TransportAddress): Future[void] {.async, closure.} =
receivedDataFuture.complete()
let sock =
if dnsServer.family == AddressFamily.IPv6:
newDatagramTransport6(datagramDataReceived)
else:
newDatagramTransport(datagramDataReceived)
try:
await sock.sendTo(dnsServer, addr sendBuf[0], sendBuf.len)
await receivedDataFuture or sleepAsync(5.seconds) #unix default
if not receivedDataFuture.finished:
raise newException(IOError, "DNS server timeout")
let rawResponse = sock.getMessage()
# parseResponse can has a raises: [Exception, ..] because of
# https://github.com/nim-lang/Nim/commit/035134de429b5d99c5607c5fae912762bebb6008
# it can't actually raise though
return exceptionToAssert: parseResponse(string.fromBytes(rawResponse))
finally:
await sock.closeWait()
method resolveIp*(
self: DnsResolver,
address: string,
port: Port,
domain: Domain = Domain.AF_UNSPEC): Future[seq[TransportAddress]] {.async.} =
trace "Resolving IP using DNS", address, servers = self.nameServers.mapIt($it), domain
for _ in 0 ..< self.nameServers.len:
let server = self.nameServers[0]
var responseFutures: seq[Future[Response]]
if domain == Domain.AF_INET or domain == Domain.AF_UNSPEC:
responseFutures.add(getDnsResponse(server, address, A))
if domain == Domain.AF_INET6 or domain == Domain.AF_UNSPEC:
let fut = getDnsResponse(server, address, AAAA)
if server.family == AddressFamily.IPv6:
trace "IPv6 DNS server, puting AAAA records first", server = $server
responseFutures.insert(fut)
else:
responseFutures.add(fut)
var
resolvedAddresses: OrderedSet[string]
resolveFailed = false
for fut in responseFutures:
try:
let resp = await fut
for answer in resp.answers:
# toString can has a raises: [Exception, ..] because of
# https://github.com/nim-lang/Nim/commit/035134de429b5d99c5607c5fae912762bebb6008
# it can't actually raise though
resolvedAddresses.incl(
exceptionToAssert(answer.toString())
)
except CancelledError as e:
raise e
except ValueError as e:
info "Invalid DNS query", address, error=e.msg
return @[]
except CatchableError as e:
info "Failed to query DNS", address, error=e.msg
resolveFailed = true
break
if resolveFailed:
self.nameServers.add(self.nameServers[0])
self.nameServers.delete(0)
continue
trace "Got IPs from DNS server", resolvedAddresses, server = $server
return resolvedAddresses.toSeq().mapIt(initTAddress(it, port))
debug "Failed to resolve address, returning empty set"
return @[]
method resolveTxt*(
self: DnsResolver,
address: string): Future[seq[string]] {.async.} =
trace "Resolving TXT using DNS", address, servers = self.nameServers.mapIt($it)
for _ in 0 ..< self.nameServers.len:
let server = self.nameServers[0]
try:
# toString can has a raises: [Exception, ..] because of
# https://github.com/nim-lang/Nim/commit/035134de429b5d99c5607c5fae912762bebb6008
# it can't actually raise though
let response = await getDnsResponse(server, address, TXT)
return exceptionToAssert:
trace "Got TXT response", server = $server, answer=response.answers.mapIt(it.toString())
response.answers.mapIt(it.toString())
except CancelledError as e:
raise e
except CatchableError as e:
info "Failed to query DNS", address, error=e.msg
self.nameServers.add(self.nameServers[0])
self.nameServers.delete(0)
continue
debug "Failed to resolve TXT, returning empty set"
return @[]
proc new*(
T: typedesc[DnsResolver],
nameServers: seq[TransportAddress]): T =
T(nameServers: nameServers)

View File

@@ -0,0 +1,46 @@
# Nim-LibP2P
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
{.push raises: [].}
import
std/tables,
chronos, chronicles
import nameresolver
export tables
logScope:
topics = "libp2p mockresolver"
type MockResolver* = ref object of NameResolver
txtResponses*: Table[string, seq[string]]
# key: address, isipv6?
ipResponses*: Table[(string, bool), seq[string]]
method resolveIp*(
self: MockResolver,
address: string,
port: Port,
domain: Domain = Domain.AF_UNSPEC): Future[seq[TransportAddress]] {.async.} =
if domain == Domain.AF_INET or domain == Domain.AF_UNSPEC:
for resp in self.ipResponses.getOrDefault((address, false)):
result.add(initTAddress(resp, port))
if domain == Domain.AF_INET6 or domain == Domain.AF_UNSPEC:
for resp in self.ipResponses.getOrDefault((address, true)):
result.add(initTAddress(resp, port))
method resolveTxt*(
self: MockResolver,
address: string): Future[seq[string]] {.async.} =
return self.txtResponses.getOrDefault(address)
proc new*(T: typedesc[MockResolver]): T = T()

View File

@@ -0,0 +1,136 @@
# Nim-LibP2P
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
{.push raises: [].}
import std/[sugar, sets, sequtils, strutils]
import
chronos,
chronicles,
stew/endians2
import ".."/[multiaddress, multicodec]
logScope:
topics = "libp2p nameresolver"
type
NameResolver* = ref object of RootObj
method resolveTxt*(
self: NameResolver,
address: string): Future[seq[string]] {.async, base.} =
## Get TXT record
##
doAssert(false, "Not implemented!")
method resolveIp*(
self: NameResolver,
address: string,
port: Port,
domain: Domain = Domain.AF_UNSPEC): Future[seq[TransportAddress]] {.async, base.} =
## Resolve the specified address
##
doAssert(false, "Not implemented!")
proc getHostname*(ma: MultiAddress): string =
let
firstPart = ma[0].valueOr: return ""
fpSplitted = ($firstPart).split('/', 2)
if fpSplitted.len > 2: fpSplitted[2]
else: ""
proc resolveOneAddress(
self: NameResolver,
ma: MultiAddress,
domain: Domain = Domain.AF_UNSPEC,
prefix = ""): Future[seq[MultiAddress]]
{.async, raises: [MaError, TransportAddressError].} =
#Resolve a single address
var pbuf: array[2, byte]
var dnsval = getHostname(ma)
if ma[1].tryGet().protoArgument(pbuf).tryGet() == 0:
raise newException(MaError, "Incorrect port number")
let
port = Port(fromBytesBE(uint16, pbuf))
resolvedAddresses = await self.resolveIp(prefix & dnsval, port, domain)
return collect(newSeqOfCap(4)):
for address in resolvedAddresses:
var createdAddress = MultiAddress.init(address).tryGet()[0].tryGet()
for part in ma:
if DNS.match(part.tryGet()): continue
createdAddress &= part.tryGet()
createdAddress
proc resolveDnsAddr*(
self: NameResolver,
ma: MultiAddress,
depth: int = 0): Future[seq[MultiAddress]] {.async.} =
if not DNSADDR.matchPartial(ma):
return @[ma]
trace "Resolving dnsaddr", ma
if depth > 6:
info "Stopping DNSADDR recursion, probably malicious", ma
return @[]
var dnsval = getHostname(ma)
let txt = await self.resolveTxt("_dnsaddr." & dnsval)
trace "txt entries", txt
var result: seq[MultiAddress]
for entry in txt:
if not entry.startsWith("dnsaddr="): continue
let entryValue = MultiAddress.init(entry[8..^1]).tryGet()
if entryValue.contains(multiCodec("p2p")).tryGet() and ma.contains(multiCodec("p2p")).tryGet():
if entryValue[multiCodec("p2p")] != ma[multiCodec("p2p")]:
continue
let resolved = await self.resolveDnsAddr(entryValue, depth + 1)
for r in resolved:
result.add(r)
if result.len == 0:
debug "Failed to resolve a DNSADDR", ma
return @[]
return result
proc resolveMAddress*(
self: NameResolver,
address: MultiAddress): Future[seq[MultiAddress]] {.async.} =
var res = initOrderedSet[MultiAddress]()
if not DNS.matchPartial(address):
res.incl(address)
else:
let code = address[0].tryGet().protoCode().tryGet()
let seq = case code:
of multiCodec("dns"):
await self.resolveOneAddress(address)
of multiCodec("dns4"):
await self.resolveOneAddress(address, Domain.AF_INET)
of multiCodec("dns6"):
await self.resolveOneAddress(address, Domain.AF_INET6)
of multiCodec("dnsaddr"):
await self.resolveDnsAddr(address)
else:
assert false
@[address]
for ad in seq:
res.incl(ad)
return res.toSeq

View File

@@ -0,0 +1,86 @@
# Nim-LibP2P
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
{.push raises: [].}
import std/[sequtils, tables, sugar]
import chronos
import multiaddress, multicodec
type
## Manages observed MultiAddresses by reomte peers. It keeps track of the most observed IP and IP/Port.
ObservedAddrManager* = ref object of RootObj
observedIPsAndPorts: seq[MultiAddress]
maxSize: int
minCount: int
proc addObservation*(self:ObservedAddrManager, observedAddr: MultiAddress): bool =
## Adds a new observed MultiAddress. If the number of observations exceeds maxSize, the oldest one is removed.
if self.observedIPsAndPorts.len >= self.maxSize:
self.observedIPsAndPorts.del(0)
self.observedIPsAndPorts.add(observedAddr)
return true
proc getProtocol(self: ObservedAddrManager, observations: seq[MultiAddress], multiCodec: MultiCodec): Opt[MultiAddress] =
var countTable = toCountTable(observations)
countTable.sort()
var orderedPairs = toSeq(countTable.pairs)
for (ma, count) in orderedPairs:
let protoCode = (ma[0].flatMap(protoCode)).valueOr: continue
if protoCode == multiCodec and count >= self.minCount:
return Opt.some(ma)
return Opt.none(MultiAddress)
proc getMostObservedProtocol(self: ObservedAddrManager, multiCodec: MultiCodec): Opt[MultiAddress] =
## Returns the most observed IP address or none if the number of observations are less than minCount.
let observedIPs = collect:
for observedIp in self.observedIPsAndPorts:
observedIp[0].valueOr: continue
return self.getProtocol(observedIPs, multiCodec)
proc getMostObservedProtoAndPort(self: ObservedAddrManager, multiCodec: MultiCodec): Opt[MultiAddress] =
## Returns the most observed IP/Port address or none if the number of observations are less than minCount.
return self.getProtocol(self.observedIPsAndPorts, multiCodec)
proc getMostObservedProtosAndPorts*(self: ObservedAddrManager): seq[MultiAddress] =
## Returns the most observed IP4/Port and IP6/Port address or an empty seq if the number of observations
## are less than minCount.
var res: seq[MultiAddress]
self.getMostObservedProtoAndPort(multiCodec("ip4")).withValue(ip4):
res.add(ip4)
self.getMostObservedProtoAndPort(multiCodec("ip6")).withValue(ip6):
res.add(ip6)
return res
proc guessDialableAddr*(
self: ObservedAddrManager,
ma: MultiAddress): MultiAddress =
## Replaces the first proto value of each listen address by the corresponding (matching the proto code) most observed value.
## If the most observed value is not available, the original MultiAddress is returned.
let
maFirst = ma[0].valueOr: return ma
maRest = ma[1..^1].valueOr: return ma
maFirstProto = maFirst.protoCode().valueOr: return ma
let observedIP = self.getMostObservedProtocol(maFirstProto).valueOr: return ma
return concat(observedIP, maRest).valueOr: ma
proc `$`*(self: ObservedAddrManager): string =
## Returns a string representation of the ObservedAddrManager.
return "IPs and Ports: " & $self.observedIPsAndPorts
proc new*(
T: typedesc[ObservedAddrManager],
maxSize = 10,
minCount = 3): T =
## Creates a new ObservedAddrManager.
return T(
observedIPsAndPorts: newSeq[MultiAddress](),
maxSize: maxSize,
minCount: minCount)

View File

@@ -1,69 +1,73 @@
## Nim-LibP2P
## Copyright (c) 2018 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-LibP2P
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
## This module implementes API for libp2p peer.
{.push raises: [Defect].}
{.push raises: [].}
{.push public.}
import
std/[hashes, strutils],
stew/[base58, results],
chronicles,
nimcrypto/utils,
utility,
./crypto/crypto, ./multicodec, ./multihash, ./vbuffer,
./protobuf/minprotobuf
export results
export results, utility
const
maxInlineKeyLength* = 42
type
PeerID* = object
PeerId* = object
data*: seq[byte]
func `$`*(pid: PeerID): string =
func `$`*(pid: PeerId): string =
## Return base58 encoded ``pid`` representation.
Base58.encode(pid.data)
# This unusual call syntax is used to avoid a strange Nim compilation error
base58.encode(Base58, pid.data)
func shortLog*(pid: PeerId): string =
## Returns compact string representation of ``pid``.
var spid = $pid
if len(spid) > 10:
spid[3] = '*'
spid.delete(4, spid.high - 6)
spid.delete(4 .. spid.high - 6)
spid
chronicles.formatIt(PeerID): shortLog(it)
chronicles.formatIt(PeerId): shortLog(it)
func toBytes*(pid: PeerID, data: var openarray[byte]): int =
## Store PeerID ``pid`` to array of bytes ``data``.
func toBytes*(pid: PeerId, data: var openArray[byte]): int =
## Store PeerId ``pid`` to array of bytes ``data``.
##
## Returns number of bytes needed to store ``pid``.
result = len(pid.data)
if len(data) >= result and result > 0:
copyMem(addr data[0], unsafeAddr pid.data[0], result)
template getBytes*(pid: PeerID): seq[byte] =
## Return PeerID ``pid`` as array of bytes.
template getBytes*(pid: PeerId): seq[byte] =
## Return PeerId ``pid`` as array of bytes.
pid.data
func hex*(pid: PeerID): string =
func hex*(pid: PeerId): string =
## Returns hexadecimal string representation of ``pid``.
toHex(pid.data)
template len*(pid: PeerID): int =
template len*(pid: PeerId): int =
## Returns length of ``pid`` binary representation.
len(pid.data)
func cmp*(a, b: PeerID): int =
func cmp*(a, b: PeerId): int =
## Compares two peer ids ``a`` and ``b``.
## Returns:
##
@@ -78,29 +82,29 @@ func cmp*(a, b: PeerID): int =
inc(i)
result = len(a.data) - len(b.data)
template `<=`*(a, b: PeerID): bool =
template `<=`*(a, b: PeerId): bool =
(cmp(a, b) <= 0)
template `<`*(a, b: PeerID): bool =
template `<`*(a, b: PeerId): bool =
(cmp(a, b) < 0)
template `>=`*(a, b: PeerID): bool =
template `>=`*(a, b: PeerId): bool =
(cmp(a, b) >= 0)
template `>`*(a, b: PeerID): bool =
template `>`*(a, b: PeerId): bool =
(cmp(a, b) > 0)
template `==`*(a, b: PeerID): bool =
template `==`*(a, b: PeerId): bool =
(cmp(a, b) == 0)
template hash*(pid: PeerID): Hash =
template hash*(pid: PeerId): Hash =
hash(pid.data)
func validate*(pid: PeerID): bool =
func validate*(pid: PeerId): bool =
## Validate check if ``pid`` is empty or not.
len(pid.data) > 0 and MultiHash.validate(pid.data)
func hasPublicKey*(pid: PeerID): bool =
func hasPublicKey*(pid: PeerId): bool =
## Returns ``true`` if ``pid`` is small enough to hold public key inside.
if len(pid.data) > 0:
var mh: MultiHash
@@ -108,8 +112,8 @@ func hasPublicKey*(pid: PeerID): bool =
if mh.mcodec == multiCodec("identity"):
result = true
func extractPublicKey*(pid: PeerID, pubkey: var PublicKey): bool =
## Returns ``true`` if public key was successfully decoded from PeerID
func extractPublicKey*(pid: PeerId, pubkey: var PublicKey): bool =
## Returns ``true`` if public key was successfully decoded from PeerId
## ``pid``and stored to ``pubkey``.
##
## Returns ``false`` otherwise.
@@ -120,16 +124,16 @@ func extractPublicKey*(pid: PeerID, pubkey: var PublicKey): bool =
let length = len(mh.data.buffer)
result = pubkey.init(mh.data.buffer.toOpenArray(mh.dpos, length - 1))
func init*(pid: var PeerID, data: openarray[byte]): bool =
func init*(pid: var PeerId, data: openArray[byte]): bool =
## Initialize peer id from raw binary representation ``data``.
##
## Returns ``true`` if peer was successfully initialiazed.
var p = PeerID(data: @data)
var p = PeerId(data: @data)
if p.validate():
pid = p
result = true
func init*(pid: var PeerID, data: string): bool =
func init*(pid: var PeerId, data: string): bool =
## Initialize peer id from base58 encoded string representation.
##
## Returns ``true`` if peer was successfully initialiazed.
@@ -137,29 +141,29 @@ func init*(pid: var PeerID, data: string): bool =
var length = 0
if Base58.decode(data, p, length) == Base58Status.Success:
p.setLen(length)
var opid: PeerID
shallowCopy(opid.data, p)
var opid: PeerId
opid.data = p
if opid.validate():
pid = opid
result = true
func init*(t: typedesc[PeerID], data: openarray[byte]): Result[PeerID, cstring] =
func init*(t: typedesc[PeerId], data: openArray[byte]): Result[PeerId, cstring] =
## Create new peer id from raw binary representation ``data``.
var res: PeerID
var res: PeerId
if not init(res, data):
err("peerid: incorrect PeerID binary form")
err("peerid: incorrect PeerId binary form")
else:
ok(res)
func init*(t: typedesc[PeerID], data: string): Result[PeerID, cstring] =
func init*(t: typedesc[PeerId], data: string): Result[PeerId, cstring] =
## Create new peer id from base58 encoded string representation ``data``.
var res: PeerID
var res: PeerId
if not init(res, data):
err("peerid: incorrect PeerID string")
err("peerid: incorrect PeerId string")
else:
ok(res)
func init*(t: typedesc[PeerID], pubkey: PublicKey): Result[PeerID, cstring] =
func init*(t: typedesc[PeerId], pubkey: PublicKey): Result[PeerId, cstring] =
## Create new peer id from public key ``pubkey``.
var pubraw = ? pubkey.getBytes().orError(
cstring("peerid: failed to get bytes from given key"))
@@ -168,62 +172,44 @@ func init*(t: typedesc[PeerID], pubkey: PublicKey): Result[PeerID, cstring] =
mh = ? MultiHash.digest("identity", pubraw)
else:
mh = ? MultiHash.digest("sha2-256", pubraw)
ok(PeerID(data: mh.data.buffer))
ok(PeerId(data: mh.data.buffer))
func init*(t: typedesc[PeerID], seckey: PrivateKey): Result[PeerID, cstring] =
func init*(t: typedesc[PeerId], seckey: PrivateKey): Result[PeerId, cstring] =
## Create new peer id from private key ``seckey``.
PeerID.init(? seckey.getKey().orError(cstring("invalid private key")))
PeerId.init(? seckey.getPublicKey().orError(cstring("invalid private key")))
func match*(pid: PeerID, pubkey: PublicKey): bool =
proc random*(t: typedesc[PeerId], rng = newRng()): Result[PeerId, cstring] =
## Create new peer id with random public key.
let randomKey = PrivateKey.random(Secp256k1, rng[])[]
PeerId.init(randomKey).orError(cstring("failed to generate random key"))
func match*(pid: PeerId, pubkey: PublicKey): bool =
## Returns ``true`` if ``pid`` matches public key ``pubkey``.
let p = PeerID.init(pubkey)
if p.isErr:
false
else:
pid == p.get()
PeerId.init(pubkey) == Result[PeerId, cstring].ok(pid)
func match*(pid: PeerID, seckey: PrivateKey): bool =
func match*(pid: PeerId, seckey: PrivateKey): bool =
## Returns ``true`` if ``pid`` matches private key ``seckey``.
let p = PeerID.init(seckey)
if p.isErr:
false
else:
pid == p.get()
PeerId.init(seckey) == Result[PeerId, cstring].ok(pid)
## Serialization/Deserialization helpers
func write*(vb: var VBuffer, pid: PeerID) =
## Write PeerID value ``peerid`` to buffer ``vb``.
func write*(vb: var VBuffer, pid: PeerId) =
## Write PeerId value ``peerid`` to buffer ``vb``.
vb.writeSeq(pid.data)
func initProtoField*(index: int, pid: PeerID): ProtoField {.deprecated.} =
## Initialize ProtoField with PeerID ``value``.
initProtoField(index, pid.data)
func getValue*(data: var ProtoBuffer, field: int, value: var PeerID): int {.
deprecated.} =
## Read ``PeerID`` from ProtoBuf's message and validate it.
var pid: PeerID
result = getLengthValue(data, field, pid.data)
if result > 0:
if not pid.validate():
result = -1
else:
value = pid
func write*(pb: var ProtoBuffer, field: int, pid: PeerID) =
## Write PeerID value ``peerid`` to object ``pb`` using ProtoBuf's encoding.
func write*(pb: var ProtoBuffer, field: int, pid: PeerId) =
## Write PeerId value ``peerid`` to object ``pb`` using ProtoBuf's encoding.
write(pb, field, pid.data)
func getField*(pb: ProtoBuffer, field: int,
pid: var PeerID): ProtoResult[bool] {.inline.} =
## Read ``PeerID`` from ProtoBuf's message and validate it
pid: var PeerId): ProtoResult[bool] {.inline.} =
## Read ``PeerId`` from ProtoBuf's message and validate it
var buffer: seq[byte]
let res = ? pb.getField(field, buffer)
if not(res):
ok(false)
else:
var peerId: PeerID
var peerId: PeerId
if peerId.init(buffer):
pid = peerId
ok(true)

View File

@@ -1,49 +1,46 @@
## Nim-LibP2P
## Copyright (c) 2019 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-LibP2P
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
{.push raises: [Defect].}
{.push raises: [].}
{.push public.}
import options, sequtils, hashes
import chronos, chronicles
import peerid, multiaddress, crypto/crypto
import std/sequtils
import pkg/[chronos, chronicles, stew/results]
import peerid, multiaddress, multicodec, crypto/crypto, routing_record, errors, utility
export peerid, multiaddress, crypto
export peerid, multiaddress, crypto, routing_record, errors, results
## A peer can be constructed in one of tree ways:
## 1) A local peer with a private key
## 2) A remote peer with a PeerID and it's public key stored
## in the ``id`` itself
## 3) A remote peer with a standalone public key, that isn't
## encoded in the ``id``
##
## Our local peer info
type
KeyType* = enum
HasPrivate,
HasPublic
PeerInfoError* = object of LPError
PeerInfo* = ref object of RootObj
peerId*: PeerID
addrs*: seq[MultiAddress]
AddressMapper* =
proc(listenAddrs: seq[MultiAddress]): Future[seq[MultiAddress]]
{.gcsafe, raises: [].}
PeerInfo* {.public.} = ref object
peerId*: PeerId
listenAddrs*: seq[MultiAddress]
addrs: seq[MultiAddress]
addressMappers*: seq[AddressMapper]
protocols*: seq[string]
protoVersion*: string
agentVersion*: string
secure*: string
case keyType*: KeyType:
of HasPrivate:
privateKey*: PrivateKey
of HasPublic:
key: Option[PublicKey]
privateKey*: PrivateKey
publicKey*: PublicKey
signedPeerRecord*: SignedPeerRecord
func shortLog*(p: PeerInfo): auto =
(
peerId: $p.peerId,
listenAddrs: mapIt(p.listenAddrs, $it),
addrs: mapIt(p.addrs, $it),
protocols: mapIt(p.protocols, $it),
protoVersion: p.protoVersion,
@@ -51,60 +48,69 @@ func shortLog*(p: PeerInfo): auto =
)
chronicles.formatIt(PeerInfo): shortLog(it)
template postInit(peerinfo: PeerInfo,
addrs: openarray[MultiAddress],
protocols: openarray[string]) =
if len(addrs) > 0:
peerinfo.addrs = @addrs
if len(protocols) > 0:
peerinfo.protocols = @protocols
proc update*(p: PeerInfo) {.async.} =
p.addrs = p.listenAddrs
for mapper in p.addressMappers:
p.addrs = await mapper(p.addrs)
proc init*(p: typedesc[PeerInfo],
key: PrivateKey,
addrs: openarray[MultiAddress] = [],
protocols: openarray[string] = []): PeerInfo {.
raises: [Defect, ResultError[cstring]].} =
result = PeerInfo(keyType: HasPrivate, peerId: PeerID.init(key).tryGet(),
privateKey: key)
result.postInit(addrs, protocols)
p.signedPeerRecord = SignedPeerRecord.init(
p.privateKey,
PeerRecord.init(p.peerId, p.addrs)
).valueOr():
info "Can't update the signed peer record"
return
proc init*(p: typedesc[PeerInfo],
peerId: PeerID,
addrs: openarray[MultiAddress] = [],
protocols: openarray[string] = []): PeerInfo =
result = PeerInfo(keyType: HasPublic, peerId: peerId)
result.postInit(addrs, protocols)
proc addrs*(p: PeerInfo): seq[MultiAddress] =
p.addrs
proc init*(p: typedesc[PeerInfo],
peerId: string,
addrs: openarray[MultiAddress] = [],
protocols: openarray[string] = []): PeerInfo {.
raises: [Defect, ResultError[cstring]].} =
result = PeerInfo(keyType: HasPublic, peerId: PeerID.init(peerId).tryGet())
result.postInit(addrs, protocols)
proc fullAddrs*(p: PeerInfo): MaResult[seq[MultiAddress]] =
let peerIdPart = ? MultiAddress.init(multiCodec("p2p"), p.peerId.data)
var res: seq[MultiAddress]
for address in p.addrs:
res.add(? concat(address, peerIdPart))
ok(res)
proc init*(p: typedesc[PeerInfo],
key: PublicKey,
addrs: openarray[MultiAddress] = [],
protocols: openarray[string] = []): PeerInfo {.
raises: [Defect, ResultError[cstring]].}=
result = PeerInfo(keyType: HasPublic,
peerId: PeerID.init(key).tryGet(),
key: some(key))
proc parseFullAddress*(ma: MultiAddress): MaResult[(PeerId, MultiAddress)] =
let p2pPart = ? ma[^1]
if ? p2pPart.protoCode != multiCodec("p2p"):
return err("Missing p2p part from multiaddress!")
result.postInit(addrs, protocols)
let res = (
? PeerId.init(? p2pPart.protoArgument()).orErr("invalid peerid"),
? ma[0 .. ^2]
)
ok(res)
proc publicKey*(p: PeerInfo): Option[PublicKey] {.
raises: [Defect, ResultError[CryptoError]].} =
if p.keyType == HasPublic:
if p.peerId.hasPublicKey():
var pubKey: PublicKey
if p.peerId.extractPublicKey(pubKey):
result = some(pubKey)
elif p.key.isSome:
result = p.key
else:
result = some(p.privateKey.getKey().tryGet())
proc parseFullAddress*(ma: string | seq[byte]): MaResult[(PeerId, MultiAddress)] =
parseFullAddress(? MultiAddress.init(ma))
func hash*(p: PeerInfo): Hash =
cast[pointer](p).hash
proc new*(
p: typedesc[PeerInfo],
key: PrivateKey,
listenAddrs: openArray[MultiAddress] = [],
protocols: openArray[string] = [],
protoVersion: string = "",
agentVersion: string = "",
addressMappers = newSeq[AddressMapper](),
): PeerInfo
{.raises: [LPError].} =
let pubkey = try:
key.getPublicKey().tryGet()
except CatchableError:
raise newException(PeerInfoError, "invalid private key")
let peerId = PeerId.init(key).tryGet()
let peerInfo = PeerInfo(
peerId: peerId,
publicKey: pubkey,
privateKey: key,
protoVersion: protoVersion,
agentVersion: agentVersion,
listenAddrs: @listenAddrs,
protocols: @protocols,
addressMappers: addressMappers
)
return peerInfo

225
libp2p/peerstore.nim Normal file
View File

@@ -0,0 +1,225 @@
# Nim-LibP2P
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
## Stores generic informations about peers.
runnableExamples:
# Will keep info of all connected peers +
# last 50 disconnected peers
let peerStore = PeerStore.new(capacity = 50)
# Create a custom book type
type MoodBook = ref object of PeerBook[string]
var somePeerId = PeerId.random().expect("get random key")
peerStore[MoodBook][somePeerId] = "Happy"
doAssert peerStore[MoodBook][somePeerId] == "Happy"
{.push raises: [].}
import
std/[tables, sets, options, macros],
chronos,
./crypto/crypto,
./protocols/identify,
./protocols/protocol,
./peerid, ./peerinfo,
./routing_record,
./multiaddress,
./stream/connection,
./multistream,
./muxers/muxer,
utility
type
#################
# Handler types #
#################
PeerBookChangeHandler* = proc(peerId: PeerId) {.gcsafe, raises: [].}
#########
# Books #
#########
# Each book contains a book (map) and event handler(s)
BasePeerBook = ref object of RootObj
changeHandlers: seq[PeerBookChangeHandler]
deletor: PeerBookChangeHandler
PeerBook*[T] {.public.} = ref object of BasePeerBook
book*: Table[PeerId, T]
SeqPeerBook*[T] = ref object of PeerBook[seq[T]]
AddressBook* {.public.} = ref object of SeqPeerBook[MultiAddress]
ProtoBook* {.public.} = ref object of SeqPeerBook[string]
KeyBook* {.public.} = ref object of PeerBook[PublicKey]
AgentBook* {.public.} = ref object of PeerBook[string]
ProtoVersionBook* {.public.} = ref object of PeerBook[string]
SPRBook* {.public.} = ref object of PeerBook[Envelope]
####################
# Peer store types #
####################
PeerStore* {.public.} = ref object
books: Table[string, BasePeerBook]
identify: Identify
capacity*: int
toClean*: seq[PeerId]
proc new*(T: type PeerStore, identify: Identify, capacity = 1000): PeerStore {.public.} =
T(
identify: identify,
capacity: capacity
)
#########################
# Generic Peer Book API #
#########################
proc `[]`*[T](peerBook: PeerBook[T],
peerId: PeerId): T {.public.} =
## Get all known metadata of a provided peer, or default(T) if missing
peerBook.book.getOrDefault(peerId)
proc `[]=`*[T](peerBook: PeerBook[T],
peerId: PeerId,
entry: T) {.public.} =
## Set metadata for a given peerId.
peerBook.book[peerId] = entry
# Notify clients
for handler in peerBook.changeHandlers:
handler(peerId)
proc del*[T](peerBook: PeerBook[T],
peerId: PeerId): bool {.public.} =
## Delete the provided peer from the book. Returns whether the peer was in the book
if peerId notin peerBook.book:
return false
else:
peerBook.book.del(peerId)
# Notify clients
for handler in peerBook.changeHandlers:
handler(peerId)
return true
proc contains*[T](peerBook: PeerBook[T], peerId: PeerId): bool {.public.} =
peerId in peerBook.book
proc addHandler*[T](peerBook: PeerBook[T], handler: PeerBookChangeHandler) {.public.} =
## Adds a callback that will be called everytime the book changes
peerBook.changeHandlers.add(handler)
proc len*[T](peerBook: PeerBook[T]): int {.public.} = peerBook.book.len
##################
# Peer Store API #
##################
macro getTypeName(t: type): untyped =
# Generate unique name in form of Module.Type
let typ = getTypeImpl(t)[1]
newLit(repr(typ.owner()) & "." & repr(typ))
proc `[]`*[T](p: PeerStore, typ: type[T]): T {.public.} =
## Get a book from the PeerStore (ex: peerStore[AddressBook])
let name = getTypeName(T)
result = T(p.books.getOrDefault(name))
if result.isNil:
result = T.new()
result.deletor = proc(pid: PeerId) =
# Manual method because generic method
# don't work
discard T(p.books.getOrDefault(name)).del(pid)
p.books[name] = result
return result
proc del*(peerStore: PeerStore,
peerId: PeerId) {.public.} =
## Delete the provided peer from every book.
for _, book in peerStore.books:
book.deletor(peerId)
proc updatePeerInfo*(
peerStore: PeerStore,
info: IdentifyInfo) =
if info.addrs.len > 0:
peerStore[AddressBook][info.peerId] = info.addrs
info.pubkey.withValue(pubkey):
peerStore[KeyBook][info.peerId] = pubkey
info.agentVersion.withValue(agentVersion):
peerStore[AgentBook][info.peerId] = agentVersion.string
info.protoVersion.withValue(protoVersion):
peerStore[ProtoVersionBook][info.peerId] = protoVersion.string
if info.protos.len > 0:
peerStore[ProtoBook][info.peerId] = info.protos
info.signedPeerRecord.withValue(signedPeerRecord):
peerStore[SPRBook][info.peerId] = signedPeerRecord
let cleanupPos = peerStore.toClean.find(info.peerId)
if cleanupPos >= 0:
peerStore.toClean.delete(cleanupPos)
proc cleanup*(
peerStore: PeerStore,
peerId: PeerId) =
if peerStore.capacity == 0:
peerStore.del(peerId)
return
elif peerStore.capacity < 0:
#infinite capacity
return
peerStore.toClean.add(peerId)
while peerStore.toClean.len > peerStore.capacity:
peerStore.del(peerStore.toClean[0])
peerStore.toClean.delete(0)
proc identify*(
peerStore: PeerStore,
muxer: Muxer) {.async.} =
# new stream for identify
var stream = await muxer.newStream()
if stream == nil:
return
try:
if (await MultistreamSelect.select(stream, peerStore.identify.codec())):
let info = await peerStore.identify.identify(stream, stream.peerId)
when defined(libp2p_agents_metrics):
var
knownAgent = "unknown"
shortAgent = info.agentVersion.get("").split("/")[0].safeToLowerAscii().get("")
if KnownLibP2PAgentsSeq.contains(shortAgent):
knownAgent = shortAgent
muxer.connection.setShortAgent(knownAgent)
peerStore.updatePeerInfo(info)
finally:
await stream.closeWithEOF()
proc getMostObservedProtosAndPorts*(self: PeerStore): seq[MultiAddress] =
return self.identify.observedAddrManager.getMostObservedProtosAndPorts()
proc guessDialableAddr*(self: PeerStore, ma: MultiAddress): MultiAddress =
return self.identify.observedAddrManager.guessDialableAddr(ma)

View File

@@ -1,21 +1,22 @@
## Nim-Libp2p
## Copyright (c) 2018 Status Research & Development GmbH
## Licensed under either of
## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
## * MIT license ([LICENSE-MIT](LICENSE-MIT))
## at your option.
## This file may not be copied, modified, or distributed except according to
## those terms.
# Nim-Libp2p
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
## This module implements minimal Google's ProtoBuf primitives.
{.push raises: [Defect].}
{.push raises: [].}
import ../varint, stew/[endians2, results]
export results
import ../varint, ../utility, stew/[endians2, results]
export results, utility
const
MaxMessageSize* = 1'u shl 22
{.push public.}
const MaxMessageSize = 1'u shl 22
type
ProtoFieldKind* = enum
@@ -32,6 +33,7 @@ type
buffer*: seq[byte]
offset*: int
length*: int
maxSize*: uint
ProtoHeader* = object
wire*: ProtoFieldKind
@@ -58,7 +60,8 @@ type
BufferOverflow,
MessageTooBig,
BadWireType,
IncorrectBlob
IncorrectBlob,
RequiredFieldMissing
ProtoResult*[T] = Result[T, ProtoError]
@@ -66,12 +69,12 @@ type
hint | hint32 | hint64 | float32 | float64
const
SupportedWireTypes* = {
int(ProtoFieldKind.Varint),
int(ProtoFieldKind.Fixed64),
int(ProtoFieldKind.Length),
int(ProtoFieldKind.Fixed32)
}
SupportedWireTypes* = @[
uint64(ProtoFieldKind.Varint),
uint64(ProtoFieldKind.Fixed64),
uint64(ProtoFieldKind.Length),
uint64(ProtoFieldKind.Fixed32)
]
template checkFieldNumber*(i: int) =
doAssert((i > 0 and i < (1 shl 29)) and not(i >= 19000 and i <= 19999),
@@ -115,61 +118,29 @@ proc vsizeof*(field: ProtoField): int {.inline.} =
else:
0
proc initProtoField*(index: int, value: SomeVarint): ProtoField {.deprecated.} =
## Initialize ProtoField with integer value.
result = ProtoField(kind: Varint, index: index)
when type(value) is uint64:
result.vint = value
else:
result.vint = cast[uint64](value)
proc initProtoField*(index: int, value: bool): ProtoField {.deprecated.} =
## Initialize ProtoField with integer value.
result = ProtoField(kind: Varint, index: index)
result.vint = byte(value)
proc initProtoField*(index: int,
value: openarray[byte]): ProtoField {.deprecated.} =
## Initialize ProtoField with bytes array.
result = ProtoField(kind: Length, index: index)
if len(value) > 0:
result.vbuffer = newSeq[byte](len(value))
copyMem(addr result.vbuffer[0], unsafeAddr value[0], len(value))
proc initProtoField*(index: int, value: string): ProtoField {.deprecated.} =
## Initialize ProtoField with string.
result = ProtoField(kind: Length, index: index)
if len(value) > 0:
result.vbuffer = newSeq[byte](len(value))
copyMem(addr result.vbuffer[0], unsafeAddr value[0], len(value))
proc initProtoField*(index: int,
value: ProtoBuffer): ProtoField {.deprecated, inline.} =
## Initialize ProtoField with nested message stored in ``value``.
##
## Note: This procedure performs shallow copy of ``value`` sequence.
result = ProtoField(kind: Length, index: index)
if len(value.buffer) > 0:
shallowCopy(result.vbuffer, value.buffer)
proc initProtoBuffer*(data: seq[byte], offset = 0,
options: set[ProtoFlags] = {}): ProtoBuffer =
options: set[ProtoFlags] = {},
maxSize = MaxMessageSize): ProtoBuffer =
## Initialize ProtoBuffer with shallow copy of ``data``.
shallowCopy(result.buffer, data)
result.buffer = data
result.offset = offset
result.options = options
result.maxSize = maxSize
proc initProtoBuffer*(data: openarray[byte], offset = 0,
options: set[ProtoFlags] = {}): ProtoBuffer =
proc initProtoBuffer*(data: openArray[byte], offset = 0,
options: set[ProtoFlags] = {},
maxSize = MaxMessageSize): ProtoBuffer =
## Initialize ProtoBuffer with copy of ``data``.
result.buffer = @data
result.offset = offset
result.options = options
result.maxSize = maxSize
proc initProtoBuffer*(options: set[ProtoFlags] = {}): ProtoBuffer =
proc initProtoBuffer*(options: set[ProtoFlags] = {}, maxSize = MaxMessageSize): ProtoBuffer =
## Initialize ProtoBuffer with new sequence of capacity ``cap``.
result.buffer = newSeq[byte]()
result.options = options
result.maxSize = maxSize
if WithVarintLength in options:
# Our buffer will start from position 10, so we can store length of buffer
# in [0, 9].
@@ -227,7 +198,7 @@ proc write*[T: ProtoScalar](pb: var ProtoBuffer,
pb.offset += sizeof(T)
proc writePacked*[T: ProtoScalar](pb: var ProtoBuffer, field: int,
value: openarray[T]) =
value: openArray[T]) =
checkFieldNumber(field)
var length = 0
let dlength =
@@ -275,7 +246,7 @@ proc writePacked*[T: ProtoScalar](pb: var ProtoBuffer, field: int,
pb.offset += sizeof(T)
proc write*[T: byte|char](pb: var ProtoBuffer, field: int,
value: openarray[T]) =
value: openArray[T]) =
checkFieldNumber(field)
var length = 0
let flength = vsizeof(getProtoHeader(field, ProtoFieldKind.Length)) +
@@ -299,55 +270,10 @@ proc write*(pb: var ProtoBuffer, field: int, value: ProtoBuffer) {.inline.} =
## ``pb`` with field number ``field``.
write(pb, field, value.buffer)
proc write*(pb: var ProtoBuffer, field: ProtoField) {.deprecated.} =
## Encode protobuf's field ``field`` and store it to protobuf's buffer ``pb``.
var length = 0
var res: VarintResult[void]
pb.buffer.setLen(len(pb.buffer) + vsizeof(field))
res = PB.putUVarint(pb.toOpenArray(), length, getProtoHeader(field))
doAssert(res.isOk())
pb.offset += length
case field.kind
of ProtoFieldKind.Varint:
res = PB.putUVarint(pb.toOpenArray(), length, field.vint)
doAssert(res.isOk())
pb.offset += length
of ProtoFieldKind.Fixed64:
doAssert(pb.isEnough(8))
var value = cast[uint64](field.vfloat64)
pb.buffer[pb.offset] = byte(value and 0xFF'u32)
pb.buffer[pb.offset + 1] = byte((value shr 8) and 0xFF'u64)
pb.buffer[pb.offset + 2] = byte((value shr 16) and 0xFF'u64)
pb.buffer[pb.offset + 3] = byte((value shr 24) and 0xFF'u64)
pb.buffer[pb.offset + 4] = byte((value shr 32) and 0xFF'u64)
pb.buffer[pb.offset + 5] = byte((value shr 40) and 0xFF'u64)
pb.buffer[pb.offset + 6] = byte((value shr 48) and 0xFF'u64)
pb.buffer[pb.offset + 7] = byte((value shr 56) and 0xFF'u64)
pb.offset += 8
of ProtoFieldKind.Fixed32:
doAssert(pb.isEnough(4))
var value = cast[uint32](field.vfloat32)
pb.buffer[pb.offset] = byte(value and 0xFF'u32)
pb.buffer[pb.offset + 1] = byte((value shr 8) and 0xFF'u32)
pb.buffer[pb.offset + 2] = byte((value shr 16) and 0xFF'u32)
pb.buffer[pb.offset + 3] = byte((value shr 24) and 0xFF'u32)
pb.offset += 4
of ProtoFieldKind.Length:
res = PB.putUVarint(pb.toOpenArray(), length, uint(len(field.vbuffer)))
doAssert(res.isOk())
pb.offset += length
doAssert(pb.isEnough(len(field.vbuffer)))
if len(field.vbuffer) > 0:
copyMem(addr pb.buffer[pb.offset], unsafeAddr field.vbuffer[0],
len(field.vbuffer))
pb.offset += len(field.vbuffer)
else:
discard
proc finish*(pb: var ProtoBuffer) =
## Prepare protobuf's buffer ``pb`` for writing to stream.
doAssert(len(pb.buffer) > 0)
if WithVarintLength in pb.options:
doAssert(len(pb.buffer) >= 10)
let size = uint(len(pb.buffer) - 10)
let pos = 10 - vsizeof(size)
var usedBytes = 0
@@ -355,14 +281,17 @@ proc finish*(pb: var ProtoBuffer) =
doAssert(res.isOk())
pb.offset = pos
elif WithUint32BeLength in pb.options:
doAssert(len(pb.buffer) >= 4)
let size = uint(len(pb.buffer) - 4)
pb.buffer[0 ..< 4] = toBytesBE(uint32(size))
pb.offset = 4
elif WithUint32LeLength in pb.options:
doAssert(len(pb.buffer) >= 4)
let size = uint(len(pb.buffer) - 4)
pb.buffer[0 ..< 4] = toBytesLE(uint32(size))
pb.offset = 4
else:
doAssert(len(pb.buffer) > 0)
pb.offset = 0
proc getHeader(data: var ProtoBuffer,
@@ -408,7 +337,7 @@ proc skipValue(data: var ProtoBuffer, header: ProtoHeader): ProtoResult[void] =
var bsize = 0'u64
if PB.getUVarint(data.toOpenArray(), length, bsize).isOk():
data.offset += length
if bsize <= uint64(MaxMessageSize):
if bsize <= uint64(data.maxSize):
if data.isEnough(int(bsize)):
data.offset += int(bsize)
ok()
@@ -463,7 +392,7 @@ proc getValue[T: ProtoScalar](data: var ProtoBuffer,
err(ProtoError.MessageIncomplete)
proc getValue[T:byte|char](data: var ProtoBuffer, header: ProtoHeader,
outBytes: var openarray[T],
outBytes: var openArray[T],
outLength: var int): ProtoResult[void] =
doAssert(header.wire == ProtoFieldKind.Length)
var length = 0
@@ -472,7 +401,7 @@ proc getValue[T:byte|char](data: var ProtoBuffer, header: ProtoHeader,
outLength = 0
if PB.getUVarint(data.toOpenArray(), length, bsize).isOk():
data.offset += length
if bsize <= uint64(MaxMessageSize):
if bsize <= uint64(data.maxSize):
if data.isEnough(int(bsize)):
outLength = int(bsize)
if len(outBytes) >= int(bsize):
@@ -500,7 +429,7 @@ proc getValue[T:seq[byte]|string](data: var ProtoBuffer, header: ProtoHeader,
if PB.getUVarint(data.toOpenArray(), length, bsize).isOk():
data.offset += length
if bsize <= uint64(MaxMessageSize):
if bsize <= uint64(data.maxSize):
if data.isEnough(int(bsize)):
outBytes.setLen(bsize)
if bsize > 0'u64:
@@ -556,7 +485,7 @@ proc getField*[T: ProtoScalar](data: ProtoBuffer, field: int,
ok(false)
proc getField*[T: byte|char](data: ProtoBuffer, field: int,
output: var openarray[T],
output: var openArray[T],
outlen: var int): ProtoResult[bool] =
checkFieldNumber(field)
var pb = data
@@ -647,15 +576,18 @@ proc getField*[T: seq[byte]|string](data: ProtoBuffer, field: int,
proc getField*(pb: ProtoBuffer, field: int,
output: var ProtoBuffer): ProtoResult[bool] {.inline.} =
var buffer: seq[byte]
let res = pb.getField(field, buffer)
if res.isOk():
if res.get():
output = initProtoBuffer(buffer)
ok(true)
else:
ok(false)
if ? pb.getField(field, buffer):
output = initProtoBuffer(buffer)
ok(true)
else:
err(res.error)
ok(false)
proc getRequiredField*[T](pb: ProtoBuffer, field: int,
output: var T): ProtoResult[void] {.inline.} =
if ? pb.getField(field, output):
ok()
else:
err(RequiredFieldMissing)
proc getRepeatedField*[T: seq[byte]|string](data: ProtoBuffer, field: int,
output: var seq[T]): ProtoResult[bool] =
@@ -733,6 +665,13 @@ proc getRepeatedField*[T: ProtoScalar](data: ProtoBuffer, field: int,
else:
ok(false)
proc getRequiredRepeatedField*[T](pb: ProtoBuffer, field: int,
output: var seq[T]): ProtoResult[void] {.inline.} =
if ? pb.getRepeatedField(field, output):
ok()
else:
err(RequiredFieldMissing)
proc getPackedRepeatedField*[T: ProtoScalar](data: ProtoBuffer, field: int,
output: var seq[T]): ProtoResult[bool] =
checkFieldNumber(field)
@@ -787,93 +726,3 @@ proc getPackedRepeatedField*[T: ProtoScalar](data: ProtoBuffer, field: int,
ok(true)
else:
ok(false)
proc getVarintValue*(data: var ProtoBuffer, field: int,
value: var SomeVarint): int {.deprecated.} =
## Get value of `Varint` type.
var length = 0
var header = 0'u64
var soffset = data.offset
if not data.isEmpty() and PB.getUVarint(data.toOpenArray(),
length, header).isOk():
data.offset += length
if header == getProtoHeader(field, Varint):
if not data.isEmpty():
when type(value) is int32 or type(value) is int64 or type(value) is int:
let res = getSVarint(data.toOpenArray(), length, value)
else:
let res = PB.getUVarint(data.toOpenArray(), length, value)
if res.isOk():
data.offset += length
result = length
return
# Restore offset on error
data.offset = soffset
proc getLengthValue*[T: string|seq[byte]](data: var ProtoBuffer, field: int,
buffer: var T): int {.deprecated.} =
## Get value of `Length` type.
var length = 0
var header = 0'u64
var ssize = 0'u64
var soffset = data.offset
result = -1
buffer.setLen(0)
if not data.isEmpty() and PB.getUVarint(data.toOpenArray(),
length, header).isOk():
data.offset += length
if header == getProtoHeader(field, Length):
if not data.isEmpty() and PB.getUVarint(data.toOpenArray(),
length, ssize).isOk():
data.offset += length
if ssize <= MaxMessageSize and data.isEnough(int(ssize)):
buffer.setLen(ssize)
# Protobuf allow zero-length values.
if ssize > 0'u64:
copyMem(addr buffer[0], addr data.buffer[data.offset], ssize)
result = int(ssize)
data.offset += int(ssize)
return
# Restore offset on error
data.offset = soffset
proc getBytes*(data: var ProtoBuffer, field: int,
buffer: var seq[byte]): int {.deprecated, inline.} =
## Get value of `Length` type as bytes.
result = getLengthValue(data, field, buffer)
proc getString*(data: var ProtoBuffer, field: int,
buffer: var string): int {.deprecated, inline.} =
## Get value of `Length` type as string.
result = getLengthValue(data, field, buffer)
proc enterSubmessage*(pb: var ProtoBuffer): int {.deprecated.} =
## Processes protobuf's sub-message and adjust internal offset to enter
## inside of sub-message. Returns field index of sub-message field or
## ``0`` on error.
var length = 0
var header = 0'u64
var msize = 0'u64
var soffset = pb.offset
if not pb.isEmpty() and PB.getUVarint(pb.toOpenArray(),
length, header).isOk():
pb.offset += length
if (header and 0x07'u64) == cast[uint64](ProtoFieldKind.Length):
if not pb.isEmpty() and PB.getUVarint(pb.toOpenArray(),
length, msize).isOk():
pb.offset += length
if msize <= MaxMessageSize and pb.isEnough(int(msize)):
pb.length = int(msize)
result = int(header shr 3)
return
# Restore offset on error
pb.offset = soffset
proc skipSubmessage*(pb: var ProtoBuffer) {.deprecated.} =
## Skip current protobuf's sub-message and adjust internal offset to the
## end of sub-message.
doAssert(pb.length != 0)
pb.offset += pb.length
pb.length = 0

View File

@@ -0,0 +1,70 @@
# Nim-LibP2P
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
{.push raises: [].}
import stew/results
import chronos, chronicles
import ../../../switch,
../../../multiaddress,
../../../peerid
import core
logScope:
topics = "libp2p autonat"
type
AutonatClient* = ref object of RootObj
proc sendDial(conn: Connection, pid: PeerId, addrs: seq[MultiAddress]) {.async.} =
let pb = AutonatDial(peerInfo: Opt.some(AutonatPeerInfo(
id: Opt.some(pid),
addrs: addrs
))).encode()
await conn.writeLp(pb.buffer)
method dialMe*(self: AutonatClient, switch: Switch, pid: PeerId, addrs: seq[MultiAddress] = newSeq[MultiAddress]()):
Future[MultiAddress] {.base, async.} =
proc getResponseOrRaise(autonatMsg: Opt[AutonatMsg]): AutonatDialResponse {.raises: [AutonatError].} =
autonatMsg.withValue(msg):
if msg.msgType == DialResponse:
msg.response.withValue(res):
if not (res.status == Ok and res.ma.isNone()):
return res
raise newException(AutonatError, "Unexpected response")
let conn =
try:
if addrs.len == 0:
await switch.dial(pid, @[AutonatCodec])
else:
await switch.dial(pid, addrs, AutonatCodec)
except CatchableError as err:
raise newException(AutonatError, "Unexpected error when dialling: " & err.msg, err)
# To bypass maxConnectionsPerPeer
let incomingConnection = switch.connManager.expectConnection(pid, In)
if incomingConnection.failed() and incomingConnection.error of AlreadyExpectingConnectionError:
raise newException(AutonatError, incomingConnection.error.msg)
defer:
await conn.close()
incomingConnection.cancel() # Safer to always try to cancel cause we aren't sure if the peer dialled us or not
if incomingConnection.completed():
await (await incomingConnection).connection.close()
trace "sending Dial", addrs = switch.peerInfo.addrs
await conn.sendDial(switch.peerInfo.peerId, switch.peerInfo.addrs)
let response = getResponseOrRaise(AutonatMsg.decode(await conn.readLp(1024)))
return case response.status:
of ResponseStatus.Ok:
response.ma.tryGet()
of ResponseStatus.DialError:
raise newException(AutonatUnreachableError, "Peer could not dial us back: " & response.text.get(""))
else:
raise newException(AutonatError, "Bad status " & $response.status & " " & response.text.get(""))

View File

@@ -0,0 +1,139 @@
# Nim-LibP2P
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
{.push raises: [].}
import stew/[results, objects]
import chronos, chronicles
import ../../../multiaddress,
../../../peerid,
../../../errors
logScope:
topics = "libp2p autonat"
const
AutonatCodec* = "/libp2p/autonat/1.0.0"
AddressLimit* = 8
type
AutonatError* = object of LPError
AutonatUnreachableError* = object of LPError
MsgType* = enum
Dial = 0
DialResponse = 1
ResponseStatus* = enum
Ok = 0
DialError = 100
DialRefused = 101
BadRequest = 200
InternalError = 300
AutonatPeerInfo* = object
id*: Opt[PeerId]
addrs*: seq[MultiAddress]
AutonatDial* = object
peerInfo*: Opt[AutonatPeerInfo]
AutonatDialResponse* = object
status*: ResponseStatus
text*: Opt[string]
ma*: Opt[MultiAddress]
AutonatMsg* = object
msgType*: MsgType
dial*: Opt[AutonatDial]
response*: Opt[AutonatDialResponse]
NetworkReachability* {.pure.} = enum
Unknown, NotReachable, Reachable
proc encode(p: AutonatPeerInfo): ProtoBuffer =
result = initProtoBuffer()
p.id.withValue(id):
result.write(1, id)
for ma in p.addrs:
result.write(2, ma.data.buffer)
result.finish()
proc encode*(d: AutonatDial): ProtoBuffer =
result = initProtoBuffer()
result.write(1, MsgType.Dial.uint)
var dial = initProtoBuffer()
d.peerInfo.withValue(pinfo):
dial.write(1, encode(pinfo))
dial.finish()
result.write(2, dial.buffer)
result.finish()
proc encode*(r: AutonatDialResponse): ProtoBuffer =
result = initProtoBuffer()
result.write(1, MsgType.DialResponse.uint)
var bufferResponse = initProtoBuffer()
bufferResponse.write(1, r.status.uint)
r.text.withValue(text):
bufferResponse.write(2, text)
r.ma.withValue(ma):
bufferResponse.write(3, ma)
bufferResponse.finish()
result.write(3, bufferResponse.buffer)
result.finish()
proc encode*(msg: AutonatMsg): ProtoBuffer =
msg.dial.withValue(dial):
return encode(dial)
msg.response.withValue(res):
return encode(res)
proc decode*(_: typedesc[AutonatMsg], buf: seq[byte]): Opt[AutonatMsg] =
var
msgTypeOrd: uint32
pbDial: ProtoBuffer
pbResponse: ProtoBuffer
msg: AutonatMsg
let pb = initProtoBuffer(buf)
if ? pb.getField(1, msgTypeOrd).toOpt() and not checkedEnumAssign(msg.msgType, msgTypeOrd):
return Opt.none(AutonatMsg)
if ? pb.getField(2, pbDial).toOpt():
var
pbPeerInfo: ProtoBuffer
dial: AutonatDial
let r4 = ? pbDial.getField(1, pbPeerInfo).toOpt()
var peerInfo: AutonatPeerInfo
if r4:
var pid: PeerId
let
r5 = ? pbPeerInfo.getField(1, pid).toOpt()
r6 = ? pbPeerInfo.getRepeatedField(2, peerInfo.addrs).toOpt()
if r5: peerInfo.id = Opt.some(pid)
dial.peerInfo = Opt.some(peerInfo)
msg.dial = Opt.some(dial)
if ? pb.getField(3, pbResponse).toOpt():
var
statusOrd: uint
text: string
ma: MultiAddress
response: AutonatDialResponse
if ? pbResponse.getField(1, statusOrd).optValue():
if not checkedEnumAssign(response.status, statusOrd):
return Opt.none(AutonatMsg)
if ? pbResponse.getField(2, text).optValue():
response.text = Opt.some(text)
if ? pbResponse.getField(3, ma).optValue():
response.ma = Opt.some(ma)
msg.response = Opt.some(response)
return Opt.some(msg)

View File

@@ -0,0 +1,160 @@
# Nim-LibP2P
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
{.push raises: [].}
import std/[sets, sequtils]
import stew/results
import chronos, chronicles
import ../../protocol,
../../../switch,
../../../multiaddress,
../../../multicodec,
../../../peerid,
../../../utils/[semaphore, future],
../../../errors
import core
export core
logScope:
topics = "libp2p autonat"
type
Autonat* = ref object of LPProtocol
sem: AsyncSemaphore
switch*: Switch
dialTimeout: Duration
proc sendDial(conn: Connection, pid: PeerId, addrs: seq[MultiAddress]) {.async.} =
let pb = AutonatDial(peerInfo: Opt.some(AutonatPeerInfo(
id: Opt.some(pid),
addrs: addrs
))).encode()
await conn.writeLp(pb.buffer)
proc sendResponseError(conn: Connection, status: ResponseStatus, text: string = "") {.async.} =
let pb = AutonatDialResponse(
status: status,
text: if text == "": Opt.none(string) else: Opt.some(text),
ma: Opt.none(MultiAddress)
).encode()
await conn.writeLp(pb.buffer)
proc sendResponseOk(conn: Connection, ma: MultiAddress) {.async.} =
let pb = AutonatDialResponse(
status: ResponseStatus.Ok,
text: Opt.some("Ok"),
ma: Opt.some(ma)
).encode()
await conn.writeLp(pb.buffer)
proc tryDial(autonat: Autonat, conn: Connection, addrs: seq[MultiAddress]) {.async.} =
await autonat.sem.acquire()
var futs: seq[Future[Opt[MultiAddress]]]
try:
# This is to bypass the per peer max connections limit
let outgoingConnection = autonat.switch.connManager.expectConnection(conn.peerId, Out)
if outgoingConnection.failed() and outgoingConnection.error of AlreadyExpectingConnectionError:
await conn.sendResponseError(DialRefused, outgoingConnection.error.msg)
return
# Safer to always try to cancel cause we aren't sure if the connection was established
defer: outgoingConnection.cancel()
# tryDial is to bypass the global max connections limit
futs = addrs.mapIt(autonat.switch.dialer.tryDial(conn.peerId, @[it]))
let fut = await anyCompleted(futs).wait(autonat.dialTimeout)
let ma = await fut
ma.withValue(maddr):
await conn.sendResponseOk(maddr)
else:
await conn.sendResponseError(DialError, "Missing observed address")
except CancelledError as exc:
raise exc
except AllFuturesFailedError as exc:
debug "All dial attempts failed", addrs, exc = exc.msg
await conn.sendResponseError(DialError, "All dial attempts failed")
except AsyncTimeoutError as exc:
debug "Dial timeout", addrs, exc = exc.msg
await conn.sendResponseError(DialError, "Dial timeout")
except CatchableError as exc:
debug "Unexpected error", addrs, exc = exc.msg
await conn.sendResponseError(DialError, "Unexpected error")
finally:
autonat.sem.release()
for f in futs:
if not f.finished():
f.cancel()
proc handleDial(autonat: Autonat, conn: Connection, msg: AutonatMsg): Future[void] =
let dial = msg.dial.valueOr:
return conn.sendResponseError(BadRequest, "Missing Dial")
let peerInfo = dial.peerInfo.valueOr:
return conn.sendResponseError(BadRequest, "Missing Peer Info")
peerInfo.id.withValue(id):
if id != conn.peerId:
return conn.sendResponseError(BadRequest, "PeerId mismatch")
let observedAddr = conn.observedAddr.valueOr:
return conn.sendResponseError(BadRequest, "Missing observed address")
var isRelayed = observedAddr.contains(multiCodec("p2p-circuit")).valueOr:
return conn.sendResponseError(DialRefused, "Invalid observed address")
if isRelayed:
return conn.sendResponseError(DialRefused, "Refused to dial a relayed observed address")
let hostIp = observedAddr[0].valueOr:
return conn.sendResponseError(InternalError, "Wrong observed address")
if not IP.match(hostIp):
return conn.sendResponseError(InternalError, "Expected an IP address")
var addrs = initHashSet[MultiAddress]()
addrs.incl(observedAddr)
trace "addrs received", addrs = peerInfo.addrs
for ma in peerInfo.addrs:
isRelayed = ma.contains(multiCodec("p2p-circuit")).valueOr: continue
let maFirst = ma[0].valueOr: continue
if not DNS_OR_IP.match(maFirst): continue
try:
addrs.incl(
if maFirst == hostIp:
ma
else:
let maEnd = ma[1..^1].valueOr: continue
hostIp & maEnd
)
except LPError as exc:
continue
if len(addrs) >= AddressLimit:
break
if len(addrs) == 0:
return conn.sendResponseError(DialRefused, "No dialable address")
let addrsSeq = toSeq(addrs)
trace "trying to dial", addrs = addrsSeq
return autonat.tryDial(conn, addrsSeq)
proc new*(T: typedesc[Autonat], switch: Switch, semSize: int = 1, dialTimeout = 15.seconds): T =
let autonat = T(switch: switch, sem: newAsyncSemaphore(semSize), dialTimeout: dialTimeout)
proc handleStream(conn: Connection, proto: string) {.async, gcsafe.} =
try:
let msg = AutonatMsg.decode(await conn.readLp(1024)).valueOr:
raise newException(AutonatError, "Received malformed message")
if msg.msgType != MsgType.Dial:
raise newException(AutonatError, "Message type should be dial")
await autonat.handleDial(conn, msg)
except CancelledError as exc:
raise exc
except CatchableError as exc:
debug "exception in autonat handler", exc = exc.msg, conn
finally:
trace "exiting autonat handler", conn
await conn.close()
autonat.handler = handleStream
autonat.codec = AutonatCodec
autonat

View File

@@ -0,0 +1,217 @@
# Nim-LibP2P
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
{.push raises: [].}
import std/[deques, sequtils]
import chronos, metrics
import ../../../switch
import ../../../wire
import client
from core import NetworkReachability, AutonatUnreachableError
import ../../../utils/heartbeat
import ../../../crypto/crypto
export core.NetworkReachability
logScope:
topics = "libp2p autonatservice"
declarePublicGauge(libp2p_autonat_reachability_confidence, "autonat reachability confidence", labels = ["reachability"])
type
AutonatService* = ref object of Service
newConnectedPeerHandler: PeerEventHandler
addressMapper: AddressMapper
scheduleHandle: Future[void]
networkReachability*: NetworkReachability
confidence: Opt[float]
answers: Deque[NetworkReachability]
autonatClient: AutonatClient
statusAndConfidenceHandler: StatusAndConfidenceHandler
rng: ref HmacDrbgContext
scheduleInterval: Opt[Duration]
askNewConnectedPeers: bool
numPeersToAsk: int
maxQueueSize: int
minConfidence: float
dialTimeout: Duration
enableAddressMapper: bool
StatusAndConfidenceHandler* = proc (networkReachability: NetworkReachability, confidence: Opt[float]): Future[void] {.gcsafe, raises: [].}
proc new*(
T: typedesc[AutonatService],
autonatClient: AutonatClient,
rng: ref HmacDrbgContext,
scheduleInterval: Opt[Duration] = Opt.none(Duration),
askNewConnectedPeers = true,
numPeersToAsk: int = 5,
maxQueueSize: int = 10,
minConfidence: float = 0.3,
dialTimeout = 30.seconds,
enableAddressMapper = true): T =
return T(
scheduleInterval: scheduleInterval,
networkReachability: Unknown,
confidence: Opt.none(float),
answers: initDeque[NetworkReachability](),
autonatClient: autonatClient,
rng: rng,
askNewConnectedPeers: askNewConnectedPeers,
numPeersToAsk: numPeersToAsk,
maxQueueSize: maxQueueSize,
minConfidence: minConfidence,
dialTimeout: dialTimeout,
enableAddressMapper: enableAddressMapper)
proc callHandler(self: AutonatService) {.async.} =
if not isNil(self.statusAndConfidenceHandler):
await self.statusAndConfidenceHandler(self.networkReachability, self.confidence)
proc hasEnoughIncomingSlots(switch: Switch): bool =
# we leave some margin instead of comparing to 0 as a peer could connect to us while we are asking for the dial back
return switch.connManager.slotsAvailable(In) >= 2
proc doesPeerHaveIncomingConn(switch: Switch, peerId: PeerId): bool =
return switch.connManager.selectMuxer(peerId, In) != nil
proc handleAnswer(self: AutonatService, ans: NetworkReachability): Future[bool] {.async.} =
if ans == Unknown:
return
let oldNetworkReachability = self.networkReachability
let oldConfidence = self.confidence
if self.answers.len == self.maxQueueSize:
self.answers.popFirst()
self.answers.addLast(ans)
self.networkReachability = Unknown
self.confidence = Opt.none(float)
const reachabilityPriority = [Reachable, NotReachable]
for reachability in reachabilityPriority:
let confidence = self.answers.countIt(it == reachability) / self.maxQueueSize
libp2p_autonat_reachability_confidence.set(value = confidence, labelValues = [$reachability])
if self.confidence.isNone and confidence >= self.minConfidence:
self.networkReachability = reachability
self.confidence = Opt.some(confidence)
debug "Current status", currentStats = $self.networkReachability, confidence = $self.confidence, answers = self.answers
# Return whether anything has changed
return self.networkReachability != oldNetworkReachability or self.confidence != oldConfidence
proc askPeer(self: AutonatService, switch: Switch, peerId: PeerId): Future[NetworkReachability] {.async.} =
logScope:
peerId = $peerId
if doesPeerHaveIncomingConn(switch, peerId):
return Unknown
if not hasEnoughIncomingSlots(switch):
debug "No incoming slots available, not asking peer", incomingSlotsAvailable=switch.connManager.slotsAvailable(In)
return Unknown
trace "Asking peer for reachability"
let ans =
try:
discard await self.autonatClient.dialMe(switch, peerId).wait(self.dialTimeout)
debug "dialMe answer is reachable"
Reachable
except AutonatUnreachableError as error:
debug "dialMe answer is not reachable", msg = error.msg
NotReachable
except AsyncTimeoutError as error:
debug "dialMe timed out", msg = error.msg
Unknown
except CatchableError as error:
debug "dialMe unexpected error", msg = error.msg
Unknown
let hasReachabilityOrConfidenceChanged = await self.handleAnswer(ans)
if hasReachabilityOrConfidenceChanged:
await self.callHandler()
await switch.peerInfo.update()
return ans
proc askConnectedPeers(self: AutonatService, switch: Switch) {.async.} =
trace "Asking peers for reachability"
var peers = switch.connectedPeers(Direction.Out)
self.rng.shuffle(peers)
var answersFromPeers = 0
for peer in peers:
if answersFromPeers >= self.numPeersToAsk:
break
if not hasEnoughIncomingSlots(switch):
debug "No incoming slots available, not asking peers", incomingSlotsAvailable=switch.connManager.slotsAvailable(In)
break
if (await askPeer(self, switch, peer)) != Unknown:
answersFromPeers.inc()
proc schedule(service: AutonatService, switch: Switch, interval: Duration) {.async.} =
heartbeat "Scheduling AutonatService run", interval:
await service.run(switch)
proc addressMapper(
self: AutonatService,
peerStore: PeerStore,
listenAddrs: seq[MultiAddress]): Future[seq[MultiAddress]] {.gcsafe, async.} =
if self.networkReachability != NetworkReachability.Reachable:
return listenAddrs
var addrs = newSeq[MultiAddress]()
for listenAddr in listenAddrs:
var processedMA = listenAddr
try:
if not listenAddr.isPublicMA() and self.networkReachability == NetworkReachability.Reachable:
processedMA = peerStore.guessDialableAddr(listenAddr) # handle manual port forwarding
except CatchableError as exc:
debug "Error while handling address mapper", msg = exc.msg
addrs.add(processedMA)
return addrs
method setup*(self: AutonatService, switch: Switch): Future[bool] {.async.} =
self.addressMapper = proc (listenAddrs: seq[MultiAddress]): Future[seq[MultiAddress]] {.gcsafe, async.} =
return await addressMapper(self, switch.peerStore, listenAddrs)
info "Setting up AutonatService"
let hasBeenSetup = await procCall Service(self).setup(switch)
if hasBeenSetup:
if self.askNewConnectedPeers:
self.newConnectedPeerHandler = proc (peerId: PeerId, event: PeerEvent): Future[void] {.async.} =
discard askPeer(self, switch, peerId)
switch.connManager.addPeerEventHandler(self.newConnectedPeerHandler, PeerEventKind.Joined)
self.scheduleInterval.withValue(interval):
self.scheduleHandle = schedule(self, switch, interval)
if self.enableAddressMapper:
switch.peerInfo.addressMappers.add(self.addressMapper)
return hasBeenSetup
method run*(self: AutonatService, switch: Switch) {.async, public.} =
trace "Running AutonatService"
await askConnectedPeers(self, switch)
method stop*(self: AutonatService, switch: Switch): Future[bool] {.async, public.} =
info "Stopping AutonatService"
let hasBeenStopped = await procCall Service(self).stop(switch)
if hasBeenStopped:
if not isNil(self.scheduleHandle):
self.scheduleHandle.cancel()
self.scheduleHandle = nil
if not isNil(self.newConnectedPeerHandler):
switch.connManager.removePeerEventHandler(self.newConnectedPeerHandler, PeerEventKind.Joined)
if self.enableAddressMapper:
switch.peerInfo.addressMappers.keepItIf(it != self.addressMapper)
await switch.peerInfo.update()
return hasBeenStopped
proc statusAndConfidenceHandler*(self: AutonatService, statusAndConfidenceHandler: StatusAndConfidenceHandler) =
self.statusAndConfidenceHandler = statusAndConfidenceHandler

View File

@@ -0,0 +1,89 @@
# Nim-LibP2P
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
{.push raises: [].}
import std/sequtils
import stew/results
import chronos, chronicles
import core
import ../../protocol,
../../../stream/connection,
../../../switch,
../../../utils/future
export DcutrError
type
DcutrClient* = ref object
connectTimeout: Duration
maxDialableAddrs: int
logScope:
topics = "libp2p dcutrclient"
proc new*(T: typedesc[DcutrClient], connectTimeout = 15.seconds, maxDialableAddrs = 8): T =
return T(connectTimeout: connectTimeout, maxDialableAddrs: maxDialableAddrs)
proc startSync*(self: DcutrClient, switch: Switch, remotePeerId: PeerId, addrs: seq[MultiAddress]) {.async.} =
logScope:
peerId = switch.peerInfo.peerId
var
peerDialableAddrs: seq[MultiAddress]
stream: Connection
try:
var ourDialableAddrs = getHolePunchableAddrs(addrs)
if ourDialableAddrs.len == 0:
debug "Dcutr initiator has no supported dialable addresses. Aborting Dcutr.", addrs
return
stream = await switch.dial(remotePeerId, DcutrCodec)
await stream.send(MsgType.Connect, addrs)
debug "Dcutr initiator has sent a Connect message."
let rttStart = Moment.now()
let connectAnswer = DcutrMsg.decode(await stream.readLp(1024))
peerDialableAddrs = getHolePunchableAddrs(connectAnswer.addrs)
if peerDialableAddrs.len == 0:
debug "Dcutr receiver has no supported dialable addresses to connect to. Aborting Dcutr.", addrs=connectAnswer.addrs
return
let rttEnd = Moment.now()
debug "Dcutr initiator has received a Connect message back.", connectAnswer
let halfRtt = (rttEnd - rttStart) div 2'i64
await stream.send(MsgType.Sync, @[])
debug "Dcutr initiator has sent a Sync message."
await sleepAsync(halfRtt)
if peerDialableAddrs.len > self.maxDialableAddrs:
peerDialableAddrs = peerDialableAddrs[0..<self.maxDialableAddrs]
var futs = peerDialableAddrs.mapIt(switch.connect(stream.peerId, @[it], forceDial = true, reuseConnection = false))
try:
discard await anyCompleted(futs).wait(self.connectTimeout)
debug "Dcutr initiator has directly connected to the remote peer."
finally:
for fut in futs: fut.cancel()
except CancelledError as err:
raise err
except AllFuturesFailedError as err:
debug "Dcutr initiator could not connect to the remote peer, all connect attempts failed", peerDialableAddrs, msg = err.msg
raise newException(DcutrError, "Dcutr initiator could not connect to the remote peer, all connect attempts failed", err)
except AsyncTimeoutError as err:
debug "Dcutr initiator could not connect to the remote peer, all connect attempts timed out", peerDialableAddrs, msg = err.msg
raise newException(DcutrError, "Dcutr initiator could not connect to the remote peer, all connect attempts timed out", err)
except CatchableError as err:
debug "Unexpected error when Dcutr initiator tried to connect to the remote peer", err = err.msg
raise newException(DcutrError, "Unexpected error when Dcutr initiator tried to connect to the remote peer", err)
finally:
if stream != nil:
await stream.close()

View File

@@ -0,0 +1,60 @@
# Nim-LibP2P
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
{.push raises: [].}
import std/sequtils
import chronos
import stew/objects
import ../../../multiaddress,
../../../errors,
../../../stream/connection
export multiaddress
const
DcutrCodec* = "/libp2p/dcutr"
type
MsgType* = enum
Connect = 100
Sync = 300
DcutrMsg* = object
msgType*: MsgType
addrs*: seq[MultiAddress]
DcutrError* = object of LPError
proc encode*(msg: DcutrMsg): ProtoBuffer =
result = initProtoBuffer()
result.write(1, msg.msgType.uint)
for addr in msg.addrs:
result.write(2, addr)
result.finish()
proc decode*(_: typedesc[DcutrMsg], buf: seq[byte]): DcutrMsg {.raises: [DcutrError].} =
var
msgTypeOrd: uint32
dcutrMsg: DcutrMsg
var pb = initProtoBuffer(buf)
var r1 = pb.getField(1, msgTypeOrd)
let r2 = pb.getRepeatedField(2, dcutrMsg.addrs)
if r1.isErr or r2.isErr or not checkedEnumAssign(dcutrMsg.msgType, msgTypeOrd):
raise newException(DcutrError, "Received malformed message")
return dcutrMsg
proc send*(conn: Connection, msgType: MsgType, addrs: seq[MultiAddress]) {.async.} =
let pb = DcutrMsg(msgType: msgType, addrs: addrs).encode()
await conn.writeLp(pb.buffer)
proc getHolePunchableAddrs*(addrs: seq[MultiAddress]): seq[MultiAddress] =
addrs.filterIt(TCP.match(it))

View File

@@ -0,0 +1,80 @@
# Nim-LibP2P
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
{.push raises: [].}
import std/[sets, sequtils]
import stew/[results, objects]
import chronos, chronicles
import core
import ../../protocol,
../../../stream/connection,
../../../switch,
../../../utils/future
export DcutrError
export chronicles
type Dcutr* = ref object of LPProtocol
logScope:
topics = "libp2p dcutr"
proc new*(T: typedesc[Dcutr], switch: Switch, connectTimeout = 15.seconds, maxDialableAddrs = 8): T =
proc handleStream(stream: Connection, proto: string) {.async, gcsafe.} =
var peerDialableAddrs: seq[MultiAddress]
try:
let connectMsg = DcutrMsg.decode(await stream.readLp(1024))
debug "Dcutr receiver received a Connect message.", connectMsg
var ourAddrs = switch.peerStore.getMostObservedProtosAndPorts() # likely empty when the peer is reachable
if ourAddrs.len == 0:
# this list should be the same as the peer's public addrs when it is reachable
ourAddrs = switch.peerInfo.listenAddrs.mapIt(switch.peerStore.guessDialableAddr(it))
var ourDialableAddrs = getHolePunchableAddrs(ourAddrs)
if ourDialableAddrs.len == 0:
debug "Dcutr receiver has no supported dialable addresses. Aborting Dcutr.", ourAddrs
return
await stream.send(MsgType.Connect, ourAddrs)
debug "Dcutr receiver has sent a Connect message back."
let syncMsg = DcutrMsg.decode(await stream.readLp(1024))
debug "Dcutr receiver has received a Sync message.", syncMsg
peerDialableAddrs = getHolePunchableAddrs(connectMsg.addrs)
if peerDialableAddrs.len == 0:
debug "Dcutr initiator has no supported dialable addresses to connect to. Aborting Dcutr.", addrs=connectMsg.addrs
return
if peerDialableAddrs.len > maxDialableAddrs:
peerDialableAddrs = peerDialableAddrs[0..<maxDialableAddrs]
var futs = peerDialableAddrs.mapIt(switch.connect(stream.peerId, @[it], forceDial = true, reuseConnection = false, upgradeDir = Direction.In))
try:
discard await anyCompleted(futs).wait(connectTimeout)
debug "Dcutr receiver has directly connected to the remote peer."
finally:
for fut in futs: fut.cancel()
except CancelledError as err:
raise err
except AllFuturesFailedError as err:
debug "Dcutr receiver could not connect to the remote peer, all connect attempts failed", peerDialableAddrs, msg = err.msg
raise newException(DcutrError, "Dcutr receiver could not connect to the remote peer, all connect attempts failed", err)
except AsyncTimeoutError as err:
debug "Dcutr receiver could not connect to the remote peer, all connect attempts timed out", peerDialableAddrs, msg = err.msg
raise newException(DcutrError, "Dcutr receiver could not connect to the remote peer, all connect attempts timed out", err)
except CatchableError as err:
warn "Unexpected error when Dcutr receiver tried to connect to the remote peer", msg = err.msg
raise newException(DcutrError, "Unexpected error when Dcutr receiver tried to connect to the remote peer", err)
let self = T()
self.handler = handleStream
self.codec = DcutrCodec
self

View File

@@ -0,0 +1,288 @@
# Nim-LibP2P
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
{.push raises: [].}
import times
import chronos, chronicles
import ./relay,
./messages,
./rconn,
./utils,
../../../peerinfo,
../../../switch,
../../../multiaddress,
../../../stream/connection
logScope:
topics = "libp2p relay relay-client"
const RelayClientMsgSize = 4096
type
RelayClientError* = object of LPError
ReservationError* = object of RelayClientError
RelayV1DialError* = object of RelayClientError
RelayV2DialError* = object of RelayClientError
RelayClientAddConn* = proc(conn: Connection,
duration: uint32,
data: uint64): Future[void] {.gcsafe, raises: [].}
RelayClient* = ref object of Relay
onNewConnection*: RelayClientAddConn
canHop: bool
Rsvp* = object
expire*: uint64 # required, Unix expiration time (UTC)
addrs*: seq[MultiAddress] # relay address for reserving peer
voucher*: Opt[Voucher] # optional, reservation voucher
limitDuration*: uint32 # seconds
limitData*: uint64 # bytes
proc sendStopError(conn: Connection, code: StatusV2) {.async.} =
trace "send stop status", status = $code & " (" & $ord(code) & ")"
let msg = StopMessage(msgType: StopMessageType.Status, status: Opt.some(code))
await conn.writeLp(encode(msg).buffer)
proc handleRelayedConnect(cl: RelayClient, conn: Connection, msg: StopMessage) {.async.} =
let
# TODO: check the go version to see in which way this could fail
# it's unclear in the spec
src = msg.peer.valueOr:
await sendStopError(conn, MalformedMessage)
return
limitDuration = msg.limit.duration
limitData = msg.limit.data
msg = StopMessage(
msgType: StopMessageType.Status,
status: Opt.some(Ok))
pb = encode(msg)
trace "incoming relay connection", src
if cl.onNewConnection == nil:
await sendStopError(conn, StatusV2.ConnectionFailed)
await conn.close()
return
await conn.writeLp(pb.buffer)
# This sound redundant but the callback could, in theory, be set to nil during
# conn.writeLp so it's safer to double check
if cl.onNewConnection != nil: await cl.onNewConnection(conn, limitDuration, limitData)
else: await conn.close()
proc reserve*(cl: RelayClient,
peerId: PeerId,
addrs: seq[MultiAddress] = @[]): Future[Rsvp] {.async.} =
let conn = await cl.switch.dial(peerId, addrs, RelayV2HopCodec)
defer: await conn.close()
let
pb = encode(HopMessage(msgType: HopMessageType.Reserve))
msg = try:
await conn.writeLp(pb.buffer)
HopMessage.decode(await conn.readLp(RelayClientMsgSize)).tryGet()
except CancelledError as exc:
raise exc
except CatchableError as exc:
trace "error writing or reading reservation message", exc=exc.msg
raise newException(ReservationError, exc.msg)
if msg.msgType != HopMessageType.Status:
raise newException(ReservationError, "Unexpected relay response type")
if msg.status.get(UnexpectedMessage) != Ok:
raise newException(ReservationError, "Reservation failed")
let reservation = msg.reservation.valueOr:
raise newException(ReservationError, "Missing reservation information")
if reservation.expire > int64.high().uint64 or
now().utc > reservation.expire.int64.fromUnix.utc:
raise newException(ReservationError, "Bad expiration date")
result.expire = reservation.expire
result.addrs = reservation.addrs
reservation.svoucher.withValue(sv):
let svoucher = SignedVoucher.decode(sv).valueOr:
raise newException(ReservationError, "Invalid voucher")
if svoucher.data.relayPeerId != peerId:
raise newException(ReservationError, "Invalid voucher PeerId")
result.voucher = Opt.some(svoucher.data)
result.limitDuration = msg.limit.duration
result.limitData = msg.limit.data
proc dialPeerV1*(
cl: RelayClient,
conn: Connection,
dstPeerId: PeerId,
dstAddrs: seq[MultiAddress]): Future[Connection] {.async.} =
var
msg = RelayMessage(
msgType: Opt.some(RelayType.Hop),
srcPeer: Opt.some(RelayPeer(peerId: cl.switch.peerInfo.peerId, addrs: cl.switch.peerInfo.addrs)),
dstPeer: Opt.some(RelayPeer(peerId: dstPeerId, addrs: dstAddrs)))
pb = encode(msg)
trace "Dial peer", msgSend=msg
try:
await conn.writeLp(pb.buffer)
except CancelledError as exc:
raise exc
except CatchableError as exc:
trace "error writing hop request", exc=exc.msg
raise exc
let msgRcvFromRelayOpt = try:
RelayMessage.decode(await conn.readLp(RelayClientMsgSize))
except CancelledError as exc:
raise exc
except CatchableError as exc:
trace "error reading stop response", exc=exc.msg
await sendStatus(conn, StatusV1.HopCantOpenDstStream)
raise exc
try:
let msgRcvFromRelay = msgRcvFromRelayOpt.valueOr:
raise newException(RelayV1DialError, "Hop can't open destination stream")
if msgRcvFromRelay.msgType.tryGet() != RelayType.Status:
raise newException(RelayV1DialError, "Hop can't open destination stream: wrong message type")
if msgRcvFromRelay.status.tryGet() != StatusV1.Success:
raise newException(RelayV1DialError, "Hop can't open destination stream: status failed")
except RelayV1DialError as exc:
await sendStatus(conn, StatusV1.HopCantOpenDstStream)
raise exc
except ValueError as exc:
await sendStatus(conn, StatusV1.HopCantOpenDstStream)
raise newException(RelayV1DialError, exc.msg)
result = conn
proc dialPeerV2*(
cl: RelayClient,
conn: RelayConnection,
dstPeerId: PeerId,
dstAddrs: seq[MultiAddress]): Future[Connection] {.async.} =
let
p = Peer(peerId: dstPeerId, addrs: dstAddrs)
pb = encode(HopMessage(msgType: HopMessageType.Connect, peer: Opt.some(p)))
trace "Dial peer", p
let msgRcvFromRelay = try:
await conn.writeLp(pb.buffer)
HopMessage.decode(await conn.readLp(RelayClientMsgSize)).tryGet()
except CancelledError as exc:
raise exc
except CatchableError as exc:
trace "error reading stop response", exc=exc.msg
raise newException(RelayV2DialError, exc.msg)
if msgRcvFromRelay.msgType != HopMessageType.Status:
raise newException(RelayV2DialError, "Unexpected stop response")
if msgRcvFromRelay.status.get(UnexpectedMessage) != Ok:
trace "Relay stop failed", msg = msgRcvFromRelay.status
raise newException(RelayV2DialError, "Relay stop failure")
conn.limitDuration = msgRcvFromRelay.limit.duration
conn.limitData = msgRcvFromRelay.limit.data
return conn
proc handleStopStreamV2(cl: RelayClient, conn: Connection) {.async, gcsafe.} =
let msg = StopMessage.decode(await conn.readLp(RelayClientMsgSize)).valueOr:
await sendHopStatus(conn, MalformedMessage)
return
trace "client circuit relay v2 handle stream", msg
if msg.msgType == StopMessageType.Connect:
await cl.handleRelayedConnect(conn, msg)
else:
trace "Unexpected client / relayv2 handshake", msgType=msg.msgType
await sendStopError(conn, MalformedMessage)
proc handleStop(cl: RelayClient, conn: Connection, msg: RelayMessage) {.async, gcsafe.} =
let src = msg.srcPeer.valueOr:
await sendStatus(conn, StatusV1.StopSrcMultiaddrInvalid)
return
let dst = msg.dstPeer.valueOr:
await sendStatus(conn, StatusV1.StopDstMultiaddrInvalid)
return
if dst.peerId != cl.switch.peerInfo.peerId:
await sendStatus(conn, StatusV1.StopDstMultiaddrInvalid)
return
trace "get a relay connection", src, conn
if cl.onNewConnection == nil:
await sendStatus(conn, StatusV1.StopRelayRefused)
await conn.close()
return
await sendStatus(conn, StatusV1.Success)
# This sound redundant but the callback could, in theory, be set to nil during
# sendStatus(Success) so it's safer to double check
if cl.onNewConnection != nil: await cl.onNewConnection(conn, 0, 0)
else: await conn.close()
proc handleStreamV1(cl: RelayClient, conn: Connection) {.async, gcsafe.} =
let msg = RelayMessage.decode(await conn.readLp(RelayClientMsgSize)).valueOr:
await sendStatus(conn, StatusV1.MalformedMessage)
return
trace "client circuit relay v1 handle stream", msg
let typ = msg.msgType.valueOr:
trace "Message type not set"
await sendStatus(conn, StatusV1.MalformedMessage)
return
case typ:
of RelayType.Hop:
if cl.canHop: await cl.handleHop(conn, msg)
else: await sendStatus(conn, StatusV1.HopCantSpeakRelay)
of RelayType.Stop: await cl.handleStop(conn, msg)
of RelayType.CanHop:
if cl.canHop: await sendStatus(conn, StatusV1.Success)
else: await sendStatus(conn, StatusV1.HopCantSpeakRelay)
else:
trace "Unexpected relay handshake", msgType=msg.msgType
await sendStatus(conn, StatusV1.MalformedMessage)
proc new*(T: typedesc[RelayClient], canHop: bool = false,
reservationTTL: times.Duration = DefaultReservationTTL,
limitDuration: uint32 = DefaultLimitDuration,
limitData: uint64 = DefaultLimitData,
heartbeatSleepTime: uint32 = DefaultHeartbeatSleepTime,
maxCircuit: int = MaxCircuit,
maxCircuitPerPeer: int = MaxCircuitPerPeer,
msgSize: int = RelayClientMsgSize,
circuitRelayV1: bool = false): T =
let cl = T(canHop: canHop,
reservationTTL: reservationTTL,
limit: Limit(duration: limitDuration, data: limitData),
heartbeatSleepTime: heartbeatSleepTime,
maxCircuit: maxCircuit,
maxCircuitPerPeer: maxCircuitPerPeer,
msgSize: msgSize,
isCircuitRelayV1: circuitRelayV1)
proc handleStream(conn: Connection, proto: string) {.async, gcsafe.} =
try:
case proto:
of RelayV1Codec: await cl.handleStreamV1(conn)
of RelayV2StopCodec: await cl.handleStopStreamV2(conn)
of RelayV2HopCodec: await cl.handleHopStreamV2(conn)
except CancelledError as exc:
raise exc
except CatchableError as exc:
trace "exception in client handler", exc = exc.msg, conn
finally:
trace "exiting client handler", conn
await conn.close()
cl.handler = handleStream
cl.codecs = if cl.canHop:
@[RelayV1Codec, RelayV2HopCodec, RelayV2StopCodec]
else:
@[RelayV1Codec, RelayV2StopCodec]
cl

View File

@@ -0,0 +1,333 @@
# Nim-LibP2P
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
{.push raises: [].}
import macros
import stew/[objects, results]
import ../../../peerinfo,
../../../signed_envelope
# Circuit Relay V1 Message
type
RelayType* {.pure.} = enum
Hop = 1
Stop = 2
Status = 3
CanHop = 4
StatusV1* {.pure.} = enum
Success = 100
HopSrcAddrTooLong = 220
HopDstAddrTooLong = 221
HopSrcMultiaddrInvalid = 250
HopDstMultiaddrInvalid = 251
HopNoConnToDst = 260
HopCantDialDst = 261
HopCantOpenDstStream = 262
HopCantSpeakRelay = 270
HopCantRelayToSelf = 280
StopSrcAddrTooLong = 320
StopDstAddrTooLong = 321
StopSrcMultiaddrInvalid = 350
StopDstMultiaddrInvalid = 351
StopRelayRefused = 390
MalformedMessage = 400
RelayPeer* = object
peerId*: PeerId
addrs*: seq[MultiAddress]
RelayMessage* = object
msgType*: Opt[RelayType]
srcPeer*: Opt[RelayPeer]
dstPeer*: Opt[RelayPeer]
status*: Opt[StatusV1]
proc encode*(msg: RelayMessage): ProtoBuffer =
result = initProtoBuffer()
msg.msgType.withValue(typ):
result.write(1, typ.ord.uint)
msg.srcPeer.withValue(srcPeer):
var peer = initProtoBuffer()
peer.write(1, srcPeer.peerId)
for ma in srcPeer.addrs:
peer.write(2, ma.data.buffer)
peer.finish()
result.write(2, peer.buffer)
msg.dstPeer.withValue(dstPeer):
var peer = initProtoBuffer()
peer.write(1, dstPeer.peerId)
for ma in dstPeer.addrs:
peer.write(2, ma.data.buffer)
peer.finish()
result.write(3, peer.buffer)
msg.status.withValue(status):
result.write(4, status.ord.uint)
result.finish()
proc decode*(_: typedesc[RelayMessage], buf: seq[byte]): Opt[RelayMessage] =
var
rMsg: RelayMessage
msgTypeOrd: uint32
src: RelayPeer
dst: RelayPeer
statusOrd: uint32
pbSrc: ProtoBuffer
pbDst: ProtoBuffer
let pb = initProtoBuffer(buf)
if ? pb.getField(1, msgTypeOrd).toOpt():
if msgTypeOrd.int notin RelayType:
return Opt.none(RelayMessage)
rMsg.msgType = Opt.some(RelayType(msgTypeOrd))
if ? pb.getField(2, pbSrc).toOpt():
discard ? pbSrc.getField(1, src.peerId).toOpt()
discard ? pbSrc.getRepeatedField(2, src.addrs).toOpt()
rMsg.srcPeer = Opt.some(src)
if ? pb.getField(3, pbDst).toOpt():
discard ? pbDst.getField(1, dst.peerId).toOpt()
discard ? pbDst.getRepeatedField(2, dst.addrs).toOpt()
rMsg.dstPeer = Opt.some(dst)
if ? pb.getField(4, statusOrd).toOpt():
var status: StatusV1
if not checkedEnumAssign(status, statusOrd):
return Opt.none(RelayMessage)
rMsg.status = Opt.some(status)
Opt.some(rMsg)
# Voucher
type
Voucher* = object
relayPeerId*: PeerId # peer ID of the relay
reservingPeerId*: PeerId # peer ID of the reserving peer
expiration*: uint64 # UNIX UTC expiration time for the reservation
proc decode*(_: typedesc[Voucher], buf: seq[byte]): Result[Voucher, ProtoError] =
let pb = initProtoBuffer(buf)
var v = Voucher()
? pb.getRequiredField(1, v.relayPeerId)
? pb.getRequiredField(2, v.reservingPeerId)
? pb.getRequiredField(3, v.expiration)
ok(v)
proc encode*(v: Voucher): seq[byte] =
var pb = initProtoBuffer()
pb.write(1, v.relayPeerId)
pb.write(2, v.reservingPeerId)
pb.write(3, v.expiration)
pb.finish()
pb.buffer
proc init*(T: typedesc[Voucher],
relayPeerId: PeerId,
reservingPeerId: PeerId,
expiration: uint64): T =
T(
relayPeerId = relayPeerId,
reservingPeerId = reservingPeerId,
expiration: expiration
)
type SignedVoucher* = SignedPayload[Voucher]
proc payloadDomain*(_: typedesc[Voucher]): string = "libp2p-relay-rsvp"
proc payloadType*(_: typedesc[Voucher]): seq[byte] = @[ (byte)0x03, (byte)0x02 ]
proc checkValid*(spr: SignedVoucher): Result[void, EnvelopeError] =
if not spr.data.relayPeerId.match(spr.envelope.publicKey):
err(EnvelopeInvalidSignature)
else:
ok()
# Circuit Relay V2 Hop Message
type
Peer* = object
peerId*: PeerId
addrs*: seq[MultiAddress]
Reservation* = object
expire*: uint64 # required, Unix expiration time (UTC)
addrs*: seq[MultiAddress] # relay address for reserving peer
svoucher*: Opt[seq[byte]] # optional, reservation voucher
Limit* = object
duration*: uint32 # seconds
data*: uint64 # bytes
StatusV2* = enum
Ok = 100
ReservationRefused = 200
ResourceLimitExceeded = 201
PermissionDenied = 202
ConnectionFailed = 203
NoReservation = 204
MalformedMessage = 400
UnexpectedMessage = 401
HopMessageType* {.pure.} = enum
Reserve = 0
Connect = 1
Status = 2
HopMessage* = object
msgType*: HopMessageType
peer*: Opt[Peer]
reservation*: Opt[Reservation]
limit*: Limit
status*: Opt[StatusV2]
proc encode*(msg: HopMessage): ProtoBuffer =
var pb = initProtoBuffer()
pb.write(1, msg.msgType.ord.uint)
msg.peer.withValue(peer):
var ppb = initProtoBuffer()
ppb.write(1, peer.peerId)
for ma in peer.addrs:
ppb.write(2, ma.data.buffer)
ppb.finish()
pb.write(2, ppb.buffer)
msg.reservation.withValue(rsrv):
var rpb = initProtoBuffer()
rpb.write(1, rsrv.expire)
for ma in rsrv.addrs:
rpb.write(2, ma.data.buffer)
rsrv.svoucher.withValue(vouch):
rpb.write(3, vouch)
rpb.finish()
pb.write(3, rpb.buffer)
if msg.limit.duration > 0 or msg.limit.data > 0:
var lpb = initProtoBuffer()
if msg.limit.duration > 0: lpb.write(1, msg.limit.duration)
if msg.limit.data > 0: lpb.write(2, msg.limit.data)
lpb.finish()
pb.write(4, lpb.buffer)
msg.status.withValue(status):
pb.write(5, status.ord.uint)
pb.finish()
pb
proc decode*(_: typedesc[HopMessage], buf: seq[byte]): Opt[HopMessage] =
var msg: HopMessage
let pb = initProtoBuffer(buf)
var msgTypeOrd: uint32
? pb.getRequiredField(1, msgTypeOrd).toOpt()
if not checkedEnumAssign(msg.msgType, msgTypeOrd):
return Opt.none(HopMessage)
var pbPeer: ProtoBuffer
if ? pb.getField(2, pbPeer).toOpt():
var peer: Peer
? pbPeer.getRequiredField(1, peer.peerId).toOpt()
discard ? pbPeer.getRepeatedField(2, peer.addrs).toOpt()
msg.peer = Opt.some(peer)
var pbReservation: ProtoBuffer
if ? pb.getField(3, pbReservation).toOpt():
var
svoucher: seq[byte]
reservation: Reservation
if ? pbReservation.getField(3, svoucher).toOpt():
reservation.svoucher = Opt.some(svoucher)
? pbReservation.getRequiredField(1, reservation.expire).toOpt()
discard ? pbReservation.getRepeatedField(2, reservation.addrs).toOpt()
msg.reservation = Opt.some(reservation)
var pbLimit: ProtoBuffer
if ? pb.getField(4, pbLimit).toOpt():
discard ? pbLimit.getField(1, msg.limit.duration).toOpt()
discard ? pbLimit.getField(2, msg.limit.data).toOpt()
var statusOrd: uint32
if ? pb.getField(5, statusOrd).toOpt():
var status: StatusV2
if not checkedEnumAssign(status, statusOrd):
return Opt.none(HopMessage)
msg.status = Opt.some(status)
Opt.some(msg)
# Circuit Relay V2 Stop Message
type
StopMessageType* {.pure.} = enum
Connect = 0
Status = 1
StopMessage* = object
msgType*: StopMessageType
peer*: Opt[Peer]
limit*: Limit
status*: Opt[StatusV2]
proc encode*(msg: StopMessage): ProtoBuffer =
var pb = initProtoBuffer()
pb.write(1, msg.msgType.ord.uint)
msg.peer.withValue(peer):
var ppb = initProtoBuffer()
ppb.write(1, peer.peerId)
for ma in peer.addrs:
ppb.write(2, ma.data.buffer)
ppb.finish()
pb.write(2, ppb.buffer)
if msg.limit.duration > 0 or msg.limit.data > 0:
var lpb = initProtoBuffer()
if msg.limit.duration > 0: lpb.write(1, msg.limit.duration)
if msg.limit.data > 0: lpb.write(2, msg.limit.data)
lpb.finish()
pb.write(3, lpb.buffer)
msg.status.withValue(status):
pb.write(4, status.ord.uint)
pb.finish()
pb
proc decode*(_: typedesc[StopMessage], buf: seq[byte]): Opt[StopMessage] =
var msg: StopMessage
let pb = initProtoBuffer(buf)
var msgTypeOrd: uint32
? pb.getRequiredField(1, msgTypeOrd).toOpt()
if msgTypeOrd.int notin StopMessageType:
return Opt.none(StopMessage)
msg.msgType = StopMessageType(msgTypeOrd)
var pbPeer: ProtoBuffer
if ? pb.getField(2, pbPeer).toOpt():
var peer: Peer
? pbPeer.getRequiredField(1, peer.peerId).toOpt()
discard ? pbPeer.getRepeatedField(2, peer.addrs).toOpt()
msg.peer = Opt.some(peer)
var pbLimit: ProtoBuffer
if ? pb.getField(3, pbLimit).toOpt():
discard ? pbLimit.getField(1, msg.limit.duration).toOpt()
discard ? pbLimit.getField(2, msg.limit.data).toOpt()
var statusOrd: uint32
if ? pb.getField(4, statusOrd).toOpt():
var status: StatusV2
if not checkedEnumAssign(status, statusOrd):
return Opt.none(StopMessage)
msg.status = Opt.some(status)
Opt.some(msg)

View File

@@ -0,0 +1,58 @@
# Nim-LibP2P
# Copyright (c) 2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
# * MIT license ([LICENSE-MIT](LICENSE-MIT))
# at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
{.push raises: [].}
import chronos
import ../../../stream/connection
type
RelayConnection* = ref object of Connection
conn*: Connection
limitDuration*: uint32
limitData*: uint64
dataSent*: uint64
method readOnce*(
self: RelayConnection,
pbytes: pointer,
nbytes: int): Future[int] {.async.} =
self.activity = true
return await self.conn.readOnce(pbytes, nbytes)
method write*(self: RelayConnection, msg: seq[byte]): Future[void] {.async.} =
self.dataSent.inc(msg.len)
if self.limitData != 0 and self.dataSent > self.limitData:
await self.close()
return
self.activity = true
await self.conn.write(msg)
method closeImpl*(self: RelayConnection): Future[void] {.async.} =
await self.conn.close()
await procCall Connection(self).closeImpl()
method getWrapped*(self: RelayConnection): Connection = self.conn
proc new*(
T: typedesc[RelayConnection],
conn: Connection,
limitDuration: uint32,
limitData: uint64): T =
let rc = T(conn: conn, limitDuration: limitDuration, limitData: limitData)
rc.initStream()
if limitDuration > 0:
proc checkDurationConnection() {.async.} =
let sleep = sleepAsync(limitDuration.seconds())
await sleep or conn.join()
if sleep.finished: await conn.close()
else: sleep.cancel()
asyncSpawn checkDurationConnection()
return rc

Some files were not shown because too many files have changed in this diff Show More