fast-sync: .bin -> .json (#448)

* serialize to json

* hex

* build script + static

* doc

Cargo.lock

@@ -782,12 +782,14 @@ dependencies = [
  "cuprate-blockchain",
  "cuprate-consensus",
  "cuprate-consensus-context",
+ "cuprate-hex",
  "cuprate-p2p",
  "cuprate-p2p-core",
  "cuprate-types",
  "hex",
  "monero-serai",
  "proptest",
+ "serde_json",
  "tempfile",
  "tokio",
  "tokio-test",

binaries/cuprated/Cargo.toml

@@ -86,5 +86,9 @@ tracing               = { workspace = true, features = ["default"] }
 [dev-dependencies]
 tempfile = { workspace = true }
 
+[build-dependencies]
+cuprate-hex = { workspace = true }
+serde_json  = { workspace = true, features = ["std"] }
+
 [lints]
 workspace = true

binaries/cuprated/build.rs

@@ -0,0 +1,33 @@
+fn main() {
+    generate_fast_sync_hashes();
+}
+
+/// Generates `fast_sync_hashes.rs` from `fast_sync_hashes.json`.
+///
+/// This creates a temporary build file with the
+/// `Debug` representation of the hashes, i.e.:
+/// ```
+/// [[0, 1, 2, ...], [0, 1, 2, ...], [0, 1, 2, ...]]
+/// ```
+///
+/// This is then used in `cuprated` with:
+/// ```rust
+/// let _: &[[u8; 32]] = &include!(...)
+/// ```
+fn generate_fast_sync_hashes() {
+    println!("cargo::rerun-if-changed=src/blockchain/fast_sync/fast_sync_hashes.json");
+
+    let hashes = serde_json::from_str::<Vec<cuprate_hex::Hex<32>>>(include_str!(
+        "src/blockchain/fast_sync/fast_sync_hashes.json"
+    ))
+    .unwrap()
+    .into_iter()
+    .map(|h| h.0)
+    .collect::<Vec<[u8; 32]>>();
+
+    std::fs::write(
+        format!("{}/fast_sync_hashes.rs", std::env::var("OUT_DIR").unwrap()),
+        format!("{hashes:?}"),
+    )
+    .unwrap();
+}

binaries/cuprated/src/blockchain/fast_sync.rs

@@ -3,16 +3,9 @@ use std::slice;
 use cuprate_helper::network::Network;
 
 /// The hashes of the compiled in fast sync file.
-static FAST_SYNC_HASHES: &[[u8; 32]] = {
-    let bytes = include_bytes!("./fast_sync/fast_sync_hashes.bin");
-
-    if bytes.len() % 32 == 0 {
-        // SAFETY: The file byte length must be perfectly divisible by 32, checked above.
-        unsafe { slice::from_raw_parts(bytes.as_ptr().cast::<[u8; 32]>(), bytes.len() / 32) }
-    } else {
-        panic!();
-    }
-};
+///
+/// See `build.rs` for how this file is generated.
+static FAST_SYNC_HASHES: &[[u8; 32]] = &include!(concat!(env!("OUT_DIR"), "/fast_sync_hashes.rs"));
 
 /// Set the fast-sync hashes according to the provided values.
 pub fn set_fast_sync_hashes(fast_sync: bool, network: Network) {
@@ -22,3 +15,14 @@ pub fn set_fast_sync_hashes(fast_sync: bool, network: Network) {
         &[]
     });
 }
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    /// Sanity check the fast sync hashes array.
+    #[test]
+    fn length() {
+        assert!(FAST_SYNC_HASHES.len() > 6642);
+    }
+}

consensus/fast-sync/Cargo.toml

@@ -12,6 +12,7 @@ path = "src/create.rs"
 cuprate-blockchain          = { workspace = true }
 cuprate-consensus           = { workspace = true }
 cuprate-consensus-context   = { workspace = true }
+cuprate-hex                 = { workspace = true }
 cuprate-types               = { workspace = true }
 cuprate-p2p                 = { workspace = true }
 cuprate-p2p-core            = { workspace = true }
@@ -22,6 +23,7 @@ monero-serai = { workspace = true }
 blake3       = { workspace = true }
 tokio        = { workspace = true, features = ["full"] }
 tower        = { workspace = true }
+serde_json   = { workspace = true }
 
 [dev-dependencies]
 proptest   = { workspace = true }

consensus/fast-sync/src/create.rs

@@ -11,6 +11,7 @@ use tower::{Service, ServiceExt};
 use cuprate_blockchain::{
     config::ConfigBuilder, cuprate_database::DbResult, service::BlockchainReadHandle,
 };
+use cuprate_hex::Hex;
 use cuprate_types::{
     blockchain::{BlockchainReadRequest, BlockchainResponse},
     Chain,
@@ -59,7 +60,7 @@ async fn main() {
     while (height + FAST_SYNC_BATCH_LEN) < height_target {
         if let Ok(block_ids) = read_batch(&mut read_handle, height).await {
             let hash = hash_of_hashes(block_ids.as_slice());
-            hashes_of_hashes.push(hash);
+            hashes_of_hashes.push(Hex(hash));
         } else {
             println!("Failed to read next batch from database");
             break;
@@ -71,8 +72,11 @@ async fn main() {
 
     drop(read_handle);
 
-    write("fast_sync_hashes.bin", hashes_of_hashes.concat().as_slice())
-        .expect("Could not write file");
+    write(
+        "fast_sync_hashes.json",
+        serde_json::to_string_pretty(&hashes_of_hashes).unwrap(),
+    )
+    .unwrap();
 
     println!("Generated hashes up to block height {height}");
 }

consensus/fast-sync/src/lib.rs

@@ -1,7 +1,9 @@
 // Used in `create.rs`
 use clap as _;
 use cuprate_blockchain as _;
+use cuprate_hex as _;
 use hex as _;
+use serde_json as _;
 use tokio as _;
 
 mod fast_sync;

misc/FAST_SYNC_HASHES.md

@@ -1,5 +1,5 @@
 # Fast sync hashes
-Cuprate has a binary that generate `fast-sync` hashes and puts them into a binary blob file.
+Cuprate has a binary that generates `fast-sync` hashes and puts them into a JSON file - this file is then used by `cuprated`.
 
 The code that does so is located at [`consensus/fast-sync`](https://github.com/Cuprate/cuprate/blob/main/consensus/fast-sync).
 
@@ -16,6 +16,6 @@ Run the binary:
 ```
 where `$HEIGHT` is the top blockchain height.
 
-The generated `fast_sync_hashes.bin` file should be in the current directory.
+The generated `fast_sync_hashes.json` file should be in the current directory.
 
-This should be moved to `binaries/cuprated/src/blockchain/fast_sync/fast_sync_hashes.bin`.
+This should be moved to `binaries/cuprated/src/blockchain/fast_sync/fast_sync_hashes.json`.