feat: intel-sgx attestation

crates/notary/server/Cargo.toml

@@ -3,6 +3,17 @@ name = "notary-server"
 version = "0.1.0-alpha.8-pre"
 edition = "2021"
 
+[features]
+tee_quote = [
+	"dep:mc-sgx-dcap-types", 
+	"dep:hex", 
+	"dep:rand_chacha", 
+	"dep:once_cell",
+	"dep:simple_asn1",
+        "dep:pem",
+	"dep:lazy_static",
+]
+
 [dependencies]
 tlsn-core = { workspace = true }
 tlsn-common = { workspace = true }
@@ -46,6 +57,15 @@ uuid = { workspace = true, features = ["v4", "fast-rng"] }
 ws_stream_tungstenite = { workspace = true, features = ["tokio_io"] }
 zeroize = { workspace = true }
 
+mc-sgx-dcap-types = { version = "0.11.0", optional = true }
+hex = { workspace = true, optional = true }
+rand_chacha = { workspace = true, optional = true }
+once_cell = { workspace = true, optional =true }
+simple_asn1 = {version = "0.6.2", optional = true }
+pem = { version =  "1.1.0", optional = true }
+lazy_static = { version = "1.4", optional = true }
+
 [build-dependencies]
 git2 = "0.19.0"
 chrono.workspace = true
+

crates/notary/server/src/domain.rs

@@ -1,7 +1,8 @@
 pub mod auth;
 pub mod cli;
 pub mod notary;
-
+#[cfg(feature = "tee_quote")]
+use crate::tee::Quote;
 use serde::{Deserialize, Serialize};
 
 /// Response object of the /info API
@@ -14,4 +15,7 @@ pub struct InfoResponse {
     pub public_key: String,
     /// Current git commit hash of notary-server
     pub git_commit_hash: String,
+    /// Hardware attestation
+    #[cfg(feature = "tee_quote")]
+    pub quote: Quote,
 }

crates/notary/server/src/lib.rs

@@ -7,6 +7,8 @@ mod server_tracing;
 mod service;
 mod settings;
 mod signing;
+#[cfg(feature = "tee_quote")]
+mod tee;
 mod util;
 
 pub use config::{

crates/notary/server/src/server.rs

@@ -46,11 +46,13 @@ use crate::{
     util::parse_csv_file,
 };
 
+#[cfg(feature = "tee_quote")]
+use crate::tee::{generate_ephemeral_keypair, quote};
+
 /// Start a TCP server (with or without TLS) to accept notarization request for
 /// both TCP and WebSocket clients
 #[tracing::instrument(skip(config))]
 pub async fn run_server(config: &NotaryServerProperties) -> Result<(), NotaryServerError> {
-    // Load the private key for notarized transcript signing
     let attestation_key = load_attestation_key(&config.notary_key).await?;
     let crypto_provider = build_crypto_provider(attestation_key);
 
@@ -139,6 +141,8 @@ pub async fn run_server(config: &NotaryServerProperties) -> Result<(), NotarySer
                         version,
                         public_key,
                         git_commit_hash,
+                        #[cfg(feature = "tee_quote")]
+                        quote: quote().await,
                     }),
                 )
                     .into_response()
@@ -229,6 +233,9 @@ fn build_crypto_provider(attestation_key: AttestationKey) -> CryptoProvider {
 
 /// Load notary signing key for attestations from static file
 async fn load_attestation_key(config: &NotarySigningKeyProperties) -> Result<AttestationKey> {
+    #[cfg(feature = "tee_quote")]
+    generate_ephemeral_keypair(&config.private_key_pem_path, &config.public_key_pem_path);
+
     debug!("Loading notary server's signing key");
 
     let mut file = File::open(&config.private_key_pem_path).await?;

crates/notary/server/src/tee.rs

@@ -0,0 +1,186 @@
+use k256::ecdsa::{SigningKey, VerifyingKey as PublicKey};
+use mc_sgx_dcap_types::{QlError, Quote3};
+use once_cell::sync::OnceCell;
+use pkcs8::{EncodePrivateKey, LineEnding};
+use rand_chacha::{
+    rand_core::{OsRng, SeedableRng},
+    ChaCha20Rng,
+};
+use serde::{Deserialize, Serialize};
+use std::{
+    fs,
+    fs::File,
+    io::{self, Read},
+    path::Path,
+};
+use tracing::{debug, error, instrument};
+
+lazy_static::lazy_static! {
+    static ref SECP256K1_OID: simple_asn1::OID = simple_asn1::oid!(1, 3, 132, 0, 10);
+    static ref ECDSA_OID: simple_asn1::OID = simple_asn1::oid!(1, 2, 840, 10045, 2, 1);
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Quote {
+    raw_quote: Option<String>,
+    mrsigner: Option<String>,
+    mrenclave: Option<String>,
+    error: Option<String>,
+}
+
+impl Default for Quote {
+    fn default() -> Quote {
+        Quote {
+            raw_quote: Some("".to_string()),
+            mrsigner: None,
+            mrenclave: None,
+            error: None,
+        }
+    }
+}
+
+impl std::fmt::Debug for QuoteError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            QuoteError::IoError(err) => write!(f, "IoError: {:?}", err),
+            QuoteError::IntelQuoteLibrary(err) => {
+                write!(f, "IntelQuoteLibrary: {}", err)
+            }
+        }
+    }
+}
+
+impl From<io::Error> for QuoteError {
+    fn from(err: io::Error) -> QuoteError {
+        QuoteError::IoError(err)
+    }
+}
+
+enum QuoteError {
+    IoError(io::Error),
+    IntelQuoteLibrary(QlError),
+}
+
+impl From<QlError> for QuoteError {
+    fn from(src: QlError) -> Self {
+        Self::IntelQuoteLibrary(src)
+    }
+}
+
+static PUBLIC_KEY: OnceCell<PublicKey> = OnceCell::new();
+
+fn pem_der_encode_with_asn1(public_point: &[u8]) -> String {
+    use simple_asn1::*;
+
+    let ecdsa_oid = ASN1Block::ObjectIdentifier(0, ECDSA_OID.clone());
+    let secp256k1_oid = ASN1Block::ObjectIdentifier(0, SECP256K1_OID.clone());
+    let alg_id = ASN1Block::Sequence(0, vec![ecdsa_oid, secp256k1_oid]);
+    let key_bytes = ASN1Block::BitString(0, public_point.len() * 8, public_point.to_vec());
+
+    let blocks = vec![alg_id, key_bytes];
+
+    let der_out = simple_asn1::to_der(&ASN1Block::Sequence(0, blocks))
+        .expect("Failed to encode ECDSA private key as DER");
+
+    pem::encode(&pem::Pem {
+        tag: "PUBLIC KEY".to_string(),
+        contents: der_out,
+    })
+}
+
+#[instrument(level = "debug", skip_all)]
+async fn gramine_quote() -> Result<Quote, QuoteError> {
+    //// Check if the the gramine pseudo-hardware exists
+    if !Path::new("/dev/attestation/quote").exists() {
+        return Ok(Quote::default());
+    }
+
+    // Reading attestation type
+    let mut attestation_file = File::open("/dev/attestation/attestation_type")?;
+    let mut attestation_type = String::new();
+    attestation_file.read_to_string(&mut attestation_type)?;
+    debug!("Detected attestation type: {}", attestation_type);
+
+    // Read `/dev/attestation/my_target_info`
+    let my_target_info = fs::read("/dev/attestation/my_target_info")?;
+
+    // Write to `/dev/attestation/target_info`
+    fs::write("/dev/attestation/target_info", my_target_info)?;
+
+    //// Writing the pubkey to bind the instance to the hw (note: this is not
+    //// mrsigner)
+    fs::write(
+        "/dev/attestation/user_report_data",
+        PUBLIC_KEY
+            .get()
+            .expect("pub_key_get")
+            .to_encoded_point(true)
+            .as_bytes(),
+    )?;
+
+    //// Reading from the gramine quote pseudo-hardware `/dev/attestation/quote`
+    let mut quote_file = File::open("/dev/attestation/quote")?;
+    let mut quote = Vec::new();
+    let _ = quote_file.read_to_end(&mut quote);
+    //// todo: wire up Qlerror and drop .expect()
+    let quote3 = Quote3::try_from(quote.as_ref()).expect("quote3 error");
+    let mrenclave = quote3.app_report_body().mr_enclave().to_string();
+    let mrsigner = quote3.app_report_body().mr_signer().to_string();
+
+    debug!("mrenclave: {}", mrenclave);
+    debug!("mrsigner: {}", mrsigner);
+
+    //// Return the Quote struct with the extracted data
+    Ok(Quote {
+        raw_quote: Some(hex::encode(quote)),
+        mrsigner: Some(mrsigner),
+        mrenclave: Some(mrenclave),
+        error: None,
+    })
+}
+
+pub fn generate_ephemeral_keypair(notary_private: &str, notary_public: &str) {
+    let mut rng = ChaCha20Rng::from_rng(OsRng).expect("os rng err!");
+    let signing_key = SigningKey::random(&mut rng);
+    let pem_string = signing_key
+        .clone()
+        .to_pkcs8_pem(LineEnding::LF)
+        .expect("to pem");
+
+    std::fs::write(notary_private, pem_string).expect("fs::write");
+
+    let der = signing_key
+        .verifying_key()
+        .to_encoded_point(true)
+        .to_bytes();
+    let pem_spki_pub = pem_der_encode_with_asn1(&der);
+    std::fs::write(notary_public, pem_spki_pub).expect("fs::write");
+    let _ = PUBLIC_KEY
+        .set(*signing_key.verifying_key())
+        .map_err(|_| "Public key has already been set");
+}
+
+pub async fn quote() -> Quote {
+    //// tee-detection logic will live here, for now its only gramine-sgx
+    match gramine_quote().await {
+        Ok(quote) => quote,
+        Err(err) => {
+            error!("Failed to retrieve quote: {:?}", err);
+            match err {
+                QuoteError::IoError(_) => Quote {
+                    raw_quote: None,
+                    mrsigner: None,
+                    mrenclave: None,
+                    error: Some("io".to_owned()),
+                },
+                QuoteError::IntelQuoteLibrary(_) => Quote {
+                    raw_quote: None,
+                    mrsigner: None,
+                    mrenclave: None,
+                    error: Some("hw".to_owned()),
+                },
+            }
+        }
+    }
+}

crates/notary/server/tee/Dockerfile

@@ -0,0 +1,25 @@
+#tlsnotary server for testing <> gramine sgx (gramine1.7, g++13, libiomp off :()
+### notaryserverbuilds.azurecr.io/prod/notary-sgx
+
+FROM            notaryserverbuilds.azurecr.io/prod/gramine AS teesdk
+
+ARG             TOOLCHAIN=1.81.0
+ENV             PATH=/root/.cargo/bin:/usr/local/musl/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+RUN             set -eux \
+                && curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain=$TOOLCHAIN \
+                && rustup target add \
+                x86_64-unknown-linux-gnu
+
+
+RUN             apt update && apt install -y libssl-dev libclang-dev
+ARG             TLSN_TAG=dev
+ARG             TLSN_FT=tee_quote
+RUN             git clone --depth 1 -b $TLSN_TAG https://github.com/tlsnotary/tlsn /tlsn  && \
+                cargo build --release --bin notary-server --features $TLSN_FT --color always --manifest-path /tlsn/Cargo.toml
+RUN             cd tlsn/crates/notary/server/tee && gramine-sgx-gen-private-key && SGX=1 make
+
+FROM            notaryserverbuilds.azurecr.io/prod/gramine AS teetime
+WORKDIR /tee
+COPY            --from=teesdk tlsn/crates/notary/server/tee .
+ENTRYPOINT      ["gramine-sgx", "notary-server"]

crates/notary/server/tee/Makefile

@@ -0,0 +1,63 @@
+# notary-server testing only
+ROOT_DIR := $(dir $(realpath $(lastword $(MAKEFILE_LIST))))
+ARCH_LIBDIR ?= /lib/$(shell $(CC) -dumpmachine)
+
+SELF_EXE = ./notary-server
+
+.PHONY: all
+all: $(SELF_EXE) notary-server.manifest
+ifeq ($(SGX),1)
+all: notary-server.manifest.sgx notary-server.sig
+endif
+
+ifeq ($(DEBUG),1)
+GRAMINE_LOG_LEVEL = debug
+else
+GRAMINE_LOG_LEVEL = error
+endif
+
+# Note that we're compiling in release mode regardless of the DEBUG setting passed
+# to Make, as compiling in debug mode results in an order of magnitude's difference in
+# performance that makes testing by running a benchmark with ab painful. The primary goal
+# of the DEBUG setting is to control Gramine's loglevel.
+-include $(SELF_EXE).d # See also: .cargo/config.toml
+$(SELF_EXE): $(ROOT_DIR)../Cargo.toml
+	cargo build --bin notary-server  --release --features tee_quote
+
+notary-server.manifest: notary-server.manifest.template
+	cp ../../../../target/release/notary-server . && \
+	gramine-manifest \
+		-Dlog_level=$(GRAMINE_LOG_LEVEL) \
+		-Darch_libdir=$(ARCH_LIBDIR) \
+		-Dself_exe=$(SELF_EXE) \
+		$< $@
+
+# Make on Ubuntu <= 20.04 doesn't support "Rules with Grouped Targets" (`&:`),
+# see the helloworld example for details on this workaround.
+notary-server.manifest.sgx notary-server.sig: sgx_sign
+	@:
+
+.INTERMEDIATE: sgx_sign
+sgx_sign: notary-server.manifest $(SELF_EXE)
+	gramine-sgx-sign \
+		--manifest $< \
+		--output $<.sgx
+
+ifeq ($(SGX),)
+GRAMINE = gramine-direct
+else
+GRAMINE = gramine-sgx
+endif
+
+.PHONY: start-gramine-server
+start-gramine-server: all
+	$(GRAMINE) notary-server
+
+.PHONY: clean
+clean:
+	$(RM) -rf *.token *.sig *.manifest.sgx *.manifest result-* OUTPUT
+
+.PHONY: distclean
+distclean: clean
+	$(RM) -rf $(SELF_EXE) Cargo.lock
+

crates/notary/server/tee/README.md

@@ -0,0 +1,21 @@
+#### gramine with intel SGX
+```bash
+SGX=1 make
+```
+```bash
+SGX=1 make start-gramine-server
+```
+#### gramine emulating SGX
+```
+make
+```
+```
+make start-gramine-server
+```
+#### generate measurement without SGX hardware
+```
+make
+```
+```
+gramine-sgx-sigstruct-view --verbose --output-format=toml notary-server.sig
+```

crates/notary/server/tee/config/config.yaml

@@ -0,0 +1,45 @@
+server:
+  name: "notary.codes"
+  host: "0.0.0.0"
+  port: 7047
+  html_info: |
+    <head>
+    <meta charset="UTF-8">
+    <meta name="description" content="mpc-tls">
+    <meta name="keywords" content="tlsnotary, mpc, free">
+    <meta name="author" content="notary">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    </head>
+    <body>
+    <svg width="86" height="88" viewBox="0 0 86 88" fill="none" xmlns="http://www.w3.org/2000/svg">
+    <path d="M25.5484 0.708986C25.5484 0.17436 26.1196 -0.167376 26.5923 0.0844205L33.6891 3.86446C33.9202 3.98756 34.0645 4.22766 34.0645 4.48902V9.44049H37.6129C38.0048 9.44049 38.3226 9.75747 38.3226 10.1485V21.4766L36.1936 20.0606V11.5645H34.0645V80.9919C34.0645 81.1134 34.0332 81.2328 33.9735 81.3388L30.4251 87.6388C30.1539 88.1204 29.459 88.1204 29.1878 87.6388L25.6394 81.3388C25.5797 81.2328 25.5484 81.1134 25.5484 80.9919V0.708986Z" fill="#243F5F"/>
+    <path d="M21.2903 25.7246V76.7012H12.7742V34.2207H0V25.7246H21.2903Z" fill="#243F5F"/>
+    <path d="M63.871 76.7012H72.3871V34.2207H76.6452V76.7012H85.1613V25.7246H63.871V76.7012Z" fill="#243F5F"/>
+    <path d="M38.3226 25.7246H59.6129V34.2207H46.8387V46.9649H59.6129V76.7012H38.3226V68.2051H51.0968V55.4609H38.3226V25.7246Z" fill="#243F5F"/>
+    </svg>
+    <h1>notary server :: at your service</h1>
+    <h3>tlsnotary {version}</h3>
+    <blink>{public_key}</blink>
+    <h4>remote attestation</h4>
+    <a href="/v{version}/info">available here</a>
+    </body>
+
+notarization:
+  max_sent_data: 4096
+  max_recv_data: 16384
+
+tls:
+  enabled: false
+  private_key_pem_path: "."
+  certificate_pem_path: "."
+
+notary_key:
+  private_key_pem_path: "/ephemeral/notary.key"
+  public_key_pem_path: "/ephemeral/notary.pub"
+
+logging:
+  level: INFO
+
+authorization:
+  enabled: false
+  whitelist_csv_path: "."

crates/notary/server/tee/notary-server.manifest.template

@@ -0,0 +1,45 @@
+libos.entrypoint = "{{ self_exe }}"
+loader.log_level = "{{ log_level }}"
+
+loader.env.LD_LIBRARY_PATH = "/lib:{{ arch_libdir }}"
+
+# See https://gramine.readthedocs.io/en/stable/performance.html#glibc-malloc-tuning
+loader.env.MALLOC_ARENA_MAX = "1"
+
+# encrypted type not used
+fs.mounts = [
+  { path = "/lib", uri = "file:{{ gramine.runtimedir() }}" },
+  { path = "{{ arch_libdir }}", uri = "file:{{ arch_libdir }}" },
+  { type = "tmpfs", path = "/ephemeral" },
+  { type = "encrypted", path = "/vault", uri = "file:vault", key_name = "_sgx_mrenclave" },
+
+]
+
+# allowed enables rw -- will rm this once notary-server drops config file
+# !!!! not hashed !!!!
+# https://gramine.readthedocs.io/en/stable/manifest-syntax.html#allowed-files-1
+sgx.allowed_files = [
+  "file:./config/config.yaml",
+]
+
+# hashed @ buildtime. at runtime => these files are +ro
+# and can be accessed if hash matches manifest
+# !!!! hashed !!!!
+# https://gramine.readthedocs.io/en/stable/manifest-syntax.html#trusted-files
+sgx.trusted_files = [
+  "file:{{ self_exe }}",
+  "file:{{ gramine.runtimedir() }}/",
+  "file:{{ arch_libdir }}/",
+]
+
+sgx.edmm_enable = false
+sgx.remote_attestation = "dcap"
+sgx.max_threads = 64
+sgx.enclave_size = "2G"
+sys.disallow_subprocesses = true
+
+
+#### tlsn rev
+sgx.isvprodid = 7
+#### F
+sgx.isvsvn = 46