diff --git a/crates/notary/server/Cargo.toml b/crates/notary/server/Cargo.toml index bbee39e87..c3178f968 100644 --- a/crates/notary/server/Cargo.toml +++ b/crates/notary/server/Cargo.toml @@ -3,6 +3,17 @@ name = "notary-server" version = "0.1.0-alpha.8-pre" edition = "2021" +[features] +tee_quote = [ + "dep:mc-sgx-dcap-types", + "dep:hex", + "dep:rand_chacha", + "dep:once_cell", + "dep:simple_asn1", + "dep:pem", + "dep:lazy_static", +] + [dependencies] tlsn-core = { workspace = true } tlsn-common = { workspace = true } @@ -46,6 +57,15 @@ uuid = { workspace = true, features = ["v4", "fast-rng"] } ws_stream_tungstenite = { workspace = true, features = ["tokio_io"] } zeroize = { workspace = true } +mc-sgx-dcap-types = { version = "0.11.0", optional = true } +hex = { workspace = true, optional = true } +rand_chacha = { workspace = true, optional = true } +once_cell = { workspace = true, optional =true } +simple_asn1 = {version = "0.6.2", optional = true } +pem = { version = "1.1.0", optional = true } +lazy_static = { version = "1.4", optional = true } + [build-dependencies] git2 = "0.19.0" chrono.workspace = true + diff --git a/crates/notary/server/src/domain.rs b/crates/notary/server/src/domain.rs index 407eb1a13..58a5dc8b1 100644 --- a/crates/notary/server/src/domain.rs +++ b/crates/notary/server/src/domain.rs @@ -1,7 +1,8 @@ pub mod auth; pub mod cli; pub mod notary; - +#[cfg(feature = "tee_quote")] +use crate::tee::Quote; use serde::{Deserialize, Serialize}; /// Response object of the /info API @@ -14,4 +15,7 @@ pub struct InfoResponse { pub public_key: String, /// Current git commit hash of notary-server pub git_commit_hash: String, + /// Hardware attestation + #[cfg(feature = "tee_quote")] + pub quote: Quote, } diff --git a/crates/notary/server/src/lib.rs b/crates/notary/server/src/lib.rs index 140a1cac4..85a873799 100644 --- a/crates/notary/server/src/lib.rs +++ b/crates/notary/server/src/lib.rs @@ -7,6 +7,8 @@ mod server_tracing; mod service; mod settings; mod signing; +#[cfg(feature = "tee_quote")] +mod tee; mod util; pub use config::{ diff --git a/crates/notary/server/src/server.rs b/crates/notary/server/src/server.rs index 9b01900ad..5a93463a1 100644 --- a/crates/notary/server/src/server.rs +++ b/crates/notary/server/src/server.rs @@ -46,11 +46,13 @@ use crate::{ util::parse_csv_file, }; +#[cfg(feature = "tee_quote")] +use crate::tee::{generate_ephemeral_keypair, quote}; + /// Start a TCP server (with or without TLS) to accept notarization request for /// both TCP and WebSocket clients #[tracing::instrument(skip(config))] pub async fn run_server(config: &NotaryServerProperties) -> Result<(), NotaryServerError> { - // Load the private key for notarized transcript signing let attestation_key = load_attestation_key(&config.notary_key).await?; let crypto_provider = build_crypto_provider(attestation_key); @@ -139,6 +141,8 @@ pub async fn run_server(config: &NotaryServerProperties) -> Result<(), NotarySer version, public_key, git_commit_hash, + #[cfg(feature = "tee_quote")] + quote: quote().await, }), ) .into_response() @@ -229,6 +233,9 @@ fn build_crypto_provider(attestation_key: AttestationKey) -> CryptoProvider { /// Load notary signing key for attestations from static file async fn load_attestation_key(config: &NotarySigningKeyProperties) -> Result { + #[cfg(feature = "tee_quote")] + generate_ephemeral_keypair(&config.private_key_pem_path, &config.public_key_pem_path); + debug!("Loading notary server's signing key"); let mut file = File::open(&config.private_key_pem_path).await?; diff --git a/crates/notary/server/src/tee.rs b/crates/notary/server/src/tee.rs new file mode 100644 index 000000000..4a4c419af --- /dev/null +++ b/crates/notary/server/src/tee.rs @@ -0,0 +1,186 @@ +use k256::ecdsa::{SigningKey, VerifyingKey as PublicKey}; +use mc_sgx_dcap_types::{QlError, Quote3}; +use once_cell::sync::OnceCell; +use pkcs8::{EncodePrivateKey, LineEnding}; +use rand_chacha::{ + rand_core::{OsRng, SeedableRng}, + ChaCha20Rng, +}; +use serde::{Deserialize, Serialize}; +use std::{ + fs, + fs::File, + io::{self, Read}, + path::Path, +}; +use tracing::{debug, error, instrument}; + +lazy_static::lazy_static! { + static ref SECP256K1_OID: simple_asn1::OID = simple_asn1::oid!(1, 3, 132, 0, 10); + static ref ECDSA_OID: simple_asn1::OID = simple_asn1::oid!(1, 2, 840, 10045, 2, 1); +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Quote { + raw_quote: Option, + mrsigner: Option, + mrenclave: Option, + error: Option, +} + +impl Default for Quote { + fn default() -> Quote { + Quote { + raw_quote: Some("".to_string()), + mrsigner: None, + mrenclave: None, + error: None, + } + } +} + +impl std::fmt::Debug for QuoteError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + QuoteError::IoError(err) => write!(f, "IoError: {:?}", err), + QuoteError::IntelQuoteLibrary(err) => { + write!(f, "IntelQuoteLibrary: {}", err) + } + } + } +} + +impl From for QuoteError { + fn from(err: io::Error) -> QuoteError { + QuoteError::IoError(err) + } +} + +enum QuoteError { + IoError(io::Error), + IntelQuoteLibrary(QlError), +} + +impl From for QuoteError { + fn from(src: QlError) -> Self { + Self::IntelQuoteLibrary(src) + } +} + +static PUBLIC_KEY: OnceCell = OnceCell::new(); + +fn pem_der_encode_with_asn1(public_point: &[u8]) -> String { + use simple_asn1::*; + + let ecdsa_oid = ASN1Block::ObjectIdentifier(0, ECDSA_OID.clone()); + let secp256k1_oid = ASN1Block::ObjectIdentifier(0, SECP256K1_OID.clone()); + let alg_id = ASN1Block::Sequence(0, vec![ecdsa_oid, secp256k1_oid]); + let key_bytes = ASN1Block::BitString(0, public_point.len() * 8, public_point.to_vec()); + + let blocks = vec![alg_id, key_bytes]; + + let der_out = simple_asn1::to_der(&ASN1Block::Sequence(0, blocks)) + .expect("Failed to encode ECDSA private key as DER"); + + pem::encode(&pem::Pem { + tag: "PUBLIC KEY".to_string(), + contents: der_out, + }) +} + +#[instrument(level = "debug", skip_all)] +async fn gramine_quote() -> Result { + //// Check if the the gramine pseudo-hardware exists + if !Path::new("/dev/attestation/quote").exists() { + return Ok(Quote::default()); + } + + // Reading attestation type + let mut attestation_file = File::open("/dev/attestation/attestation_type")?; + let mut attestation_type = String::new(); + attestation_file.read_to_string(&mut attestation_type)?; + debug!("Detected attestation type: {}", attestation_type); + + // Read `/dev/attestation/my_target_info` + let my_target_info = fs::read("/dev/attestation/my_target_info")?; + + // Write to `/dev/attestation/target_info` + fs::write("/dev/attestation/target_info", my_target_info)?; + + //// Writing the pubkey to bind the instance to the hw (note: this is not + //// mrsigner) + fs::write( + "/dev/attestation/user_report_data", + PUBLIC_KEY + .get() + .expect("pub_key_get") + .to_encoded_point(true) + .as_bytes(), + )?; + + //// Reading from the gramine quote pseudo-hardware `/dev/attestation/quote` + let mut quote_file = File::open("/dev/attestation/quote")?; + let mut quote = Vec::new(); + let _ = quote_file.read_to_end(&mut quote); + //// todo: wire up Qlerror and drop .expect() + let quote3 = Quote3::try_from(quote.as_ref()).expect("quote3 error"); + let mrenclave = quote3.app_report_body().mr_enclave().to_string(); + let mrsigner = quote3.app_report_body().mr_signer().to_string(); + + debug!("mrenclave: {}", mrenclave); + debug!("mrsigner: {}", mrsigner); + + //// Return the Quote struct with the extracted data + Ok(Quote { + raw_quote: Some(hex::encode(quote)), + mrsigner: Some(mrsigner), + mrenclave: Some(mrenclave), + error: None, + }) +} + +pub fn generate_ephemeral_keypair(notary_private: &str, notary_public: &str) { + let mut rng = ChaCha20Rng::from_rng(OsRng).expect("os rng err!"); + let signing_key = SigningKey::random(&mut rng); + let pem_string = signing_key + .clone() + .to_pkcs8_pem(LineEnding::LF) + .expect("to pem"); + + std::fs::write(notary_private, pem_string).expect("fs::write"); + + let der = signing_key + .verifying_key() + .to_encoded_point(true) + .to_bytes(); + let pem_spki_pub = pem_der_encode_with_asn1(&der); + std::fs::write(notary_public, pem_spki_pub).expect("fs::write"); + let _ = PUBLIC_KEY + .set(*signing_key.verifying_key()) + .map_err(|_| "Public key has already been set"); +} + +pub async fn quote() -> Quote { + //// tee-detection logic will live here, for now its only gramine-sgx + match gramine_quote().await { + Ok(quote) => quote, + Err(err) => { + error!("Failed to retrieve quote: {:?}", err); + match err { + QuoteError::IoError(_) => Quote { + raw_quote: None, + mrsigner: None, + mrenclave: None, + error: Some("io".to_owned()), + }, + QuoteError::IntelQuoteLibrary(_) => Quote { + raw_quote: None, + mrsigner: None, + mrenclave: None, + error: Some("hw".to_owned()), + }, + } + } + } +} diff --git a/crates/notary/server/tee/Dockerfile b/crates/notary/server/tee/Dockerfile new file mode 100644 index 000000000..bc7607638 --- /dev/null +++ b/crates/notary/server/tee/Dockerfile @@ -0,0 +1,25 @@ +#tlsnotary server for testing <> gramine sgx (gramine1.7, g++13, libiomp off :() +### notaryserverbuilds.azurecr.io/prod/notary-sgx + +FROM notaryserverbuilds.azurecr.io/prod/gramine AS teesdk + +ARG TOOLCHAIN=1.81.0 +ENV PATH=/root/.cargo/bin:/usr/local/musl/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +RUN set -eux \ + && curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain=$TOOLCHAIN \ + && rustup target add \ + x86_64-unknown-linux-gnu + + +RUN apt update && apt install -y libssl-dev libclang-dev +ARG TLSN_TAG=dev +ARG TLSN_FT=tee_quote +RUN git clone --depth 1 -b $TLSN_TAG https://github.com/tlsnotary/tlsn /tlsn && \ + cargo build --release --bin notary-server --features $TLSN_FT --color always --manifest-path /tlsn/Cargo.toml +RUN cd tlsn/crates/notary/server/tee && gramine-sgx-gen-private-key && SGX=1 make + +FROM notaryserverbuilds.azurecr.io/prod/gramine AS teetime +WORKDIR /tee +COPY --from=teesdk tlsn/crates/notary/server/tee . +ENTRYPOINT ["gramine-sgx", "notary-server"] diff --git a/crates/notary/server/tee/Makefile b/crates/notary/server/tee/Makefile new file mode 100644 index 000000000..b3a35341e --- /dev/null +++ b/crates/notary/server/tee/Makefile @@ -0,0 +1,63 @@ +# notary-server testing only +ROOT_DIR := $(dir $(realpath $(lastword $(MAKEFILE_LIST)))) +ARCH_LIBDIR ?= /lib/$(shell $(CC) -dumpmachine) + +SELF_EXE = ./notary-server + +.PHONY: all +all: $(SELF_EXE) notary-server.manifest +ifeq ($(SGX),1) +all: notary-server.manifest.sgx notary-server.sig +endif + +ifeq ($(DEBUG),1) +GRAMINE_LOG_LEVEL = debug +else +GRAMINE_LOG_LEVEL = error +endif + +# Note that we're compiling in release mode regardless of the DEBUG setting passed +# to Make, as compiling in debug mode results in an order of magnitude's difference in +# performance that makes testing by running a benchmark with ab painful. The primary goal +# of the DEBUG setting is to control Gramine's loglevel. +-include $(SELF_EXE).d # See also: .cargo/config.toml +$(SELF_EXE): $(ROOT_DIR)../Cargo.toml + cargo build --bin notary-server --release --features tee_quote + +notary-server.manifest: notary-server.manifest.template + cp ../../../../target/release/notary-server . && \ + gramine-manifest \ + -Dlog_level=$(GRAMINE_LOG_LEVEL) \ + -Darch_libdir=$(ARCH_LIBDIR) \ + -Dself_exe=$(SELF_EXE) \ + $< $@ + +# Make on Ubuntu <= 20.04 doesn't support "Rules with Grouped Targets" (`&:`), +# see the helloworld example for details on this workaround. +notary-server.manifest.sgx notary-server.sig: sgx_sign + @: + +.INTERMEDIATE: sgx_sign +sgx_sign: notary-server.manifest $(SELF_EXE) + gramine-sgx-sign \ + --manifest $< \ + --output $<.sgx + +ifeq ($(SGX),) +GRAMINE = gramine-direct +else +GRAMINE = gramine-sgx +endif + +.PHONY: start-gramine-server +start-gramine-server: all + $(GRAMINE) notary-server + +.PHONY: clean +clean: + $(RM) -rf *.token *.sig *.manifest.sgx *.manifest result-* OUTPUT + +.PHONY: distclean +distclean: clean + $(RM) -rf $(SELF_EXE) Cargo.lock + diff --git a/crates/notary/server/tee/README.md b/crates/notary/server/tee/README.md new file mode 100644 index 000000000..d62b0f9dd --- /dev/null +++ b/crates/notary/server/tee/README.md @@ -0,0 +1,21 @@ +#### gramine with intel SGX +```bash +SGX=1 make +``` +```bash +SGX=1 make start-gramine-server +``` +#### gramine emulating SGX +``` +make +``` +``` +make start-gramine-server +``` +#### generate measurement without SGX hardware +``` +make +``` +``` +gramine-sgx-sigstruct-view --verbose --output-format=toml notary-server.sig +``` diff --git a/crates/notary/server/tee/config/config.yaml b/crates/notary/server/tee/config/config.yaml new file mode 100644 index 000000000..d85859fed --- /dev/null +++ b/crates/notary/server/tee/config/config.yaml @@ -0,0 +1,45 @@ +server: + name: "notary.codes" + host: "0.0.0.0" + port: 7047 + html_info: | + + + + + + + + + + + + + + +

notary server :: at your service

+

tlsnotary {version}

+ {public_key} +

remote attestation

+ available here + + +notarization: + max_sent_data: 4096 + max_recv_data: 16384 + +tls: + enabled: false + private_key_pem_path: "." + certificate_pem_path: "." + +notary_key: + private_key_pem_path: "/ephemeral/notary.key" + public_key_pem_path: "/ephemeral/notary.pub" + +logging: + level: INFO + +authorization: + enabled: false + whitelist_csv_path: "." diff --git a/crates/notary/server/tee/notary-server.manifest.template b/crates/notary/server/tee/notary-server.manifest.template new file mode 100644 index 000000000..40ab8a354 --- /dev/null +++ b/crates/notary/server/tee/notary-server.manifest.template @@ -0,0 +1,45 @@ +libos.entrypoint = "{{ self_exe }}" +loader.log_level = "{{ log_level }}" + +loader.env.LD_LIBRARY_PATH = "/lib:{{ arch_libdir }}" + +# See https://gramine.readthedocs.io/en/stable/performance.html#glibc-malloc-tuning +loader.env.MALLOC_ARENA_MAX = "1" + +# encrypted type not used +fs.mounts = [ + { path = "/lib", uri = "file:{{ gramine.runtimedir() }}" }, + { path = "{{ arch_libdir }}", uri = "file:{{ arch_libdir }}" }, + { type = "tmpfs", path = "/ephemeral" }, + { type = "encrypted", path = "/vault", uri = "file:vault", key_name = "_sgx_mrenclave" }, + +] + +# allowed enables rw -- will rm this once notary-server drops config file +# !!!! not hashed !!!! +# https://gramine.readthedocs.io/en/stable/manifest-syntax.html#allowed-files-1 +sgx.allowed_files = [ + "file:./config/config.yaml", +] + +# hashed @ buildtime. at runtime => these files are +ro +# and can be accessed if hash matches manifest +# !!!! hashed !!!! +# https://gramine.readthedocs.io/en/stable/manifest-syntax.html#trusted-files +sgx.trusted_files = [ + "file:{{ self_exe }}", + "file:{{ gramine.runtimedir() }}/", + "file:{{ arch_libdir }}/", +] + +sgx.edmm_enable = false +sgx.remote_attestation = "dcap" +sgx.max_threads = 64 +sgx.enclave_size = "2G" +sys.disallow_subprocesses = true + + +#### tlsn rev +sgx.isvprodid = 7 +#### F +sgx.isvsvn = 46