From 9e0f79125b3c478e30e7c518c8e16797212e06ae Mon Sep 17 00:00:00 2001
From: dan <themighty1@users.noreply.github.com>
Date: Tue, 22 Apr 2025 14:03:23 +0200
Subject: [PATCH] misc(notary): improve error msg when tls is expected (#776)

* misc(notary): improve error msg when tls is expected

* change wording

* fix nested if

* process hyper error

* refactor into a fn

* fix error msg

Co-authored-by: yuroitaki <25913766+yuroitaki@users.noreply.github.com>

* do not catch hyper error

---------

Co-authored-by: yuroitaki <25913766+yuroitaki@users.noreply.github.com>
---
 crates/notary/client/src/client.rs | 21 +++++++++++++++++++--
 crates/notary/server/src/server.rs | 11 ++++++++++-
 2 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/crates/notary/client/src/client.rs b/crates/notary/client/src/client.rs
index 5b432b3d9..560ff93f0 100644
--- a/crates/notary/client/src/client.rs
+++ b/crates/notary/client/src/client.rs
@@ -24,7 +24,7 @@ use tokio::{
 };
 use tokio_rustls::{
     client::TlsStream,
-    rustls::{ClientConfig, OwnedTrustAnchor, RootCertStore},
+    rustls::{self, ClientConfig, OwnedTrustAnchor, RootCertStore},
     TlsConnector,
 };
 use tracing::{debug, error};
@@ -192,7 +192,12 @@ impl NotaryClient {
                     notary_socket,
                 )
                 .await
-                .map_err(|err| ClientError::new(ErrorKind::TlsSetup, Some(Box::new(err))))?;
+                .map_err(|err| {
+                    if is_tls_mismatch_error(&err) {
+                        error!("Perhaps the notary server is not accepting our TLS connection");
+                    }
+                    ClientError::new(ErrorKind::TlsSetup, Some(Box::new(err)))
+                })?;
 
             self.send_request(notary_tls_socket, notarization_request)
                 .await
@@ -469,6 +474,18 @@ fn default_root_store() -> RootCertStore {
     root_store
 }
 
+// Checks whether the error is potentially related to a mismatch in TLS
+// configuration between the client and the server.
+fn is_tls_mismatch_error(err: &std::io::Error) -> bool {
+    if let Some(rustls::Error::InvalidMessage(rustls::InvalidMessage::InvalidContentType)) = err
+        .get_ref()
+        .and_then(|inner| inner.downcast_ref::<rustls::Error>())
+    {
+        return true;
+    }
+    false
+}
+
 // Attempts to parse the value of the "Retry-After" header from the given
 // `response`.
 fn parse_retry_after(response: &Response<Incoming>) -> Result<u64, ClientError> {
diff --git a/crates/notary/server/src/server.rs b/crates/notary/server/src/server.rs
index c2f0f5ae4..c91afd8cb 100644
--- a/crates/notary/server/src/server.rs
+++ b/crates/notary/server/src/server.rs
@@ -26,7 +26,7 @@ use std::{
 };
 use tlsn_core::CryptoProvider;
 use tokio::{fs::File, io::AsyncReadExt, net::TcpListener};
-use tokio_rustls::TlsAcceptor;
+use tokio_rustls::{rustls, TlsAcceptor};
 use tower_http::cors::CorsLayer;
 use tower_service::Service;
 use tracing::{debug, error, info};
@@ -214,6 +214,15 @@ pub async fn run_server(config: &NotaryServerProperties) -> Result<(), NotarySer
 
                     Err(err) => {
                         error!("{}", NotaryServerError::Connection(err.to_string()));
+
+                        if let Some(rustls::Error::InvalidMessage(
+                            rustls::InvalidMessage::InvalidContentType,
+                        )) = err
+                            .get_ref()
+                            .and_then(|inner| inner.downcast_ref::<rustls::Error>())
+                        {
+                            error!("Perhaps the client is connecting without TLS");
+                        }
                     }
                 }
             } else {