From 7acbfc1bde88deb47000246853844c90e62fafcd Mon Sep 17 00:00:00 2001 From: Parker Diamond Date: Tue, 14 Nov 2023 13:59:21 -0500 Subject: [PATCH 1/2] Add Application-level Keepalive --- ExternalIO/client.py | 34 +++++++++++++++++++++++++++++ Networking/sockets.cpp | 49 ++++++++++++++++++++++++++++++++++++++++++ deps/SimplestOT_C | 1 - 3 files changed, 83 insertions(+), 1 deletion(-) delete mode 160000 deps/SimplestOT_C diff --git a/ExternalIO/client.py b/ExternalIO/client.py index 17f2e3ea..7942d85c 100644 --- a/ExternalIO/client.py +++ b/ExternalIO/client.py @@ -1,7 +1,34 @@ +import platform import socket, ssl import struct import time +# The following function is either taken directly or derived from: +# https://stackoverflow.com/questions/12248132/how-to-change-tcp-keepalive-timer-using-python-script +def set_keepalive_linux(sock, after_idle_sec=1, interval_sec=3, max_fails=5): + """Set TCP keepalive on an open socket. + + It activates after 1 second (after_idle_sec) of idleness, + then sends a keepalive ping once every 3 seconds (interval_sec), + and closes the connection after 5 failed ping (max_fails), or 15 seconds + """ + sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, after_idle_sec) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, interval_sec) + sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, max_fails) + +# The following function is either taken directly or derived from: +# https://stackoverflow.com/questions/12248132/how-to-change-tcp-keepalive-timer-using-python-script +def set_keepalive_osx(sock, after_idle_sec=1, interval_sec=3, max_fails=5): + """Set TCP keepalive on an open socket. + + sends a keepalive ping once every 3 seconds (interval_sec) + """ + # scraped from /usr/include, not exported by python's socket module + TCP_KEEPALIVE = 0x10 + sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + sock.setsockopt(socket.IPPROTO_TCP, TCP_KEEPALIVE, interval_sec) + class Client: def __init__(self, hostnames, port_base, my_client_id): ctx = ssl.SSLContext() @@ -22,6 +49,13 @@ class Client: time.sleep(1) else: raise + + if platform.system() == "Linux": + set_keepalive_linux(plain_socket) + elif platform.system() == "Darwin": + set_keepalive_osx(plain_socket) + + set_keepalive_linux(plain_socket) octetStream(b'%d' % my_client_id).Send(plain_socket) self.sockets.append(ctx.wrap_socket(plain_socket, server_hostname='P%d' % i)) diff --git a/Networking/sockets.cpp b/Networking/sockets.cpp index b03613d4..3de7d176 100644 --- a/Networking/sockets.cpp +++ b/Networking/sockets.cpp @@ -120,6 +120,55 @@ void set_up_client_socket(int& mysocket,const char* hostname,int Portnum) int one=1; fl= setsockopt(mysocket, IPPROTO_TCP, TCP_NODELAY, (char*)&one, sizeof(int)); if (fl<0) { error("set_up_socket:setsockopt"); } + + + /* + * The following code block is either taken directly from or derived from the solutions posted at: + * https://stackoverflow.com/questions/20188718/configuring-tcp-keep-alive-with-boostasio + * https://stackoverflow.com/questions/23669005/tcp-keepalive-protocol-not-available + */ + unsigned int timeout_milli = 10000; + + #if (defined _WIN32 || defined WIN32 || defined OS_WIN64 || defined _WIN64 || defined WIN64 || defined WINNT) + int32_t timeout = timeout_milli; + fl = setsockopt(mysocket, SOL_SOCKET, SO_RCVTIMEO, (const char*)&timeout, sizeof(timeout)); + if (fl<0) { error("set_tcp_keepalive:setsockopt(SOL_SOCKET, SO_RCVTIMEO)"); } + + fl = setsockopt(mysocket, SOL_SOCKET, SO_SNDTIMEO, (const char*)&timeout, sizeof(timeout)); + if (fl<0) { error("set_tcp_keepalive:setsockopt(SOL_SOCKET, SO_SNDTIMEO)"); } + + #else + struct timeval tv; + tv.tv_sec = timeout_milli / 1000; + tv.tv_usec = (timeout_milli % 1000) * 1000; + + fl = setsockopt(mysocket, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); + if (fl<0) { error("set_tcp_keepalive:setsockopt(SOL_SOCKET, SO_RCVTIMEO)"); } + + fl = setsockopt(mysocket, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv)); + if (fl<0) { error("set_tcp_keepalive:setsockopt(SOL_SOCKET, SO_RCVTIMEO)"); } + + int enable_keepalive = 1; + int keepalive_strobe_interval_secs = 3; + int num_keepalive_strobes = 5; + + fl = setsockopt(mysocket, SOL_SOCKET, SO_KEEPALIVE,(char *)&enable_keepalive, sizeof(enable_keepalive)); + if (fl<0) { error("set_tcp_keepalive:setsockopt(SOL_SOCKET, SO_KEEPALIVE)"); } + + #ifdef TCP_KEEPIDLE + int keepalive_idle_time_secs = 1; + fl = setsockopt(mysocket, IPPROTO_TCP, TCP_KEEPIDLE, (char *)&keepalive_idle_time_secs, sizeof(keepalive_idle_time_secs)); + if (fl<0) { error("set_tcp_keepalive:setsockopt(IPPROTO_TCP, TCP_KEEPIDLE)"); } + #endif + + fl = setsockopt(mysocket, IPPROTO_TCP, TCP_KEEPINTVL, (char *)&keepalive_strobe_interval_secs, sizeof(keepalive_strobe_interval_secs)); + if (fl<0) { error("set_tcp_keepalive:setsockopt(IPPROTO_TCP, TCP_KEEPINTVL)"); } + + setsockopt(mysocket, IPPROTO_TCP, TCP_KEEPCNT, (char *)&num_keepalive_strobes, sizeof(num_keepalive_strobes)); + if (fl<0) { error("set_tcp_keepalive:setsockopt(IPPROTO_TCP, TCP_KEEPCNT)"); } + + #endif + /* End third-party code */ #ifdef __APPLE__ int flags = fcntl(mysocket, F_GETFL, 0); diff --git a/deps/SimplestOT_C b/deps/SimplestOT_C deleted file mode 160000 index 033e2428..00000000 --- a/deps/SimplestOT_C +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 033e24280ae32d599c5b037b44edd7c7e8228b35 From 2093ba520fa83ff473c56bee5cb7e39cdf353eae Mon Sep 17 00:00:00 2001 From: Parker Diamond Date: Tue, 14 Nov 2023 14:15:34 -0500 Subject: [PATCH 2/2] Add back in SimplestOT_C --- deps/SimplestOT_C | 1 + 1 file changed, 1 insertion(+) create mode 160000 deps/SimplestOT_C diff --git a/deps/SimplestOT_C b/deps/SimplestOT_C new file mode 160000 index 00000000..033e2428 --- /dev/null +++ b/deps/SimplestOT_C @@ -0,0 +1 @@ +Subproject commit 033e24280ae32d599c5b037b44edd7c7e8228b35