fix(backend): Fix structured logging for cloud environments (#11227)

- Resolves #11226

### Changes 🏗️

- Drop use of `CloudLoggingHandler` which docs state isn't for use in
GKE
- For cloud logging, output only structured log entries to `stdout`

### Checklist 📋

#### For code changes:
- [x] I have clearly listed my changes in the PR description
- [x] I have made a test plan
- [x] I have tested my changes according to the test plan:
  - [x] Test deploy to dev and check logs

autogpt_platform/autogpt_libs/autogpt_libs/logging/config.py

@@ -94,42 +94,36 @@ def configure_logging(force_cloud_logging: bool = False) -> None:
     config = LoggingConfig()
     log_handlers: list[logging.Handler] = []
 
+    structured_logging = config.enable_cloud_logging or force_cloud_logging
+
     # Console output handlers
-    stdout = logging.StreamHandler(stream=sys.stdout)
-    stdout.setLevel(config.level)
-    stdout.addFilter(BelowLevelFilter(logging.WARNING))
-    if config.level == logging.DEBUG:
-        stdout.setFormatter(AGPTFormatter(DEBUG_LOG_FORMAT))
-    else:
-        stdout.setFormatter(AGPTFormatter(SIMPLE_LOG_FORMAT))
+    if not structured_logging:
+        stdout = logging.StreamHandler(stream=sys.stdout)
+        stdout.setLevel(config.level)
+        stdout.addFilter(BelowLevelFilter(logging.WARNING))
+        if config.level == logging.DEBUG:
+            stdout.setFormatter(AGPTFormatter(DEBUG_LOG_FORMAT))
+        else:
+            stdout.setFormatter(AGPTFormatter(SIMPLE_LOG_FORMAT))
 
-    stderr = logging.StreamHandler()
-    stderr.setLevel(logging.WARNING)
-    if config.level == logging.DEBUG:
-        stderr.setFormatter(AGPTFormatter(DEBUG_LOG_FORMAT))
-    else:
-        stderr.setFormatter(AGPTFormatter(SIMPLE_LOG_FORMAT))
+        stderr = logging.StreamHandler()
+        stderr.setLevel(logging.WARNING)
+        if config.level == logging.DEBUG:
+            stderr.setFormatter(AGPTFormatter(DEBUG_LOG_FORMAT))
+        else:
+            stderr.setFormatter(AGPTFormatter(SIMPLE_LOG_FORMAT))
 
-    log_handlers += [stdout, stderr]
+        log_handlers += [stdout, stderr]
 
     # Cloud logging setup
-    if config.enable_cloud_logging or force_cloud_logging:
-        import google.cloud.logging
-        from google.cloud.logging.handlers import CloudLoggingHandler
-        from google.cloud.logging_v2.handlers.transports import (
-            BackgroundThreadTransport,
-        )
+    else:
+        # Use Google Cloud Structured Log Handler. Log entries are printed to stdout
+        # in a JSON format which is automatically picked up by Google Cloud Logging.
+        from google.cloud.logging.handlers import StructuredLogHandler
 
-        client = google.cloud.logging.Client()
-        # Use BackgroundThreadTransport to prevent blocking the main thread
-        # and deadlocks when gRPC calls to Google Cloud Logging hang
-        cloud_handler = CloudLoggingHandler(
-            client,
-            name="autogpt_logs",
-            transport=BackgroundThreadTransport,
-        )
-        cloud_handler.setLevel(config.level)
-        log_handlers.append(cloud_handler)
+        structured_log_handler = StructuredLogHandler(stream=sys.stdout)
+        structured_log_handler.setLevel(config.level)
+        log_handlers.append(structured_log_handler)
 
     # File logging setup
     if config.enable_file_logging:
@@ -185,7 +179,13 @@ def configure_logging(force_cloud_logging: bool = False) -> None:
 
     # Configure the root logger
     logging.basicConfig(
-        format=DEBUG_LOG_FORMAT if config.level == logging.DEBUG else SIMPLE_LOG_FORMAT,
+        format=(
+            "%(levelname)s  %(message)s"
+            if structured_logging
+            else (
+                DEBUG_LOG_FORMAT if config.level == logging.DEBUG else SIMPLE_LOG_FORMAT
+            )
+        ),
         level=config.level,
         handlers=log_handlers,
     )

autogpt_platform/backend/backend/executor/utils.py

@@ -42,7 +42,7 @@ from backend.util.clients import (
     get_integration_credentials_store,
 )
 from backend.util.exceptions import GraphValidationError, NotFoundError
-from backend.util.logging import TruncatedLogger
+from backend.util.logging import TruncatedLogger, is_structured_logging_enabled
 from backend.util.settings import Config
 from backend.util.type import convert
 
@@ -99,7 +99,11 @@ class LogMetadata(TruncatedLogger):
             "node_id": node_id,
             "block_name": block_name,
         }
-        prefix = f"[ExecutionManager|uid:{user_id}|gid:{graph_id}|nid:{node_id}]|geid:{graph_eid}|neid:{node_eid}|{block_name}]"
+        prefix = (
+            "[ExecutionManager]"
+            if is_structured_logging_enabled()
+            else f"[ExecutionManager|uid:{user_id}|gid:{graph_id}|nid:{node_id}]|geid:{graph_eid}|neid:{node_eid}|{block_name}]"  # noqa
+        )
         super().__init__(
             logger,
             max_length=max_length,

autogpt_platform/backend/backend/util/logging.py

@@ -8,10 +8,7 @@ settings = Settings()
 def configure_logging():
     import autogpt_libs.logging.config
 
-    if (
-        settings.config.behave_as == BehaveAs.LOCAL
-        or settings.config.app_env == AppEnvironment.LOCAL
-    ):
+    if not is_structured_logging_enabled():
         autogpt_libs.logging.config.configure_logging(force_cloud_logging=False)
     else:
         autogpt_libs.logging.config.configure_logging(force_cloud_logging=True)
@@ -20,6 +17,14 @@ def configure_logging():
     logging.getLogger("httpx").setLevel(logging.WARNING)
 
 
+def is_structured_logging_enabled() -> bool:
+    """Check if structured logging (cloud logging) is enabled."""
+    return not (
+        settings.config.behave_as == BehaveAs.LOCAL
+        or settings.config.app_env == AppEnvironment.LOCAL
+    )
+
+
 class TruncatedLogger:
     def __init__(
         self,