mirror of
https://github.com/Significant-Gravitas/AutoGPT.git
synced 2026-01-09 15:17:59 -05:00
fix(logging): remove uvicorn log config to prevent startup deadlock (#10638)
## Problem After applying the CloudLoggingHandler fix to use BackgroundThreadTransport (#10634), scheduler pods entered a new deadlock during startup when uvicorn reconfigures logging. ## Root Cause When uvicorn starts with a log_config parameter, it calls `logging.config.dictConfig()` which: 1. Calls `_clearExistingHandlers()` 2. Which calls `logging.shutdown()` 3. Which tries to `flush()` all handlers including CloudLoggingHandler 4. CloudLoggingHandler with BackgroundThreadTransport tries to flush its queue 5. The background worker thread tries to acquire the logging module lock to check log levels 6. **Deadlock**: shutdown holds lock waiting for flush to complete, worker thread needs lock to continue ## Thread Dump Evidence From py-spy analysis of the stuck pod: - **Thread 21 (FastAPI)**: Stuck in `flush()` waiting for background thread to drain queue - **Thread 13 (google.cloud.logging.Worker)**: Waiting for logging lock in `isEnabledFor()` - **Thread 1 (MainThread)**: Waiting for logging lock in `getLogger()` during SQLAlchemy import - **Threads 30, 31 (Sentry)**: Also waiting for logging lock ## Solution Set `log_config=None` for all uvicorn servers. This prevents uvicorn from calling `dictConfig()` and avoids the deadlock entirely. **Trade-off**: Uvicorn will use its default logging configuration which may produce duplicate log entries (one from uvicorn, one from the app), but the application will start successfully without deadlocks. ## Changes - Set `log_config=None` in all uvicorn.Config() calls - Remove unused `generate_uvicorn_config` imports ## Testing - [x] Verified scheduler pods can start and become healthy - [x] Health checks respond properly - [x] No deadlocks during startup - [x] Application logs still appear (though may be duplicated) ## Related Issues - Fixes the startup deadlock introduced after #10634
This commit is contained in:
@@ -1,39 +1,5 @@
|
||||
import logging
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
import uvicorn.config
|
||||
from colorama import Fore
|
||||
|
||||
|
||||
def remove_color_codes(s: str) -> str:
|
||||
return re.sub(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", s)
|
||||
|
||||
|
||||
def fmt_kwargs(kwargs: dict) -> str:
|
||||
return ", ".join(f"{n}={repr(v)}" for n, v in kwargs.items())
|
||||
|
||||
|
||||
def print_attribute(
|
||||
title: str, value: Any, title_color: str = Fore.GREEN, value_color: str = ""
|
||||
) -> None:
|
||||
logger = logging.getLogger()
|
||||
logger.info(
|
||||
str(value),
|
||||
extra={
|
||||
"title": f"{title.rstrip(':')}:",
|
||||
"title_color": title_color,
|
||||
"color": value_color,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def generate_uvicorn_config():
|
||||
"""
|
||||
Generates a uvicorn logging config that silences uvicorn's default logging and tells it to use the native logging module.
|
||||
"""
|
||||
log_config = dict(uvicorn.config.LOGGING_CONFIG)
|
||||
log_config["loggers"]["uvicorn"] = {"handlers": []}
|
||||
log_config["loggers"]["uvicorn.error"] = {"handlers": []}
|
||||
log_config["loggers"]["uvicorn.access"] = {"handlers": []}
|
||||
return log_config
|
||||
|
||||
@@ -9,7 +9,6 @@ import fastapi.responses
|
||||
import pydantic
|
||||
import starlette.middleware.cors
|
||||
import uvicorn
|
||||
from autogpt_libs.logging.utils import generate_uvicorn_config
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.routing import APIRoute
|
||||
|
||||
@@ -247,7 +246,7 @@ class AgentServer(backend.util.service.AppProcess):
|
||||
server_app,
|
||||
host=backend.util.settings.Config().agent_api_host,
|
||||
port=backend.util.settings.Config().agent_api_port,
|
||||
log_config=generate_uvicorn_config(),
|
||||
log_config=None,
|
||||
)
|
||||
|
||||
def cleanup(self):
|
||||
|
||||
@@ -6,7 +6,6 @@ from typing import Protocol
|
||||
import pydantic
|
||||
import uvicorn
|
||||
from autogpt_libs.auth import parse_jwt_token
|
||||
from autogpt_libs.logging.utils import generate_uvicorn_config
|
||||
from fastapi import Depends, FastAPI, WebSocket, WebSocketDisconnect
|
||||
from starlette.middleware.cors import CORSMiddleware
|
||||
|
||||
@@ -309,7 +308,7 @@ class WebsocketServer(AppProcess):
|
||||
server_app,
|
||||
host=Config().websocket_server_host,
|
||||
port=Config().websocket_server_port,
|
||||
log_config=generate_uvicorn_config(),
|
||||
log_config=None,
|
||||
)
|
||||
|
||||
def cleanup(self):
|
||||
|
||||
@@ -24,7 +24,6 @@ from typing import (
|
||||
|
||||
import httpx
|
||||
import uvicorn
|
||||
from autogpt_libs.logging.utils import generate_uvicorn_config
|
||||
from fastapi import FastAPI, Request, responses
|
||||
from pydantic import BaseModel, TypeAdapter, create_model
|
||||
|
||||
@@ -266,7 +265,7 @@ class AppService(BaseAppService, ABC):
|
||||
self.fastapi_app,
|
||||
host=api_host,
|
||||
port=self.get_port(),
|
||||
log_config=generate_uvicorn_config(),
|
||||
log_config=None, # Explicitly None to avoid uvicorn replacing the logger.
|
||||
log_level=self.log_level,
|
||||
)
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user