limit pythagora.log to 20k of lines

core/config/__init__.py

@@ -5,6 +5,8 @@ from typing import Any, Literal, Optional, Union
 from pydantic import BaseModel, ConfigDict, Field, field_validator
 from typing_extensions import Annotated
 
+from core.config.constants import LOGS_LINE_LIMIT
+
 ROOT_DIR = abspath(join(dirname(__file__), "..", ".."))
 DEFAULT_IGNORE_PATHS = [
     ".git",
@@ -218,6 +220,10 @@ class LogConfig(_StrictModel):
         "pythagora.log",
         description="Output file for logs (if not specified, logs are printed to stderr)",
     )
+    max_lines: int = Field(
+        LOGS_LINE_LIMIT,
+        description="Maximum number of lines to keep in the log file",
+    )
 
 
 class DBConfig(_StrictModel):

core/config/constants.py

@@ -1 +1,2 @@
 CONVO_ITERATIONS_LIMIT = 8
+LOGS_LINE_LIMIT = 20000

core/log/__init__.py

@@ -1,6 +1,65 @@
+import os
+from collections import deque
 from logging import FileHandler, Formatter, Logger, StreamHandler, getLogger
 
 from core.config import LogConfig
+from core.config.constants import LOGS_LINE_LIMIT
+
+
+class LineCountLimitedFileHandler(FileHandler):
+    """
+    A file handler that limits the number of lines in the log file.
+    It keeps a fixed number of the most recent log lines.
+    """
+
+    def __init__(self, filename, max_lines=LOGS_LINE_LIMIT, mode="a", encoding=None, delay=False):
+        """
+        Initialize the handler with the file and max lines.
+
+        :param filename: Log file path
+        :param max_lines: Maximum number of lines to keep in the file
+        :param mode: File open mode
+        :param encoding: File encoding
+        :param delay: Delay file opening until first emit
+        """
+        super().__init__(filename, mode, encoding, delay)
+        self.max_lines = max_lines
+        self.line_buffer = deque(maxlen=max_lines)
+        self._load_existing_lines()
+
+    def _load_existing_lines(self):
+        """Load existing lines from the file into the buffer if the file exists."""
+        if os.path.exists(self.baseFilename):
+            try:
+                with open(self.baseFilename, "r", encoding=self.encoding) as f:
+                    for line in f:
+                        if len(self.line_buffer) < self.max_lines:
+                            self.line_buffer.append(line)
+                        else:
+                            self.line_buffer.popleft()
+                            self.line_buffer.append(line)
+            except Exception:
+                # If there's an error reading the file, we'll just start with an empty buffer
+                self.line_buffer.clear()
+
+    def emit(self, record):
+        """
+        Emit a record and maintain the line count limit.
+
+        :param record: Log record to emit
+        """
+        try:
+            msg = self.format(record)
+            line = msg + self.terminator
+            self.line_buffer.append(line)
+
+            # Rewrite the entire file with the current buffer
+            with open(self.baseFilename, "w", encoding=self.encoding) as f:
+                f.writelines(self.line_buffer)
+
+            self.flush()
+        except Exception:
+            self.handleError(record)
 
 
 def setup(config: LogConfig, force: bool = False):
@@ -27,7 +86,9 @@ def setup(config: LogConfig, force: bool = False):
     formatter = Formatter(config.format)
 
     if config.output:
-        handler = FileHandler(config.output, encoding="utf-8")
+        # Use our custom handler that limits line count
+        max_lines = getattr(config, "max_lines", LOGS_LINE_LIMIT)
+        handler = LineCountLimitedFileHandler(config.output, max_lines=max_lines, encoding="utf-8")
     else:
         handler = StreamHandler()