From 738c8ffff0a57220809fbbe9563272676cbc483a Mon Sep 17 00:00:00 2001
From: Reinier van der Leer <pwuts@agpt.co>
Date: Thu, 30 May 2024 17:16:17 +0200
Subject: [PATCH] fix(benchmark): Improve output and debug logging of pytest
 evals

---
 benchmark/agbenchmark/challenges/builtin.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/benchmark/agbenchmark/challenges/builtin.py b/benchmark/agbenchmark/challenges/builtin.py
index ecd9cf2b55..b93dc1140e 100644
--- a/benchmark/agbenchmark/challenges/builtin.py
+++ b/benchmark/agbenchmark/challenges/builtin.py
@@ -339,8 +339,11 @@ class BuiltinChallenge(BaseChallenge):
                     capture_output=True,
                     text=True,
                 )
+                logger.debug(f"EXIT CODE: {result.returncode}")
+                logger.debug(f"STDOUT: {result.stdout}")
+                logger.debug(f"STDERR: {result.stderr}")
                 if "error" in result.stderr or result.returncode != 0:
-                    yield "pytest", f"Error: {result.stderr}\n"
+                    yield "pytest", f"Error: {result.stderr.strip() or result.stdout}\n"
                 else:
                     yield "pytest", f"Output: {result.stdout}\n"