From 4743fccbaf5fa0bcfca2f7b26c0fd9b99d1451a6 Mon Sep 17 00:00:00 2001
From: Senko Rasic <senko@senko.net>
Date: Tue, 11 Jun 2024 14:16:42 +0200
Subject: [PATCH 1/9] store whether the system dependencies are installed in
 telemetry

---
 core/agents/architect.py         | 29 ++-----------------
 core/agents/mixins.py            | 49 ++++++++++++++++++++++++++++++++
 core/agents/orchestrator.py      |  2 +-
 core/agents/spec_writer.py       | 18 +++++-------
 tests/agents/test_spec_writer.py | 14 +++++++--
 5 files changed, 72 insertions(+), 40 deletions(-)

diff --git a/core/agents/architect.py b/core/agents/architect.py
index 111e8478..712c7099 100644
--- a/core/agents/architect.py
+++ b/core/agents/architect.py
@@ -4,6 +4,7 @@ from pydantic import BaseModel, Field
 
 from core.agents.base import BaseAgent
 from core.agents.convo import AgentConvo
+from core.agents.mixins import SystemDependencyCheckerMixin
 from core.agents.response import AgentResponse
 from core.llm.parser import JSONParser
 from core.telemetry import telemetry
@@ -67,7 +68,7 @@ class Architecture(BaseModel):
     )
 
 
-class Architect(BaseAgent):
+class Architect(SystemDependencyCheckerMixin, BaseAgent):
     agent_type = "architect"
     display_name = "Architect"
 
@@ -81,7 +82,6 @@ class Architect(BaseAgent):
         arch: Architecture = await llm(convo, parser=JSONParser(Architecture))
 
         await self.check_compatibility(arch)
-        await self.check_system_dependencies(arch.system_dependencies)
 
         spec = self.current_state.specification.clone()
         spec.architecture = arch.architecture
@@ -90,14 +90,7 @@ class Architect(BaseAgent):
         spec.template = arch.template.value if arch.template else None
 
         self.next_state.specification = spec
-        telemetry.set(
-            "architecture",
-            {
-                "description": spec.architecture,
-                "system_dependencies": spec.system_dependencies,
-                "package_dependencies": spec.package_dependencies,
-            },
-        )
+        await self.check_system_dependencies(spec)
         telemetry.set("template", spec.template)
         self.next_state.action = ARCHITECTURE_STEP_NAME
         return AgentResponse.done(self)
@@ -129,19 +122,3 @@ class Architect(BaseAgent):
         # return AgentResponse.revise_spec()
         # that SpecWriter should catch and allow the user to reword the initial spec.
         return True
-
-    async def check_system_dependencies(self, deps: list[SystemDependency]):
-        """
-        Check whether the required system dependencies are installed.
-        """
-
-        for dep in deps:
-            status_code, _, _ = await self.process_manager.run_command(dep.test)
-            if status_code != 0:
-                if dep.required_locally:
-                    remedy = "Please install it before proceeding with your app."
-                else:
-                    remedy = "If you would like to use it locally, please install it before proceeding."
-                await self.send_message(f"❌ {dep.name} is not available. {remedy}")
-            else:
-                await self.send_message(f"✅ {dep.name} is available.")
diff --git a/core/agents/mixins.py b/core/agents/mixins.py
index 5ea0aae7..77f10a2f 100644
--- a/core/agents/mixins.py
+++ b/core/agents/mixins.py
@@ -1,6 +1,8 @@
 from typing import Optional
 
 from core.agents.convo import AgentConvo
+from core.db.models.specification import Specification
+from core.telemetry import telemetry
 
 
 class IterationPromptMixin:
@@ -35,3 +37,50 @@ class IterationPromptMixin:
         )
         llm_solution: str = await llm(convo)
         return llm_solution
+
+
+class SystemDependencyCheckerMixin:
+    """
+    Provides a method to check whether the required system dependencies are installed.
+
+    Used by Architect and SpecWriter agents. Assumes the agent has access to UI
+    and ProcessManager.
+    """
+
+    async def check_system_dependencies(self, spec: Specification):
+        """
+        Check whether the required system dependencies are installed.
+
+        This also stores the app architecture telemetry data, including the
+        information about whether each system dependency is installed.
+
+        :param spec: Project specification.
+        """
+        deps = spec.system_dependencies
+
+        for dep in deps:
+            status_code, _, _ = await self.process_manager.run_command(dep["test"])
+            dep["installed"] = bool(status_code == 0)
+            if status_code != 0:
+                if dep["required_locally"]:
+                    remedy = "Please install it before proceeding with your app."
+                else:
+                    remedy = "If you would like to use it locally, please install it before proceeding."
+                await self.send_message(f"❌ {dep['name']} is not available. {remedy}")
+                await self.ask_question(
+                    f"Once you have installed {dep['name']}, please press Continue.",
+                    buttons={"continue": "Continue"},
+                    buttons_only=True,
+                    default="continue",
+                )
+            else:
+                await self.send_message(f"✅ {dep['name']} is available.")
+
+        telemetry.set(
+            "architecture",
+            {
+                "description": spec.architecture,
+                "system_dependencies": deps,
+                "package_dependencies": spec.package_dependencies,
+            },
+        )
diff --git a/core/agents/orchestrator.py b/core/agents/orchestrator.py
index 9bf4d561..4ce425fa 100644
--- a/core/agents/orchestrator.py
+++ b/core/agents/orchestrator.py
@@ -186,7 +186,7 @@ class Orchestrator(BaseAgent):
                 return Importer(self.state_manager, self.ui)
             else:
                 # New project: ask the Spec Writer to refine and save the project specification
-                return SpecWriter(self.state_manager, self.ui)
+                return SpecWriter(self.state_manager, self.ui, process_manager=self.process_manager)
         elif not state.specification.architecture:
             # Ask the Architect to design the project architecture and determine dependencies
             return Architect(self.state_manager, self.ui, process_manager=self.process_manager)
diff --git a/core/agents/spec_writer.py b/core/agents/spec_writer.py
index 706d6e56..01ff9515 100644
--- a/core/agents/spec_writer.py
+++ b/core/agents/spec_writer.py
@@ -1,5 +1,6 @@
 from core.agents.base import BaseAgent
 from core.agents.convo import AgentConvo
+from core.agents.mixins import SystemDependencyCheckerMixin
 from core.agents.response import AgentResponse
 from core.db.models import Complexity
 from core.llm.parser import StringParser
@@ -19,7 +20,7 @@ INITIAL_PROJECT_HOWTO_URL = (
 SPEC_STEP_NAME = "Create specification"
 
 
-class SpecWriter(BaseAgent):
+class SpecWriter(SystemDependencyCheckerMixin, BaseAgent):
     agent_type = "spec-writer"
     display_name = "Spec Writer"
 
@@ -43,7 +44,7 @@ class SpecWriter(BaseAgent):
         if response.button == "example":
             await self.send_message("Starting example project with description:")
             await self.send_message(EXAMPLE_PROJECT_DESCRIPTION)
-            self.prepare_example_project()
+            await self.prepare_example_project()
             return AgentResponse.done(self)
         elif response.button == "continue":
             # FIXME: Workaround for the fact that VSCode "continue" button does
@@ -73,7 +74,7 @@ class SpecWriter(BaseAgent):
         llm_response: str = await llm(convo, temperature=0, parser=StringParser())
         return llm_response.lower()
 
-    def prepare_example_project(self):
+    async def prepare_example_project(self):
         spec = self.current_state.specification.clone()
         spec.description = EXAMPLE_PROJECT_DESCRIPTION
         spec.architecture = EXAMPLE_PROJECT_ARCHITECTURE["architecture"]
@@ -84,14 +85,9 @@ class SpecWriter(BaseAgent):
         telemetry.set("initial_prompt", spec.description.strip())
         telemetry.set("is_complex_app", False)
         telemetry.set("template", spec.template)
-        telemetry.set(
-            "architecture",
-            {
-                "architecture": spec.architecture,
-                "system_dependencies": spec.system_dependencies,
-                "package_dependencies": spec.package_dependencies,
-            },
-        )
+
+        await self.check_system_dependencies(spec)
+
         self.next_state.specification = spec
 
         self.next_state.epics = [
diff --git a/tests/agents/test_spec_writer.py b/tests/agents/test_spec_writer.py
index 1a5ab30e..88f94959 100644
--- a/tests/agents/test_spec_writer.py
+++ b/tests/agents/test_spec_writer.py
@@ -1,18 +1,24 @@
+from unittest.mock import AsyncMock
+
 import pytest
 
 from core.agents.response import ResponseType
 from core.agents.spec_writer import SpecWriter
 from core.db.models import Complexity
+from core.telemetry import telemetry
 from core.ui.base import UserInput
 
 
 @pytest.mark.asyncio
 async def test_start_example_project(agentcontext):
-    sm, _, ui, _ = agentcontext
+    sm, pm, ui, _ = agentcontext
 
     ui.ask_question.return_value = UserInput(button="example")
+    pm.run_command = AsyncMock(return_value=(0, "", ""))
 
-    sw = SpecWriter(sm, ui)
+    telemetry.start()
+
+    sw = SpecWriter(sm, ui, process_manager=pm)
     response = await sw.run()
     assert response.type == ResponseType.DONE
 
@@ -23,6 +29,10 @@ async def test_start_example_project(agentcontext):
     assert sm.current_state.specification.complexity == Complexity.SIMPLE
     assert sm.current_state.epics != []
     assert sm.current_state.tasks != []
+    pm.run_command.assert_awaited_once_with("node --version")
+
+    assert telemetry.data["initial_prompt"] == sm.current_state.specification.description.strip()
+    assert telemetry.data["architecture"]["system_dependencies"][0]["installed"] is True
 
 
 @pytest.mark.asyncio

From ca77cc5055398fab1b615dd86da4526505d7d93f Mon Sep 17 00:00:00 2001
From: Senko Rasic <senko@senko.net>
Date: Tue, 11 Jun 2024 16:17:33 +0200
Subject: [PATCH 2/9] disable telemetry in automated tests

---
 core/telemetry/__init__.py        |  5 +++--
 tests/conftest.py                 |  7 +++++++
 tests/telemetry/test_telemetry.py | 21 +++++++++++++++------
 3 files changed, 25 insertions(+), 8 deletions(-)

diff --git a/core/telemetry/__init__.py b/core/telemetry/__init__.py
index db10dbc9..cbf957ad 100644
--- a/core/telemetry/__init__.py
+++ b/core/telemetry/__init__.py
@@ -2,6 +2,7 @@ import sys
 import time
 import traceback
 from copy import deepcopy
+from os import getenv
 from pathlib import Path
 from typing import Any
 
@@ -320,7 +321,7 @@ class Telemetry:
 
         Note: this method clears all telemetry data after sending it.
         """
-        if not self.enabled:
+        if not self.enabled or getenv("DISABLE_TELEMETRY"):
             log.debug("Telemetry.send(): telemetry is disabled, not sending data")
             return
 
@@ -362,7 +363,7 @@ class Telemetry:
         :param name: name of the event
         :param data: data to send with the event
         """
-        if not self.enabled:
+        if not self.enabled or getenv("DISABLE_TELEMETRY"):
             return
 
         payload = {
diff --git a/tests/conftest.py b/tests/conftest.py
index c1bb695d..4c6093b1 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,6 +1,8 @@
+import os
 from typing import Callable
 from unittest.mock import AsyncMock, MagicMock, patch
 
+import pytest
 import pytest_asyncio
 
 from core.config import DBConfig
@@ -9,6 +11,11 @@ from core.db.session import SessionManager
 from core.state.state_manager import StateManager
 
 
+@pytest.fixture(autouse=True)
+def disable_test_telemetry(monkeypatch):
+    os.environ["DISABLE_TELEMETRY"] = "1"
+
+
 @pytest_asyncio.fixture
 async def testmanager():
     """
diff --git a/tests/telemetry/test_telemetry.py b/tests/telemetry/test_telemetry.py
index c2c9d947..19888c0a 100644
--- a/tests/telemetry/test_telemetry.py
+++ b/tests/telemetry/test_telemetry.py
@@ -98,14 +98,15 @@ def test_inc_ignores_unknown_data_field(mock_settings):
     assert "unknown_field" not in telemetry.data
 
 
+@patch("core.telemetry.getenv")
 @patch("core.telemetry.time")
 @patch("core.telemetry.settings")
-def test_start_with_telemetry_enabled(mock_settings, mock_time):
+def test_start_with_telemetry_enabled(mock_settings, mock_time, mock_getenv):
     mock_settings.telemetry = MagicMock(id="test-id", endpoint="test-endpoint", enabled=True)
     mock_time.time.return_value = 1234.0
+    mock_getenv.return_value = None  # override DISABLE_TELEMETRY test env var
 
     telemetry = Telemetry()
-
     telemetry.start()
     assert telemetry.start_time == 1234.0
 
@@ -134,9 +135,11 @@ def test_stop_calculates_elapsed_time(mock_settings, mock_time):
 
 
 @pytest.mark.asyncio
+@patch("core.telemetry.getenv")
 @patch("core.telemetry.settings")
-async def test_send_enabled_and_successful(mock_settings, mock_httpx_post):
+async def test_send_enabled_and_successful(mock_settings, mock_getenv, mock_httpx_post):
     mock_settings.telemetry = MagicMock(id="test-id", endpoint="test-endpoint", enabled=True)
+    mock_getenv.return_value = None  # override DISABLE_TELEMETRY test env var
 
     telemetry = Telemetry()
     with patch.object(telemetry, "calculate_statistics"):
@@ -151,10 +154,12 @@ async def test_send_enabled_and_successful(mock_settings, mock_httpx_post):
 
 
 @pytest.mark.asyncio
+@patch("core.telemetry.getenv")
 @patch("core.telemetry.settings")
-async def test_send_enabled_but_post_fails(mock_settings, mock_httpx_post):
+async def test_send_enabled_but_post_fails(mock_settings, mock_getenv, mock_httpx_post):
     mock_settings.telemetry = MagicMock(id="test-id", endpoint="test-endpoint", enabled=True)
     mock_httpx_post.side_effect = httpx.RequestError("Connection error")
+    mock_getenv.return_value = None  # override DISABLE_TELEMETRY test env var
 
     telemetry = Telemetry()
     with patch.object(telemetry, "calculate_statistics"):
@@ -180,9 +185,11 @@ async def test_send_not_enabled(mock_settings, mock_httpx_post):
 
 
 @pytest.mark.asyncio
+@patch("core.telemetry.getenv")
 @patch("core.telemetry.settings")
-async def test_send_no_endpoint_configured(mock_settings, mock_httpx_post):
+async def test_send_no_endpoint_configured(mock_settings, mock_getenv, mock_httpx_post):
     mock_settings.telemetry = MagicMock(id="test-id", endpoint=None, enabled=True)
+    mock_getenv.return_value = None  # override DISABLE_TELEMETRY test env var
 
     telemetry = Telemetry()
     await telemetry.send()
@@ -191,9 +198,11 @@ async def test_send_no_endpoint_configured(mock_settings, mock_httpx_post):
 
 
 @pytest.mark.asyncio
+@patch("core.telemetry.getenv")
 @patch("core.telemetry.settings")
-async def test_send_clears_counters_after_sending(mock_settings, mock_httpx_post):
+async def test_send_clears_counters_after_sending(mock_settings, mock_getenv, mock_httpx_post):
     mock_settings.telemetry = MagicMock(id="test-id", endpoint="test-endpoint", enabled=True)
+    mock_getenv.return_value = None  # override DISABLE_TELEMETRY test env var
 
     telemetry = Telemetry()
     telemetry.data["model"] = "test-model"

From 515ccee93ce48c9fa887dc2b2ae035e02c348bc9 Mon Sep 17 00:00:00 2001
From: Senko Rasic <senko@senko.net>
Date: Tue, 11 Jun 2024 16:51:29 +0200
Subject: [PATCH 3/9] annotate trace events with app_id if we have it

---
 core/telemetry/__init__.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/core/telemetry/__init__.py b/core/telemetry/__init__.py
index cbf957ad..9c848590 100644
--- a/core/telemetry/__init__.py
+++ b/core/telemetry/__init__.py
@@ -366,6 +366,9 @@ class Telemetry:
         if not self.enabled or getenv("DISABLE_TELEMETRY"):
             return
 
+        if not data.get("app_id") and self.data("app_id"):
+            data = {**data, "app_id": self.data["app_id"]}
+
         payload = {
             "pathId": self.telemetry_id,
             "event": f"trace-{name}",

From a4b920d6fa94473b75f548adff6f30ea083a60f0 Mon Sep 17 00:00:00 2001
From: Senko Rasic <senko@senko.net>
Date: Tue, 11 Jun 2024 16:52:12 +0200
Subject: [PATCH 4/9] add trace events for project creation and specification

---
 core/agents/spec_writer.py  | 20 ++++++++++++++++++++
 core/state/state_manager.py |  1 +
 2 files changed, 21 insertions(+)

diff --git a/core/agents/spec_writer.py b/core/agents/spec_writer.py
index 01ff9515..1e2a9a24 100644
--- a/core/agents/spec_writer.py
+++ b/core/agents/spec_writer.py
@@ -54,6 +54,14 @@ class SpecWriter(SystemDependencyCheckerMixin, BaseAgent):
         spec = response.text
 
         complexity = await self.check_prompt_complexity(spec)
+        await telemetry.trace_code_event(
+            "project-description",
+            {
+                "initial_prompt": spec,
+                "complexity": complexity,
+            },
+        )
+
         if len(spec) < ANALYZE_THRESHOLD and complexity != Complexity.SIMPLE:
             spec = await self.analyze_spec(spec)
             spec = await self.review_spec(spec)
@@ -111,6 +119,8 @@ class SpecWriter(SystemDependencyCheckerMixin, BaseAgent):
 
         llm = self.get_llm()
         convo = AgentConvo(self).template("ask_questions").user(spec)
+        n_questions = 0
+        n_answers = 0
 
         while True:
             response: str = await llm(convo)
@@ -125,12 +135,21 @@ class SpecWriter(SystemDependencyCheckerMixin, BaseAgent):
                     buttons={"continue": "continue"},
                 )
                 if confirm.cancelled or confirm.button == "continue" or confirm.text == "":
+                    await self.telemetry.trace_code_event(
+                        "spec-writer-questions",
+                        {
+                            "num_questions": n_questions,
+                            "num_answers": n_answers,
+                            "new_spec": spec,
+                        },
+                    )
                     return spec
                 convo.user(confirm.text)
 
             else:
                 convo.assistant(response)
 
+                n_questions += 1
                 user_response = await self.ask_question(
                     response,
                     buttons={"skip": "Skip questions"},
@@ -143,6 +162,7 @@ class SpecWriter(SystemDependencyCheckerMixin, BaseAgent):
                     response: str = await llm(convo)
                     return response
 
+                n_answers += 1
                 convo.user(user_response.text)
 
     async def review_spec(self, spec: str) -> str:
diff --git a/core/state/state_manager.py b/core/state/state_manager.py
index bcdc405d..a7eac3a9 100644
--- a/core/state/state_manager.py
+++ b/core/state/state_manager.py
@@ -77,6 +77,7 @@ class StateManager:
             f'with default branch "{branch.name}" (id={branch.id}) '
             f"and initial state id={state.id} (step_index={state.step_index})"
         )
+        await telemetry.trace_code_event("create-project", {"name": name})
 
         self.current_session = session
         self.current_state = state

From 1645cdd27733208e3a401b18b3bf11215d07f61b Mon Sep 17 00:00:00 2001
From: Goran Peretin <goran.peretin@gmail.com>
Date: Tue, 11 Jun 2024 19:03:02 +0200
Subject: [PATCH 5/9] Add remaining telemetry entries.

---
 core/agents/developer.py      | 2 ++
 core/agents/mixins.py         | 4 ++++
 core/agents/task_completer.py | 4 ++++
 core/agents/tech_lead.py      | 2 ++
 core/agents/troubleshooter.py | 1 +
 core/telemetry/__init__.py    | 2 +-
 6 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/core/agents/developer.py b/core/agents/developer.py
index 75960f19..80666c61 100644
--- a/core/agents/developer.py
+++ b/core/agents/developer.py
@@ -10,6 +10,7 @@ from core.agents.response import AgentResponse, ResponseType
 from core.db.models.project_state import TaskStatus
 from core.llm.parser import JSONParser
 from core.log import get_logger
+from core.telemetry import telemetry
 
 log = get_logger(__name__)
 
@@ -195,6 +196,7 @@ class Developer(BaseAgent):
         self.next_state.modified_files = {}
         self.set_next_steps(response, source)
         self.next_state.action = f"Task #{current_task_index + 1} start"
+        await telemetry.trace_code_event("task-start", {"task-num": current_task_index + 1})
         return AgentResponse.done(self)
 
     async def get_relevant_files(
diff --git a/core/agents/mixins.py b/core/agents/mixins.py
index 77f10a2f..bcaebddb 100644
--- a/core/agents/mixins.py
+++ b/core/agents/mixins.py
@@ -57,6 +57,7 @@ class SystemDependencyCheckerMixin:
         :param spec: Project specification.
         """
         deps = spec.system_dependencies
+        checked = {}
 
         for dep in deps:
             status_code, _, _ = await self.process_manager.run_command(dep["test"])
@@ -73,8 +74,10 @@ class SystemDependencyCheckerMixin:
                     buttons_only=True,
                     default="continue",
                 )
+                checked[dep["name"]] = "missing"
             else:
                 await self.send_message(f"✅ {dep['name']} is available.")
+                checked[dep["name"]] = "present"
 
         telemetry.set(
             "architecture",
@@ -82,5 +85,6 @@ class SystemDependencyCheckerMixin:
                 "description": spec.architecture,
                 "system_dependencies": deps,
                 "package_dependencies": spec.package_dependencies,
+                "checked_system_dependencies": checked,
             },
         )
diff --git a/core/agents/task_completer.py b/core/agents/task_completer.py
index 20ec3bf9..75b5be9d 100644
--- a/core/agents/task_completer.py
+++ b/core/agents/task_completer.py
@@ -1,6 +1,7 @@
 from core.agents.base import BaseAgent
 from core.agents.response import AgentResponse
 from core.log import get_logger
+from core.telemetry import telemetry
 
 log = get_logger(__name__)
 
@@ -25,5 +26,8 @@ class TaskCompleter(BaseAgent):
             self.current_state.get_source_index(source),
             tasks,
         )
+        await telemetry.trace_code_event(
+            "task-end", {"task-num": current_task_index1, "num-iterations": len(self.current_state.iterations)}
+        )
 
         return AgentResponse.done(self)
diff --git a/core/agents/tech_lead.py b/core/agents/tech_lead.py
index fcf9ca6d..1dc36a6f 100644
--- a/core/agents/tech_lead.py
+++ b/core/agents/tech_lead.py
@@ -10,6 +10,7 @@ from core.db.models import Complexity
 from core.db.models.project_state import TaskStatus
 from core.llm.parser import JSONParser
 from core.log import get_logger
+from core.telemetry import telemetry
 from core.templates.registry import apply_project_template, get_template_description, get_template_summary
 from core.ui.base import ProjectStage, success_source
 
@@ -151,6 +152,7 @@ class TechLead(BaseAgent):
             }
             for task in response.plan
         ]
+        await telemetry.trace_code_event("development-plan", {"num-tasks": len(response.plan)})
         return AgentResponse.done(self)
 
     async def update_epic(self) -> AgentResponse:
diff --git a/core/agents/troubleshooter.py b/core/agents/troubleshooter.py
index 5b3b5de7..05dafb54 100644
--- a/core/agents/troubleshooter.py
+++ b/core/agents/troubleshooter.py
@@ -182,6 +182,7 @@ class Troubleshooter(IterationPromptMixin, BaseAgent):
             return False, False, ""
 
         if user_response.button == "loop":
+            await telemetry.trace_code_event("stuck-in-loop", {"clicked": True})
             return True, True, ""
 
         return True, False, user_response.text
diff --git a/core/telemetry/__init__.py b/core/telemetry/__init__.py
index 9c848590..2270a83a 100644
--- a/core/telemetry/__init__.py
+++ b/core/telemetry/__init__.py
@@ -366,7 +366,7 @@ class Telemetry:
         if not self.enabled or getenv("DISABLE_TELEMETRY"):
             return
 
-        if not data.get("app_id") and self.data("app_id"):
+        if not data.get("app_id") and self.data["app_id"]:
             data = {**data, "app_id": self.data["app_id"]}
 
         payload = {

From 7d225bbfe203dc40c2e1b77a7ef5fc485b3e1243 Mon Sep 17 00:00:00 2001
From: Senko Rasic <senko@senko.net>
Date: Thu, 13 Jun 2024 10:48:37 +0200
Subject: [PATCH 6/9] refactor example project so each agent does its job in
 turn

---
 core/agents/architect.py                      | 82 ++++++++++++++++---
 core/agents/external_docs.py                  | 12 ++-
 core/agents/mixins.py                         | 53 ------------
 core/agents/spec_writer.py                    | 52 +++++-------
 core/agents/tech_lead.py                      | 22 ++++-
 core/db/migrations/README                     | 11 ++-
 ...f891d366761_add_example_project_to_spec.py | 34 ++++++++
 core/db/models/specification.py               |  2 +
 core/telemetry/__init__.py                    |  4 +-
 core/templates/example_project.py             | 14 ++++
 tests/agents/test_spec_writer.py              | 10 +--
 11 files changed, 185 insertions(+), 111 deletions(-)
 create mode 100644 core/db/migrations/versions/ff891d366761_add_example_project_to_spec.py

diff --git a/core/agents/architect.py b/core/agents/architect.py
index 712c7099..c1ce78d7 100644
--- a/core/agents/architect.py
+++ b/core/agents/architect.py
@@ -4,10 +4,12 @@ from pydantic import BaseModel, Field
 
 from core.agents.base import BaseAgent
 from core.agents.convo import AgentConvo
-from core.agents.mixins import SystemDependencyCheckerMixin
 from core.agents.response import AgentResponse
+from core.db.models import Specification
 from core.llm.parser import JSONParser
+from core.log import get_logger
 from core.telemetry import telemetry
+from core.templates.example_project import EXAMPLE_PROJECTS
 from core.templates.registry import PROJECT_TEMPLATES, ProjectTemplateEnum
 from core.ui.base import ProjectStage
 
@@ -16,6 +18,8 @@ WARN_SYSTEM_DEPS = ["docker", "kubernetes", "microservices"]
 WARN_FRAMEWORKS = ["next.js", "vue", "vue.js", "svelte", "angular"]
 WARN_FRAMEWORKS_URL = "https://github.com/Pythagora-io/gpt-pilot/wiki/Using-GPT-Pilot-with-frontend-frameworks"
 
+log = get_logger(__name__)
+
 
 # FIXME: all the reponse pydantic models should be strict (see config._StrictModel), also check if we
 # can disallow adding custom Python attributes to the model
@@ -68,33 +72,41 @@ class Architecture(BaseModel):
     )
 
 
-class Architect(SystemDependencyCheckerMixin, BaseAgent):
+class Architect(BaseAgent):
     agent_type = "architect"
     display_name = "Architect"
 
     async def run(self) -> AgentResponse:
         await self.ui.send_project_stage(ProjectStage.ARCHITECTURE)
 
+        spec = self.current_state.specification.clone()
+
+        if spec.example_project:
+            self.prepare_example_project(spec)
+        else:
+            await self.plan_architecture(spec)
+
+        await self.check_system_dependencies(spec)
+
+        self.next_state.specification = spec
+        telemetry.set("template", spec.template)
+        self.next_state.action = ARCHITECTURE_STEP_NAME
+        return AgentResponse.done(self)
+
+    async def plan_architecture(self, spec: Specification):
+        await self.send_message("Planning project architecture ...")
+
         llm = self.get_llm()
         convo = AgentConvo(self).template("technologies", templates=PROJECT_TEMPLATES).require_schema(Architecture)
-
-        await self.send_message("Planning project architecture ...")
         arch: Architecture = await llm(convo, parser=JSONParser(Architecture))
 
         await self.check_compatibility(arch)
 
-        spec = self.current_state.specification.clone()
         spec.architecture = arch.architecture
         spec.system_dependencies = [d.model_dump() for d in arch.system_dependencies]
         spec.package_dependencies = [d.model_dump() for d in arch.package_dependencies]
         spec.template = arch.template.value if arch.template else None
 
-        self.next_state.specification = spec
-        await self.check_system_dependencies(spec)
-        telemetry.set("template", spec.template)
-        self.next_state.action = ARCHITECTURE_STEP_NAME
-        return AgentResponse.done(self)
-
     async def check_compatibility(self, arch: Architecture) -> bool:
         warn_system_deps = [dep.name for dep in arch.system_dependencies if dep.name.lower() in WARN_SYSTEM_DEPS]
         warn_package_deps = [dep.name for dep in arch.package_dependencies if dep.name.lower() in WARN_FRAMEWORKS]
@@ -122,3 +134,51 @@ class Architect(SystemDependencyCheckerMixin, BaseAgent):
         # return AgentResponse.revise_spec()
         # that SpecWriter should catch and allow the user to reword the initial spec.
         return True
+
+    def prepare_example_project(self, spec: Specification):
+        log.debug(f"Setting architecture for example project: {spec.example_project}")
+        arch = EXAMPLE_PROJECTS[spec.example_project]["architecture"]
+
+        spec.architecture = arch["architecture"]
+        spec.system_dependencies = arch["system_dependencies"]
+        spec.package_dependencies = arch["package_dependencies"]
+        spec.template = arch["template"]
+        telemetry.set("template", spec.template)
+
+    async def check_system_dependencies(self, spec: Specification):
+        """
+        Check whether the required system dependencies are installed.
+
+        This also stores the app architecture telemetry data, including the
+        information about whether each system dependency is installed.
+
+        :param spec: Project specification.
+        """
+        deps = spec.system_dependencies
+
+        for dep in deps:
+            status_code, _, _ = await self.process_manager.run_command(dep["test"])
+            dep["installed"] = bool(status_code == 0)
+            if status_code != 0:
+                if dep["required_locally"]:
+                    remedy = "Please install it before proceeding with your app."
+                else:
+                    remedy = "If you would like to use it locally, please install it before proceeding."
+                await self.send_message(f"❌ {dep['name']} is not available. {remedy}")
+                await self.ask_question(
+                    f"Once you have installed {dep['name']}, please press Continue.",
+                    buttons={"continue": "Continue"},
+                    buttons_only=True,
+                    default="continue",
+                )
+            else:
+                await self.send_message(f"✅ {dep['name']} is available.")
+
+        telemetry.set(
+            "architecture",
+            {
+                "description": spec.architecture,
+                "system_dependencies": deps,
+                "package_dependencies": spec.package_dependencies,
+            },
+        )
diff --git a/core/agents/external_docs.py b/core/agents/external_docs.py
index 6e818df9..0da9bc5a 100644
--- a/core/agents/external_docs.py
+++ b/core/agents/external_docs.py
@@ -26,7 +26,7 @@ class SelectedDocsets(BaseModel):
 class ExternalDocumentation(BaseAgent):
     """Agent in charge of collecting and storing additional documentation.
 
-    Docs are per task and are stores in the `tasks` variable in the project state.
+    Docs are per task and are stores in the `docs` variable in the project state.
     This agent ensures documentation is collected only once per task.
 
     Agent does 2 LLM interactions:
@@ -44,7 +44,12 @@ class ExternalDocumentation(BaseAgent):
     display_name = "Documentation"
 
     async def run(self) -> AgentResponse:
-        available_docsets = await self._get_available_docsets()
+        if self.current_state.specification.example_project:
+            log.debug("Example project detected, no documentation selected.")
+            available_docsets = []
+        else:
+            available_docsets = await self._get_available_docsets()
+
         selected_docsets = await self._select_docsets(available_docsets)
         await telemetry.trace_code_event("docsets_used", selected_docsets)
 
@@ -149,6 +154,8 @@ class ExternalDocumentation(BaseAgent):
         Documentation snippets are stored as a list of dictionaries:
         {"key": docset-key, "desc": documentation-description, "snippets": list-of-snippets}
 
+        :param snippets: List of tuples: (docset_key, snippets)
+        :param available_docsets: List of available docsets from the API.
         """
 
         docsets_dict = dict(available_docsets)
@@ -157,4 +164,3 @@ class ExternalDocumentation(BaseAgent):
             docs.append({"key": docset_key, "desc": docsets_dict[docset_key], "snippets": snip})
 
         self.next_state.docs = docs
-        self.next_state.flag_tasks_as_modified()
diff --git a/core/agents/mixins.py b/core/agents/mixins.py
index bcaebddb..5ea0aae7 100644
--- a/core/agents/mixins.py
+++ b/core/agents/mixins.py
@@ -1,8 +1,6 @@
 from typing import Optional
 
 from core.agents.convo import AgentConvo
-from core.db.models.specification import Specification
-from core.telemetry import telemetry
 
 
 class IterationPromptMixin:
@@ -37,54 +35,3 @@ class IterationPromptMixin:
         )
         llm_solution: str = await llm(convo)
         return llm_solution
-
-
-class SystemDependencyCheckerMixin:
-    """
-    Provides a method to check whether the required system dependencies are installed.
-
-    Used by Architect and SpecWriter agents. Assumes the agent has access to UI
-    and ProcessManager.
-    """
-
-    async def check_system_dependencies(self, spec: Specification):
-        """
-        Check whether the required system dependencies are installed.
-
-        This also stores the app architecture telemetry data, including the
-        information about whether each system dependency is installed.
-
-        :param spec: Project specification.
-        """
-        deps = spec.system_dependencies
-        checked = {}
-
-        for dep in deps:
-            status_code, _, _ = await self.process_manager.run_command(dep["test"])
-            dep["installed"] = bool(status_code == 0)
-            if status_code != 0:
-                if dep["required_locally"]:
-                    remedy = "Please install it before proceeding with your app."
-                else:
-                    remedy = "If you would like to use it locally, please install it before proceeding."
-                await self.send_message(f"❌ {dep['name']} is not available. {remedy}")
-                await self.ask_question(
-                    f"Once you have installed {dep['name']}, please press Continue.",
-                    buttons={"continue": "Continue"},
-                    buttons_only=True,
-                    default="continue",
-                )
-                checked[dep["name"]] = "missing"
-            else:
-                await self.send_message(f"✅ {dep['name']} is available.")
-                checked[dep["name"]] = "present"
-
-        telemetry.set(
-            "architecture",
-            {
-                "description": spec.architecture,
-                "system_dependencies": deps,
-                "package_dependencies": spec.package_dependencies,
-                "checked_system_dependencies": checked,
-            },
-        )
diff --git a/core/agents/spec_writer.py b/core/agents/spec_writer.py
index 1e2a9a24..9e2f7373 100644
--- a/core/agents/spec_writer.py
+++ b/core/agents/spec_writer.py
@@ -1,14 +1,13 @@
 from core.agents.base import BaseAgent
 from core.agents.convo import AgentConvo
-from core.agents.mixins import SystemDependencyCheckerMixin
 from core.agents.response import AgentResponse
 from core.db.models import Complexity
 from core.llm.parser import StringParser
+from core.log import get_logger
 from core.telemetry import telemetry
 from core.templates.example_project import (
-    EXAMPLE_PROJECT_ARCHITECTURE,
-    EXAMPLE_PROJECT_DESCRIPTION,
-    EXAMPLE_PROJECT_PLAN,
+    DEFAULT_EXAMPLE_PROJECT,
+    EXAMPLE_PROJECTS,
 )
 
 # If the project description is less than this, perform an analysis using LLM
@@ -19,8 +18,10 @@ INITIAL_PROJECT_HOWTO_URL = (
 )
 SPEC_STEP_NAME = "Create specification"
 
+log = get_logger(__name__)
 
-class SpecWriter(SystemDependencyCheckerMixin, BaseAgent):
+
+class SpecWriter(BaseAgent):
     agent_type = "spec-writer"
     display_name = "Spec Writer"
 
@@ -42,16 +43,15 @@ class SpecWriter(SystemDependencyCheckerMixin, BaseAgent):
             return AgentResponse.import_project(self)
 
         if response.button == "example":
-            await self.send_message("Starting example project with description:")
-            await self.send_message(EXAMPLE_PROJECT_DESCRIPTION)
-            await self.prepare_example_project()
+            await self.prepare_example_project(DEFAULT_EXAMPLE_PROJECT)
             return AgentResponse.done(self)
+
         elif response.button == "continue":
             # FIXME: Workaround for the fact that VSCode "continue" button does
             # nothing but repeat the question. We reproduce this bug for bug here.
             return AgentResponse.done(self)
 
-        spec = response.text
+        spec = response.text.strip()
 
         complexity = await self.check_prompt_complexity(spec)
         await telemetry.trace_code_event(
@@ -82,31 +82,21 @@ class SpecWriter(SystemDependencyCheckerMixin, BaseAgent):
         llm_response: str = await llm(convo, temperature=0, parser=StringParser())
         return llm_response.lower()
 
-    async def prepare_example_project(self):
+    async def prepare_example_project(self, example_name: str):
+        example_description = EXAMPLE_PROJECTS[example_name]["description"].strip()
+
+        log.debug(f"Starting example project: {example_name}")
+        await self.send_message(f"Starting example project with description:\n\n{example_description}")
+
         spec = self.current_state.specification.clone()
-        spec.description = EXAMPLE_PROJECT_DESCRIPTION
-        spec.architecture = EXAMPLE_PROJECT_ARCHITECTURE["architecture"]
-        spec.system_dependencies = EXAMPLE_PROJECT_ARCHITECTURE["system_dependencies"]
-        spec.package_dependencies = EXAMPLE_PROJECT_ARCHITECTURE["package_dependencies"]
-        spec.template = EXAMPLE_PROJECT_ARCHITECTURE["template"]
-        spec.complexity = Complexity.SIMPLE
-        telemetry.set("initial_prompt", spec.description.strip())
-        telemetry.set("is_complex_app", False)
-        telemetry.set("template", spec.template)
-
-        await self.check_system_dependencies(spec)
-
+        spec.example_project = example_name
+        spec.description = example_description
+        spec.complexity = EXAMPLE_PROJECTS[example_name]["complexity"]
         self.next_state.specification = spec
 
-        self.next_state.epics = [
-            {
-                "name": "Initial Project",
-                "description": EXAMPLE_PROJECT_DESCRIPTION,
-                "completed": False,
-                "complexity": Complexity.SIMPLE,
-            }
-        ]
-        self.next_state.tasks = EXAMPLE_PROJECT_PLAN
+        telemetry.set("initial_prompt", spec.description)
+        telemetry.set("example_project", example_name)
+        telemetry.set("is_complex_app", spec.complexity != Complexity.SIMPLE)
 
     async def analyze_spec(self, spec: str) -> str:
         msg = (
diff --git a/core/agents/tech_lead.py b/core/agents/tech_lead.py
index 1dc36a6f..4c11d558 100644
--- a/core/agents/tech_lead.py
+++ b/core/agents/tech_lead.py
@@ -11,6 +11,7 @@ from core.db.models.project_state import TaskStatus
 from core.llm.parser import JSONParser
 from core.log import get_logger
 from core.telemetry import telemetry
+from core.templates.example_project import EXAMPLE_PROJECTS
 from core.templates.registry import apply_project_template, get_template_description, get_template_summary
 from core.ui.base import ProjectStage, success_source
 
@@ -42,8 +43,10 @@ class TechLead(BaseAgent):
             return await self.update_epic()
 
         if len(self.current_state.epics) == 0:
-            self.create_initial_project_epic()
-            # Orchestrator will rerun us to break down the initial project epic
+            if self.current_state.specification.example_project:
+                self.plan_example_project()
+            else:
+                self.create_initial_project_epic()
             return AgentResponse.done(self)
 
         await self.ui.send_project_stage(ProjectStage.CODING)
@@ -203,3 +206,18 @@ class TechLead(BaseAgent):
         ]
         log.debug(f"Updated development plan for {epic['name']}, {len(response.plan)} tasks remaining")
         return AgentResponse.done(self)
+
+    def plan_example_project(self):
+        example_name = self.current_state.specification.example_project
+        log.debug(f"Planning example project: {example_name}")
+
+        example = EXAMPLE_PROJECTS[example_name]
+        self.next_state.epics = [
+            {
+                "name": "Initial Project",
+                "description": example["description"],
+                "completed": False,
+                "complexity": example["complexity"],
+            }
+        ]
+        self.next_state.tasks = example["plan"]
diff --git a/core/db/migrations/README b/core/db/migrations/README
index 98e4f9c4..5cdc7408 100644
--- a/core/db/migrations/README
+++ b/core/db/migrations/README
@@ -1 +1,10 @@
-Generic single-database configuration.
\ No newline at end of file
+Pythagora uses Alembic for database migrations.
+
+After changing any of the database models, create a new migration:
+
+    alembic -c core/db/alembic.ini revision --autogenerate -m "description"
+
+Migrations are applied automatically when the application starts, but can also be
+run manually with:
+
+    alembic -c core/db/alembic.ini upgrade head
diff --git a/core/db/migrations/versions/ff891d366761_add_example_project_to_spec.py b/core/db/migrations/versions/ff891d366761_add_example_project_to_spec.py
new file mode 100644
index 00000000..f788761e
--- /dev/null
+++ b/core/db/migrations/versions/ff891d366761_add_example_project_to_spec.py
@@ -0,0 +1,34 @@
+"""add example project to spec
+
+Revision ID: ff891d366761
+Revises: b760f66138c0
+Create Date: 2024-06-13 09:38:33.329161
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "ff891d366761"
+down_revision: Union[str, None] = "b760f66138c0"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("specifications", schema=None) as batch_op:
+        batch_op.add_column(sa.Column("example_project", sa.String(), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table("specifications", schema=None) as batch_op:
+        batch_op.drop_column("example_project")
+
+    # ### end Alembic commands ###
diff --git a/core/db/models/specification.py b/core/db/models/specification.py
index 9e2eb7c9..e711f387 100644
--- a/core/db/models/specification.py
+++ b/core/db/models/specification.py
@@ -31,6 +31,7 @@ class Specification(Base):
     package_dependencies: Mapped[list[dict]] = mapped_column(default=list)
     template: Mapped[Optional[str]] = mapped_column()
     complexity: Mapped[str] = mapped_column(server_default=Complexity.HARD)
+    example_project: Mapped[Optional[str]] = mapped_column()
 
     # Relationships
     project_states: Mapped[list["ProjectState"]] = relationship(back_populates="specification", lazy="raise")
@@ -46,6 +47,7 @@ class Specification(Base):
             package_dependencies=self.package_dependencies,
             template=self.template,
             complexity=self.complexity,
+            example_project=self.example_project,
         )
         return clone
 
diff --git a/core/telemetry/__init__.py b/core/telemetry/__init__.py
index 2270a83a..37a38b3f 100644
--- a/core/telemetry/__init__.py
+++ b/core/telemetry/__init__.py
@@ -73,7 +73,7 @@ class Telemetry:
             "python_version": sys.version,
             # GPT Pilot version
             "pilot_version": get_version(),
-            # GPT Pilot Extension version
+            # Pythagora VSCode Extension version
             "extension_version": None,
             # Is extension used
             "is_extension": False,
@@ -86,6 +86,8 @@ class Telemetry:
             "is_complex_app": None,
             # Optional template used for the project
             "template": None,
+            # Optional, example project selected by the user
+            "example_project": None,
             # Optional user contact email
             "user_contact": None,
             # Unique project ID (app_id)
diff --git a/core/templates/example_project.py b/core/templates/example_project.py
index 6703a906..7cb47a6e 100644
--- a/core/templates/example_project.py
+++ b/core/templates/example_project.py
@@ -1,3 +1,5 @@
+from core.db.models import Complexity
+
 EXAMPLE_PROJECT_DESCRIPTION = """
 The application is a simple ToDo app built using React. Its primary function is to allow users to manage a list of tasks (todos). Each task has a description and a state (open or completed, with the default state being open). The application is frontend-only, with no user sign-up or authentication process. The goal is to provide a straightforward and user-friendly interface for task management.
 
@@ -64,3 +66,15 @@ EXAMPLE_PROJECT_PLAN = [
         "status": "todo",
     }
 ]
+
+
+EXAMPLE_PROJECTS = {
+    "example-project": {
+        "description": EXAMPLE_PROJECT_DESCRIPTION,
+        "architecture": EXAMPLE_PROJECT_ARCHITECTURE,
+        "complexity": Complexity.SIMPLE,
+        "plan": EXAMPLE_PROJECT_PLAN,
+    }
+}
+
+DEFAULT_EXAMPLE_PROJECT = "example-project"
diff --git a/tests/agents/test_spec_writer.py b/tests/agents/test_spec_writer.py
index 88f94959..2140dee2 100644
--- a/tests/agents/test_spec_writer.py
+++ b/tests/agents/test_spec_writer.py
@@ -23,16 +23,8 @@ async def test_start_example_project(agentcontext):
     assert response.type == ResponseType.DONE
 
     assert sm.current_state.specification.description != ""
-    assert sm.current_state.specification.architecture != ""
-    assert sm.current_state.specification.system_dependencies != []
-    assert sm.current_state.specification.package_dependencies != []
     assert sm.current_state.specification.complexity == Complexity.SIMPLE
-    assert sm.current_state.epics != []
-    assert sm.current_state.tasks != []
-    pm.run_command.assert_awaited_once_with("node --version")
-
-    assert telemetry.data["initial_prompt"] == sm.current_state.specification.description.strip()
-    assert telemetry.data["architecture"]["system_dependencies"][0]["installed"] is True
+    assert telemetry.data["initial_prompt"] == sm.current_state.specification.description
 
 
 @pytest.mark.asyncio

From 9b1b33fb9c884b0573f40c8b69c936d63b628f30 Mon Sep 17 00:00:00 2001
From: Senko Rasic <senko@senko.net>
Date: Thu, 13 Jun 2024 10:50:36 +0200
Subject: [PATCH 7/9] fix orchestrator tests

---
 tests/agents/test_orchestrator.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/tests/agents/test_orchestrator.py b/tests/agents/test_orchestrator.py
index 416835b4..ae60504b 100644
--- a/tests/agents/test_orchestrator.py
+++ b/tests/agents/test_orchestrator.py
@@ -8,9 +8,9 @@ from core.state.state_manager import StateManager
 
 @pytest.mark.asyncio
 async def test_offline_changes_check_restores_if_workspace_empty():
-    sm = Mock(spec=StateManager)
-    sm.workspace_is_empty.return_value = True
-    ui = Mock()
+    sm = AsyncMock(spec=StateManager)
+    sm.workspace_is_empty = Mock(return_value=False)
+    ui = AsyncMock()
     orca = Orchestrator(state_manager=sm, ui=ui)
     await orca.offline_changes_check()
     assert sm.restore_files.assert_called_once
@@ -19,7 +19,8 @@ async def test_offline_changes_check_restores_if_workspace_empty():
 @pytest.mark.asyncio
 async def test_offline_changes_check_imports_changes_from_disk():
     sm = AsyncMock()
-    sm.workspace_is_empty.return_value = False
+    sm.workspace_is_empty = Mock(return_value=False)
+    sm.import_files = AsyncMock(return_value=([], []))
     ui = AsyncMock()
     ui.ask_question.return_value.button = "yes"
     orca = Orchestrator(state_manager=sm, ui=ui)
@@ -31,7 +32,7 @@ async def test_offline_changes_check_imports_changes_from_disk():
 @pytest.mark.asyncio
 async def test_offline_changes_check_restores_changes_from_db():
     sm = AsyncMock()
-    sm.workspace_is_empty.return_value = False
+    sm.workspace_is_empty = Mock(return_value=False)
     ui = AsyncMock()
     ui.ask_question.return_value.button = "no"
     orca = Orchestrator(state_manager=sm, ui=ui)

From 50d5a23620d01f50578442fdad599f5f002e3a29 Mon Sep 17 00:00:00 2001
From: Senko Rasic <senko@senko.net>
Date: Thu, 13 Jun 2024 13:52:15 +0200
Subject: [PATCH 8/9] add more telemetry data

---
 core/agents/developer.py      |  9 ++++++++-
 core/agents/task_completer.py |  8 +++++++-
 core/agents/tech_lead.py      |  8 +++++++-
 core/agents/troubleshooter.py | 16 +++++++++++++++-
 core/telemetry/__init__.py    | 11 ++++++-----
 5 files changed, 43 insertions(+), 9 deletions(-)

diff --git a/core/agents/developer.py b/core/agents/developer.py
index 80666c61..25b9670b 100644
--- a/core/agents/developer.py
+++ b/core/agents/developer.py
@@ -196,7 +196,14 @@ class Developer(BaseAgent):
         self.next_state.modified_files = {}
         self.set_next_steps(response, source)
         self.next_state.action = f"Task #{current_task_index + 1} start"
-        await telemetry.trace_code_event("task-start", {"task-num": current_task_index + 1})
+        await telemetry.trace_code_event(
+            "task-start",
+            {
+                "task_index": current_task_index + 1,
+                "num_tasks": len(self.current_state.tasks),
+                "num_epics": len(self.current_state.epics),
+            },
+        )
         return AgentResponse.done(self)
 
     async def get_relevant_files(
diff --git a/core/agents/task_completer.py b/core/agents/task_completer.py
index 75b5be9d..1bbfeb22 100644
--- a/core/agents/task_completer.py
+++ b/core/agents/task_completer.py
@@ -27,7 +27,13 @@ class TaskCompleter(BaseAgent):
             tasks,
         )
         await telemetry.trace_code_event(
-            "task-end", {"task-num": current_task_index1, "num-iterations": len(self.current_state.iterations)}
+            "task-end",
+            {
+                "task_index": current_task_index1,
+                "num_tasks": len(self.current_state.tasks),
+                "num_epics": len(self.current_state.epics),
+                "num_iterations": len(self.current_state.iterations),
+            },
         )
 
         return AgentResponse.done(self)
diff --git a/core/agents/tech_lead.py b/core/agents/tech_lead.py
index 4c11d558..e8cb953e 100644
--- a/core/agents/tech_lead.py
+++ b/core/agents/tech_lead.py
@@ -155,7 +155,13 @@ class TechLead(BaseAgent):
             }
             for task in response.plan
         ]
-        await telemetry.trace_code_event("development-plan", {"num-tasks": len(response.plan)})
+        await telemetry.trace_code_event(
+            "development-plan",
+            {
+                "num_tasks": len(self.current_state.tasks),
+                "num_epics": len(self.current_state.epics),
+            },
+        )
         return AgentResponse.done(self)
 
     async def update_epic(self) -> AgentResponse:
diff --git a/core/agents/troubleshooter.py b/core/agents/troubleshooter.py
index 05dafb54..a04437be 100644
--- a/core/agents/troubleshooter.py
+++ b/core/agents/troubleshooter.py
@@ -182,7 +182,21 @@ class Troubleshooter(IterationPromptMixin, BaseAgent):
             return False, False, ""
 
         if user_response.button == "loop":
-            await telemetry.trace_code_event("stuck-in-loop", {"clicked": True})
+            await telemetry.trace_code_event(
+                "stuck-in-loop",
+                {
+                    "clicked": True,
+                    "task_index": self.current_state.tasks.index(self.current_state.current_task) + 1,
+                    "num_tasks": len(self.current_state.tasks),
+                    "num_epics": len(self.current_state.epics),
+                    "num_iterations": len(self.current_state.iterations),
+                    "num_steps": len(self.current_state.steps),
+                    "architecture": {
+                        "system_dependencies": self.current_state.specification.system_dependencies,
+                        "app_dependencies": self.current_state.specification.package_dependencies,
+                    },
+                },
+            )
             return True, True, ""
 
         return True, False, user_response.text
diff --git a/core/telemetry/__init__.py b/core/telemetry/__init__.py
index 37a38b3f..89bd68a8 100644
--- a/core/telemetry/__init__.py
+++ b/core/telemetry/__init__.py
@@ -368,8 +368,9 @@ class Telemetry:
         if not self.enabled or getenv("DISABLE_TELEMETRY"):
             return
 
-        if not data.get("app_id") and self.data["app_id"]:
-            data = {**data, "app_id": self.data["app_id"]}
+        data = deepcopy(data)
+        for item in ["app_id", "user_contact", "platform", "pilot_version", "model"]:
+            data[item] = self.data[item]
 
         payload = {
             "pathId": self.telemetry_id,
@@ -377,13 +378,13 @@ class Telemetry:
             "data": data,
         }
 
-        log.debug(f"Sending trace event {name} to {self.endpoint}")
+        log.debug(f"Sending trace event {name} to {self.endpoint}: {repr(payload)}")
 
         try:
             async with httpx.AsyncClient() as client:
                 await client.post(self.endpoint, json=payload)
-        except httpx.RequestError:
-            pass
+        except httpx.RequestError as e:
+            log.error(f"Failed to send trace event {name}: {e}", exc_info=True)
 
     async def trace_loop(self, name: str, task_with_loop: dict):
         payload = deepcopy(self.data)

From b3ea0bbd592e68e25c61ee889ceec94069994f68 Mon Sep 17 00:00:00 2001
From: Senko Rasic <senko@senko.net>
Date: Thu, 13 Jun 2024 14:07:02 +0200
Subject: [PATCH 9/9] update telemetry readme with up-to-date info

---
 docs/TELEMETRY.md | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/docs/TELEMETRY.md b/docs/TELEMETRY.md
index 72e5ac81..204f644e 100644
--- a/docs/TELEMETRY.md
+++ b/docs/TELEMETRY.md
@@ -1,30 +1,34 @@
-## Telemetry in GPT Pilot
+## Telemetry in Pythagora
 
-At GPT Pilot, we are dedicated to improving your experience and the overall quality of our software. To achieve this, we gather anonymous telemetry data which helps us understand how the tool is being used and identify areas for improvement.
+At Pythagora, we are dedicated to improving your experience and the overall quality of our software. To achieve this, we gather anonymous telemetry data which helps us understand how the tool is being used and identify areas for improvement.
 
 ### What We Collect
 
 The telemetry data we collect includes:
 
-- **Total Runtime**: The total time GPT Pilot was active and running.
+- **Total Runtime**: The total time Pythagora was active and running.
 - **Command Runs**: How many commands were executed during a session.
 - **Development Steps**: The number of development steps that were performed.
 - **LLM Requests**: The number of LLM requests made.
 - **User Inputs**: The number of times you provide input to the tool.
 - **Operating System**: The operating system you are using (and Linux distro if applicable).
 - **Python Version**: The version of Python you are using.
-- **GPT Pilot Version**: The version of GPT Pilot you are using.
-- **LLM Model**: LLM model used for the session.
+- **GPT Pilot Version**: The version of Pythagora you are using.
+- **LLM Model**: LLM model(s) used for the session.
 - **Time**: How long it took to generate a project.
 - **Initial prompt**: App description used to create app (after Specification Writer Agent).
+- **Architecture**: Architecture designed by Pythagora for the app.
+- **Documentation**: Pythagora documentation that was used while creating the app.
+- **User Email**: User email (if using Pythagora VSCode Extgension, or if explicitly provided when running Pythagora from the command line).
+- **Pythagora Tasks/Steps**: Information about the development tasks and steps Pythagora does while coding the app.
 
-All the data points are listed in [pilot.utils.telemetry:Telemetry.clear_data()](../pilot/utils/telemetry.py).
+All the data points are listed in [core.telemetry:Telemetry.clear_data()](../core/telemetry/__init__.py).
 
 ### How We Use This Data
 
 We use this data to:
 
-- Monitor the performance and reliability of GPT Pilot.
+- Monitor the performance and reliability of Pythagora.
 - Understand usage patterns to guide our development and feature prioritization.
 - Identify common workflows and improve the user experience.
 - Ensure the scalability and efficiency of our language model interactions.
@@ -37,9 +41,9 @@ Your privacy is important to us. The data collected is purely for internal analy
 
 We believe in transparency and control. If you prefer not to send telemetry data, you can opt-out at any time by setting `telemetry.enabled` to `false` in your `~/.gpt-pilot/config.json` configuration file.
 
-After you update this setting, GPT Pilot will no longer collect telemetry data from your machine.
+After you update this setting, Pythagora will no longer collect telemetry data from your machine.
 
 ### Questions and Feedback
 If you have questions about our telemetry practices or would like to provide feedback, please open an issue in our repository, and we will be happy to engage with you.
 
-Thank you for supporting GPT Pilot and helping us make it better for everyone.
+Thank you for supporting Pythagora and helping us make it better for everyone.