track tasks progress and report if it goes in loop

pilot/const/ipc.py

@@ -13,6 +13,7 @@ MESSAGE_TYPE = {
     'ipc': 'ipc',                          # Regular print message that is for extension only
     'openFile': 'openFile',                # Open a file in extension
     'loadingFinished': 'loadingFinished',  # Marks end of loading project
+    'loopTrigger': 'loopTrigger',          # Trigger loop feedback popup in extension
 }
 
 LOCAL_IGNORE_MESSAGE_TYPES = [
@@ -24,4 +25,5 @@ LOCAL_IGNORE_MESSAGE_TYPES = [
     'ipc',
     'openFile',
     'loadingFinished',
+    'loopTrigger',
 ]

pilot/const/telemetry.py

@@ -1,2 +1,3 @@
 LARGE_REQUEST_THRESHOLD = 50000  # tokens
 SLOW_REQUEST_THRESHOLD = 300  # seconds
+LOOP_THRESHOLD = 50  # number of steps in task to be considered a loop

pilot/helpers/Debugger.py

@@ -34,6 +34,7 @@ class Debugger:
         """
         logger.info('Debugging %s', command)
         self.recursion_layer += 1
+        self.agent.project.current_task.add_debugging_task(self.recursion_layer, command, user_input, issue_description)
         if self.recursion_layer > MAX_RECUSION_LAYER:
             self.recursion_layer = 0
             raise TooDeepRecursionError()
@@ -53,6 +54,7 @@ class Debugger:
                     return True
                 if answer and answer.lower() not in AFFIRMATIVE_ANSWERS:
                     user_input = answer
+                    self.agent.project.current_task.add_user_input_to_debugging_task(user_input)
 
             llm_response = convo.send_message('dev_ops/debug.prompt',
                 {

pilot/helpers/Project.py

@@ -31,6 +31,7 @@ from utils.llm_connection import test_api_access
 from utils.ignore import IgnoreMatcher
 
 from utils.telemetry import telemetry
+from utils.task import Task
 
 class Project:
     def __init__(
@@ -56,6 +57,7 @@ class Project:
         self.llm_req_num = 0
         self.command_runs_count = 0
         self.user_inputs_count = 0
+        self.current_task = Task()
         self.checkpoints = {
             'last_user_input': None,
             'last_command_run': None,

pilot/helpers/agents/Developer.py

@@ -67,6 +67,7 @@ class Developer(Agent):
                     self.project.technical_writer.document_project(current_progress_percent)
                     documented_thresholds.add(threshold)
 
+            self.project.current_task.start_new_task(dev_task['description'], i + 1)
             self.implement_task(i, dev_task)
             telemetry.inc("num_tasks")
 
@@ -376,6 +377,10 @@ class Developer(Agent):
             if i < continue_from_step:
                 continue
             logger.info('---------- execute_task() step #%d: %s', i, step)
+            # this if statement is for current way of loading app,
+            # once we move to backwards compatibility if statement can be removed
+            if not self.project.skip_steps:
+                self.project.current_task.inc('steps')
 
             result = None
             step_implementation_try = 0

pilot/main.py

@@ -1,5 +1,6 @@
 # main.py
 import builtins
+import json
 import os
 
 import sys
@@ -114,6 +115,7 @@ if __name__ == "__main__":
 
     finally:
         if project is not None:
+            project.current_task.exit()
             project.finish_loading()
         if run_exit_fn:
             exit_gpt_pilot(project, ask_feedback)

pilot/utils/task.py

@@ -0,0 +1,147 @@
+import json
+from uuid import uuid4
+
+from utils.telemetry import telemetry
+from utils.exit import trace_code_event
+from const.telemetry import LOOP_THRESHOLD
+
+
+class Task:
+    """
+    Task data structure to store information about the current task. The task data structure is sent to telemetry.
+    Currently used to trace big loops in the code.
+
+    >>> from utils.task import Task
+
+    To set up a new task:
+
+    >>> task = Task()
+
+    To set a value:
+
+    >>> task.set('task_description', 'test')
+
+    To increment a value:
+
+    >>> task.inc('steps')
+
+    To start a new task:
+
+    >>> task.start_new_task('test', 1)
+
+    When debugging recursion happens inside a task (see pilot/helpers/Debugger.py) we add a debugging task to the
+    task data structure. To add a debugging task:
+
+    >>> task.add_debugging_task(1, {'command': 'test'}, 'This is not working', 'Command is not working')
+
+    To clear the task:
+
+    >>> task.clear()
+
+    To send the task:
+
+    >>> task.send()
+
+    Note: the task will be sent automatically if the number of steps exceeds the threshold
+    """
+
+    def __init__(self):
+        self.initial_data = {
+            'task_description': '',
+            'task_number': 0,
+            'steps': 0,
+            'debugging': [],
+        }
+        self.data = self.initial_data.copy()
+        self.ping_extension = True
+
+    def set(self, key: str, value: any):
+        """
+        Set a value in the task data
+
+        :param key: key to set
+        :param value: value to set
+        """
+        self.data[key] = value
+
+    def inc(self, key: str, value: int = 1):
+        """
+        Increment a value in the task data
+
+        :param key: key to increment
+        :param value: value to increment by
+        """
+        self.data[key] += value
+        if key == 'steps' and self.data[key] == LOOP_THRESHOLD + 1:
+            self.send()
+
+    def start_new_task(self, task_description: str, i: int):
+        """
+        Start a new task
+
+        :param task_description: description of the task
+        :param i: task number
+        """
+        self.send(name='loop-end')
+        self.clear()
+        self.set('task_description', task_description)
+        self.set('task_number', i)
+        self.set('loopId', f"{uuid4()}")
+
+    def add_debugging_task(self, recursion_layer: int = None,
+                           command: dict = None,
+                           user_input: str = None,
+                           issue_description: str = None):
+        """
+        Add a debugging task to the task data structure
+
+        :param recursion_layer: recursion layer
+        :param command: command to debug
+        :param user_input: user input
+        :param issue_description: description of the issue
+        """
+        self.data['debugging'].append({
+            'recursion_layer': recursion_layer,
+            'command': command,
+            'user_inputs': [user_input] if user_input is not None else [],
+            'issue_description': issue_description,
+        })
+
+    def add_user_input_to_debugging_task(self, user_input: str):
+        """
+        Add user input to the last debugging task
+
+        :param user_input: user input
+        """
+        if self.data.get('debugging') and len(self.data['debugging']) > 0:
+            self.data['debugging'][-1]['user_inputs'].append(user_input)
+
+    def clear(self):
+        """
+        Clear all the task data
+        """
+        self.data = self.initial_data.copy()
+
+    def send(self, name: str = 'loop-start'):
+        """
+        Send the task data to telemetry
+
+        :param name: name of the event
+        """
+        if self.data['steps'] > LOOP_THRESHOLD:
+            full_data = telemetry.data.copy()
+            full_data['task_with_loop'] = self.data.copy()
+            trace_code_event(name=name, data=full_data)
+            if self.ping_extension:
+                print(json.dumps({
+                    'pathId': telemetry.telemetry_id,
+                    'data': full_data,
+                }), type='loopTrigger')
+                # TODO: see if we want to ping the extension multiple times
+                self.ping_extension = False
+
+    def exit(self):
+        """
+        Send the task data to telemetry and exit the process
+        """
+        self.send(name='loop-end')