update prompt

examples/run.py

@@ -33,7 +33,7 @@ def main():
     parser.add_argument('--config_path', type=str, default='config.json', help='openAI config file path')
     parser.add_argument('--query', type=str, default=None, help='user query')
     parser.add_argument('--query_file_path', type=str, default='', help='user query file path')
-    parser.add_argument('--task_id', type=str, default="6af95c8f-8cbf-4c12-b02c-f9a23cc1ecb9", help='GAIA dataset task_id')
+    parser.add_argument('--task_id', type=str, default="5e76cfc3-f33d-4416-b8ae-3ef292830207", help='GAIA dataset task_id')
     parser.add_argument('--cache_dir', type=str, default=None, help='GAIA dataset cache dir path')
     parser.add_argument('--logging_filedir', type=str, default='log/test_level1', help='GAIA dataset cache dir path')
     args = parser.parse_args()

jarvis/agent/prompt.py

@@ -381,7 +381,6 @@ prompt = {
         25. When the task involves retrieving a certain detailed content, then after decomposing the API subtask using '/tools/bing/searchv2', you also need to decompose an API subtask using '/tools/bing/load_pagev2', using for more detailed content.
         26. If the attached file is a picture file, the task must be broken down into two sub-tasks. The first is a API subtask, which uses image caption API to extract detail information of the picture file. The second is a QA subtask, which analyzes and completes task based on the return from API subtask.
         27. Please note that all available APIs are only in the API List. You should not make up APIs that are not in the API List.
-        28. Please note that if the problem involves the database, you can use the Bing Search API to obtain relevant results.
         ''',
         '_USER_TASK_DECOMPOSE_PROMPT' : '''
         User's information are as follows:

jarvis/core/openapi.json

@@ -283,6 +283,81 @@
           }
         }
       }
+    },
+    "/tools/image_caption": {
+      "post": {
+        "summary": "You don't have eyes, so if you want to know what the local picture(not on the internet) is about, you have to use the Image Caption tool, who converts the picture into natural language text that you can understand.For local images you want to understand, you need to only give the image_file without url.In most time,you should give the 'query' paramter to tell the question you want to ask in the picture",
+        "operationId": "image_search_tools_image_caption_post",
+        "parameters": [
+          {
+            "name": "query",
+            "in": "query",
+            "required": false,
+            "schema": {
+              "anyOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "default": "What's in this image?",
+              "title": "Query"
+            }
+          },
+          {
+            "name": "url",
+            "in": "query",
+            "required": false,
+            "schema": {
+              "anyOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "title": "Url"
+            }
+          }
+        ],
+        "requestBody": {
+          "content": {
+            "multipart/form-data": {
+              "schema": {
+                "allOf": [
+                  {
+                    "$ref": "#/components/schemas/Body_image_search_tools_image_caption_post"
+                  }
+                ],
+                "title": "Body"
+              }
+            }
+          }
+        },
+        "responses": {
+          "200": {
+            "description": "Successful Response",
+            "content": {
+              "application/json": {
+                "schema": {}
+              }
+            }
+          },
+          "422": {
+            "description": "Validation Error",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/HTTPValidationError"
+                }
+              }
+            }
+          }
+        }
+      }
     }
   },
   "components": {

jarvis/core/tool_request_util.py

@@ -19,9 +19,9 @@ class ToolRequestUtil:
             # 判断请求方法
             if method.lower() == "get":
                 if content_type == "application/json":
-                    result = self.session.get(url=url, json=params, headers=self.headers).json()
+                    result = self.session.get(url=url, json=params, headers=self.headers, timeout=60).json()
                 else: 
-                    result = self.session.get(url=url, params=params, headers=self.headers).json()
+                    result = self.session.get(url=url, params=params, headers=self.headers, timeout=60).json()
             elif method.lower() == "post":
                 if content_type == "multipart/form-data":
                     result = self.session.post(url=url, files=files, data=params, headers=self.headers).json()

jarvis/environment/env.py

@@ -18,7 +18,7 @@ class Env:
 
     def __init__(self) -> None:
         self._name: str = self.__class__.__name__
-        self.timeout: int = 20
+        self.timeout: int = 60
         self.working_dir = os.path.abspath(os.path.join(__file__, "..", "..", "..", "working_dir"))
         if not os.path.exists(self.working_dir):
             os.makedirs(self.working_dir)

working_dir/code/audio2text.py

@@ -0,0 +1,26 @@
+from jarvis.core.tool_request_util import ToolRequestUtil
+
+# Initialize the ToolRequestUtil
+tool_request_util = ToolRequestUtil()
+
+# Define the API path
+api_path = '/tools/audio2text'
+
+# Define the method to be used for the request
+method = 'post'
+
+# Since the API documentation does not specify any additional parameters, we will set params to None
+params = None
+
+# Set the content type for the request
+content_type = 'multipart/form-data'
+
+# Specify the file path and the file parameter name as per the API documentation
+file_path = '/home/heroding/.cache/huggingface/datasets/downloads/30628ff4e5650083191b5763452662b2c0818a8ca2daef455a91860cc34ef490.mp3'
+files = {'file': ('audio.mp3', open(file_path, 'rb'))}
+
+# Make the API request and get the response
+response = tool_request_util.request(api_path=api_path, method=method, params=params, files=files, content_type=content_type)
+
+# Print the response from the API
+print(response)

working_dir/code/bing_search.py

@@ -0,0 +1,26 @@
+from jarvis.core.tool_request_util import ToolRequestUtil
+
+# Initialize the ToolRequestUtil
+tool_request_util = ToolRequestUtil()
+
+# Define the API path
+api_path = '/tools/bing/searchv2'
+
+# Define the method to be used for the API request
+method = 'get'
+
+# Define the parameters for the API request
+# The query is constructed to include information about koi fish, the watershed id, and the year
+params = {
+    "query": "koi fish 02040203",
+    "top_k": None  # Assuming we want the default number of results
+}
+
+# Define the content type for the API request
+content_type = 'application/json'
+
+# Make the API request and store the response
+response = tool_request_util.request(api_path, method, params=params, content_type=content_type)
+
+# Print the response from the API
+print(response)