mirror of
https://github.com/Significant-Gravitas/AutoGPT.git
synced 2026-04-08 03:00:28 -04:00
fix(copilot): reference SendAuthenticatedWebRequestBlock in tool discovery + fix CI
This commit is contained in:
@@ -123,9 +123,9 @@ When the user asks to interact with a service or API, follow this order:
|
||||
|
||||
2. **run_mcp_tool** — If no matching block exists, check if a hosted MCP server is available for the service. Only use known MCP server URLs from the registry.
|
||||
|
||||
3. **Authenticated web request** — If no block or MCP server exists, check if we have existing credentials (OAuth tokens, API keys) for the service via `connect_integration`, then use authenticated HTTP requests.
|
||||
3. **SendAuthenticatedWebRequestBlock** — If no block or MCP server exists, use `SendAuthenticatedWebRequestBlock` with existing host-scoped credentials. Check available credentials via `connect_integration`.
|
||||
|
||||
4. **Manual API call** — As a last resort, guide the user to set up credentials and use direct API calls.
|
||||
4. **Manual API call** — As a last resort, guide the user to set up credentials and use `SendAuthenticatedWebRequestBlock` with direct API calls.
|
||||
|
||||
**Never skip step 1.** Built-in blocks are more reliable, tested, and user-friendly than MCP or raw API calls.
|
||||
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
"""Prompt regression tests for the dry-run verification loop.
|
||||
|
||||
These tests verify that the create -> dry-run -> fix iterative workflow is
|
||||
properly communicated through system prompts, tool descriptions, the
|
||||
prompting supplement, and the agent building guide.
|
||||
properly communicated through tool descriptions, the prompting supplement,
|
||||
and the agent building guide.
|
||||
|
||||
After deduplication, the full dry-run workflow lives in the
|
||||
agent_generation_guide.md only. The system prompt and individual tool
|
||||
descriptions no longer repeat it — they keep a minimal footprint.
|
||||
|
||||
**Intentionally brittle**: the assertions check for specific substrings so
|
||||
that accidental removal or rewording of key instructions is caught. If you
|
||||
@@ -22,55 +26,22 @@ from backend.copilot.tools import TOOL_REGISTRY
|
||||
_SDK_SUPPLEMENT = get_sdk_supplement(use_e2b=False, cwd="/tmp/test")
|
||||
|
||||
|
||||
class TestSystemPromptDryRunLoop:
|
||||
"""Verify the system prompt includes a brief dry-run loop reference.
|
||||
class TestSystemPromptBasics:
|
||||
"""Verify the system prompt includes essential baseline content.
|
||||
|
||||
The detailed workflow lives in the supplement / guide; the system prompt
|
||||
only carries a short pointer to keep it minimal.
|
||||
After deduplication, the dry-run workflow lives only in the guide.
|
||||
The system prompt carries tone and personality only.
|
||||
"""
|
||||
|
||||
def test_mentions_dry_run(self):
|
||||
assert (
|
||||
"dry-run" in DEFAULT_SYSTEM_PROMPT.lower()
|
||||
or "dry_run" in DEFAULT_SYSTEM_PROMPT
|
||||
)
|
||||
def test_mentions_automations(self):
|
||||
assert "automations" in DEFAULT_SYSTEM_PROMPT.lower()
|
||||
|
||||
def test_mentions_never_skip(self):
|
||||
assert "NEVER skip" in DEFAULT_SYSTEM_PROMPT
|
||||
|
||||
def test_references_tool_notes(self):
|
||||
assert "tool notes" in DEFAULT_SYSTEM_PROMPT.lower()
|
||||
|
||||
def test_mentions_repeat_until(self):
|
||||
assert "repeat until" in DEFAULT_SYSTEM_PROMPT.lower()
|
||||
def test_mentions_action_oriented(self):
|
||||
assert "action-oriented" in DEFAULT_SYSTEM_PROMPT.lower()
|
||||
|
||||
|
||||
class TestToolDescriptionsDryRunLoop:
|
||||
"""Verify tool descriptions guide the LLM through the dry-run loop."""
|
||||
|
||||
def test_create_agent_mentions_dry_run(self):
|
||||
desc = TOOL_REGISTRY["create_agent"].description
|
||||
assert "dry_run" in desc or "dry-run" in desc.lower()
|
||||
assert "run_agent" in desc
|
||||
|
||||
def test_edit_agent_mentions_dry_run_verification(self):
|
||||
desc = TOOL_REGISTRY["edit_agent"].description
|
||||
assert "dry_run" in desc or "dry-run" in desc.lower()
|
||||
assert "dry-run testing" in desc.lower() or "wiring errors" in desc.lower()
|
||||
|
||||
def test_run_agent_mentions_dry_run_for_testing(self):
|
||||
desc = TOOL_REGISTRY["run_agent"].description
|
||||
assert "dry_run=True" in desc
|
||||
desc_lower = desc.lower()
|
||||
assert "test agent wiring" in desc_lower or "simulates execution" in desc_lower
|
||||
|
||||
def test_run_agent_dry_run_param_mentions_workflow(self):
|
||||
schema = TOOL_REGISTRY["run_agent"].as_openai_tool()
|
||||
params = cast(dict[str, Any], schema["function"].get("parameters", {}))
|
||||
dry_run_desc = params["properties"]["dry_run"]["description"]
|
||||
assert "create_agent" in dry_run_desc or "edit_agent" in dry_run_desc
|
||||
assert "wait_for_result" in dry_run_desc
|
||||
assert "repeat" in dry_run_desc.lower()
|
||||
"""Verify tool descriptions and parameters related to the dry-run loop."""
|
||||
|
||||
def test_get_agent_building_guide_mentions_workflow(self):
|
||||
desc = TOOL_REGISTRY["get_agent_building_guide"].description
|
||||
@@ -82,32 +53,36 @@ class TestToolDescriptionsDryRunLoop:
|
||||
assert "dry_run" in params["properties"]
|
||||
assert params["properties"]["dry_run"]["type"] == "boolean"
|
||||
|
||||
def test_run_agent_dry_run_param_references_guide(self):
|
||||
"""After deduplication the dry_run param description points to the guide."""
|
||||
schema = TOOL_REGISTRY["run_agent"].as_openai_tool()
|
||||
params = cast(dict[str, Any], schema["function"].get("parameters", {}))
|
||||
dry_run_desc = params["properties"]["dry_run"]["description"]
|
||||
assert "agent_generation_guide" in dry_run_desc
|
||||
|
||||
class TestPromptingSupplementDryRunLoop:
|
||||
"""Verify the prompting supplement (via get_sdk_supplement) includes the
|
||||
iterative workflow. Tests use the public ``get_sdk_supplement`` API
|
||||
rather than importing the private ``_SHARED_TOOL_NOTES`` directly.
|
||||
def test_run_agent_dry_run_param_mentions_simulates(self):
|
||||
schema = TOOL_REGISTRY["run_agent"].as_openai_tool()
|
||||
params = cast(dict[str, Any], schema["function"].get("parameters", {}))
|
||||
dry_run_desc = params["properties"]["dry_run"]["description"]
|
||||
assert "simulates" in dry_run_desc.lower()
|
||||
|
||||
|
||||
class TestPromptingSupplementContent:
|
||||
"""Verify the prompting supplement (via get_sdk_supplement) includes
|
||||
essential shared tool notes. After deduplication, the dry-run workflow
|
||||
lives only in the guide; the supplement carries storage, file-handling,
|
||||
and tool-discovery notes.
|
||||
"""
|
||||
|
||||
def test_includes_dry_run_section_header(self):
|
||||
assert "Iterative agent development" in _SDK_SUPPLEMENT
|
||||
|
||||
def test_includes_create_dry_run_fix_workflow(self):
|
||||
assert "create -> dry-run -> fix" in _SDK_SUPPLEMENT.lower()
|
||||
|
||||
def test_includes_error_inspection_guidance(self):
|
||||
"""The supplement should reference error patterns (detailed list
|
||||
lives in the guide; the supplement carries at least a summary)."""
|
||||
lower = _SDK_SUPPLEMENT.lower()
|
||||
assert "errors" in lower or "failed" in lower
|
||||
assert "null" in lower or "empty" in lower
|
||||
|
||||
def test_includes_repeat_until(self):
|
||||
assert "repeat until" in _SDK_SUPPLEMENT.lower()
|
||||
|
||||
def test_includes_tool_discovery_priority(self):
|
||||
assert "Tool Discovery Priority" in _SDK_SUPPLEMENT
|
||||
|
||||
def test_includes_find_block_first(self):
|
||||
assert "find_block first" in _SDK_SUPPLEMENT or "find_block" in _SDK_SUPPLEMENT
|
||||
|
||||
def test_includes_send_authenticated_web_request(self):
|
||||
assert "SendAuthenticatedWebRequestBlock" in _SDK_SUPPLEMENT
|
||||
|
||||
|
||||
class TestAgentBuildingGuideDryRunLoop:
|
||||
"""Verify the agent building guide includes the dry-run loop."""
|
||||
|
||||
Reference in New Issue
Block a user