fix(copilot): reference SendAuthenticatedWebRequestBlock in tool discovery + fix CI

This commit is contained in:
Zamil Majdy
2026-03-27 13:44:25 +07:00
parent 7767c97f50
commit 65108c31dc
2 changed files with 41 additions and 66 deletions

View File

@@ -123,9 +123,9 @@ When the user asks to interact with a service or API, follow this order:
2. **run_mcp_tool** — If no matching block exists, check if a hosted MCP server is available for the service. Only use known MCP server URLs from the registry.
3. **Authenticated web request** — If no block or MCP server exists, check if we have existing credentials (OAuth tokens, API keys) for the service via `connect_integration`, then use authenticated HTTP requests.
3. **SendAuthenticatedWebRequestBlock** — If no block or MCP server exists, use `SendAuthenticatedWebRequestBlock` with existing host-scoped credentials. Check available credentials via `connect_integration`.
4. **Manual API call** — As a last resort, guide the user to set up credentials and use direct API calls.
4. **Manual API call** — As a last resort, guide the user to set up credentials and use `SendAuthenticatedWebRequestBlock` with direct API calls.
**Never skip step 1.** Built-in blocks are more reliable, tested, and user-friendly than MCP or raw API calls.

View File

@@ -1,8 +1,12 @@
"""Prompt regression tests for the dry-run verification loop.
These tests verify that the create -> dry-run -> fix iterative workflow is
properly communicated through system prompts, tool descriptions, the
prompting supplement, and the agent building guide.
properly communicated through tool descriptions, the prompting supplement,
and the agent building guide.
After deduplication, the full dry-run workflow lives in the
agent_generation_guide.md only. The system prompt and individual tool
descriptions no longer repeat it — they keep a minimal footprint.
**Intentionally brittle**: the assertions check for specific substrings so
that accidental removal or rewording of key instructions is caught. If you
@@ -22,55 +26,22 @@ from backend.copilot.tools import TOOL_REGISTRY
_SDK_SUPPLEMENT = get_sdk_supplement(use_e2b=False, cwd="/tmp/test")
class TestSystemPromptDryRunLoop:
"""Verify the system prompt includes a brief dry-run loop reference.
class TestSystemPromptBasics:
"""Verify the system prompt includes essential baseline content.
The detailed workflow lives in the supplement / guide; the system prompt
only carries a short pointer to keep it minimal.
After deduplication, the dry-run workflow lives only in the guide.
The system prompt carries tone and personality only.
"""
def test_mentions_dry_run(self):
assert (
"dry-run" in DEFAULT_SYSTEM_PROMPT.lower()
or "dry_run" in DEFAULT_SYSTEM_PROMPT
)
def test_mentions_automations(self):
assert "automations" in DEFAULT_SYSTEM_PROMPT.lower()
def test_mentions_never_skip(self):
assert "NEVER skip" in DEFAULT_SYSTEM_PROMPT
def test_references_tool_notes(self):
assert "tool notes" in DEFAULT_SYSTEM_PROMPT.lower()
def test_mentions_repeat_until(self):
assert "repeat until" in DEFAULT_SYSTEM_PROMPT.lower()
def test_mentions_action_oriented(self):
assert "action-oriented" in DEFAULT_SYSTEM_PROMPT.lower()
class TestToolDescriptionsDryRunLoop:
"""Verify tool descriptions guide the LLM through the dry-run loop."""
def test_create_agent_mentions_dry_run(self):
desc = TOOL_REGISTRY["create_agent"].description
assert "dry_run" in desc or "dry-run" in desc.lower()
assert "run_agent" in desc
def test_edit_agent_mentions_dry_run_verification(self):
desc = TOOL_REGISTRY["edit_agent"].description
assert "dry_run" in desc or "dry-run" in desc.lower()
assert "dry-run testing" in desc.lower() or "wiring errors" in desc.lower()
def test_run_agent_mentions_dry_run_for_testing(self):
desc = TOOL_REGISTRY["run_agent"].description
assert "dry_run=True" in desc
desc_lower = desc.lower()
assert "test agent wiring" in desc_lower or "simulates execution" in desc_lower
def test_run_agent_dry_run_param_mentions_workflow(self):
schema = TOOL_REGISTRY["run_agent"].as_openai_tool()
params = cast(dict[str, Any], schema["function"].get("parameters", {}))
dry_run_desc = params["properties"]["dry_run"]["description"]
assert "create_agent" in dry_run_desc or "edit_agent" in dry_run_desc
assert "wait_for_result" in dry_run_desc
assert "repeat" in dry_run_desc.lower()
"""Verify tool descriptions and parameters related to the dry-run loop."""
def test_get_agent_building_guide_mentions_workflow(self):
desc = TOOL_REGISTRY["get_agent_building_guide"].description
@@ -82,32 +53,36 @@ class TestToolDescriptionsDryRunLoop:
assert "dry_run" in params["properties"]
assert params["properties"]["dry_run"]["type"] == "boolean"
def test_run_agent_dry_run_param_references_guide(self):
"""After deduplication the dry_run param description points to the guide."""
schema = TOOL_REGISTRY["run_agent"].as_openai_tool()
params = cast(dict[str, Any], schema["function"].get("parameters", {}))
dry_run_desc = params["properties"]["dry_run"]["description"]
assert "agent_generation_guide" in dry_run_desc
class TestPromptingSupplementDryRunLoop:
"""Verify the prompting supplement (via get_sdk_supplement) includes the
iterative workflow. Tests use the public ``get_sdk_supplement`` API
rather than importing the private ``_SHARED_TOOL_NOTES`` directly.
def test_run_agent_dry_run_param_mentions_simulates(self):
schema = TOOL_REGISTRY["run_agent"].as_openai_tool()
params = cast(dict[str, Any], schema["function"].get("parameters", {}))
dry_run_desc = params["properties"]["dry_run"]["description"]
assert "simulates" in dry_run_desc.lower()
class TestPromptingSupplementContent:
"""Verify the prompting supplement (via get_sdk_supplement) includes
essential shared tool notes. After deduplication, the dry-run workflow
lives only in the guide; the supplement carries storage, file-handling,
and tool-discovery notes.
"""
def test_includes_dry_run_section_header(self):
assert "Iterative agent development" in _SDK_SUPPLEMENT
def test_includes_create_dry_run_fix_workflow(self):
assert "create -> dry-run -> fix" in _SDK_SUPPLEMENT.lower()
def test_includes_error_inspection_guidance(self):
"""The supplement should reference error patterns (detailed list
lives in the guide; the supplement carries at least a summary)."""
lower = _SDK_SUPPLEMENT.lower()
assert "errors" in lower or "failed" in lower
assert "null" in lower or "empty" in lower
def test_includes_repeat_until(self):
assert "repeat until" in _SDK_SUPPLEMENT.lower()
def test_includes_tool_discovery_priority(self):
assert "Tool Discovery Priority" in _SDK_SUPPLEMENT
def test_includes_find_block_first(self):
assert "find_block first" in _SDK_SUPPLEMENT or "find_block" in _SDK_SUPPLEMENT
def test_includes_send_authenticated_web_request(self):
assert "SendAuthenticatedWebRequestBlock" in _SDK_SUPPLEMENT
class TestAgentBuildingGuideDryRunLoop:
"""Verify the agent building guide includes the dry-run loop."""