mirror of
https://github.com/Significant-Gravitas/AutoGPT.git
synced 2026-02-04 11:55:11 -05:00
ffc3eff7e2
We need stricter URL validation for the hostname we can request in the block code. ### Changes 🏗️ * Canonicalization: Ensures \ are converted to /, adds http:// if missing, and normalizes the input URL. * Scheme Check: Only http or https are allowed. * Hostname Validation: - Ensures a hostname exists. - Converts it to an IDNA ASCII form to prevent Unicode spoofing. - Verifies that the hostname matches a safe DNS pattern. * Trusted Origins Check: Allows certain hostnames explicitly if needed. * IP Resolution and Blocking: - Resolves the hostname to its IP addresses. - Checks against a list of private/reserved IP networks to prevent SSRF to internal services. ### Checklist 📋 #### For code changes: - [ ] I have clearly listed my changes in the PR description - [ ] I have made a test plan - [ ] I have tested my changes according to the test plan: <!-- Put your test plan here: --> - [ ] ... <details> <summary>Example test plan</summary> - [ ] Create from scratch and execute an agent with at least 3 blocks - [ ] Import an agent from file upload, and confirm it executes correctly - [ ] Upload agent to marketplace - [ ] Import an agent from marketplace and confirm it executes correctly - [ ] Edit an agent from monitor, and confirm it executes correctly </details> #### For configuration changes: - [ ] `.env.example` is updated or already compatible with my changes - [ ] `docker-compose.yml` is updated or already compatible with my changes - [ ] I have included a list of my configuration changes in the PR description (under **Changes**) <details> <summary>Examples of configuration changes</summary> - Changing ports - Adding new services that need to communicate with each other - Secrets or environment variable changes - New or infrastructure changes such as databases </details>
80 lines
2.7 KiB
Python
80 lines
2.7 KiB
Python
import pytest
|
|
|
|
from backend.util.request import validate_url
|
|
|
|
|
|
def test_validate_url():
|
|
# Rejected IP ranges
|
|
with pytest.raises(ValueError):
|
|
validate_url("localhost", [])
|
|
|
|
with pytest.raises(ValueError):
|
|
validate_url("192.168.1.1", [])
|
|
|
|
with pytest.raises(ValueError):
|
|
validate_url("127.0.0.1", [])
|
|
|
|
with pytest.raises(ValueError):
|
|
validate_url("0.0.0.0", [])
|
|
|
|
# Normal URLs
|
|
assert validate_url("google.com/a?b=c", []) == "http://google.com/a?b=c"
|
|
assert validate_url("github.com?key=!@!@", []) == "http://github.com?key=!@!@"
|
|
|
|
# Scheme Enforcement
|
|
with pytest.raises(ValueError):
|
|
validate_url("ftp://example.com", [])
|
|
with pytest.raises(ValueError):
|
|
validate_url("file://example.com", [])
|
|
|
|
# International domain that converts to punycode - should be allowed if public
|
|
assert validate_url("http://xn--exmple-cua.com", []) == "http://xn--exmple-cua.com"
|
|
# If the domain fails IDNA encoding or is invalid, it should raise an error
|
|
with pytest.raises(ValueError):
|
|
validate_url("http://exa◌mple.com", [])
|
|
|
|
# IPv6 Addresses
|
|
with pytest.raises(ValueError):
|
|
validate_url("::1", []) # IPv6 loopback should be blocked
|
|
with pytest.raises(ValueError):
|
|
validate_url("http://[::1]", []) # IPv6 loopback in URL form
|
|
|
|
# Suspicious Characters in Hostname
|
|
with pytest.raises(ValueError):
|
|
validate_url("http://example_underscore.com", [])
|
|
with pytest.raises(ValueError):
|
|
validate_url("http://exa mple.com", []) # Space in hostname
|
|
|
|
# Malformed URLs
|
|
with pytest.raises(ValueError):
|
|
validate_url("http://", []) # No hostname
|
|
with pytest.raises(ValueError):
|
|
validate_url("://missing-scheme", []) # Missing proper scheme
|
|
|
|
# Trusted Origins
|
|
trusted = ["internal-api.company.com", "10.0.0.5"]
|
|
assert (
|
|
validate_url("internal-api.company.com", trusted)
|
|
== "http://internal-api.company.com"
|
|
)
|
|
assert validate_url("10.0.0.5", ["10.0.0.5"]) == "http://10.0.0.5"
|
|
|
|
# Special Characters in Path or Query
|
|
assert (
|
|
validate_url("example.com/path%20with%20spaces", [])
|
|
== "http://example.com/path%20with%20spaces"
|
|
)
|
|
|
|
# Backslashes should be replaced with forward slashes
|
|
assert (
|
|
validate_url("http://example.com\\backslash", [])
|
|
== "http://example.com/backslash"
|
|
)
|
|
|
|
# Check defaulting scheme behavior for valid domains
|
|
assert validate_url("example.com", []) == "http://example.com"
|
|
assert validate_url("https://secure.com", []) == "https://secure.com"
|
|
|
|
# Non-ASCII Characters in Query/Fragment
|
|
assert validate_url("example.com?param=äöü", []) == "http://example.com?param=äöü"
|