fix(web_search): case-fold only scheme and host when matching fetched URLs

This commit is contained in:
Ahmed Allam
2026-09-04 17:08:15 +00:00
parent b1c15ec524
commit 37df57748e
2 changed files with 18 additions and 1 deletions

View File

@@ -6,6 +6,7 @@ import asyncio
import json
import logging
from typing import TYPE_CHECKING, Any, cast
from urllib.parse import urlsplit, urlunsplit
import requests
from agents import RunContextWrapper, function_tool
@@ -133,7 +134,11 @@ def _exa_content(api_key: str, query: str, search_type: str, num_results: int) -
def _normalize_url(url: str) -> str:
return url.strip().rstrip("/").lower()
"""Canonical form for matching: case-fold scheme and host only, drop a trailing slash."""
parts = urlsplit(url.strip())
return urlunsplit(
(parts.scheme.lower(), parts.netloc.lower(), parts.path.rstrip("/"), parts.query, "")
)
def _exa_page_text(api_key: str, urls: list[str]) -> tuple[str, set[str]]:

View File

@@ -343,6 +343,18 @@ def test_do_get_contents_reports_urls_exa_did_not_return(
assert "blocked.example" not in result["content"]
def test_normalize_url_folds_only_scheme_and_host() -> None:
assert tool._normalize_url("HTTPS://Ex.Example/Path/") == tool._normalize_url(
"https://ex.example/Path"
)
assert tool._normalize_url("https://ex.example/Path") != tool._normalize_url(
"https://ex.example/path"
)
assert tool._normalize_url("https://ex.example/p?Q=A") != tool._normalize_url(
"https://ex.example/p?q=a"
)
def test_do_get_contents_omits_the_warning_when_every_page_returns(
monkeypatch: pytest.MonkeyPatch,
) -> None: