mirror of
https://hubproxy.babadafafafafa.cn/https://github.com/usestrix/strix.git
synced 2026-09-20 08:03:42 +08:00
fix(web_search): case-fold only scheme and host when matching fetched URLs
This commit is contained in:
@@ -6,6 +6,7 @@ import asyncio
|
||||
import json
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
|
||||
import requests
|
||||
from agents import RunContextWrapper, function_tool
|
||||
@@ -133,7 +134,11 @@ def _exa_content(api_key: str, query: str, search_type: str, num_results: int) -
|
||||
|
||||
|
||||
def _normalize_url(url: str) -> str:
|
||||
return url.strip().rstrip("/").lower()
|
||||
"""Canonical form for matching: case-fold scheme and host only, drop a trailing slash."""
|
||||
parts = urlsplit(url.strip())
|
||||
return urlunsplit(
|
||||
(parts.scheme.lower(), parts.netloc.lower(), parts.path.rstrip("/"), parts.query, "")
|
||||
)
|
||||
|
||||
|
||||
def _exa_page_text(api_key: str, urls: list[str]) -> tuple[str, set[str]]:
|
||||
|
||||
@@ -343,6 +343,18 @@ def test_do_get_contents_reports_urls_exa_did_not_return(
|
||||
assert "blocked.example" not in result["content"]
|
||||
|
||||
|
||||
def test_normalize_url_folds_only_scheme_and_host() -> None:
|
||||
assert tool._normalize_url("HTTPS://Ex.Example/Path/") == tool._normalize_url(
|
||||
"https://ex.example/Path"
|
||||
)
|
||||
assert tool._normalize_url("https://ex.example/Path") != tool._normalize_url(
|
||||
"https://ex.example/path"
|
||||
)
|
||||
assert tool._normalize_url("https://ex.example/p?Q=A") != tool._normalize_url(
|
||||
"https://ex.example/p?q=a"
|
||||
)
|
||||
|
||||
|
||||
def test_do_get_contents_omits_the_warning_when_every_page_returns(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
|
||||
Reference in New Issue
Block a user