fix(indexer): keep music site category on rust parsing

This commit is contained in:
jxxghp
2026-09-02 08:55:02 +08:00
parent 535d3adf16
commit 8795e30c39
3 changed files with 72 additions and 17 deletions

View File

@@ -1,24 +1,22 @@
import datetime
import re
import traceback
from typing import Any, Optional
from typing import List
from urllib.parse import quote, urlparse, parse_qs
from typing import Any, List, Optional
from urllib.parse import parse_qs, quote, urlparse
from jinja2 import Template
from pyquery import PyQuery
from app.runtime.execution import run_in_threadpool
from app.runtime.settings import get_runtime_setting
from app.runtime.log import logger
from app.schemas.types import MediaType
from app.adapters.network.http import AsyncRequestUtils, RequestUtils
from app.adapters.system import rust as rust_accel
from app.adapters.network.http import RequestUtils, AsyncRequestUtils
from app.foundation import size as size_tools
from app.foundation import temporal as time_tools
from app.foundation import url as url_tools
from app.foundation.url import UrlUtils
from app.runtime.execution import run_in_threadpool
from app.runtime.log import logger
from app.runtime.settings import get_runtime_setting
from app.schemas.types import MediaType
def select_media_categories(category: Optional[dict], mtype: Optional[MediaType]) -> list[dict]:
@@ -79,7 +77,7 @@ class SiteSpider:
@property
def __dir__(self):
"""拒绝外部列举 Spider 的受保护属性。"""
raise AttributeError(f"Cannot read protected attribute!")
raise AttributeError("Cannot read protected attribute!")
def __init__(self,
indexer: dict,
@@ -810,17 +808,28 @@ class SiteSpider:
resolved_type = self.site_media_type
self.torrents_info['category'] = resolved_type.value
def __apply_requested_result_media_type(self, torrents: Optional[List[dict]]) -> List[dict]:
def __apply_result_media_type(self, torrents: Optional[List[dict]]) -> List[dict]:
"""
已由站点查询条件约束类型的结果补充统一媒体类型。
解析结果补充统一媒体类型。
仅在站点配置显式声明 result_media_type=requested 时生效,避免把普通混合搜索结果误分类。
显式请求类型始终覆盖结果;纯媒体站点在分类缺失或未知时兜底,
保持 Python 与 Rust 两条解析路径的分类行为一致。
"""
results = torrents or []
if not self.requested_result_media_type:
if self.requested_result_media_type:
for torrent in results:
torrent["category"] = self.requested_result_media_type.value
return results
if not self.site_media_type:
return results
for torrent in results:
torrent["category"] = self.requested_result_media_type.value
if torrent.get("category") in (
None,
"",
MediaType.UNKNOWN,
MediaType.UNKNOWN.value,
):
torrent["category"] = self.site_media_type.value
return results
def __get_subtitle_field(self, torrent: Any, field_name: str):
@@ -1117,7 +1126,7 @@ class SiteSpider:
result_num=self.result_num
)
if rust_torrents is not None:
return self.__apply_requested_result_media_type(rust_torrents)
return self.__apply_result_media_type(rust_torrents)
# 清空旧结果
self.torrents_info_array = []
@@ -1148,7 +1157,7 @@ class SiteSpider:
torrent_query.clear()
del torrent_query
# 返回数组的副本,防止被后续清理操作影响
return self.__apply_requested_result_media_type(self.torrents_info_array.copy())
return self.__apply_result_media_type(self.torrents_info_array.copy())
except Exception as err:
self.is_error = True
logger.warn(f"错误:{self.indexername} {str(err)}")

View File

@@ -1,5 +1,6 @@
from typing import Any, NoReturn
from unittest.mock import patch
from urllib.parse import parse_qs, urlparse
import pytest
@@ -45,6 +46,9 @@ def _dicmusic_indexer() -> dict[str, Any]:
"searchstr": "{keyword}",
},
},
"browse": {
"path": "torrents.php?searchsubmit=1&group_results=0&page={page}",
},
"torrents": {
"list": {"selector": "table#torrent_table > tbody > tr.torrent"},
"fields": {
@@ -79,6 +83,22 @@ def _dicmusic_indexer() -> dict[str, Any]:
}
def test_dicmusic_browse_requests_ungrouped_torrent_rows() -> None:
"""DIC Music 无关键词浏览也应显式关闭 Gazelle 结果分组。"""
spider = SiteSpider(_dicmusic_indexer(), page=2)
browse_url = spider._SiteSpider__get_search_url()
parsed_url = urlparse(browse_url)
query = parse_qs(parsed_url.query)
assert parsed_url.path == "/torrents.php"
assert query == {
"searchsubmit": ["1"],
"group_results": ["0"],
"page": ["2"],
}
def test_dicmusic_search_uses_complete_music_title_in_python_fallback() -> None:
"""DIC Music Python 兜底解析应从非分组行保留完整音乐名。"""
with patch(
@@ -91,6 +111,7 @@ def test_dicmusic_search_uses_complete_music_title_in_python_fallback() -> None:
assert len(results) == 1
assert results[0]["title"] == "BTS - ARIRANG"
assert results[0]["category"] == MediaType.MUSIC.value
assert results[0]["page_url"].endswith("torrents.php?id=123&torrentid=456")
assert results[0]["enclosure"].endswith("torrents.php?action=download&id=456")
@@ -116,3 +137,4 @@ def test_dicmusic_search_uses_complete_music_title_in_rust_parser(
assert len(results) == 1
assert results[0]["title"] == "BTS - ARIRANG"
assert results[0]["category"] == MediaType.MUSIC.value

View File

@@ -1,3 +1,5 @@
from unittest.mock import patch
from app.modules.indexer.spider import (
SiteSpider,
resolve_category_media_type,
@@ -63,6 +65,28 @@ def test_site_level_music_type_fills_missing_torrent_category():
assert spider.torrents_info["category"] == MediaType.MUSIC.value
def test_site_level_music_type_fills_missing_rust_result_category():
"""Rust 解析未返回分类时应使用纯音乐站点的站点级类型。"""
indexer = {
"id": "music",
"name": "Music",
"domain": "https://music.example/",
"media_type": "music",
"search": {"paths": [{"path": "torrents.php"}]},
"torrents": {"list": {}, "fields": {}},
}
with patch(
"app.modules.indexer.spider.rust_accel.parse_indexer_torrents",
return_value=[{"title": "Artist - Album", "category": None}],
):
results = SiteSpider(indexer=indexer, mtype=MediaType.MUSIC).parse(
"<html><body></body></html>"
)
assert results[0]["category"] == MediaType.MUSIC.value
def test_requested_result_media_type_overrides_unrepresentable_site_category():
"""音乐专属查询使用非主分类筛选时,应按显式契约把结果标记为音乐。"""
spider = SiteSpider(