mirror of
https://hubproxy.babadafafafafa.cn/https://github.com/jxxghp/MoviePilot.git
synced 2026-09-20 08:03:34 +08:00
fix(indexer): keep music site category on rust parsing
This commit is contained in:
@@ -1,24 +1,22 @@
|
||||
import datetime
|
||||
import re
|
||||
import traceback
|
||||
from typing import Any, Optional
|
||||
from typing import List
|
||||
from urllib.parse import quote, urlparse, parse_qs
|
||||
from typing import Any, List, Optional
|
||||
from urllib.parse import parse_qs, quote, urlparse
|
||||
|
||||
from jinja2 import Template
|
||||
from pyquery import PyQuery
|
||||
|
||||
from app.runtime.execution import run_in_threadpool
|
||||
from app.runtime.settings import get_runtime_setting
|
||||
|
||||
from app.runtime.log import logger
|
||||
from app.schemas.types import MediaType
|
||||
from app.adapters.network.http import AsyncRequestUtils, RequestUtils
|
||||
from app.adapters.system import rust as rust_accel
|
||||
from app.adapters.network.http import RequestUtils, AsyncRequestUtils
|
||||
from app.foundation import size as size_tools
|
||||
from app.foundation import temporal as time_tools
|
||||
from app.foundation import url as url_tools
|
||||
from app.foundation.url import UrlUtils
|
||||
from app.runtime.execution import run_in_threadpool
|
||||
from app.runtime.log import logger
|
||||
from app.runtime.settings import get_runtime_setting
|
||||
from app.schemas.types import MediaType
|
||||
|
||||
|
||||
def select_media_categories(category: Optional[dict], mtype: Optional[MediaType]) -> list[dict]:
|
||||
@@ -79,7 +77,7 @@ class SiteSpider:
|
||||
@property
|
||||
def __dir__(self):
|
||||
"""拒绝外部列举 Spider 的受保护属性。"""
|
||||
raise AttributeError(f"Cannot read protected attribute!")
|
||||
raise AttributeError("Cannot read protected attribute!")
|
||||
|
||||
def __init__(self,
|
||||
indexer: dict,
|
||||
@@ -810,17 +808,28 @@ class SiteSpider:
|
||||
resolved_type = self.site_media_type
|
||||
self.torrents_info['category'] = resolved_type.value
|
||||
|
||||
def __apply_requested_result_media_type(self, torrents: Optional[List[dict]]) -> List[dict]:
|
||||
def __apply_result_media_type(self, torrents: Optional[List[dict]]) -> List[dict]:
|
||||
"""
|
||||
为已由站点查询条件约束类型的结果补充统一媒体类型。
|
||||
为解析结果补充统一媒体类型。
|
||||
|
||||
仅在站点配置显式声明 result_media_type=requested 时生效,避免把普通混合搜索结果误分类。
|
||||
显式请求类型始终覆盖结果;纯媒体站点在分类缺失或未知时兜底,
|
||||
保持 Python 与 Rust 两条解析路径的分类行为一致。
|
||||
"""
|
||||
results = torrents or []
|
||||
if not self.requested_result_media_type:
|
||||
if self.requested_result_media_type:
|
||||
for torrent in results:
|
||||
torrent["category"] = self.requested_result_media_type.value
|
||||
return results
|
||||
if not self.site_media_type:
|
||||
return results
|
||||
for torrent in results:
|
||||
torrent["category"] = self.requested_result_media_type.value
|
||||
if torrent.get("category") in (
|
||||
None,
|
||||
"",
|
||||
MediaType.UNKNOWN,
|
||||
MediaType.UNKNOWN.value,
|
||||
):
|
||||
torrent["category"] = self.site_media_type.value
|
||||
return results
|
||||
|
||||
def __get_subtitle_field(self, torrent: Any, field_name: str):
|
||||
@@ -1117,7 +1126,7 @@ class SiteSpider:
|
||||
result_num=self.result_num
|
||||
)
|
||||
if rust_torrents is not None:
|
||||
return self.__apply_requested_result_media_type(rust_torrents)
|
||||
return self.__apply_result_media_type(rust_torrents)
|
||||
|
||||
# 清空旧结果
|
||||
self.torrents_info_array = []
|
||||
@@ -1148,7 +1157,7 @@ class SiteSpider:
|
||||
torrent_query.clear()
|
||||
del torrent_query
|
||||
# 返回数组的副本,防止被后续清理操作影响
|
||||
return self.__apply_requested_result_media_type(self.torrents_info_array.copy())
|
||||
return self.__apply_result_media_type(self.torrents_info_array.copy())
|
||||
except Exception as err:
|
||||
self.is_error = True
|
||||
logger.warn(f"错误:{self.indexername} {str(err)}")
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from typing import Any, NoReturn
|
||||
from unittest.mock import patch
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -45,6 +46,9 @@ def _dicmusic_indexer() -> dict[str, Any]:
|
||||
"searchstr": "{keyword}",
|
||||
},
|
||||
},
|
||||
"browse": {
|
||||
"path": "torrents.php?searchsubmit=1&group_results=0&page={page}",
|
||||
},
|
||||
"torrents": {
|
||||
"list": {"selector": "table#torrent_table > tbody > tr.torrent"},
|
||||
"fields": {
|
||||
@@ -79,6 +83,22 @@ def _dicmusic_indexer() -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def test_dicmusic_browse_requests_ungrouped_torrent_rows() -> None:
|
||||
"""DIC Music 无关键词浏览也应显式关闭 Gazelle 结果分组。"""
|
||||
spider = SiteSpider(_dicmusic_indexer(), page=2)
|
||||
|
||||
browse_url = spider._SiteSpider__get_search_url()
|
||||
parsed_url = urlparse(browse_url)
|
||||
query = parse_qs(parsed_url.query)
|
||||
|
||||
assert parsed_url.path == "/torrents.php"
|
||||
assert query == {
|
||||
"searchsubmit": ["1"],
|
||||
"group_results": ["0"],
|
||||
"page": ["2"],
|
||||
}
|
||||
|
||||
|
||||
def test_dicmusic_search_uses_complete_music_title_in_python_fallback() -> None:
|
||||
"""DIC Music Python 兜底解析应从非分组行保留完整音乐名。"""
|
||||
with patch(
|
||||
@@ -91,6 +111,7 @@ def test_dicmusic_search_uses_complete_music_title_in_python_fallback() -> None:
|
||||
|
||||
assert len(results) == 1
|
||||
assert results[0]["title"] == "BTS - ARIRANG"
|
||||
assert results[0]["category"] == MediaType.MUSIC.value
|
||||
assert results[0]["page_url"].endswith("torrents.php?id=123&torrentid=456")
|
||||
assert results[0]["enclosure"].endswith("torrents.php?action=download&id=456")
|
||||
|
||||
@@ -116,3 +137,4 @@ def test_dicmusic_search_uses_complete_music_title_in_rust_parser(
|
||||
|
||||
assert len(results) == 1
|
||||
assert results[0]["title"] == "BTS - ARIRANG"
|
||||
assert results[0]["category"] == MediaType.MUSIC.value
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.modules.indexer.spider import (
|
||||
SiteSpider,
|
||||
resolve_category_media_type,
|
||||
@@ -63,6 +65,28 @@ def test_site_level_music_type_fills_missing_torrent_category():
|
||||
assert spider.torrents_info["category"] == MediaType.MUSIC.value
|
||||
|
||||
|
||||
def test_site_level_music_type_fills_missing_rust_result_category():
|
||||
"""Rust 解析未返回分类时应使用纯音乐站点的站点级类型。"""
|
||||
indexer = {
|
||||
"id": "music",
|
||||
"name": "Music",
|
||||
"domain": "https://music.example/",
|
||||
"media_type": "music",
|
||||
"search": {"paths": [{"path": "torrents.php"}]},
|
||||
"torrents": {"list": {}, "fields": {}},
|
||||
}
|
||||
|
||||
with patch(
|
||||
"app.modules.indexer.spider.rust_accel.parse_indexer_torrents",
|
||||
return_value=[{"title": "Artist - Album", "category": None}],
|
||||
):
|
||||
results = SiteSpider(indexer=indexer, mtype=MediaType.MUSIC).parse(
|
||||
"<html><body></body></html>"
|
||||
)
|
||||
|
||||
assert results[0]["category"] == MediaType.MUSIC.value
|
||||
|
||||
|
||||
def test_requested_result_media_type_overrides_unrepresentable_site_category():
|
||||
"""音乐专属查询使用非主分类筛选时,应按显式契约把结果标记为音乐。"""
|
||||
spider = SiteSpider(
|
||||
|
||||
Reference in New Issue
Block a user