mirror of
https://github.com/ZhuLinsen/daily_stock_analysis
synced 2026-09-20 10:53:33 +08:00
eb1333a678
* feat: add one-shot index analysis via --stocks with structured AnalysisTarget pipeline * fix: render index data sources in aggregate reports * fix: refresh index registry before --stocks parsing and filter indices on CN holidays --------- Co-authored-by: zhulinsen <42829555+ZhuLinsen@users.noreply.github.com>
4807 lines
202 KiB
Python
4807 lines
202 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""
|
||
===================================
|
||
数据源基类与管理器
|
||
===================================
|
||
|
||
设计模式：策略模式 (Strategy Pattern)
|
||
- BaseFetcher: 抽象基类，定义统一接口
|
||
- DataFetcherManager: 策略管理器，实现自动切换
|
||
|
||
防封禁策略：
|
||
1. 每个 Fetcher 内置流控逻辑
|
||
2. 失败自动切换到下一个数据源
|
||
3. 指数退避重试机制
|
||
"""
|
||
|
||
import logging
|
||
import random
|
||
import time
|
||
from threading import BoundedSemaphore, RLock, Thread
|
||
from abc import ABC, abstractmethod
|
||
from datetime import datetime, timezone
|
||
from typing import Callable, Optional, List, Tuple, Dict, Any
|
||
|
||
import pandas as pd
|
||
import numpy as np
|
||
from src.data.stock_index_loader import get_index_stock_name
|
||
from src.data.stock_mapping import STOCK_NAME_MAP, is_meaningful_stock_name
|
||
from src.services.market_symbol_utils import is_suffix_market_symbol
|
||
from src.services.run_diagnostics import record_provider_run, record_provider_run_started
|
||
from src.services.stock_list_parser import AnalysisTarget, ParseStatus, parse_analysis_target
|
||
from .fundamental_adapter import AkshareFundamentalAdapter
|
||
from .yfinance_fundamental_adapter import YfinanceFundamentalAdapter
|
||
from .realtime_types import CircuitBreaker
|
||
|
||
# 配置日志
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
# === 标准化列名定义 ===
|
||
STANDARD_COLUMNS = ['date', 'open', 'high', 'low', 'close', 'volume', 'amount', 'pct_chg']
|
||
|
||
|
||
def unwrap_exception(exc: Exception) -> Exception:
|
||
"""
|
||
Follow chained exceptions and return the deepest non-cyclic cause.
|
||
"""
|
||
current = exc
|
||
visited = set()
|
||
|
||
while current is not None and id(current) not in visited:
|
||
visited.add(id(current))
|
||
next_exc = current.__cause__ or current.__context__
|
||
if next_exc is None:
|
||
break
|
||
current = next_exc
|
||
|
||
return current
|
||
|
||
|
||
def summarize_exception(exc: Exception) -> Tuple[str, str]:
|
||
"""
|
||
Build a stable summary for logs while preserving the application-layer message.
|
||
"""
|
||
root = unwrap_exception(exc)
|
||
error_type = type(root).__name__
|
||
message = str(exc).strip() or str(root).strip() or error_type
|
||
return error_type, " ".join(message.split())
|
||
|
||
|
||
def normalize_stock_code(stock_code: str) -> str:
|
||
"""
|
||
Normalize stock code by stripping exchange prefixes/suffixes.
|
||
|
||
Accepted formats and their normalized results:
|
||
- '600519' -> '600519' (already clean)
|
||
- 'SH600519' -> '600519' (strip SH prefix)
|
||
- 'SH.600519' -> '600519' (strip SH. prefix)
|
||
- 'SZ000001' -> '000001' (strip SZ prefix)
|
||
- 'SS600519' -> '600519' (strip legacy Yahoo Shanghai prefix)
|
||
- 'SZ.000001' -> '000001' (strip SZ. prefix)
|
||
- 'BJ920748' -> '920748' (strip BJ prefix, BSE)
|
||
- 'BJ.920748' -> '920748' (strip BJ. prefix, BSE)
|
||
- 'sh600519' -> '600519' (case-insensitive)
|
||
- '600519.SH' -> '600519' (strip .SH suffix)
|
||
- '000001.SZ' -> '000001' (strip .SZ suffix)
|
||
- '920748.BJ' -> '920748' (strip .BJ suffix, BSE)
|
||
- 'HK00700' -> 'HK00700' (keep HK prefix for HK stocks)
|
||
- '1810.HK' -> 'HK01810' (normalize HK suffix to canonical prefix form)
|
||
- '7203.T' -> '7203.T' (keep Japan Yahoo suffix form)
|
||
- '005930.KS' -> '005930.KS' (keep Korea Yahoo suffix form)
|
||
- '2330.TW' -> '2330.TW' (keep Taiwan TWSE Yahoo suffix form)
|
||
- '6505.TWO' -> '6505.TWO' (keep Taiwan TPEx Yahoo suffix form)
|
||
- 'AAPL' -> 'AAPL' (keep US stock ticker as-is)
|
||
|
||
This function is applied at the DataProviderManager layer so that
|
||
all individual fetchers receive a clean 6-digit code (for A-shares/ETFs).
|
||
"""
|
||
code = stock_code.strip()
|
||
upper = code.upper()
|
||
|
||
# Normalize HK prefix to a canonical 5-digit form (e.g. hk1810 -> HK01810)
|
||
if upper.startswith('HK') and not upper.startswith('HK.'):
|
||
candidate = upper[2:]
|
||
if candidate.isdigit() and 1 <= len(candidate) <= 5:
|
||
return f"HK{candidate.zfill(5)}"
|
||
|
||
# Strip SH/SZ/SS prefix (e.g. SH600519 -> 600519, SS600519 -> 600519)
|
||
if upper.startswith(('SH', 'SZ', 'SS')) and not upper.startswith(('SH.', 'SZ.', 'SS.')):
|
||
candidate = code[2:]
|
||
# Only strip if the remainder looks like a valid numeric code
|
||
if candidate.isdigit() and len(candidate) in (5, 6):
|
||
return candidate
|
||
|
||
# Strip dotted SH/SZ/SS prefix (e.g. SH.600519 -> 600519)
|
||
if upper.startswith(('SH.', 'SZ.', 'SS.')):
|
||
candidate = code[3:]
|
||
if candidate.isdigit() and len(candidate) in (5, 6):
|
||
return candidate
|
||
|
||
# Strip BJ prefix (e.g. BJ920748 -> 920748)
|
||
if upper.startswith('BJ') and not upper.startswith('BJ.'):
|
||
candidate = code[2:]
|
||
if candidate.isdigit() and len(candidate) == 6:
|
||
return candidate
|
||
|
||
# Strip dotted BJ prefix (e.g. BJ.920748 -> 920748)
|
||
if upper.startswith('BJ.'):
|
||
candidate = code[3:]
|
||
if candidate.isdigit() and len(candidate) == 6:
|
||
return candidate
|
||
|
||
# Strip .SH/.SZ/.BJ suffix (e.g. 600519.SH -> 600519, 920748.BJ -> 920748)
|
||
# while preserving explicit Yahoo suffix forms for JP/KR/TW.
|
||
if '.' in code:
|
||
base, suffix = code.rsplit('.', 1)
|
||
if suffix.upper() == 'T' and base.isdigit() and len(base) in (4, 5):
|
||
return f"{base}.{suffix.upper()}"
|
||
if suffix.upper() in ('KS', 'KQ') and base.isdigit() and len(base) == 6:
|
||
return f"{base}.{suffix.upper()}"
|
||
if suffix.upper() in ('TW', 'TWO') and base.isdigit() and 4 <= len(base) <= 6:
|
||
return f"{base}.{suffix.upper()}"
|
||
if suffix.upper() == 'HK' and base.isdigit() and 1 <= len(base) <= 5:
|
||
return f"HK{base.zfill(5)}"
|
||
if base.upper() in ('SH', 'SS', 'SZ', 'BJ') and suffix.isdigit():
|
||
return suffix
|
||
if suffix.upper() in ('SH', 'SZ', 'SS', 'BJ') and base.isdigit():
|
||
return base
|
||
|
||
return code
|
||
|
||
|
||
ETF_PREFIXES = ("51", "52", "56", "58", "15", "16", "18")
|
||
|
||
|
||
def _is_us_market(code: str) -> bool:
|
||
"""判断是否为美股/美股指数代码（不含中文前后缀）。"""
|
||
from .us_index_mapping import is_us_stock_code, is_us_index_code
|
||
|
||
normalized = (code or "").strip().upper()
|
||
return is_us_index_code(normalized) or is_us_stock_code(normalized)
|
||
|
||
|
||
def _is_hk_market(code: str) -> bool:
|
||
"""
|
||
判定是否为港股代码。
|
||
|
||
支持 ``.HK`` 后缀、``HK00700`` 前缀形式，以及 4-5 位纯数字裸码
|
||
（A 股 ETF/股票为 6 位，与港股 4-5 位裸数字不冲突）。``YfinanceFetcher``
|
||
与 ``AkshareFetcher`` / ``LongbridgeFetcher`` 的 ``_is_hk_code`` 与本
|
||
函数对裸港股码的位数范围保持一致。
|
||
"""
|
||
normalized = (code or "").strip().upper()
|
||
if normalized.endswith(".HK"):
|
||
base = normalized[:-3]
|
||
return base.isdigit() and 1 <= len(base) <= 5
|
||
if normalized.startswith("HK"):
|
||
digits = normalized[2:]
|
||
return digits.isdigit() and 1 <= len(digits) <= 5
|
||
if normalized.isdigit() and 4 <= len(normalized) <= 5:
|
||
return True
|
||
return False
|
||
|
||
|
||
def _is_jp_market(code: str) -> bool:
|
||
"""判定是否为日本 Yahoo Finance suffix 代码（如 7203.T）。"""
|
||
return is_suffix_market_symbol(code, "jp")
|
||
|
||
|
||
def _is_kr_market(code: str) -> bool:
|
||
"""判定是否为韩国 Yahoo Finance suffix 代码（如 005930.KS / 035720.KQ）。"""
|
||
return is_suffix_market_symbol(code, "kr")
|
||
|
||
|
||
def _is_tw_market(code: str) -> bool:
|
||
"""判定是否为台湾 Yahoo Finance suffix 代码（TWSE 上市 2330.TW / TPEx 上柜 6505.TWO）。
|
||
|
||
台股 base 为 4-6 位（普通股 4 位，ETF/其他至 6 位，如 00878 / 006208）。
|
||
仅带 .TW/.TWO 后缀的代码才识别为台股，裸 6 位代码仍按 A 股语义处理。
|
||
"""
|
||
return is_suffix_market_symbol(code, "tw")
|
||
|
||
|
||
def _is_etf_code(code: str) -> bool:
|
||
"""判定 A 股 ETF 基金代码（保守规则）。"""
|
||
normalized = normalize_stock_code(code)
|
||
return (
|
||
normalized.isdigit()
|
||
and len(normalized) == 6
|
||
and normalized.startswith(ETF_PREFIXES)
|
||
)
|
||
|
||
|
||
def _coerce_chip_metric(value: Any) -> Optional[float]:
|
||
try:
|
||
if value is None:
|
||
return None
|
||
numeric = float(value)
|
||
if np.isnan(numeric):
|
||
return None
|
||
return numeric
|
||
except (TypeError, ValueError):
|
||
return None
|
||
|
||
|
||
def _is_meaningful_chip_distribution(chip: Any) -> bool:
|
||
"""Validate that a provider returned usable core chip metrics."""
|
||
if chip is None:
|
||
return False
|
||
avg_cost = _coerce_chip_metric(getattr(chip, "avg_cost", None))
|
||
concentration_90 = _coerce_chip_metric(getattr(chip, "concentration_90", None))
|
||
concentration_70 = _coerce_chip_metric(getattr(chip, "concentration_70", None))
|
||
return (
|
||
avg_cost is not None
|
||
and avg_cost > 0
|
||
and (
|
||
(concentration_90 is not None and concentration_90 >= 0)
|
||
or (concentration_70 is not None and concentration_70 >= 0)
|
||
)
|
||
)
|
||
|
||
|
||
def _market_tag(code: str) -> str:
|
||
"""返回市场标签: cn/us/hk/jp/kr/tw."""
|
||
if _is_us_market(code):
|
||
return "us"
|
||
if _is_hk_market(code):
|
||
return "hk"
|
||
if _is_jp_market(code):
|
||
return "jp"
|
||
if _is_kr_market(code):
|
||
return "kr"
|
||
if _is_tw_market(code):
|
||
return "tw"
|
||
return "cn"
|
||
|
||
|
||
def is_bse_code(code: str) -> bool:
|
||
"""
|
||
Check if the code is a Beijing Stock Exchange (BSE) A-share code.
|
||
|
||
BSE rules (2026):
|
||
- New format (2024+): 92xxxx main trading codes
|
||
- Historical ranges: 43xxxx, 83xxxx, 87xxxx, 88xxxx
|
||
- Special instruments: 81xxxx convertible bonds, 82xxxx preferred shares
|
||
- Subscription codes: 889xxx
|
||
Note: 900xxx are Shanghai B-shares and must return False.
|
||
"""
|
||
c = (code or "").strip().split(".")[0]
|
||
if len(c) != 6 or not c.isdigit():
|
||
return False
|
||
|
||
if c.startswith("900"):
|
||
return False
|
||
|
||
return c.startswith(("92", "43", "81", "82", "83", "87", "88"))
|
||
|
||
def is_st_stock(name: str) -> bool:
|
||
"""
|
||
Check if the stock is an ST or *ST stock based on its name.
|
||
|
||
ST stocks have special trading rules and typically a ±5% limit.
|
||
"""
|
||
n = (name or "").upper()
|
||
return 'ST' in n
|
||
|
||
def is_kc_cy_stock(code: str) -> bool:
|
||
"""
|
||
Check if the stock is a STAR Market (科创板) or ChiNext (创业板) stock based on its code.
|
||
|
||
- STAR Market: Codes starting with 688
|
||
- ChiNext: Codes starting with 300
|
||
Both have a ±20% limit.
|
||
"""
|
||
c = (code or "").strip().split(".")[0]
|
||
return c.startswith("688") or c.startswith("30")
|
||
|
||
|
||
def canonical_stock_code(code: str) -> str:
|
||
"""
|
||
Return the canonical (uppercase) form of a stock code.
|
||
|
||
This is a display/storage layer concern, distinct from normalize_stock_code
|
||
which strips exchange prefixes. Apply at system input boundaries to ensure
|
||
consistent case across BOT, WEB UI, API, and CLI paths (Issue #355).
|
||
|
||
Examples:
|
||
'aapl' -> 'AAPL'
|
||
'AAPL' -> 'AAPL'
|
||
'600519' -> '600519' (digits are unchanged)
|
||
'hk00700' -> 'HK00700'
|
||
"""
|
||
return (code or "").strip().upper()
|
||
|
||
|
||
class DataFetchError(Exception):
|
||
"""数据获取异常基类"""
|
||
pass
|
||
|
||
|
||
class RateLimitError(DataFetchError):
|
||
"""API 速率限制异常"""
|
||
pass
|
||
|
||
|
||
class DataSourceUnavailableError(DataFetchError):
|
||
"""数据源不可用异常"""
|
||
pass
|
||
|
||
|
||
class BaseFetcher(ABC):
|
||
"""
|
||
数据源抽象基类
|
||
|
||
职责：
|
||
1. 定义统一的数据获取接口
|
||
2. 提供数据标准化方法
|
||
3. 实现通用的技术指标计算
|
||
|
||
子类实现：
|
||
- _fetch_raw_data(): 从具体数据源获取原始数据
|
||
- _normalize_data(): 将原始数据转换为标准格式
|
||
"""
|
||
|
||
name: str = "BaseFetcher"
|
||
priority: int = 99 # 优先级数字越小越优先
|
||
allow_empty_daily_data: bool = False
|
||
|
||
@abstractmethod
|
||
def _fetch_raw_data(self, stock_code: str, start_date: str, end_date: str) -> pd.DataFrame:
|
||
"""
|
||
从数据源获取原始数据（子类必须实现）
|
||
|
||
Args:
|
||
stock_code: 股票代码，如 '600519', '000001'
|
||
start_date: 开始日期，格式 'YYYY-MM-DD'
|
||
end_date: 结束日期，格式 'YYYY-MM-DD'
|
||
|
||
Returns:
|
||
原始数据 DataFrame（列名因数据源而异）
|
||
"""
|
||
pass
|
||
|
||
@abstractmethod
|
||
def _normalize_data(self, df: pd.DataFrame, stock_code: str) -> pd.DataFrame:
|
||
"""
|
||
标准化数据列名（子类必须实现）
|
||
|
||
将不同数据源的列名统一为：
|
||
['date', 'open', 'high', 'low', 'close', 'volume', 'amount', 'pct_chg']
|
||
"""
|
||
pass
|
||
|
||
def get_main_indices(self, region: str = "cn") -> Optional[List[Dict[str, Any]]]:
|
||
"""
|
||
获取主要指数实时行情
|
||
|
||
Args:
|
||
region: 市场区域，cn=A股 us=美股
|
||
|
||
Returns:
|
||
List[Dict]: 指数列表，每个元素为字典，包含:
|
||
- code: 指数代码
|
||
- name: 指数名称
|
||
- current: 当前点位
|
||
- change: 涨跌点数
|
||
- change_pct: 涨跌幅(%)
|
||
- volume: 成交量
|
||
- amount: 成交额
|
||
"""
|
||
return None
|
||
|
||
def get_market_stats(self) -> Optional[Dict[str, Any]]:
|
||
"""
|
||
获取市场涨跌统计
|
||
|
||
Returns:
|
||
Dict: 包含:
|
||
- up_count: 上涨家数
|
||
- down_count: 下跌家数
|
||
- flat_count: 平盘家数
|
||
- limit_up_count: 涨停家数
|
||
- limit_down_count: 跌停家数
|
||
- total_amount: 两市成交额
|
||
"""
|
||
return None
|
||
|
||
def get_sector_rankings(self, n: int = 5) -> Optional[Tuple[List[Dict], List[Dict]]]:
|
||
"""
|
||
获取板块涨跌榜
|
||
|
||
Args:
|
||
n: 返回前n个
|
||
|
||
Returns:
|
||
Tuple: (领涨板块列表, 领跌板块列表)
|
||
"""
|
||
return None
|
||
|
||
def get_concept_rankings(self, n: int = 5) -> Optional[Tuple[List[Dict], List[Dict]]]:
|
||
"""
|
||
获取概念/题材涨跌榜。
|
||
|
||
Returns:
|
||
Tuple: (领涨概念列表, 领跌概念列表)
|
||
"""
|
||
return None
|
||
|
||
def get_hot_stocks(self, n: int = 10) -> Optional[List[Dict[str, Any]]]:
|
||
"""
|
||
获取市场人气股榜。
|
||
|
||
Returns:
|
||
List[Dict]: 人气股列表
|
||
"""
|
||
return None
|
||
|
||
def get_limit_up_pool(
|
||
self,
|
||
date: Optional[str] = None,
|
||
n: int = 20,
|
||
) -> Optional[List[Dict[str, Any]]]:
|
||
"""
|
||
获取涨停池/连板梯队。
|
||
|
||
Args:
|
||
date: YYYYMMDD，默认由具体数据源决定
|
||
n: 返回条数
|
||
"""
|
||
return None
|
||
|
||
def get_daily_data(
|
||
self,
|
||
stock_code: str,
|
||
start_date: Optional[str] = None,
|
||
end_date: Optional[str] = None,
|
||
days: int = 30
|
||
) -> pd.DataFrame:
|
||
"""
|
||
获取日线数据（统一入口）
|
||
|
||
流程：
|
||
1. 计算日期范围
|
||
2. 调用子类获取原始数据
|
||
3. 标准化列名
|
||
4. 计算技术指标
|
||
|
||
Args:
|
||
stock_code: 股票代码
|
||
start_date: 开始日期（可选）
|
||
end_date: 结束日期（可选，默认今天）
|
||
days: 获取天数（当 start_date 未指定时使用）
|
||
|
||
Returns:
|
||
标准化的 DataFrame，包含技术指标
|
||
"""
|
||
# 计算日期范围
|
||
if end_date is None:
|
||
end_date = datetime.now().strftime('%Y-%m-%d')
|
||
|
||
if start_date is None:
|
||
# 默认获取最近 30 个交易日（按日历日估算，多取一些）
|
||
from datetime import timedelta
|
||
start_dt = datetime.strptime(end_date, '%Y-%m-%d') - timedelta(days=days * 2)
|
||
start_date = start_dt.strftime('%Y-%m-%d')
|
||
|
||
request_start = time.time()
|
||
logger.info(f"[{self.name}] 开始获取 {stock_code} 日线数据: 范围={start_date} ~ {end_date}")
|
||
|
||
try:
|
||
# Step 1: 获取原始数据
|
||
raw_df = self._fetch_raw_data(stock_code, start_date, end_date)
|
||
|
||
if raw_df is None:
|
||
raise DataFetchError(f"[{self.name}] 未获取到 {stock_code} 的数据")
|
||
if raw_df.empty:
|
||
elapsed = time.time() - request_start
|
||
logger.info(
|
||
f"[{self.name}] {stock_code} 返回空日线结果: 范围={start_date} ~ {end_date}, "
|
||
f"elapsed={elapsed:.2f}s"
|
||
)
|
||
if self.allow_empty_daily_data:
|
||
return pd.DataFrame(columns=STANDARD_COLUMNS)
|
||
raise DataFetchError(f"[{self.name}] 未获取到 {stock_code} 的数据")
|
||
|
||
# Step 2: 标准化列名
|
||
df = self._normalize_data(raw_df, stock_code)
|
||
|
||
# Step 3: 数据清洗
|
||
df = self._clean_data(df)
|
||
|
||
# Step 4: 计算技术指标
|
||
df = self._calculate_indicators(df)
|
||
|
||
elapsed = time.time() - request_start
|
||
logger.info(
|
||
f"[{self.name}] {stock_code} 获取成功: 范围={start_date} ~ {end_date}, "
|
||
f"rows={len(df)}, elapsed={elapsed:.2f}s"
|
||
)
|
||
return df
|
||
|
||
except Exception as e:
|
||
elapsed = time.time() - request_start
|
||
error_type, error_reason = summarize_exception(e)
|
||
logger.error(
|
||
f"[{self.name}] {stock_code} 获取失败: 范围={start_date} ~ {end_date}, "
|
||
f"error_type={error_type}, elapsed={elapsed:.2f}s, reason={error_reason}"
|
||
)
|
||
raise DataFetchError(f"[{self.name}] {stock_code}: {error_reason}") from e
|
||
|
||
def _clean_data(self, df: pd.DataFrame) -> pd.DataFrame:
|
||
"""
|
||
数据清洗
|
||
|
||
处理：
|
||
1. 确保日期列格式正确
|
||
2. 数值类型转换
|
||
3. 去除空值行
|
||
4. 按日期排序
|
||
"""
|
||
df = df.copy()
|
||
|
||
# 确保日期列为 datetime 类型
|
||
if 'date' in df.columns:
|
||
df['date'] = pd.to_datetime(df['date'])
|
||
|
||
# 数值列类型转换
|
||
numeric_cols = ['open', 'high', 'low', 'close', 'volume', 'amount', 'pct_chg']
|
||
for col in numeric_cols:
|
||
if col in df.columns:
|
||
df[col] = pd.to_numeric(df[col], errors='coerce')
|
||
|
||
# 去除关键列为空的行
|
||
df = df.dropna(subset=['close', 'volume'])
|
||
|
||
# 按日期升序排序
|
||
df = df.sort_values('date', ascending=True).reset_index(drop=True)
|
||
|
||
return df
|
||
|
||
def _calculate_indicators(self, df: pd.DataFrame) -> pd.DataFrame:
|
||
"""
|
||
计算技术指标
|
||
|
||
计算指标：
|
||
- MA5, MA10, MA20: 移动平均线
|
||
- Volume_Ratio: 量比（今日成交量 / 5日平均成交量）
|
||
"""
|
||
df = df.copy()
|
||
|
||
# 移动平均线
|
||
df['ma5'] = df['close'].rolling(window=5, min_periods=1).mean()
|
||
df['ma10'] = df['close'].rolling(window=10, min_periods=1).mean()
|
||
df['ma20'] = df['close'].rolling(window=20, min_periods=1).mean()
|
||
|
||
# 量比：当日成交量 / 5日平均成交量
|
||
# 注意：此处的 volume_ratio 是“日线成交量 / 前5日均量(shift 1)”的相对倍数，
|
||
# 与部分交易软件口径的“分时量比（同一时刻对比）”不同，含义更接近“放量倍数”。
|
||
# 该行为目前保留（按需求不改逻辑）。
|
||
avg_volume_5 = df['volume'].rolling(window=5, min_periods=1).mean()
|
||
df['volume_ratio'] = df['volume'] / avg_volume_5.shift(1)
|
||
df['volume_ratio'] = df['volume_ratio'].fillna(1.0)
|
||
|
||
# 保留2位小数
|
||
for col in ['ma5', 'ma10', 'ma20', 'volume_ratio']:
|
||
if col in df.columns:
|
||
df[col] = df[col].round(2)
|
||
|
||
return df
|
||
|
||
@staticmethod
|
||
def random_sleep(min_seconds: float = 1.0, max_seconds: float = 3.0) -> None:
|
||
"""
|
||
智能随机休眠（Jitter）
|
||
|
||
防封禁策略：模拟人类行为的随机延迟
|
||
在请求之间加入不规则的等待时间
|
||
"""
|
||
sleep_time = random.uniform(min_seconds, max_seconds)
|
||
logger.debug(f"随机休眠 {sleep_time:.2f} 秒...")
|
||
time.sleep(sleep_time)
|
||
|
||
|
||
class DataFetcherManager:
|
||
"""
|
||
数据源策略管理器
|
||
|
||
职责：
|
||
1. 管理多个数据源（按优先级排序）
|
||
2. 自动故障切换（Failover）
|
||
3. 提供统一的数据获取接口
|
||
|
||
切换策略：
|
||
- 优先使用高优先级数据源
|
||
- 失败后自动切换到下一个
|
||
- 所有数据源都失败时抛出异常
|
||
"""
|
||
|
||
_DAILY_MARKET_FETCHER_SUPPORT = {
|
||
"EfinanceFetcher": {"cn"},
|
||
"TencentFetcher": {"cn"},
|
||
"AkshareFetcher": {"cn", "hk"},
|
||
"TushareFetcher": {"cn", "hk"},
|
||
"TickFlowFetcher": {"cn"},
|
||
"PytdxFetcher": {"cn"},
|
||
"BaostockFetcher": {"cn"},
|
||
"YfinanceFetcher": {"cn", "hk", "us", "jp", "kr", "tw"},
|
||
"LongbridgeFetcher": {"hk", "us"},
|
||
"FutuFetcher": {"hk"},
|
||
"FinnhubFetcher": {"us"},
|
||
"AlphaVantageFetcher": {"us"},
|
||
}
|
||
_daily_source_health = CircuitBreaker(failure_threshold=3, cooldown_seconds=300.0)
|
||
_CN_INDEX_DAILY_SOURCE_ORDER = (
|
||
"TencentFetcher",
|
||
"AkshareFetcher",
|
||
"TickFlowFetcher",
|
||
"YfinanceFetcher",
|
||
)
|
||
_CN_INDEX_REALTIME_SOURCE_ORDER = (
|
||
("AkshareFetcher", "tencent"),
|
||
("AkshareFetcher", "sina"),
|
||
("EfinanceFetcher", "index"),
|
||
("TickFlowFetcher", "tickflow"),
|
||
)
|
||
_CN_INDEX_NAME_SOURCE_ORDER = (
|
||
"TencentFetcher",
|
||
"AkshareFetcher",
|
||
"TickFlowFetcher",
|
||
)
|
||
_CN_INDEX_BARE_CODE_CONFLICTS = frozenset({"000001", "000016", "000688"})
|
||
_CONCEPT_RANKINGS_CACHE_TTL_SECONDS = 300.0
|
||
_CONCEPT_RANKINGS_EMPTY_CACHE_TTL_SECONDS = 30.0
|
||
_concept_rankings_cache_lock = RLock()
|
||
_concept_rankings_cache: Dict[int, Tuple[float, List[Dict], List[Dict]]] = {}
|
||
|
||
def __init__(self, fetchers: Optional[List[BaseFetcher]] = None):
|
||
"""
|
||
初始化管理器
|
||
|
||
Args:
|
||
fetchers: 数据源列表（可选，默认按优先级自动创建）
|
||
"""
|
||
self._fetchers: List[BaseFetcher] = []
|
||
self._fetchers_lock = RLock()
|
||
self._fetchers_by_name: Dict[str, BaseFetcher] = {}
|
||
self._fetcher_call_locks: Dict[int, RLock] = {}
|
||
self._fetcher_call_locks_lock = RLock()
|
||
self._stock_name_cache: Dict[str, str] = {}
|
||
self._stock_name_cache_lock = RLock()
|
||
|
||
if fetchers:
|
||
# 按优先级排序
|
||
self._fetchers = sorted(fetchers, key=lambda f: f.priority)
|
||
self._refresh_fetcher_indexes_locked()
|
||
else:
|
||
# 默认数据源将在首次使用时延迟加载
|
||
self._init_default_fetchers()
|
||
self._fundamental_adapter = AkshareFundamentalAdapter()
|
||
self._yfinance_fundamental_adapter = YfinanceFundamentalAdapter()
|
||
self._futu_fundamental_fetcher = None
|
||
self._tickflow_fetcher = None
|
||
self._tickflow_api_key: Optional[str] = None
|
||
self._tickflow_lock = RLock()
|
||
self._fundamental_cache: Dict[str, Dict[str, Any]] = {}
|
||
self._fundamental_cache_lock = RLock()
|
||
self._fundamental_timeout_worker_limit = 8
|
||
self._fundamental_timeout_slots = BoundedSemaphore(self._fundamental_timeout_worker_limit)
|
||
|
||
def _ensure_concurrency_guards(self) -> None:
|
||
"""Lazily initialize thread-safety primitives for test scaffolds using __new__."""
|
||
if not hasattr(self, "_fetchers_lock") or self._fetchers_lock is None:
|
||
self._fetchers_lock = RLock()
|
||
if not hasattr(self, "_fetchers_by_name") or self._fetchers_by_name is None:
|
||
self._fetchers_by_name = {}
|
||
if not hasattr(self, "_fetcher_call_locks") or self._fetcher_call_locks is None:
|
||
self._fetcher_call_locks = {}
|
||
if not hasattr(self, "_fetcher_call_locks_lock") or self._fetcher_call_locks_lock is None:
|
||
self._fetcher_call_locks_lock = RLock()
|
||
if not hasattr(self, "_stock_name_cache") or self._stock_name_cache is None:
|
||
self._stock_name_cache = {}
|
||
if not hasattr(self, "_stock_name_cache_lock") or self._stock_name_cache_lock is None:
|
||
self._stock_name_cache_lock = RLock()
|
||
|
||
def _get_fetchers_snapshot(self) -> List[BaseFetcher]:
|
||
self._ensure_concurrency_guards()
|
||
with self._fetchers_lock:
|
||
return list(getattr(self, "_fetchers", []))
|
||
|
||
def _refresh_fetcher_indexes_locked(self) -> None:
|
||
self._fetchers_by_name = {fetcher.name: fetcher for fetcher in self._fetchers}
|
||
|
||
def _get_fetcher_by_name(self, fetcher_name: str, capability: str = "") -> Optional[BaseFetcher]:
|
||
self._ensure_concurrency_guards()
|
||
with self._fetchers_lock:
|
||
fetcher = self._fetchers_by_name.get(fetcher_name)
|
||
if fetcher is None and self._fetchers:
|
||
self._refresh_fetcher_indexes_locked()
|
||
fetcher = self._fetchers_by_name.get(fetcher_name)
|
||
if fetcher is None:
|
||
return None
|
||
if not self._is_fetcher_available(fetcher, capability=capability):
|
||
return None
|
||
return fetcher
|
||
|
||
@staticmethod
|
||
def _call_availability_probe(fetcher: BaseFetcher, probe_name: str, capability: str) -> Optional[bool]:
|
||
probe = getattr(fetcher, probe_name, None)
|
||
if not callable(probe):
|
||
return None
|
||
try:
|
||
if probe_name == "is_available_for_request":
|
||
return bool(probe(capability))
|
||
return bool(probe())
|
||
except TypeError:
|
||
return bool(probe())
|
||
except Exception as exc:
|
||
logger.debug(
|
||
"[数据源可用性] %s.%s 检查失败(capability=%s): %s",
|
||
fetcher.name,
|
||
probe_name,
|
||
capability or "default",
|
||
exc,
|
||
)
|
||
return False
|
||
|
||
@classmethod
|
||
def _is_fetcher_available(cls, fetcher: BaseFetcher, capability: str = "") -> bool:
|
||
for probe_name in ("is_available_for_request", "is_available", "_is_available"):
|
||
result = cls._call_availability_probe(fetcher, probe_name, capability)
|
||
if result is not None:
|
||
return result
|
||
return True
|
||
|
||
def _get_fetcher_call_lock(self, fetcher: BaseFetcher) -> RLock:
|
||
self._ensure_concurrency_guards()
|
||
fetcher_id = id(fetcher)
|
||
with self._fetcher_call_locks_lock:
|
||
lock = self._fetcher_call_locks.get(fetcher_id)
|
||
if lock is None:
|
||
lock = RLock()
|
||
self._fetcher_call_locks[fetcher_id] = lock
|
||
return lock
|
||
|
||
def _call_fetcher_method(self, fetcher: BaseFetcher, method_name: str, *args, **kwargs):
|
||
"""Serialize shared fetcher state access through manager-owned per-instance locks."""
|
||
method = getattr(fetcher, method_name)
|
||
with self._get_fetcher_call_lock(fetcher):
|
||
return method(*args, **kwargs)
|
||
|
||
@classmethod
|
||
def _filter_daily_fetchers_for_market(
|
||
cls,
|
||
fetchers: List[BaseFetcher],
|
||
market: str,
|
||
) -> List[BaseFetcher]:
|
||
"""Skip built-in daily fetchers that are known not to support a market."""
|
||
|
||
kept: List[BaseFetcher] = []
|
||
skipped: List[str] = []
|
||
for fetcher in fetchers:
|
||
supported = cls._DAILY_MARKET_FETCHER_SUPPORT.get(fetcher.name)
|
||
if supported is not None and market not in supported:
|
||
skipped.append(fetcher.name)
|
||
else:
|
||
kept.append(fetcher)
|
||
|
||
if skipped:
|
||
logger.info(
|
||
"[数据源路由] %s 日线跳过不支持的数据源: %s",
|
||
market,
|
||
", ".join(skipped),
|
||
)
|
||
return kept
|
||
|
||
@classmethod
|
||
def _filter_fetchers_by_capability(
|
||
cls,
|
||
fetchers: List[BaseFetcher],
|
||
capability: str,
|
||
) -> List[BaseFetcher]:
|
||
"""Skip request-time unavailable fetchers before entering route-specific loops."""
|
||
kept: List[BaseFetcher] = []
|
||
skipped: List[str] = []
|
||
|
||
for fetcher in fetchers:
|
||
if cls._is_fetcher_available(fetcher, capability=capability):
|
||
kept.append(fetcher)
|
||
else:
|
||
skipped.append(fetcher.name)
|
||
|
||
if skipped:
|
||
logger.info(
|
||
"[数据源路由] %s 跳过暂不可用的数据源: %s",
|
||
capability or "request",
|
||
", ".join(skipped),
|
||
)
|
||
|
||
return kept
|
||
|
||
@classmethod
|
||
def _daily_health_key(cls, fetcher: BaseFetcher, market: str) -> str:
|
||
return f"daily_data:{market}:{fetcher.name}"
|
||
|
||
@classmethod
|
||
def _is_daily_source_available(
|
||
cls,
|
||
fetcher: BaseFetcher,
|
||
market: str,
|
||
) -> bool:
|
||
key = cls._daily_health_key(fetcher, market)
|
||
if cls._daily_source_health.is_available(key):
|
||
return True
|
||
logger.info(
|
||
"[数据源健康度] %s 日线跳过短期熔断的数据源: %s",
|
||
market,
|
||
fetcher.name,
|
||
)
|
||
return False
|
||
|
||
@staticmethod
|
||
def _daily_source_unavailable_error(fetcher: BaseFetcher) -> str:
|
||
return f"[{fetcher.name}] (CircuitOpen) 数据源短期熔断"
|
||
|
||
@classmethod
|
||
def _record_daily_source_success(cls, fetcher: BaseFetcher, market: str) -> None:
|
||
cls._daily_source_health.record_success(cls._daily_health_key(fetcher, market))
|
||
|
||
@classmethod
|
||
def _record_daily_source_failure(cls, fetcher: BaseFetcher, market: str, error: str) -> None:
|
||
cls._daily_source_health.record_failure(cls._daily_health_key(fetcher, market), error=error)
|
||
|
||
@classmethod
|
||
def reset_daily_source_health(cls) -> None:
|
||
"""Reset daily source health state for tests/admin diagnostics."""
|
||
cls._daily_source_health.reset()
|
||
|
||
def _get_cached_stock_name(self, stock_code: str) -> Optional[str]:
|
||
self._ensure_concurrency_guards()
|
||
with self._stock_name_cache_lock:
|
||
return self._stock_name_cache.get(stock_code)
|
||
|
||
def _cache_stock_name(self, stock_code: str, name: Optional[str]) -> Optional[str]:
|
||
if name is None:
|
||
return None
|
||
self._ensure_concurrency_guards()
|
||
with self._stock_name_cache_lock:
|
||
self._stock_name_cache[stock_code] = name
|
||
return name
|
||
|
||
def _discard_cached_stock_name(self, stock_code: str) -> None:
|
||
self._ensure_concurrency_guards()
|
||
with self._stock_name_cache_lock:
|
||
self._stock_name_cache.pop(stock_code, None)
|
||
|
||
@classmethod
|
||
def _warn_bare_index_conflict(cls, target: AnalysisTarget) -> None:
|
||
if target.asset_type != ParseStatus.STOCK or target.normalized_prefix is not None:
|
||
return
|
||
bare_code = target.normalized_code or (target.raw_input or "").strip()
|
||
if (
|
||
target.matched_index is None
|
||
and bare_code not in cls._CN_INDEX_BARE_CODE_CONFLICTS
|
||
):
|
||
return
|
||
logger.warning(
|
||
"[指数路由] 裸代码 %s 存在股票/指数歧义，按股票路由处理",
|
||
bare_code,
|
||
)
|
||
|
||
@staticmethod
|
||
def _cn_index_provider_symbol(target: AnalysisTarget, fetcher_name: str) -> str:
|
||
entry = target.matched_index
|
||
if entry is None:
|
||
return ""
|
||
exchange = entry.exchange.upper()
|
||
if exchange == "CSI":
|
||
# CSI indices are only supported by AkShare (``csi{code}``); the
|
||
# other providers in the fixed daily chain return an empty symbol
|
||
# so the caller records an ``unsupported`` provider-run and skips.
|
||
if fetcher_name == "AkshareFetcher":
|
||
return f"csi{entry.bare_code}"
|
||
return ""
|
||
if exchange not in {"SH", "SZ"}:
|
||
return ""
|
||
if fetcher_name in {"TencentFetcher", "AkshareFetcher"}:
|
||
return f"{exchange.lower()}{entry.bare_code}"
|
||
if fetcher_name == "TickFlowFetcher":
|
||
return f"{entry.bare_code}.{exchange}"
|
||
if fetcher_name == "YfinanceFetcher":
|
||
suffix = "SS" if exchange == "SH" else "SZ"
|
||
return f"{entry.bare_code}.{suffix}"
|
||
return ""
|
||
|
||
@classmethod
|
||
def _is_meaningful_cn_index_name(
|
||
cls, name: Optional[str], target: AnalysisTarget
|
||
) -> bool:
|
||
if not is_meaningful_stock_name(name, target.canonical_id):
|
||
return False
|
||
|
||
aliases = {target.canonical_id}
|
||
entry = target.matched_index
|
||
if entry is not None:
|
||
if normalize_stock_code(str(name).strip()) == entry.bare_code:
|
||
return False
|
||
aliases.add(entry.bare_code)
|
||
aliases.update(entry.aliases)
|
||
display_code = (target.display_code or "").strip()
|
||
registry_name = (entry.display_name or "").strip()
|
||
# The parser currently uses display_code for the human label too.
|
||
if display_code and (
|
||
display_code != registry_name
|
||
or normalize_stock_code(display_code) == entry.bare_code
|
||
):
|
||
aliases.add(display_code)
|
||
for source_name in cls._CN_INDEX_DAILY_SOURCE_ORDER:
|
||
aliases.add(cls._cn_index_provider_symbol(target, source_name))
|
||
|
||
candidate = str(name).strip().upper()
|
||
code_aliases = {
|
||
str(alias).strip().upper()
|
||
for alias in aliases
|
||
if alias is not None and str(alias).strip()
|
||
}
|
||
return candidate not in code_aliases
|
||
|
||
def _get_cn_index_daily_data(
|
||
self,
|
||
target: AnalysisTarget,
|
||
start_date: Optional[str],
|
||
end_date: Optional[str],
|
||
days: int,
|
||
) -> Tuple[pd.DataFrame, str]:
|
||
source_order = self._CN_INDEX_DAILY_SOURCE_ORDER
|
||
fetchers_by_name = {
|
||
fetcher.name: fetcher for fetcher in self._get_fetchers_snapshot()
|
||
}
|
||
errors: List[str] = []
|
||
request_start = time.time()
|
||
|
||
for index, source_name in enumerate(source_order):
|
||
fallback_to = source_order[index + 1] if index + 1 < len(source_order) else None
|
||
fetcher = fetchers_by_name.get(source_name)
|
||
provider_symbol = self._cn_index_provider_symbol(target, source_name)
|
||
|
||
if not provider_symbol:
|
||
reason = (
|
||
"unsupported index provider symbol: "
|
||
f"{target.canonical_id} -> {source_name}"
|
||
)
|
||
record_provider_run(
|
||
data_type="daily_data",
|
||
provider=source_name,
|
||
operation="get_daily_data",
|
||
success=False,
|
||
latency_ms=0,
|
||
error_type="unsupported",
|
||
error_message=reason,
|
||
fallback_to=fallback_to,
|
||
record_count=0,
|
||
)
|
||
logger.warning(
|
||
"[指数数据源不支持 %d/%d] [%s] %s: %s",
|
||
index + 1,
|
||
len(source_order),
|
||
source_name,
|
||
target.canonical_id,
|
||
reason,
|
||
)
|
||
errors.append(f"[{source_name}] {reason}")
|
||
continue
|
||
|
||
if fetcher is None or not self._is_fetcher_available(
|
||
fetcher, capability="daily_data"
|
||
):
|
||
reason = "数据源未配置或暂不可用"
|
||
record_provider_run(
|
||
data_type="daily_data",
|
||
provider=source_name,
|
||
operation="get_daily_data",
|
||
success=False,
|
||
latency_ms=0,
|
||
error_type="unavailable",
|
||
error_message=reason,
|
||
fallback_to=fallback_to,
|
||
record_count=0,
|
||
)
|
||
logger.warning(
|
||
"[指数数据源失败 %d/%d] [%s] %s: %s",
|
||
index + 1,
|
||
len(source_order),
|
||
source_name,
|
||
target.canonical_id,
|
||
reason,
|
||
)
|
||
errors.append(f"[{source_name}] {reason}")
|
||
continue
|
||
|
||
if not self._is_daily_source_available(fetcher, "cn_index"):
|
||
reason = self._daily_source_unavailable_error(fetcher)
|
||
record_provider_run(
|
||
data_type="daily_data",
|
||
provider=source_name,
|
||
operation="get_daily_data",
|
||
success=False,
|
||
latency_ms=0,
|
||
error_type="CircuitOpen",
|
||
error_message=reason,
|
||
fallback_to=fallback_to,
|
||
record_count=0,
|
||
)
|
||
logger.warning(
|
||
"[指数数据源失败 %d/%d] [%s] %s: %s",
|
||
index + 1,
|
||
len(source_order),
|
||
source_name,
|
||
target.canonical_id,
|
||
reason,
|
||
)
|
||
errors.append(reason)
|
||
continue
|
||
|
||
attempt_start = time.time()
|
||
try:
|
||
logger.info(
|
||
"[指数数据源尝试 %d/%d] [%s] %s -> %s",
|
||
index + 1,
|
||
len(source_order),
|
||
source_name,
|
||
target.canonical_id,
|
||
provider_symbol,
|
||
)
|
||
record_provider_run_started(
|
||
data_type="daily_data",
|
||
provider=source_name,
|
||
operation="get_daily_data",
|
||
)
|
||
df = self._call_fetcher_method(
|
||
fetcher,
|
||
"get_daily_data",
|
||
stock_code=provider_symbol,
|
||
start_date=start_date,
|
||
end_date=end_date,
|
||
days=days,
|
||
)
|
||
duration_ms = int((time.time() - attempt_start) * 1000)
|
||
if df is not None and not df.empty:
|
||
record_provider_run(
|
||
data_type="daily_data",
|
||
provider=source_name,
|
||
operation="get_daily_data",
|
||
success=True,
|
||
latency_ms=duration_ms,
|
||
record_count=len(df),
|
||
)
|
||
self._record_daily_source_success(fetcher, "cn_index")
|
||
logger.info(
|
||
"[指数数据源完成] %s 使用 [%s] 获取成功: rows=%d, elapsed=%.2fs",
|
||
target.canonical_id,
|
||
source_name,
|
||
len(df),
|
||
time.time() - request_start,
|
||
)
|
||
return df, source_name
|
||
|
||
reason = "empty result"
|
||
record_provider_run(
|
||
data_type="daily_data",
|
||
provider=source_name,
|
||
operation="get_daily_data",
|
||
success=False,
|
||
latency_ms=duration_ms,
|
||
error_type="empty",
|
||
error_message=reason,
|
||
fallback_to=fallback_to,
|
||
record_count=0,
|
||
)
|
||
if df is not None and df.empty:
|
||
self._record_daily_source_success(fetcher, "cn_index")
|
||
else:
|
||
self._record_daily_source_failure(fetcher, "cn_index", reason)
|
||
logger.warning(
|
||
"[指数数据源失败 %d/%d] [%s] %s: %s",
|
||
index + 1,
|
||
len(source_order),
|
||
source_name,
|
||
target.canonical_id,
|
||
reason,
|
||
)
|
||
errors.append(f"[{source_name}] {reason}")
|
||
except Exception as exc:
|
||
error_type, error_reason = summarize_exception(exc)
|
||
duration_ms = int((time.time() - attempt_start) * 1000)
|
||
record_provider_run(
|
||
data_type="daily_data",
|
||
provider=source_name,
|
||
operation="get_daily_data",
|
||
success=False,
|
||
latency_ms=duration_ms,
|
||
error_type=error_type,
|
||
error_message=error_reason,
|
||
fallback_to=fallback_to,
|
||
record_count=0,
|
||
)
|
||
self._record_daily_source_failure(
|
||
fetcher, "cn_index", error_reason
|
||
)
|
||
logger.warning(
|
||
"[指数数据源失败 %d/%d] [%s] %s: error_type=%s, reason=%s",
|
||
index + 1,
|
||
len(source_order),
|
||
source_name,
|
||
target.canonical_id,
|
||
error_type,
|
||
error_reason,
|
||
)
|
||
errors.append(f"[{source_name}] ({error_type}) {error_reason}")
|
||
|
||
logger.warning(
|
||
"[指数数据源终止] %s 所有指数日线数据源均失败: elapsed=%.2fs; %s",
|
||
target.canonical_id,
|
||
time.time() - request_start,
|
||
"; ".join(errors) or "暂无可用数据源",
|
||
)
|
||
return pd.DataFrame(columns=STANDARD_COLUMNS), ""
|
||
|
||
def _get_cn_index_realtime_quote(
|
||
self,
|
||
target: AnalysisTarget,
|
||
*,
|
||
log_final_failure: bool = True,
|
||
):
|
||
"""Fetch a realtime quote for a registered CN index via a fixed chain.
|
||
|
||
Chain (Story 1.5):
|
||
1. Tencent via AkshareFetcher (``sh000016``/``sz399001`` prefixed symbol)
|
||
2. Sina via AkshareFetcher (same prefixed symbol)
|
||
3. Eastmoney single-stock secid via EfinanceFetcher (SH/SZ/CSI)
|
||
4. TickFlow (``000016.SH`` symbol; SH/SZ only)
|
||
|
||
The explicit index identity is preserved end-to-end: provider symbols
|
||
are derived from the registry entry, never from ``normalize_stock_code``,
|
||
so ``sh000016`` can never degrade into the stock ``000016`` path.
|
||
"""
|
||
fetchers_by_name = {
|
||
fetcher.name: fetcher for fetcher in self._get_fetchers_snapshot()
|
||
}
|
||
errors: List[str] = []
|
||
request_start = time.time()
|
||
source_order = self._CN_INDEX_REALTIME_SOURCE_ORDER
|
||
|
||
for index, (source_name, source_kind) in enumerate(source_order):
|
||
fallback_to = (
|
||
source_order[index + 1][0] if index + 1 < len(source_order) else None
|
||
)
|
||
fetcher = fetchers_by_name.get(source_name)
|
||
provider_symbol = self._cn_index_realtime_provider_symbol(
|
||
target, source_name, source_kind
|
||
)
|
||
|
||
if not provider_symbol:
|
||
reason = (
|
||
"unsupported index provider symbol: "
|
||
f"{target.canonical_id} -> {source_name}"
|
||
)
|
||
record_provider_run(
|
||
data_type="realtime_quote",
|
||
provider=source_name,
|
||
operation="get_realtime_quote",
|
||
success=False,
|
||
latency_ms=0,
|
||
error_type="unsupported",
|
||
error_message=reason,
|
||
fallback_to=fallback_to,
|
||
record_count=0,
|
||
)
|
||
logger.warning(
|
||
"[指数实时行情不支持 %d/%d] [%s] %s: %s",
|
||
index + 1,
|
||
len(source_order),
|
||
source_name,
|
||
target.canonical_id,
|
||
reason,
|
||
)
|
||
errors.append(f"[{source_name}] {reason}")
|
||
continue
|
||
|
||
if fetcher is None or not self._is_fetcher_available(
|
||
fetcher, capability="realtime_quote"
|
||
):
|
||
reason = "数据源未配置或暂不可用"
|
||
record_provider_run(
|
||
data_type="realtime_quote",
|
||
provider=source_name,
|
||
operation="get_realtime_quote",
|
||
success=False,
|
||
latency_ms=0,
|
||
error_type="unavailable",
|
||
error_message=reason,
|
||
fallback_to=fallback_to,
|
||
record_count=0,
|
||
)
|
||
logger.warning(
|
||
"[指数实时行情失败 %d/%d] [%s] %s: %s",
|
||
index + 1,
|
||
len(source_order),
|
||
source_name,
|
||
target.canonical_id,
|
||
reason,
|
||
)
|
||
errors.append(f"[{source_name}] {reason}")
|
||
continue
|
||
|
||
attempt_start = time.time()
|
||
try:
|
||
logger.info(
|
||
"[指数实时行情尝试 %d/%d] [%s] %s -> %s",
|
||
index + 1,
|
||
len(source_order),
|
||
source_name,
|
||
target.canonical_id,
|
||
provider_symbol,
|
||
)
|
||
record_provider_run_started(
|
||
data_type="realtime_quote",
|
||
provider=source_name,
|
||
operation="get_realtime_quote",
|
||
)
|
||
if source_kind == "index":
|
||
quote = self._call_fetcher_method(
|
||
fetcher,
|
||
"get_index_realtime_quote",
|
||
target.canonical_id,
|
||
)
|
||
elif source_kind == "tickflow":
|
||
quote = self._call_fetcher_method(
|
||
fetcher,
|
||
"get_realtime_quote",
|
||
provider_symbol,
|
||
)
|
||
else:
|
||
quote = self._call_fetcher_method(
|
||
fetcher,
|
||
"get_realtime_quote",
|
||
provider_symbol,
|
||
source=source_kind,
|
||
)
|
||
duration_ms = int((time.time() - attempt_start) * 1000)
|
||
if quote is not None and quote.has_basic_data():
|
||
record_provider_run(
|
||
data_type="realtime_quote",
|
||
provider=source_name,
|
||
operation="get_realtime_quote",
|
||
success=True,
|
||
latency_ms=duration_ms,
|
||
record_count=1,
|
||
)
|
||
logger.info(
|
||
"[指数实时行情完成] %s 使用 [%s] 获取成功: elapsed=%.2fs",
|
||
target.canonical_id,
|
||
source_name,
|
||
time.time() - request_start,
|
||
)
|
||
from src.config import get_config as _get_config_safe
|
||
|
||
return self._enrich_realtime_quote(
|
||
quote,
|
||
realtime_cache_ttl=getattr(
|
||
_get_config_safe(), "realtime_cache_ttl", None
|
||
),
|
||
)
|
||
|
||
reason = "empty or incomplete quote"
|
||
record_provider_run(
|
||
data_type="realtime_quote",
|
||
provider=source_name,
|
||
operation="get_realtime_quote",
|
||
success=False,
|
||
latency_ms=duration_ms,
|
||
error_type="empty",
|
||
error_message=reason,
|
||
fallback_to=fallback_to,
|
||
record_count=0,
|
||
)
|
||
logger.warning(
|
||
"[指数实时行情失败 %d/%d] [%s] %s: %s",
|
||
index + 1,
|
||
len(source_order),
|
||
source_name,
|
||
target.canonical_id,
|
||
reason,
|
||
)
|
||
errors.append(f"[{source_name}] {reason}")
|
||
except Exception as exc:
|
||
error_type, error_reason = summarize_exception(exc)
|
||
duration_ms = int((time.time() - attempt_start) * 1000)
|
||
record_provider_run(
|
||
data_type="realtime_quote",
|
||
provider=source_name,
|
||
operation="get_realtime_quote",
|
||
success=False,
|
||
latency_ms=duration_ms,
|
||
error_type=error_type,
|
||
error_message=error_reason,
|
||
fallback_to=fallback_to,
|
||
record_count=0,
|
||
)
|
||
logger.warning(
|
||
"[指数实时行情失败 %d/%d] [%s] %s: error_type=%s, reason=%s",
|
||
index + 1,
|
||
len(source_order),
|
||
source_name,
|
||
target.canonical_id,
|
||
error_type,
|
||
error_reason,
|
||
)
|
||
errors.append(f"[{source_name}] ({error_type}) {error_reason}")
|
||
|
||
if log_final_failure:
|
||
logger.warning(
|
||
"[指数实时行情终止] %s 所有指数实时行情数据源均失败: elapsed=%.2fs; %s",
|
||
target.canonical_id,
|
||
time.time() - request_start,
|
||
"; ".join(errors) or "暂无可用数据源",
|
||
)
|
||
return None
|
||
|
||
@classmethod
|
||
def _cn_index_realtime_provider_symbol(
|
||
cls,
|
||
target: AnalysisTarget,
|
||
fetcher_name: str,
|
||
source_kind: str,
|
||
) -> str:
|
||
"""Derive the provider symbol for the CN index realtime chain."""
|
||
entry = target.matched_index
|
||
if entry is None:
|
||
return ""
|
||
exchange = entry.exchange.upper()
|
||
if exchange == "CSI":
|
||
# CSI indices are only supported by the Eastmoney single-stock
|
||
# secid endpoint (EfinanceFetcher); other providers return an
|
||
# empty symbol so the caller records ``unsupported`` and skips.
|
||
if fetcher_name == "EfinanceFetcher" and source_kind == "index":
|
||
return target.canonical_id
|
||
return ""
|
||
if exchange not in {"SH", "SZ"}:
|
||
return ""
|
||
if fetcher_name == "AkshareFetcher" and source_kind in ("tencent", "sina"):
|
||
return f"{exchange.lower()}{entry.bare_code}"
|
||
if fetcher_name == "EfinanceFetcher" and source_kind == "index":
|
||
return target.canonical_id
|
||
if fetcher_name == "TickFlowFetcher" and source_kind == "tickflow":
|
||
return f"{entry.bare_code}.{exchange}"
|
||
return ""
|
||
|
||
def _get_cn_index_name(self, target: AnalysisTarget) -> str:
|
||
cache_key = target.canonical_id
|
||
entry = target.matched_index
|
||
display_name = entry.display_name if entry is not None else ""
|
||
if self._is_meaningful_cn_index_name(display_name, target):
|
||
return self._cache_stock_name(cache_key, display_name) or display_name
|
||
|
||
cached_name = self._get_cached_stock_name(cache_key)
|
||
if cached_name is not None:
|
||
if self._is_meaningful_cn_index_name(cached_name, target):
|
||
return cached_name
|
||
self._discard_cached_stock_name(cache_key)
|
||
|
||
fetchers_by_name = {
|
||
fetcher.name: fetcher for fetcher in self._get_fetchers_snapshot()
|
||
}
|
||
for source_name in self._CN_INDEX_NAME_SOURCE_ORDER:
|
||
provider_symbol = self._cn_index_provider_symbol(target, source_name)
|
||
if not provider_symbol:
|
||
logger.warning(
|
||
"[指数名称] [%s] 不支持 provider symbol: %s",
|
||
source_name,
|
||
cache_key,
|
||
)
|
||
continue
|
||
fetcher = fetchers_by_name.get(source_name)
|
||
if (
|
||
fetcher is None
|
||
or not hasattr(fetcher, "get_stock_name")
|
||
or not self._is_fetcher_available(fetcher, capability="stock_name")
|
||
):
|
||
logger.warning(
|
||
"[指数名称] [%s] 未配置或暂不可用: %s",
|
||
source_name,
|
||
cache_key,
|
||
)
|
||
continue
|
||
try:
|
||
name = self._call_fetcher_method(
|
||
fetcher, "get_stock_name", provider_symbol
|
||
)
|
||
except Exception as exc:
|
||
logger.warning(
|
||
"[指数名称] [%s] 获取 %s 失败: %s",
|
||
source_name,
|
||
cache_key,
|
||
exc,
|
||
)
|
||
continue
|
||
if self._is_meaningful_cn_index_name(name, target):
|
||
self._cache_stock_name(cache_key, name)
|
||
logger.info(
|
||
"[指数名称] 从 %s 获取: %s -> %s",
|
||
source_name,
|
||
cache_key,
|
||
name,
|
||
)
|
||
return name
|
||
logger.warning(
|
||
"[指数名称] [%s] 获取 %s 返回空结果",
|
||
source_name,
|
||
cache_key,
|
||
)
|
||
|
||
static_name = STOCK_NAME_MAP.get(cache_key)
|
||
if static_name and self._is_meaningful_cn_index_name(static_name, target):
|
||
self._cache_stock_name(cache_key, static_name)
|
||
logger.info(
|
||
"[指数名称] 从静态映射获取: %s -> %s",
|
||
cache_key,
|
||
static_name,
|
||
)
|
||
return static_name
|
||
|
||
logger.warning("[指数名称] 所有数据源都无法获取 %s 的名称", cache_key)
|
||
return cache_key
|
||
|
||
def _get_tickflow_fetcher(self):
|
||
"""Lazily create a TickFlow fetcher for market-review-only calls."""
|
||
from src.config import get_config
|
||
|
||
config = get_config()
|
||
api_key = (getattr(config, "tickflow_api_key", None) or "").strip()
|
||
|
||
if not hasattr(self, "_tickflow_lock") or self._tickflow_lock is None:
|
||
self._tickflow_lock = RLock()
|
||
|
||
with self._tickflow_lock:
|
||
current_fetcher = getattr(self, "_tickflow_fetcher", None)
|
||
current_key = getattr(self, "_tickflow_api_key", None)
|
||
|
||
if not api_key:
|
||
if current_fetcher is not None and hasattr(current_fetcher, "close"):
|
||
try:
|
||
current_fetcher.close()
|
||
except Exception as exc:
|
||
logger.debug("[TickFlowFetcher] 关闭旧实例失败: %s", exc)
|
||
self._tickflow_fetcher = None
|
||
self._tickflow_api_key = None
|
||
return None
|
||
|
||
configured_fetcher = self._get_fetcher_by_name("TickFlowFetcher")
|
||
if configured_fetcher is not None:
|
||
return configured_fetcher
|
||
|
||
if current_fetcher is not None and current_key == api_key:
|
||
return current_fetcher
|
||
|
||
if current_fetcher is not None and hasattr(current_fetcher, "close"):
|
||
try:
|
||
current_fetcher.close()
|
||
except Exception as exc:
|
||
logger.debug("[TickFlowFetcher] 切换实例时关闭失败: %s", exc)
|
||
|
||
try:
|
||
from .tickflow_fetcher import TickFlowFetcher
|
||
|
||
fetcher = TickFlowFetcher(
|
||
api_key=api_key,
|
||
kline_adjust=getattr(config, "tickflow_kline_adjust", "none"),
|
||
batch_daily_enabled=getattr(config, "tickflow_batch_daily_enabled", True),
|
||
batch_size=getattr(config, "tickflow_batch_size", 100),
|
||
priority=getattr(config, "tickflow_priority", 2),
|
||
)
|
||
self._tickflow_fetcher = fetcher
|
||
self._tickflow_api_key = api_key
|
||
return fetcher
|
||
except Exception as exc:
|
||
logger.warning("[TickFlowFetcher] 初始化失败: %s", exc)
|
||
self._tickflow_fetcher = None
|
||
self._tickflow_api_key = None
|
||
return None
|
||
|
||
def close(self) -> None:
|
||
"""Best-effort release of manager-owned resources."""
|
||
if not hasattr(self, "_tickflow_lock") or self._tickflow_lock is None:
|
||
self._tickflow_lock = RLock()
|
||
|
||
with self._tickflow_lock:
|
||
current_fetcher = getattr(self, "_tickflow_fetcher", None)
|
||
self._tickflow_fetcher = None
|
||
self._tickflow_api_key = None
|
||
|
||
if current_fetcher is not None and hasattr(current_fetcher, "close"):
|
||
try:
|
||
current_fetcher.close()
|
||
except Exception as exc:
|
||
logger.debug("[TickFlowFetcher] 关闭管理器资源失败: %s", exc)
|
||
|
||
# The HK Futu fundamental path lazily creates and caches its own
|
||
# FutuFetcher (an OpenQuoteContext-backed connection) on
|
||
# _futu_fundamental_fetcher; release it here so explicit close /
|
||
# reload paths do not leak the OpenD connection.
|
||
futu_fundamental_fetcher = getattr(self, "_futu_fundamental_fetcher", None)
|
||
if futu_fundamental_fetcher is not None:
|
||
self._futu_fundamental_fetcher = None
|
||
close_futu = getattr(futu_fundamental_fetcher, "close", None)
|
||
if callable(close_futu):
|
||
try:
|
||
close_futu()
|
||
except Exception as exc:
|
||
logger.debug("[FutuFetcher] 关闭管理器资源失败: %s", exc)
|
||
|
||
for fetcher in self._get_fetchers_snapshot():
|
||
close = getattr(fetcher, "close", None)
|
||
if callable(close):
|
||
try:
|
||
close()
|
||
except Exception as exc:
|
||
logger.debug("[%s] close failed: %s", fetcher.name, exc)
|
||
|
||
def __del__(self) -> None:
|
||
try:
|
||
self.close()
|
||
except Exception:
|
||
# Best-effort cleanup during interpreter shutdown.
|
||
pass
|
||
|
||
def _get_fundamental_cache_key(self, stock_code: str, budget_seconds: Optional[float] = None) -> str:
|
||
"""生成基本面缓存 key（包含预算分桶以避免低预算结果污染高预算请求）。"""
|
||
normalized_code = normalize_stock_code(stock_code)
|
||
if budget_seconds is None:
|
||
return f"{normalized_code}|budget=default"
|
||
try:
|
||
budget = max(0.0, float(budget_seconds))
|
||
except (TypeError, ValueError):
|
||
budget = 0.0
|
||
# 100ms bucket to balance cache reuse and scenario isolation.
|
||
budget_bucket = int(round(budget * 10))
|
||
return f"{normalized_code}|budget={budget_bucket}"
|
||
|
||
def _prune_fundamental_cache(self, ttl_seconds: int, max_entries: int) -> None:
|
||
"""Prune expired and overflow fundamental cache items."""
|
||
with self._fundamental_cache_lock:
|
||
if not self._fundamental_cache:
|
||
return
|
||
|
||
now_ts = time.time()
|
||
if ttl_seconds > 0:
|
||
cache_items = list(self._fundamental_cache.items())
|
||
expired_keys = [
|
||
key
|
||
for key, value in cache_items
|
||
if now_ts - float(value.get("ts", 0)) > ttl_seconds
|
||
]
|
||
for key in expired_keys:
|
||
self._fundamental_cache.pop(key, None)
|
||
|
||
if max_entries > 0 and len(self._fundamental_cache) > max_entries:
|
||
overflow = len(self._fundamental_cache) - max_entries
|
||
sorted_items = sorted(
|
||
list(self._fundamental_cache.items()),
|
||
key=lambda item: float(item[1].get("ts", 0)),
|
||
)
|
||
for key, _ in sorted_items[:overflow]:
|
||
self._fundamental_cache.pop(key, None)
|
||
|
||
@staticmethod
|
||
def _try_scalar_isna(value: Any, context: str) -> Optional[bool]:
|
||
"""Return scalar ``pd.isna`` result, or ``None`` when callers should use fallback logic."""
|
||
if isinstance(value, (dict, list, tuple, set, pd.DataFrame, pd.Series, pd.Index)):
|
||
return None
|
||
|
||
if isinstance(value, np.ndarray):
|
||
if value.ndim != 0:
|
||
return None
|
||
value = value.item()
|
||
|
||
try:
|
||
isna_result = pd.isna(value)
|
||
except (TypeError, ValueError) as exc:
|
||
if hasattr(value, "__array__"):
|
||
logger.debug(
|
||
"[%s] pd.isna failed for array-like object; re-raise: value_type=%s error_type=%s",
|
||
context,
|
||
type(value).__name__,
|
||
type(exc).__name__,
|
||
)
|
||
raise
|
||
logger.debug(
|
||
"[%s] pd.isna fallback: value_type=%s error_type=%s",
|
||
context,
|
||
type(value).__name__,
|
||
type(exc).__name__,
|
||
)
|
||
return None
|
||
|
||
if isinstance(isna_result, (bool, np.bool_)):
|
||
return bool(isna_result)
|
||
|
||
if isinstance(isna_result, np.ndarray):
|
||
if isna_result.ndim == 0:
|
||
return bool(isna_result.item())
|
||
logger.debug(
|
||
"[%s] pd.isna returned non-scalar result: value_type=%s result_type=%s",
|
||
context,
|
||
type(value).__name__,
|
||
type(isna_result).__name__,
|
||
)
|
||
return None
|
||
|
||
logger.debug(
|
||
"[%s] pd.isna returned unexpected result type: value_type=%s result_type=%s",
|
||
context,
|
||
type(value).__name__,
|
||
type(isna_result).__name__,
|
||
)
|
||
return None
|
||
|
||
@staticmethod
|
||
def _is_missing_board_value(value: Any) -> bool:
|
||
"""Return True when a board field value should be treated as missing."""
|
||
if value is None:
|
||
return True
|
||
is_missing = DataFetcherManager._try_scalar_isna(value, "board_value")
|
||
if is_missing is True:
|
||
return True
|
||
text = str(value).strip()
|
||
return text == "" or text.lower() in {"nan", "none", "null", "na", "n/a"}
|
||
|
||
@staticmethod
|
||
def _normalize_belong_boards(raw_data: Any) -> List[Dict[str, Any]]:
|
||
"""Normalize belong-board results from heterogeneous providers."""
|
||
if DataFetcherManager._is_missing_board_value(raw_data):
|
||
return []
|
||
|
||
normalized: List[Dict[str, Any]] = []
|
||
dedupe = set()
|
||
|
||
if isinstance(raw_data, pd.DataFrame):
|
||
if raw_data.empty:
|
||
return []
|
||
name_col = next(
|
||
(
|
||
col
|
||
for col in raw_data.columns
|
||
if str(col) in {"板块名称", "板块", "所属板块", "板块名", "name", "industry"}
|
||
),
|
||
None,
|
||
)
|
||
code_col = next(
|
||
(
|
||
col
|
||
for col in raw_data.columns
|
||
if str(col) in {"板块代码", "代码", "code"}
|
||
),
|
||
None,
|
||
)
|
||
type_col = next(
|
||
(
|
||
col
|
||
for col in raw_data.columns
|
||
if str(col) in {"板块类型", "类别", "type"}
|
||
),
|
||
None,
|
||
)
|
||
if name_col is None:
|
||
return []
|
||
for _, row in raw_data.iterrows():
|
||
board_name_raw = row.get(name_col, "")
|
||
if DataFetcherManager._is_missing_board_value(board_name_raw):
|
||
continue
|
||
board_name = str(board_name_raw).strip()
|
||
if board_name in dedupe:
|
||
continue
|
||
dedupe.add(board_name)
|
||
item = {"name": board_name}
|
||
if code_col is not None:
|
||
board_code_raw = row.get(code_col, "")
|
||
if not DataFetcherManager._is_missing_board_value(board_code_raw):
|
||
item["code"] = str(board_code_raw).strip()
|
||
if type_col is not None:
|
||
board_type_raw = row.get(type_col, "")
|
||
if not DataFetcherManager._is_missing_board_value(board_type_raw):
|
||
item["type"] = str(board_type_raw).strip()
|
||
normalized.append(item)
|
||
return normalized
|
||
|
||
if isinstance(raw_data, dict):
|
||
raw_data = [raw_data]
|
||
|
||
if isinstance(raw_data, (list, tuple, set)):
|
||
for item in raw_data:
|
||
if isinstance(item, dict):
|
||
board_name_raw = (
|
||
item.get("name")
|
||
or item.get("board_name")
|
||
or item.get("板块名称")
|
||
or item.get("板块")
|
||
or item.get("所属板块")
|
||
or item.get("板块名")
|
||
or item.get("industry")
|
||
or item.get("行业")
|
||
)
|
||
if DataFetcherManager._is_missing_board_value(board_name_raw):
|
||
continue
|
||
board_name = str(board_name_raw).strip()
|
||
if board_name in dedupe:
|
||
continue
|
||
dedupe.add(board_name)
|
||
normalized_item: Dict[str, Any] = {"name": board_name}
|
||
code_raw = (
|
||
item.get("code")
|
||
or item.get("板块代码")
|
||
or item.get("代码")
|
||
)
|
||
if not DataFetcherManager._is_missing_board_value(code_raw):
|
||
normalized_item["code"] = str(code_raw).strip()
|
||
type_raw = (
|
||
item.get("type")
|
||
or item.get("板块类型")
|
||
or item.get("类别")
|
||
)
|
||
if not DataFetcherManager._is_missing_board_value(type_raw):
|
||
normalized_item["type"] = str(type_raw).strip()
|
||
normalized.append(normalized_item)
|
||
continue
|
||
if DataFetcherManager._is_missing_board_value(item):
|
||
continue
|
||
board_name = str(item).strip()
|
||
if board_name in dedupe:
|
||
continue
|
||
dedupe.add(board_name)
|
||
normalized.append({"name": board_name})
|
||
return normalized
|
||
|
||
if not DataFetcherManager._is_missing_board_value(raw_data):
|
||
board_name = str(raw_data).strip()
|
||
return [{"name": board_name}]
|
||
return []
|
||
|
||
def _init_default_fetchers(self) -> None:
|
||
"""
|
||
初始化默认数据源列表
|
||
|
||
优先级动态调整逻辑：
|
||
- 如果配置了 TUSHARE_TOKEN：实例化 TushareFetcher，并按其内部逻辑提升优先级
|
||
- 如果配置了 Longbridge OAuth 或 Legacy 凭据：实例化 LongbridgeFetcher 作为美股/港股兜底
|
||
- 未配置的可选数据源不实例化，避免在批量拉取时反复探测无效源
|
||
- 默认优先级：
|
||
0. EfinanceFetcher (Priority 0) - 最高优先级
|
||
1. AkshareFetcher (Priority 1)
|
||
2. PytdxFetcher (Priority 2) - 通达信
|
||
3. BaostockFetcher (Priority 3)
|
||
4. YfinanceFetcher (Priority 4)
|
||
5. TencentFetcher (Priority 5) - A 股最终兜底
|
||
"""
|
||
from src.config import get_config
|
||
from .efinance_fetcher import EfinanceFetcher
|
||
from .tencent_fetcher import TencentFetcher
|
||
from .akshare_fetcher import AkshareFetcher
|
||
from .tushare_fetcher import TushareFetcher
|
||
from .tickflow_fetcher import TickFlowFetcher
|
||
from .pytdx_fetcher import PytdxFetcher
|
||
from .baostock_fetcher import BaostockFetcher
|
||
from .yfinance_fetcher import YfinanceFetcher
|
||
from .longbridge_fetcher import LongbridgeFetcher
|
||
from .futu_fetcher import FutuFetcher
|
||
config = get_config()
|
||
# 创建所有数据源实例（优先级在各 Fetcher 的 __init__ 中确定）
|
||
efinance = EfinanceFetcher()
|
||
tencent = TencentFetcher()
|
||
akshare = AkshareFetcher()
|
||
pytdx = PytdxFetcher() # 通达信数据源（可配 PYTDX_HOST/PYTDX_PORT）
|
||
baostock = BaostockFetcher()
|
||
yfinance = YfinanceFetcher()
|
||
optional_fetchers: List[BaseFetcher] = []
|
||
|
||
tushare_token = (getattr(config, "tushare_token", None) or "").strip()
|
||
if tushare_token:
|
||
optional_fetchers.append(TushareFetcher()) # 会根据 Token 配置自动调整优先级
|
||
else:
|
||
logger.debug("[数据源初始化] 跳过未配置的 TushareFetcher")
|
||
|
||
tickflow_api_key = (getattr(config, "tickflow_api_key", None) or "").strip()
|
||
if tickflow_api_key:
|
||
optional_fetchers.append(
|
||
TickFlowFetcher(
|
||
api_key=tickflow_api_key,
|
||
kline_adjust=getattr(config, "tickflow_kline_adjust", "none"),
|
||
batch_daily_enabled=getattr(config, "tickflow_batch_daily_enabled", True),
|
||
batch_size=getattr(config, "tickflow_batch_size", 100),
|
||
priority=getattr(config, "tickflow_priority", 2),
|
||
)
|
||
)
|
||
else:
|
||
logger.debug("[data source init] skip TickFlowFetcher because TICKFLOW_API_KEY is not configured")
|
||
|
||
if LongbridgeFetcher.has_configured_credentials(config):
|
||
optional_fetchers.append(LongbridgeFetcher()) # 长桥（美股/港股兜底，懒加载）
|
||
else:
|
||
logger.debug("[数据源初始化] 跳过未配置的 LongbridgeFetcher")
|
||
|
||
if FutuFetcher.has_configured_endpoint():
|
||
optional_fetchers.append(FutuFetcher()) # 富途（港股，依赖 OpenD）
|
||
else:
|
||
logger.debug("[数据源初始化] 跳过未配置的 FutuFetcher")
|
||
|
||
finnhub_api_key = (getattr(config, "finnhub_api_key", None) or "").strip()
|
||
if finnhub_api_key:
|
||
from .finnhub_fetcher import FinnhubFetcher
|
||
optional_fetchers.append(FinnhubFetcher())
|
||
else:
|
||
logger.debug("[数据源初始化] 跳过未配置的 FinnhubFetcher")
|
||
|
||
alphavantage_api_key = (getattr(config, "alphavantage_api_key", None) or "").strip()
|
||
if alphavantage_api_key:
|
||
from .alphavantage_fetcher import AlphaVantageFetcher
|
||
optional_fetchers.append(AlphaVantageFetcher())
|
||
else:
|
||
logger.debug("[数据源初始化] 跳过未配置的 AlphaVantageFetcher")
|
||
|
||
# 初始化数据源列表
|
||
self._ensure_concurrency_guards()
|
||
with self._fetchers_lock:
|
||
self._fetchers = [
|
||
efinance,
|
||
akshare,
|
||
pytdx,
|
||
baostock,
|
||
yfinance,
|
||
tencent,
|
||
*optional_fetchers,
|
||
]
|
||
|
||
# 按优先级排序（Tushare 如果配置了 Token 且初始化成功，优先级为 0）
|
||
self._fetchers.sort(key=lambda f: f.priority)
|
||
self._refresh_fetcher_indexes_locked()
|
||
|
||
# 构建优先级说明
|
||
priority_info = ", ".join([f"{f.name}(P{f.priority})" for f in self._get_fetchers_snapshot()])
|
||
logger.info(f"已初始化 {len(self._fetchers)} 个数据源（按优先级）: {priority_info}")
|
||
|
||
def add_fetcher(self, fetcher: BaseFetcher) -> None:
|
||
"""添加数据源并重新排序"""
|
||
self._ensure_concurrency_guards()
|
||
with self._fetchers_lock:
|
||
self._fetchers.append(fetcher)
|
||
self._fetchers.sort(key=lambda f: f.priority)
|
||
self._refresh_fetcher_indexes_locked()
|
||
|
||
def get_daily_data(
|
||
self,
|
||
stock_code: str,
|
||
start_date: Optional[str] = None,
|
||
end_date: Optional[str] = None,
|
||
days: int = 30
|
||
) -> Tuple[pd.DataFrame, str]:
|
||
"""
|
||
获取日线数据（自动切换数据源）
|
||
|
||
故障切换策略：
|
||
1. 美股指数/美股股票直接路由到 YfinanceFetcher
|
||
2. 当前注册表已识别的 A 股指数使用固定指数数据源链
|
||
3. 其他代码从最高优先级数据源开始尝试
|
||
4. 捕获异常后自动切换到下一个并记录失败原因
|
||
5. 指数全源失败返回标准空结果；非指数全源失败抛出详细异常
|
||
|
||
Args:
|
||
stock_code: 股票代码
|
||
start_date: 开始日期
|
||
end_date: 结束日期
|
||
days: 获取天数
|
||
|
||
Returns:
|
||
Tuple[DataFrame, str]: (数据, 成功的数据源名称)
|
||
|
||
Raises:
|
||
DataFetchError: 非指数代码的所有数据源都失败时抛出
|
||
"""
|
||
from .us_index_mapping import is_us_index_code, is_us_stock_code
|
||
|
||
raw_stock_code = (stock_code or "").strip()
|
||
target = parse_analysis_target(raw_stock_code)
|
||
self._warn_bare_index_conflict(target)
|
||
if target.asset_type == ParseStatus.UNSUPPORTED:
|
||
reason = target.unsupported_reason or "unsupported analysis target"
|
||
raise DataFetchError(f"{raw_stock_code}: {reason}")
|
||
if target.asset_type == ParseStatus.INDEX:
|
||
return self._get_cn_index_daily_data(
|
||
target,
|
||
start_date=start_date,
|
||
end_date=end_date,
|
||
days=days,
|
||
)
|
||
|
||
# Normalize code (strip SH/SZ prefix etc.)
|
||
stock_code = normalize_stock_code(stock_code)
|
||
|
||
fetchers = self._get_fetchers_snapshot()
|
||
errors = []
|
||
request_start = time.time()
|
||
|
||
# 快速路径：美股使用专用数据源路由；港股先过滤不支持港股日线的数据源
|
||
# - 配置长桥凭据后: Longbridge 为首选, YFinance/AkShare 兜底
|
||
# - 未配置长桥: YFinance 为首选（美股）, 通用 fetcher 循环（港股）
|
||
# - 美股指数: 始终 YFinance 为首选（Longbridge 不提供指数K线）
|
||
is_us_index = is_us_index_code(stock_code)
|
||
is_us = is_us_index or is_us_stock_code(stock_code)
|
||
is_hk = (not is_us) and _is_hk_market(stock_code)
|
||
is_jp = (not is_us) and (not is_hk) and _is_jp_market(stock_code)
|
||
is_kr = (not is_us) and (not is_hk) and _is_kr_market(stock_code)
|
||
is_tw = (not is_us) and (not is_hk) and _is_tw_market(stock_code)
|
||
market = "us" if is_us else "hk" if is_hk else "jp" if is_jp else "kr" if is_kr else "tw" if is_tw else "cn"
|
||
if market != "cn":
|
||
fetchers = self._filter_daily_fetchers_for_market(fetchers, market)
|
||
fetchers = self._filter_fetchers_by_capability(fetchers, capability="daily_data")
|
||
total_fetchers = len(fetchers)
|
||
|
||
if total_fetchers == 0:
|
||
market_label = "美股指数" if is_us_index else "美股" if is_us else "港股" if is_hk else "台股" if is_tw else "A股"
|
||
error_summary = f"{market_label} {stock_code} 获取失败:\n暂无可用数据源"
|
||
logger.error(f"[数据源终止] {stock_code} 获取失败: {error_summary}")
|
||
raise DataFetchError(error_summary)
|
||
|
||
# 美股（含美股指数）使用专用路由；港股走下方通用数据源循环
|
||
# Failover chain: Finnhub(P2) -> AlphaVantage(P3) -> Yfinance(P4) -> Longbridge(P5)
|
||
# When Longbridge preferred: Longbridge -> Finnhub -> AlphaVantage -> Yfinance
|
||
if is_us:
|
||
prefer_lb = self._longbridge_preferred(capability="daily_data") and not is_us_index
|
||
if is_us_index:
|
||
# 指数始终 YFinance 首选（Longbridge 不提供指数K线）
|
||
source_order = ["YfinanceFetcher", "FinnhubFetcher"]
|
||
elif prefer_lb:
|
||
source_order = ["LongbridgeFetcher", "FinnhubFetcher", "AlphaVantageFetcher", "YfinanceFetcher"]
|
||
else:
|
||
source_order = ["FinnhubFetcher", "AlphaVantageFetcher", "YfinanceFetcher", "LongbridgeFetcher"]
|
||
market_label = "美股指数" if is_us_index else "美股"
|
||
|
||
for order_index, src_name in enumerate(source_order):
|
||
fallback_to = (
|
||
source_order[order_index + 1]
|
||
if order_index + 1 < len(source_order)
|
||
else None
|
||
)
|
||
for attempt, fetcher in enumerate(fetchers, start=1):
|
||
if fetcher.name != src_name:
|
||
continue
|
||
if not self._is_daily_source_available(fetcher, market):
|
||
errors.append(self._daily_source_unavailable_error(fetcher))
|
||
break
|
||
attempt_start = time.time()
|
||
try:
|
||
role = "首选" if src_name == source_order[0] else "兜底"
|
||
logger.info(
|
||
f"[数据源尝试 {attempt}/{total_fetchers}] [{fetcher.name}] "
|
||
f"{market_label} {stock_code} {role}路由..."
|
||
)
|
||
record_provider_run_started(
|
||
data_type="daily_data",
|
||
provider=fetcher.name,
|
||
operation="get_daily_data",
|
||
)
|
||
df = self._call_fetcher_method(
|
||
fetcher,
|
||
"get_daily_data",
|
||
stock_code=stock_code,
|
||
start_date=start_date,
|
||
end_date=end_date,
|
||
days=days,
|
||
)
|
||
if df is not None and not df.empty:
|
||
duration_ms = int((time.time() - attempt_start) * 1000)
|
||
record_provider_run(
|
||
data_type="daily_data",
|
||
provider=fetcher.name,
|
||
operation="get_daily_data",
|
||
success=True,
|
||
latency_ms=duration_ms,
|
||
record_count=len(df),
|
||
)
|
||
elapsed = time.time() - request_start
|
||
logger.info(
|
||
f"[数据源完成] {stock_code} 使用 [{fetcher.name}] 获取成功: "
|
||
f"rows={len(df)}, elapsed={elapsed:.2f}s"
|
||
)
|
||
self._record_daily_source_success(fetcher, market)
|
||
return df, fetcher.name
|
||
duration_ms = int((time.time() - attempt_start) * 1000)
|
||
record_provider_run(
|
||
data_type="daily_data",
|
||
provider=fetcher.name,
|
||
operation="get_daily_data",
|
||
success=False,
|
||
latency_ms=duration_ms,
|
||
error_type="empty",
|
||
error_message="empty result",
|
||
fallback_to=fallback_to,
|
||
record_count=0,
|
||
)
|
||
if df is not None and df.empty:
|
||
self._record_daily_source_success(fetcher, market)
|
||
except Exception as e:
|
||
error_type, error_reason = summarize_exception(e)
|
||
error_msg = f"[{fetcher.name}] ({error_type}) {error_reason}"
|
||
duration_ms = int((time.time() - attempt_start) * 1000)
|
||
record_provider_run(
|
||
data_type="daily_data",
|
||
provider=fetcher.name,
|
||
operation="get_daily_data",
|
||
success=False,
|
||
latency_ms=duration_ms,
|
||
error_type=error_type,
|
||
error_message=error_reason,
|
||
fallback_to=fallback_to,
|
||
)
|
||
logger.warning(
|
||
f"[数据源失败 {attempt}/{total_fetchers}] [{fetcher.name}] {stock_code}: "
|
||
f"error_type={error_type}, reason={error_reason}"
|
||
)
|
||
self._record_daily_source_failure(fetcher, market, error_reason)
|
||
errors.append(error_msg)
|
||
break
|
||
|
||
error_summary = f"{market_label} {stock_code} 获取失败:\n" + "\n".join(errors)
|
||
elapsed = time.time() - request_start
|
||
logger.error(f"[数据源终止] {stock_code} 获取失败: elapsed={elapsed:.2f}s\n{error_summary}")
|
||
raise DataFetchError(error_summary)
|
||
|
||
for attempt, fetcher in enumerate(fetchers, start=1):
|
||
if not self._is_daily_source_available(fetcher, market):
|
||
errors.append(self._daily_source_unavailable_error(fetcher))
|
||
continue
|
||
attempt_start = time.time()
|
||
fallback_to = fetchers[attempt].name if attempt < total_fetchers else None
|
||
try:
|
||
logger.info(f"[数据源尝试 {attempt}/{total_fetchers}] [{fetcher.name}] 获取 {stock_code}...")
|
||
record_provider_run_started(
|
||
data_type="daily_data",
|
||
provider=fetcher.name,
|
||
operation="get_daily_data",
|
||
)
|
||
df = self._call_fetcher_method(
|
||
fetcher,
|
||
"get_daily_data",
|
||
stock_code=stock_code,
|
||
start_date=start_date,
|
||
end_date=end_date,
|
||
days=days
|
||
)
|
||
|
||
if df is not None and not df.empty:
|
||
duration_ms = int((time.time() - attempt_start) * 1000)
|
||
record_provider_run(
|
||
data_type="daily_data",
|
||
provider=fetcher.name,
|
||
operation="get_daily_data",
|
||
success=True,
|
||
latency_ms=duration_ms,
|
||
record_count=len(df),
|
||
)
|
||
elapsed = time.time() - request_start
|
||
logger.info(
|
||
f"[数据源完成] {stock_code} 使用 [{fetcher.name}] 获取成功: "
|
||
f"rows={len(df)}, elapsed={elapsed:.2f}s"
|
||
)
|
||
self._record_daily_source_success(fetcher, market)
|
||
return df, fetcher.name
|
||
duration_ms = int((time.time() - attempt_start) * 1000)
|
||
record_provider_run(
|
||
data_type="daily_data",
|
||
provider=fetcher.name,
|
||
operation="get_daily_data",
|
||
success=False,
|
||
latency_ms=duration_ms,
|
||
error_type="empty",
|
||
error_message="empty result",
|
||
fallback_to=fallback_to,
|
||
record_count=0,
|
||
)
|
||
if df is not None and df.empty:
|
||
self._record_daily_source_success(fetcher, market)
|
||
|
||
except Exception as e:
|
||
error_type, error_reason = summarize_exception(e)
|
||
error_msg = f"[{fetcher.name}] ({error_type}) {error_reason}"
|
||
duration_ms = int((time.time() - attempt_start) * 1000)
|
||
record_provider_run(
|
||
data_type="daily_data",
|
||
provider=fetcher.name,
|
||
operation="get_daily_data",
|
||
success=False,
|
||
latency_ms=duration_ms,
|
||
error_type=error_type,
|
||
error_message=error_reason,
|
||
fallback_to=fallback_to,
|
||
)
|
||
logger.warning(
|
||
f"[数据源失败 {attempt}/{total_fetchers}] [{fetcher.name}] {stock_code}: "
|
||
f"error_type={error_type}, reason={error_reason}"
|
||
)
|
||
self._record_daily_source_failure(fetcher, market, error_reason)
|
||
errors.append(error_msg)
|
||
if attempt < total_fetchers:
|
||
next_fetcher = fetchers[attempt]
|
||
logger.info(f"[数据源切换] {stock_code}: [{fetcher.name}] -> [{next_fetcher.name}]")
|
||
# 继续尝试下一个数据源
|
||
continue
|
||
|
||
# 所有数据源都失败
|
||
error_summary = f"所有数据源获取 {stock_code} 失败:\n" + "\n".join(errors)
|
||
elapsed = time.time() - request_start
|
||
logger.error(f"[数据源终止] {stock_code} 获取失败: elapsed={elapsed:.2f}s\n{error_summary}")
|
||
raise DataFetchError(error_summary)
|
||
|
||
@property
|
||
def available_fetchers(self) -> List[str]:
|
||
"""返回可用数据源名称列表"""
|
||
return [f.name for f in self._get_fetchers_snapshot()]
|
||
|
||
def prefetch_realtime_quotes(self, stock_codes: List[str]) -> int:
|
||
"""
|
||
批量预取实时行情数据（在分析开始前调用）
|
||
|
||
策略：
|
||
1. 检查优先级中是否包含适合预取的数据源（efinance/akshare_em/tushare/tickflow）
|
||
2. 如果不包含，跳过预取（新浪/腾讯是单股票查询，无需预取）
|
||
3. 如果自选股数量 >= 5 且使用可预取数据源，则预取填充缓存
|
||
|
||
这样做的好处：
|
||
- 使用新浪/腾讯时：每只股票独立查询，无全量拉取问题
|
||
- 使用 efinance/东财/Tushare 时：预取一次，后续缓存命中
|
||
- 使用 TickFlow 时：按当前自选股批量预取，避免逐股重复请求
|
||
|
||
Args:
|
||
stock_codes: 待分析的股票代码列表
|
||
|
||
Returns:
|
||
预取的股票数量（0 表示跳过预取）
|
||
"""
|
||
# Normalize all codes, preserving explicit index identities
|
||
# (``sh000016`` / ``csi930955``) so the prefetch never degrades an
|
||
# index into the colliding stock bucket (Story 1.5).
|
||
normalized_codes: List[str] = []
|
||
for code in stock_codes:
|
||
target = parse_analysis_target(code)
|
||
if target.asset_type == ParseStatus.INDEX:
|
||
normalized_codes.append(target.canonical_id)
|
||
else:
|
||
normalized_codes.append(normalize_stock_code(code))
|
||
stock_codes = normalized_codes
|
||
|
||
from src.config import get_config
|
||
|
||
config = get_config()
|
||
|
||
# Issue #455: PREFETCH_REALTIME_QUOTES=false 可禁用预取，避免全市场拉取
|
||
if not getattr(config, "prefetch_realtime_quotes", True):
|
||
logger.debug("[预取] component=realtime_prefetch action=skip reason=disabled")
|
||
return 0
|
||
|
||
# 如果实时行情被禁用，跳过预取
|
||
if not config.enable_realtime_quote:
|
||
logger.debug("[预取] component=realtime_prefetch action=skip reason=realtime_quote_disabled")
|
||
return 0
|
||
|
||
# 检查优先级中是否包含适合批量预取的数据源
|
||
# efinance/akshare_em/tushare 通过一次调用填充全市场缓存；
|
||
# tickflow 通过 symbols 批量接口预取当前自选股缓存。
|
||
priority = config.realtime_source_priority.lower()
|
||
prefetch_sources = ['efinance', 'akshare_em', 'tushare', 'tickflow']
|
||
|
||
# 如果优先级中前两个都不是可预取数据源，跳过预取
|
||
# 因为新浪/腾讯是单股票查询，不需要预取
|
||
priority_list = [s.strip() for s in priority.split(',')]
|
||
first_prefetch_source_index = None
|
||
for i, source in enumerate(priority_list):
|
||
if source in prefetch_sources:
|
||
first_prefetch_source_index = i
|
||
break
|
||
|
||
# 如果没有可预取数据源，或者它排在第 3 位之后，跳过预取
|
||
if first_prefetch_source_index is None or first_prefetch_source_index >= 2:
|
||
logger.info(
|
||
"[预取] component=realtime_prefetch action=skip reason=no_early_prefetch_source priority=%s",
|
||
priority,
|
||
)
|
||
return 0
|
||
|
||
# 如果股票数量少于 5 个，不进行批量预取（逐个查询更高效）
|
||
if len(stock_codes) < 5:
|
||
logger.info(
|
||
"[预取] component=realtime_prefetch action=skip reason=small_batch "
|
||
"stock_count=%d threshold=5 prefetch_source=%s",
|
||
len(stock_codes),
|
||
priority_list[first_prefetch_source_index],
|
||
)
|
||
return 0
|
||
|
||
prefetch_source = priority_list[first_prefetch_source_index]
|
||
logger.info(
|
||
"[预取] component=realtime_prefetch action=start stock_count=%d prefetch_source=%s first_code=%s",
|
||
len(stock_codes),
|
||
prefetch_source,
|
||
stock_codes[0],
|
||
)
|
||
|
||
# TickFlow 使用 symbols 批量接口；其他可预取源通过首次查询触发自身缓存。
|
||
if prefetch_source == "tickflow":
|
||
fetcher = self._get_fetcher_by_name("TickFlowFetcher", capability="realtime_quote")
|
||
if fetcher is None or not hasattr(fetcher, "prefetch_realtime_quotes"):
|
||
logger.info(
|
||
"[prefetch] component=realtime_prefetch action=skip reason=tickflow_unavailable"
|
||
)
|
||
return 0
|
||
try:
|
||
return int(
|
||
self._call_fetcher_method(
|
||
fetcher,
|
||
"prefetch_realtime_quotes",
|
||
stock_codes,
|
||
batch_size=getattr(config, "tickflow_batch_size", 100),
|
||
)
|
||
or 0
|
||
)
|
||
except Exception as exc:
|
||
logger.warning("[TickFlowFetcher] realtime prefetch failed: %s", exc)
|
||
return 0
|
||
|
||
try:
|
||
# 用第一只股票触发全量拉取
|
||
first_code = stock_codes[0]
|
||
quote = self.get_realtime_quote(first_code)
|
||
|
||
if quote:
|
||
logger.info(
|
||
"[预取] component=realtime_prefetch action=complete status=success "
|
||
"stock_count=%d prefetch_source=%s",
|
||
len(stock_codes),
|
||
prefetch_source,
|
||
)
|
||
return len(stock_codes)
|
||
else:
|
||
logger.warning(
|
||
"[预取] component=realtime_prefetch action=complete status=failed "
|
||
"stock_count=%d prefetch_source=%s fallback=per_stock",
|
||
len(stock_codes),
|
||
prefetch_source,
|
||
)
|
||
return 0
|
||
|
||
except Exception as e:
|
||
logger.error(
|
||
"[预取] component=realtime_prefetch action=complete status=error "
|
||
"stock_count=%d prefetch_source=%s error=%s",
|
||
len(stock_codes),
|
||
prefetch_source,
|
||
e,
|
||
)
|
||
return 0
|
||
|
||
def prefetch_daily_klines(self, stock_codes: List[str], days: int = 30) -> int:
|
||
"""Batch-prefetch TickFlow daily K-lines without changing per-stock callers."""
|
||
fetcher = self._get_fetcher_by_name("TickFlowFetcher", capability="daily_data")
|
||
if fetcher is None or not hasattr(fetcher, "prefetch_daily_klines"):
|
||
return 0
|
||
|
||
try:
|
||
return int(
|
||
self._call_fetcher_method(
|
||
fetcher,
|
||
"prefetch_daily_klines",
|
||
stock_codes,
|
||
days=days,
|
||
)
|
||
or 0
|
||
)
|
||
except Exception as exc:
|
||
logger.warning("[TickFlowFetcher] daily K-line prefetch failed: %s", exc)
|
||
return 0
|
||
|
||
@staticmethod
|
||
def _utc_now_iso() -> str:
|
||
return datetime.now(timezone.utc).isoformat()
|
||
|
||
@staticmethod
|
||
def _parse_realtime_timestamp(value: Any) -> Optional[datetime]:
|
||
if value in (None, ""):
|
||
return None
|
||
if isinstance(value, datetime):
|
||
parsed = value
|
||
else:
|
||
text = str(value).strip()
|
||
if not text:
|
||
return None
|
||
if text.endswith("Z"):
|
||
text = text[:-1] + "+00:00"
|
||
try:
|
||
parsed = datetime.fromisoformat(text)
|
||
except ValueError:
|
||
return None
|
||
if parsed.tzinfo is None:
|
||
return parsed.replace(tzinfo=timezone.utc)
|
||
return parsed.astimezone(timezone.utc)
|
||
|
||
@staticmethod
|
||
def _realtime_fetcher_token(fetcher_name: str, **kw) -> str:
|
||
if fetcher_name == "AkshareFetcher" and kw.get("source") == "hk":
|
||
return "akshare_hk"
|
||
mapping = {
|
||
"LongbridgeFetcher": "longbridge",
|
||
"FutuFetcher": "futu",
|
||
"YfinanceFetcher": "yfinance",
|
||
"AkshareFetcher": "akshare",
|
||
"FinnhubFetcher": "finnhub",
|
||
"AlphaVantageFetcher": "alphavantage",
|
||
"EfinanceFetcher": "efinance",
|
||
"TushareFetcher": "tushare",
|
||
}
|
||
return mapping.get(fetcher_name, fetcher_name.replace("Fetcher", "").lower())
|
||
|
||
def _enrich_realtime_quote(
|
||
self,
|
||
quote,
|
||
*,
|
||
fallback_from: Optional[str] = None,
|
||
realtime_cache_ttl: Optional[int] = None,
|
||
):
|
||
"""Attach runtime metadata without inventing provider-side timestamps."""
|
||
if quote is None:
|
||
return None
|
||
|
||
fetched_at = self._utc_now_iso()
|
||
setattr(quote, "fetched_at", fetched_at)
|
||
if fallback_from:
|
||
setattr(quote, "fallback_from", str(fallback_from))
|
||
|
||
provider_dt = self._parse_realtime_timestamp(
|
||
getattr(quote, "provider_timestamp", None)
|
||
)
|
||
if provider_dt is None:
|
||
setattr(quote, "provider_timestamp", None)
|
||
setattr(quote, "stale_seconds", None)
|
||
setattr(quote, "is_stale", None)
|
||
return quote
|
||
|
||
setattr(quote, "provider_timestamp", provider_dt.isoformat())
|
||
fetched_dt = self._parse_realtime_timestamp(fetched_at) or datetime.now(timezone.utc)
|
||
stale_seconds = max(0, int((fetched_dt - provider_dt).total_seconds()))
|
||
ttl = realtime_cache_ttl if realtime_cache_ttl is not None else 600
|
||
setattr(quote, "stale_seconds", stale_seconds)
|
||
setattr(quote, "is_stale", stale_seconds > int(ttl))
|
||
return quote
|
||
|
||
def get_realtime_quote(self, stock_code: str, *, log_final_failure: bool = True):
|
||
"""
|
||
获取实时行情数据（自动故障切换）
|
||
|
||
故障切换策略（按配置的优先级）：
|
||
1. 美股：使用 YfinanceFetcher.get_realtime_quote()
|
||
2. EfinanceFetcher.get_realtime_quote()
|
||
3. AkshareFetcher.get_realtime_quote(source="em") - 东财
|
||
4. AkshareFetcher.get_realtime_quote(source="sina") - 新浪
|
||
5. AkshareFetcher.get_realtime_quote(source="tencent") - 腾讯
|
||
6. 返回 None（降级兜底）
|
||
|
||
Args:
|
||
stock_code: 股票代码
|
||
log_final_failure: Whether to emit the final "all sources failed"
|
||
summary log when no realtime quote is available.
|
||
|
||
Returns:
|
||
UnifiedRealtimeQuote 对象，所有数据源都失败则返回 None
|
||
"""
|
||
raw_stock_code = (stock_code or "").strip()
|
||
# Normalize code (strip SH/SZ prefix etc.)
|
||
stock_code = normalize_stock_code(stock_code)
|
||
|
||
from .akshare_fetcher import _is_us_code
|
||
from .us_index_mapping import is_us_index_code
|
||
from src.config import get_config
|
||
|
||
config = get_config()
|
||
|
||
# 如果实时行情功能被禁用，直接返回 None
|
||
if not config.enable_realtime_quote:
|
||
logger.debug(f"[实时行情] 功能已禁用，跳过 {stock_code}")
|
||
return None
|
||
|
||
# ----------------------------------------------------------
|
||
# 已登记 A 股指数 — 固定实时行情链（Story 1.5）
|
||
# 显式指数身份在 normalize 前解析，避免 sh000016 被剥成
|
||
# 股票 000016 后误取同名股票行情。
|
||
# ----------------------------------------------------------
|
||
index_target = parse_analysis_target(raw_stock_code)
|
||
if index_target.asset_type == ParseStatus.INDEX:
|
||
return self._get_cn_index_realtime_quote(
|
||
index_target, log_final_failure=log_final_failure
|
||
)
|
||
|
||
# ----------------------------------------------------------
|
||
# 美股 (指数 + 个股) / 港股 — 专用双源路由
|
||
# 配置长桥后: Longbridge 首选, YFinance/AkShare 补充
|
||
# 未配置长桥: YFinance/AkShare 首选, Longbridge 补充
|
||
# 美股指数: 始终 YFinance 首选（Longbridge 不提供指数行情）
|
||
# ----------------------------------------------------------
|
||
is_us_index = is_us_index_code(stock_code)
|
||
is_us = is_us_index or _is_us_code(stock_code)
|
||
is_hk = (not is_us) and _is_hk_market(stock_code)
|
||
is_jp = (not is_us) and (not is_hk) and _is_jp_market(stock_code)
|
||
is_kr = (not is_us) and (not is_hk) and _is_kr_market(stock_code)
|
||
is_tw = (not is_us) and (not is_hk) and _is_tw_market(stock_code)
|
||
|
||
if is_jp or is_kr or is_tw:
|
||
market_label = "日股" if is_jp else "韩股" if is_kr else "台股"
|
||
quote = self._try_fetcher_quote(stock_code, "YfinanceFetcher")
|
||
if quote is not None:
|
||
logger.info(f"[实时行情] {market_label} {stock_code} 成功获取 (来源: YfinanceFetcher)")
|
||
return self._enrich_realtime_quote(
|
||
quote,
|
||
realtime_cache_ttl=getattr(config, "realtime_cache_ttl", None),
|
||
)
|
||
if log_final_failure:
|
||
logger.info(f"[实时行情] {market_label} {stock_code} 无可用数据源")
|
||
return None
|
||
|
||
if is_us or is_hk:
|
||
prefer_lb = self._longbridge_preferred() and not is_us_index
|
||
if is_us:
|
||
primary_src = "LongbridgeFetcher" if prefer_lb else "YfinanceFetcher"
|
||
secondary_src = "YfinanceFetcher" if prefer_lb else "LongbridgeFetcher"
|
||
market_label = "美股指数" if is_us_index else "美股"
|
||
primary_kw: dict = {}
|
||
secondary_kw: dict = {}
|
||
else:
|
||
hk_priority = [
|
||
source.strip().lower()
|
||
for source in getattr(
|
||
config,
|
||
"futu_hk_realtime_source_priority",
|
||
"futu,longbridge,akshare,yfinance",
|
||
).split(",")
|
||
if source.strip()
|
||
]
|
||
source_map = {
|
||
"futu": ("FutuFetcher", {}),
|
||
"longbridge": ("LongbridgeFetcher", {}),
|
||
"akshare": ("AkshareFetcher", {"source": "hk"}),
|
||
"yfinance": ("YfinanceFetcher", {}),
|
||
}
|
||
primary_quote = None
|
||
primary_token = None
|
||
primary_src_index = -1
|
||
fallback_from = None
|
||
# Futu only participates when an OpenD endpoint is configured.
|
||
# Skipping an unconfigured source here (instead of letting
|
||
# _try_fetcher_quote fail on it) avoids recording a never-enabled
|
||
# source as the failed primary, which would wrongly mark a
|
||
# successful quote from the next enabled source as fallback.
|
||
futu_enabled = False
|
||
try:
|
||
from data_provider.futu_fetcher import FutuFetcher
|
||
futu_enabled = FutuFetcher.has_configured_endpoint()
|
||
except Exception: # noqa: BLE001 - fail closed: treat futu as disabled
|
||
futu_enabled = False
|
||
for index, source in enumerate(hk_priority):
|
||
mapped = source_map.get(source)
|
||
if mapped is None:
|
||
logger.warning("[实时行情] 忽略未知港股数据源: %s", source)
|
||
continue
|
||
fetcher_name, fetcher_kw = mapped
|
||
if fetcher_name == "FutuFetcher" and not futu_enabled:
|
||
logger.info(
|
||
"[实时行情] 港股 %s 未配置 FUTU_OPEND_HOST，跳过 futu 源", stock_code
|
||
)
|
||
continue
|
||
quote = self._try_fetcher_quote(stock_code, fetcher_name, **fetcher_kw)
|
||
if quote is not None:
|
||
primary_quote = quote
|
||
primary_token = self._realtime_fetcher_token(fetcher_name, **fetcher_kw)
|
||
primary_src_index = index
|
||
logger.info("[实时行情] 港股 %s 成功获取 (来源: %s)", stock_code, fetcher_name)
|
||
break
|
||
# 该源失败：记住它的 token，供后续成功源作为 fallback_from 使用。
|
||
if fallback_from is None:
|
||
fallback_from = self._realtime_fetcher_token(fetcher_name, **fetcher_kw)
|
||
if primary_quote is not None:
|
||
# 用后续数据源补充缺失字段（volume_ratio / turnover_rate / 估值 / 市值），
|
||
# 保持与美股路径一致的 _supplement_quote 补字段能力。
|
||
for source in hk_priority[primary_src_index + 1:]:
|
||
mapped = source_map.get(source)
|
||
if mapped is None:
|
||
continue
|
||
if not self._quote_needs_supplement(primary_quote):
|
||
break
|
||
fetcher_name, fetcher_kw = mapped
|
||
self._supplement_quote(stock_code, primary_quote, fetcher_name, **fetcher_kw)
|
||
return self._enrich_realtime_quote(
|
||
primary_quote,
|
||
fallback_from=fallback_from,
|
||
realtime_cache_ttl=getattr(config, "realtime_cache_ttl", None),
|
||
)
|
||
if log_final_failure:
|
||
logger.info("[实时行情] 港股 %s 无可用数据源", stock_code)
|
||
return None
|
||
primary_token = self._realtime_fetcher_token(primary_src, **primary_kw)
|
||
primary_quote = self._try_fetcher_quote(stock_code, primary_src, **primary_kw)
|
||
fallback_from = primary_token if primary_quote is None else None
|
||
if primary_quote is not None:
|
||
logger.info(f"[实时行情] {market_label} {stock_code} 成功获取 (来源: {primary_src})")
|
||
primary_quote = self._supplement_quote(
|
||
stock_code, primary_quote, secondary_src, **secondary_kw,
|
||
)
|
||
if is_us and not is_us_index and primary_quote is not None:
|
||
for extra_src in ["FinnhubFetcher", "AlphaVantageFetcher"]:
|
||
primary_quote = self._supplement_quote(
|
||
stock_code, primary_quote, extra_src,
|
||
)
|
||
if primary_quote is not None:
|
||
return self._enrich_realtime_quote(
|
||
primary_quote,
|
||
fallback_from=fallback_from,
|
||
realtime_cache_ttl=getattr(config, "realtime_cache_ttl", None),
|
||
)
|
||
if log_final_failure:
|
||
logger.info(f"[实时行情] {market_label} {stock_code} 无可用数据源")
|
||
return None
|
||
|
||
# 获取配置的数据源优先级
|
||
source_priority = [
|
||
source.strip().lower()
|
||
for source in config.realtime_source_priority.split(',')
|
||
if source.strip()
|
||
]
|
||
|
||
errors = []
|
||
failed_sources: List[str] = []
|
||
# primary_quote holds the first successful result; we may supplement
|
||
# missing fields (volume_ratio, turnover_rate, etc.) from later sources.
|
||
primary_quote = None
|
||
primary_fallback_from: Optional[str] = None
|
||
|
||
for source_index, source in enumerate(source_priority):
|
||
attempt_start = time.time()
|
||
fallback_to = source_priority[source_index + 1] if source_index + 1 < len(source_priority) else None
|
||
fetcher = None
|
||
try:
|
||
quote = None
|
||
|
||
if source == "efinance":
|
||
fetcher = self._get_fetcher_by_name("EfinanceFetcher", capability="realtime_quote")
|
||
if fetcher is not None and hasattr(fetcher, 'get_realtime_quote'):
|
||
record_provider_run_started(
|
||
data_type="realtime_quote",
|
||
provider=fetcher.name,
|
||
operation="get_realtime_quote",
|
||
)
|
||
quote = self._call_fetcher_method(fetcher, 'get_realtime_quote', stock_code)
|
||
|
||
elif source == "akshare_em":
|
||
fetcher = self._get_fetcher_by_name("AkshareFetcher", capability="realtime_quote")
|
||
if fetcher is not None and hasattr(fetcher, 'get_realtime_quote'):
|
||
record_provider_run_started(
|
||
data_type="realtime_quote",
|
||
provider=fetcher.name,
|
||
operation="get_realtime_quote",
|
||
)
|
||
quote = self._call_fetcher_method(fetcher, 'get_realtime_quote', stock_code, source="em")
|
||
|
||
elif source == "akshare_sina":
|
||
fetcher = self._get_fetcher_by_name("AkshareFetcher", capability="realtime_quote")
|
||
if fetcher is not None and hasattr(fetcher, 'get_realtime_quote'):
|
||
record_provider_run_started(
|
||
data_type="realtime_quote",
|
||
provider=fetcher.name,
|
||
operation="get_realtime_quote",
|
||
)
|
||
quote = self._call_fetcher_method(fetcher, 'get_realtime_quote', stock_code, source="sina")
|
||
|
||
elif source in ("tencent", "akshare_qq"):
|
||
fetcher = self._get_fetcher_by_name("AkshareFetcher", capability="realtime_quote")
|
||
if fetcher is not None and hasattr(fetcher, 'get_realtime_quote'):
|
||
record_provider_run_started(
|
||
data_type="realtime_quote",
|
||
provider=fetcher.name,
|
||
operation="get_realtime_quote",
|
||
)
|
||
quote = self._call_fetcher_method(fetcher, 'get_realtime_quote', stock_code, source="tencent")
|
||
|
||
elif source == "tushare":
|
||
fetcher = self._get_fetcher_by_name("TushareFetcher", capability="realtime_quote")
|
||
if fetcher is not None and hasattr(fetcher, 'get_realtime_quote'):
|
||
record_provider_run_started(
|
||
data_type="realtime_quote",
|
||
provider=fetcher.name,
|
||
operation="get_realtime_quote",
|
||
)
|
||
quote = self._call_fetcher_method(fetcher, 'get_realtime_quote', raw_stock_code or stock_code)
|
||
|
||
elif source == "tickflow":
|
||
fetcher = self._get_fetcher_by_name("TickFlowFetcher", capability="realtime_quote")
|
||
if fetcher is not None and hasattr(fetcher, 'get_realtime_quote'):
|
||
record_provider_run_started(
|
||
data_type="realtime_quote",
|
||
provider=fetcher.name,
|
||
operation="get_realtime_quote",
|
||
)
|
||
quote = self._call_fetcher_method(fetcher, 'get_realtime_quote', raw_stock_code or stock_code)
|
||
|
||
provider_name = fetcher.name if fetcher is not None else source
|
||
|
||
if quote is not None and quote.has_basic_data():
|
||
record_provider_run(
|
||
data_type="realtime_quote",
|
||
provider=provider_name,
|
||
operation="get_realtime_quote",
|
||
success=True,
|
||
latency_ms=int((time.time() - attempt_start) * 1000),
|
||
fallback_to=fallback_to if primary_quote is None and self._quote_needs_supplement(quote) else None,
|
||
record_count=1,
|
||
)
|
||
if primary_quote is None:
|
||
# First successful source becomes primary
|
||
primary_quote = quote
|
||
primary_fallback_from = failed_sources[0] if failed_sources else None
|
||
logger.info(f"[实时行情] {stock_code} 成功获取 (来源: {source})")
|
||
# If all key supplementary fields are present, return early
|
||
if not self._quote_needs_supplement(primary_quote):
|
||
return self._enrich_realtime_quote(
|
||
primary_quote,
|
||
fallback_from=primary_fallback_from,
|
||
realtime_cache_ttl=getattr(config, "realtime_cache_ttl", None),
|
||
)
|
||
# Otherwise, continue to try later sources for missing fields
|
||
logger.debug(f"[实时行情] {stock_code} 部分字段缺失，尝试从后续数据源补充")
|
||
supplement_attempts = 0
|
||
else:
|
||
# Supplement missing fields from this source (limit attempts)
|
||
supplement_attempts += 1
|
||
if supplement_attempts > 1:
|
||
logger.debug(f"[实时行情] {stock_code} 补充尝试已达上限，停止继续")
|
||
break
|
||
merged = self._merge_quote_fields(primary_quote, quote)
|
||
if merged:
|
||
logger.info(f"[实时行情] {stock_code} 从 {source} 补充了缺失字段: {merged}")
|
||
# Stop supplementing once all key fields are filled
|
||
if not self._quote_needs_supplement(primary_quote):
|
||
break
|
||
else:
|
||
record_provider_run(
|
||
data_type="realtime_quote",
|
||
provider=provider_name,
|
||
operation="get_realtime_quote",
|
||
success=False,
|
||
latency_ms=int((time.time() - attempt_start) * 1000),
|
||
error_type="empty",
|
||
error_message="empty or incomplete quote",
|
||
fallback_to=fallback_to,
|
||
record_count=0,
|
||
)
|
||
if primary_quote is None:
|
||
failed_sources.append(source)
|
||
|
||
except Exception as e:
|
||
error_msg = f"[{source}] 失败: {str(e)}"
|
||
error_type, error_reason = summarize_exception(e)
|
||
record_provider_run(
|
||
data_type="realtime_quote",
|
||
provider=getattr(fetcher, "name", source),
|
||
operation="get_realtime_quote",
|
||
success=False,
|
||
latency_ms=int((time.time() - attempt_start) * 1000),
|
||
error_type=error_type,
|
||
error_message=error_reason,
|
||
fallback_to=fallback_to,
|
||
)
|
||
logger.info(f"[实时行情] {stock_code} {error_msg}，继续尝试下一个数据源")
|
||
errors.append(error_msg)
|
||
if primary_quote is None:
|
||
failed_sources.append(source)
|
||
continue
|
||
|
||
# Return primary even if some fields are still missing
|
||
if primary_quote is not None:
|
||
return self._enrich_realtime_quote(
|
||
primary_quote,
|
||
fallback_from=primary_fallback_from,
|
||
realtime_cache_ttl=getattr(config, "realtime_cache_ttl", None),
|
||
)
|
||
|
||
# 所有数据源都失败，返回 None（降级兜底）
|
||
if log_final_failure:
|
||
if errors:
|
||
logger.info(f"[实时行情] {stock_code} 所有数据源均失败: {'; '.join(errors)}")
|
||
else:
|
||
logger.info(f"[实时行情] {stock_code} 无可用数据源")
|
||
|
||
return None
|
||
|
||
# Fields worth supplementing from secondary sources when the primary
|
||
# source returns None for them. Ordered by importance.
|
||
_SUPPLEMENT_FIELDS = [
|
||
'volume_ratio', 'turnover_rate',
|
||
'pe_ratio', 'pb_ratio', 'total_mv', 'circ_mv',
|
||
'amplitude',
|
||
]
|
||
|
||
@classmethod
|
||
def _quote_needs_supplement(cls, quote) -> bool:
|
||
"""Check if any key supplementary field is still None."""
|
||
for f in cls._SUPPLEMENT_FIELDS:
|
||
if getattr(quote, f, None) is None:
|
||
return True
|
||
return False
|
||
|
||
@classmethod
|
||
def _merge_quote_fields(cls, primary, secondary) -> list:
|
||
"""
|
||
Copy non-None fields from *secondary* into *primary* where
|
||
*primary* has None. Returns list of field names that were filled.
|
||
"""
|
||
filled = []
|
||
for f in cls._SUPPLEMENT_FIELDS:
|
||
if getattr(primary, f, None) is None:
|
||
val = getattr(secondary, f, None)
|
||
if val is not None:
|
||
setattr(primary, f, val)
|
||
filled.append(f)
|
||
return filled
|
||
|
||
def _longbridge_preferred(self, capability: str = "realtime_quote") -> bool:
|
||
"""Return True when Longbridge keys are configured and available.
|
||
|
||
When True, non-A-share routing (US & HK) uses Longbridge as the
|
||
primary data source with Yfinance/AkShare as fallback.
|
||
"""
|
||
return self._get_fetcher_by_name(
|
||
"LongbridgeFetcher",
|
||
capability=capability,
|
||
) is not None
|
||
|
||
def _try_fetcher_quote(self, stock_code: str, fetcher_name: str, **kw):
|
||
"""Try to get a realtime quote from a named fetcher; returns quote or None."""
|
||
fetcher = self._get_fetcher_by_name(fetcher_name, capability="realtime_quote")
|
||
if fetcher is None or not hasattr(fetcher, 'get_realtime_quote'):
|
||
record_provider_run(
|
||
data_type="realtime_quote",
|
||
provider=fetcher_name,
|
||
operation="get_realtime_quote",
|
||
success=False,
|
||
error_type="unavailable",
|
||
error_message="fetcher unavailable",
|
||
)
|
||
return None
|
||
attempt_start = time.time()
|
||
try:
|
||
record_provider_run_started(
|
||
data_type="realtime_quote",
|
||
provider=fetcher.name,
|
||
operation="get_realtime_quote",
|
||
)
|
||
q = self._call_fetcher_method(fetcher, 'get_realtime_quote', stock_code, **kw)
|
||
if q is not None and q.has_basic_data():
|
||
record_provider_run(
|
||
data_type="realtime_quote",
|
||
provider=fetcher.name,
|
||
operation="get_realtime_quote",
|
||
success=True,
|
||
latency_ms=int((time.time() - attempt_start) * 1000),
|
||
record_count=1,
|
||
)
|
||
return q
|
||
record_provider_run(
|
||
data_type="realtime_quote",
|
||
provider=fetcher.name,
|
||
operation="get_realtime_quote",
|
||
success=False,
|
||
latency_ms=int((time.time() - attempt_start) * 1000),
|
||
error_type="empty",
|
||
error_message="empty or incomplete quote",
|
||
record_count=0,
|
||
)
|
||
except Exception as e:
|
||
error_type, error_reason = summarize_exception(e)
|
||
record_provider_run(
|
||
data_type="realtime_quote",
|
||
provider=fetcher.name,
|
||
operation="get_realtime_quote",
|
||
success=False,
|
||
latency_ms=int((time.time() - attempt_start) * 1000),
|
||
error_type=error_type,
|
||
error_message=error_reason,
|
||
)
|
||
logger.debug(f"[实时行情] {stock_code} {fetcher_name} 获取失败: {e}")
|
||
return None
|
||
|
||
def _supplement_quote(self, stock_code: str, primary_quote, fetcher_name: str, **kw):
|
||
"""Supplement *primary_quote* with data from *fetcher_name*.
|
||
|
||
If *primary_quote* is None, try *fetcher_name* as the sole source.
|
||
Returns the (potentially enriched) quote, or None.
|
||
"""
|
||
if primary_quote is not None:
|
||
if not self._quote_needs_supplement(primary_quote):
|
||
return primary_quote
|
||
try:
|
||
secondary = self._try_fetcher_quote(stock_code, fetcher_name, **kw)
|
||
if secondary is not None:
|
||
filled = self._merge_quote_fields(primary_quote, secondary)
|
||
if filled:
|
||
logger.info(f"[实时行情] {stock_code} 从 {fetcher_name} 补充了: {filled}")
|
||
except Exception as e:
|
||
logger.debug(f"[实时行情] {stock_code} {fetcher_name} 补充失败: {e}")
|
||
return primary_quote
|
||
|
||
q = self._try_fetcher_quote(stock_code, fetcher_name, **kw)
|
||
if q is not None:
|
||
logger.info(f"[实时行情] {stock_code} 从 {fetcher_name} 获取成功 (独立数据源)")
|
||
return q
|
||
|
||
def _supplement_from_longbridge(self, stock_code: str, primary_quote):
|
||
"""Shortcut kept for backward-compat with A-share general loop."""
|
||
return self._supplement_quote(stock_code, primary_quote, "LongbridgeFetcher")
|
||
|
||
def get_chip_distribution(self, stock_code: str):
|
||
"""
|
||
获取筹码分布数据（带熔断和多数据源降级）
|
||
|
||
策略：
|
||
1. 检查配置开关
|
||
2. 检查熔断器状态
|
||
3. 依次尝试多个数据源：数据源优先级与获取daily的数据优先级一致
|
||
4. 所有数据源失败则返回 None（降级兜底）
|
||
|
||
Args:
|
||
stock_code: 股票代码
|
||
|
||
Returns:
|
||
ChipDistribution 对象，失败则返回 None
|
||
"""
|
||
# Normalize code (strip SH/SZ prefix etc.)
|
||
stock_code = normalize_stock_code(stock_code)
|
||
|
||
from .realtime_types import get_chip_circuit_breaker
|
||
from src.config import get_config
|
||
|
||
config = get_config()
|
||
|
||
# 如果筹码分布功能被禁用，直接返回 None
|
||
if not config.enable_chip_distribution:
|
||
logger.debug(f"[筹码分布] 功能已禁用，跳过 {stock_code}")
|
||
return None
|
||
|
||
circuit_breaker = get_chip_circuit_breaker()
|
||
|
||
candidate_fetchers = []
|
||
# 直接遍历管理器已经按 priority 排好序的数据源列表
|
||
for fetcher in self._get_fetchers_snapshot():
|
||
# 只处理实现了筹码分布逻辑的数据源
|
||
if not hasattr(fetcher, 'get_chip_distribution'):
|
||
continue
|
||
|
||
fetcher_name = fetcher.name
|
||
# 动态生成熔断器的 key，例如 "TushareFetcher" -> "tushare_chip"
|
||
source_key = f"{fetcher_name.replace('Fetcher', '').lower()}_chip"
|
||
|
||
# 检查熔断器状态
|
||
if not circuit_breaker.is_available(source_key):
|
||
logger.debug(f"[熔断] {fetcher_name} 筹码接口处于熔断状态，尝试下一个")
|
||
continue
|
||
|
||
candidate_fetchers.append((fetcher, fetcher_name, source_key))
|
||
|
||
for index, (fetcher, fetcher_name, source_key) in enumerate(candidate_fetchers):
|
||
fallback_to = (
|
||
candidate_fetchers[index + 1][1]
|
||
if index + 1 < len(candidate_fetchers)
|
||
else None
|
||
)
|
||
attempt_start = time.time()
|
||
try:
|
||
record_provider_run_started(
|
||
data_type="chip",
|
||
provider=fetcher_name,
|
||
operation="get_chip_distribution",
|
||
)
|
||
chip = self._call_fetcher_method(fetcher, 'get_chip_distribution', stock_code)
|
||
latency_ms = int((time.time() - attempt_start) * 1000)
|
||
if _is_meaningful_chip_distribution(chip):
|
||
record_provider_run(
|
||
data_type="chip",
|
||
provider=fetcher_name,
|
||
operation="get_chip_distribution",
|
||
success=True,
|
||
latency_ms=latency_ms,
|
||
record_count=1,
|
||
)
|
||
circuit_breaker.record_success(source_key)
|
||
logger.info(f"[筹码分布] {stock_code} 成功获取 (来源: {fetcher_name})")
|
||
return chip
|
||
else:
|
||
record_provider_run(
|
||
data_type="chip",
|
||
provider=fetcher_name,
|
||
operation="get_chip_distribution",
|
||
success=False,
|
||
latency_ms=latency_ms,
|
||
error_type="empty",
|
||
error_message="empty or incomplete chip distribution",
|
||
fallback_to=fallback_to,
|
||
record_count=0,
|
||
)
|
||
if chip is not None:
|
||
logger.warning(
|
||
"[筹码分布] %s 返回字段不完整或占位值，继续尝试下一个数据源",
|
||
fetcher_name,
|
||
)
|
||
# 空结果或占位结果：释放 HALF_OPEN 探测名额，避免卡死
|
||
circuit_breaker.record_inconclusive(source_key)
|
||
except Exception as e:
|
||
error_type, error_reason = summarize_exception(e)
|
||
record_provider_run(
|
||
data_type="chip",
|
||
provider=fetcher_name,
|
||
operation="get_chip_distribution",
|
||
success=False,
|
||
latency_ms=int((time.time() - attempt_start) * 1000),
|
||
error_type=error_type,
|
||
error_message=error_reason,
|
||
fallback_to=fallback_to,
|
||
)
|
||
logger.warning(f"[筹码分布] {fetcher_name} 获取 {stock_code} 失败: {e}")
|
||
circuit_breaker.record_failure(source_key, str(e))
|
||
continue
|
||
|
||
logger.warning(f"[筹码分布] {stock_code} 所有数据源均失败")
|
||
return None
|
||
|
||
def get_stock_name(self, stock_code: str, allow_realtime: bool = True) -> Optional[str]:
|
||
"""
|
||
获取股票中文名称（自动切换数据源）
|
||
|
||
尝试从多个数据源获取股票名称：
|
||
1. 先从内存缓存中获取（如果有）
|
||
2. 再尝试本地维护映射与 stocks.index.json 索引
|
||
3. 然后按需查询实时行情
|
||
4. 依次尝试各个数据源的 get_stock_name 方法
|
||
|
||
Args:
|
||
stock_code: 股票代码
|
||
allow_realtime: Whether to query realtime quote first. Set False when
|
||
caller only wants lightweight prefetch without triggering heavy
|
||
realtime source calls.
|
||
|
||
Returns:
|
||
股票中文名称，所有数据源都失败则返回 None
|
||
"""
|
||
raw_stock_code = (stock_code or "").strip()
|
||
target = parse_analysis_target(raw_stock_code)
|
||
self._warn_bare_index_conflict(target)
|
||
if target.asset_type == ParseStatus.UNSUPPORTED:
|
||
logger.warning(
|
||
"[股票名称] 跳过不支持的输入 %s: %s",
|
||
raw_stock_code,
|
||
target.unsupported_reason or "unsupported analysis target",
|
||
)
|
||
return ""
|
||
if target.asset_type == ParseStatus.INDEX:
|
||
return self._get_cn_index_name(target)
|
||
|
||
# Normalize code (strip SH/SZ prefix etc.)
|
||
stock_code = normalize_stock_code(stock_code)
|
||
static_name = STOCK_NAME_MAP.get(stock_code)
|
||
|
||
# 1. 先检查缓存
|
||
cached_name = self._get_cached_stock_name(stock_code)
|
||
if cached_name is not None:
|
||
return cached_name
|
||
|
||
if is_meaningful_stock_name(static_name, stock_code):
|
||
return self._cache_stock_name(stock_code, static_name) or static_name
|
||
|
||
index_name = get_index_stock_name(stock_code)
|
||
if is_meaningful_stock_name(index_name, stock_code):
|
||
return self._cache_stock_name(stock_code, index_name) or index_name
|
||
|
||
# 2. 尝试从实时行情中获取（最快，可按需禁用）
|
||
if allow_realtime:
|
||
quote = self.get_realtime_quote(raw_stock_code or stock_code, log_final_failure=False)
|
||
if quote and hasattr(quote, 'name') and is_meaningful_stock_name(getattr(quote, 'name', ''), stock_code):
|
||
name = quote.name
|
||
self._cache_stock_name(stock_code, name)
|
||
logger.info(f"[股票名称] 从实时行情获取: {stock_code} -> {name}")
|
||
return name
|
||
|
||
# 3. 依次尝试各个数据源
|
||
from .akshare_fetcher import _is_us_code
|
||
is_us = _is_us_code(stock_code)
|
||
_US_CAPABLE_FETCHERS = {"YfinanceFetcher", "LongbridgeFetcher", "FinnhubFetcher", "AlphaVantageFetcher"}
|
||
for fetcher in self._get_fetchers_snapshot():
|
||
if not hasattr(fetcher, 'get_stock_name'):
|
||
continue
|
||
if is_us and fetcher.name not in _US_CAPABLE_FETCHERS:
|
||
continue
|
||
if not self._is_fetcher_available(fetcher, capability="stock_name"):
|
||
continue
|
||
try:
|
||
name = self._call_fetcher_method(fetcher, 'get_stock_name', stock_code)
|
||
if is_meaningful_stock_name(name, stock_code):
|
||
self._cache_stock_name(stock_code, name)
|
||
logger.info(f"[股票名称] 从 {fetcher.name} 获取: {stock_code} -> {name}")
|
||
return name
|
||
except Exception as e:
|
||
logger.debug(f"[股票名称] {fetcher.name} 获取失败: {e}")
|
||
continue
|
||
|
||
# 4. 所有数据源都失败
|
||
logger.warning(f"[股票名称] 所有数据源都无法获取 {stock_code} 的名称")
|
||
return ""
|
||
|
||
def get_belong_boards(self, stock_code: str) -> List[Dict[str, Any]]:
|
||
"""
|
||
Get stock membership boards through capability probing.
|
||
|
||
Keep this at manager layer to avoid changing BaseFetcher abstraction.
|
||
"""
|
||
stock_code = normalize_stock_code(stock_code)
|
||
if _market_tag(stock_code) != "cn":
|
||
return []
|
||
candidate_fetchers = [
|
||
fetcher
|
||
for fetcher in self._fetchers
|
||
if hasattr(fetcher, "get_belong_board")
|
||
]
|
||
for index, fetcher in enumerate(candidate_fetchers):
|
||
fallback_to = (
|
||
candidate_fetchers[index + 1].name
|
||
if index + 1 < len(candidate_fetchers)
|
||
else None
|
||
)
|
||
start = time.time()
|
||
try:
|
||
record_provider_run_started(
|
||
data_type="belong_boards",
|
||
provider=fetcher.name,
|
||
operation="get_belong_board",
|
||
)
|
||
raw_data = fetcher.get_belong_board(stock_code)
|
||
boards = self._normalize_belong_boards(raw_data)
|
||
if boards:
|
||
record_provider_run(
|
||
data_type="belong_boards",
|
||
provider=fetcher.name,
|
||
operation="get_belong_board",
|
||
success=True,
|
||
latency_ms=int((time.time() - start) * 1000),
|
||
record_count=len(boards),
|
||
)
|
||
logger.info(f"[{fetcher.name}] 获取所属板块成功: {stock_code}, count={len(boards)}")
|
||
return boards
|
||
record_provider_run(
|
||
data_type="belong_boards",
|
||
provider=fetcher.name,
|
||
operation="get_belong_board",
|
||
success=False,
|
||
latency_ms=int((time.time() - start) * 1000),
|
||
error_type="empty",
|
||
error_message="empty belong boards",
|
||
fallback_to=fallback_to,
|
||
record_count=0,
|
||
)
|
||
except Exception as e:
|
||
error_type, error_reason = summarize_exception(e)
|
||
record_provider_run(
|
||
data_type="belong_boards",
|
||
provider=fetcher.name,
|
||
operation="get_belong_board",
|
||
success=False,
|
||
latency_ms=int((time.time() - start) * 1000),
|
||
error_type=error_type,
|
||
error_message=error_reason,
|
||
fallback_to=fallback_to,
|
||
)
|
||
logger.debug(f"[{fetcher.name}] 获取所属板块失败: {e}")
|
||
continue
|
||
return []
|
||
|
||
def prefetch_stock_names(self, stock_codes: List[str], use_bulk: bool = False) -> None:
|
||
"""
|
||
Pre-fetch stock names into cache before parallel analysis (Issue #455).
|
||
|
||
When use_bulk=False, only calls get_stock_name per code (no get_stock_list),
|
||
avoiding full-market fetch. Sequential execution to avoid rate limits.
|
||
|
||
Args:
|
||
stock_codes: Stock codes to prefetch.
|
||
use_bulk: If True, may use get_stock_list (full fetch). Default False.
|
||
"""
|
||
if not stock_codes:
|
||
return
|
||
normalized_codes: List[str] = []
|
||
for code in stock_codes:
|
||
target = parse_analysis_target(code)
|
||
if target.asset_type == ParseStatus.UNSUPPORTED:
|
||
logger.warning(
|
||
"[股票名称] 预取跳过不支持的输入 %s: %s",
|
||
code,
|
||
target.unsupported_reason or "unsupported analysis target",
|
||
)
|
||
continue
|
||
normalized_codes.append(
|
||
target.canonical_id
|
||
if target.asset_type == ParseStatus.INDEX
|
||
else normalize_stock_code(code)
|
||
)
|
||
stock_codes = normalized_codes
|
||
if use_bulk:
|
||
self.batch_get_stock_names(stock_codes)
|
||
return
|
||
for code in stock_codes:
|
||
# Skip realtime lookup to avoid triggering expensive full-market quote
|
||
# requests during the prefetch phase.
|
||
self.get_stock_name(code, allow_realtime=False)
|
||
|
||
def batch_get_stock_names(self, stock_codes: List[str]) -> Dict[str, str]:
|
||
"""
|
||
批量获取股票中文名称
|
||
|
||
先尝试从支持批量查询的数据源获取股票列表，
|
||
然后再逐个查询缺失的股票名称。
|
||
|
||
Args:
|
||
stock_codes: 股票代码列表
|
||
|
||
Returns:
|
||
{股票代码: 股票名称} 字典
|
||
"""
|
||
result = {}
|
||
missing_codes = set(stock_codes)
|
||
|
||
# 1. 先检查缓存
|
||
self._ensure_concurrency_guards()
|
||
with self._stock_name_cache_lock:
|
||
for code in stock_codes:
|
||
cached_name = self._stock_name_cache.get(code)
|
||
if cached_name is not None:
|
||
result[code] = cached_name
|
||
missing_codes.discard(code)
|
||
|
||
if not missing_codes:
|
||
return result
|
||
|
||
# 2. 尝试批量获取股票列表
|
||
for fetcher in self._get_fetchers_snapshot():
|
||
if not hasattr(fetcher, 'get_stock_list') or not missing_codes:
|
||
continue
|
||
if not self._is_fetcher_available(fetcher, capability="stock_list"):
|
||
continue
|
||
try:
|
||
stock_list = self._call_fetcher_method(fetcher, 'get_stock_list')
|
||
if stock_list is not None and not stock_list.empty:
|
||
cache_updates: Dict[str, str] = {}
|
||
for _, row in stock_list.iterrows():
|
||
code = row.get('code')
|
||
name = row.get('name')
|
||
if code and name:
|
||
cache_updates[code] = name
|
||
if code in missing_codes:
|
||
result[code] = name
|
||
missing_codes.discard(code)
|
||
|
||
if cache_updates:
|
||
with self._stock_name_cache_lock:
|
||
self._stock_name_cache.update(cache_updates)
|
||
|
||
if not missing_codes:
|
||
break
|
||
|
||
logger.info(f"[股票名称] 从 {fetcher.name} 批量获取完成，剩余 {len(missing_codes)} 个待查")
|
||
except Exception as e:
|
||
logger.debug(f"[股票名称] {fetcher.name} 批量获取失败: {e}")
|
||
continue
|
||
|
||
# 3. 逐个获取剩余的
|
||
for code in list(missing_codes):
|
||
name = self.get_stock_name(code)
|
||
if name:
|
||
result[code] = name
|
||
missing_codes.discard(code)
|
||
|
||
logger.info(f"[股票名称] 批量获取完成，成功 {len(result)}/{len(stock_codes)}")
|
||
return result
|
||
|
||
def get_main_indices(self, region: str = "cn") -> List[Dict[str, Any]]:
|
||
"""获取主要指数实时行情（自动切换数据源）"""
|
||
if region == "cn":
|
||
tickflow_fetcher = self._get_tickflow_fetcher()
|
||
if tickflow_fetcher is not None:
|
||
try:
|
||
data = tickflow_fetcher.get_main_indices(region=region)
|
||
if data:
|
||
logger.info("[TickFlowFetcher] 获取指数行情成功")
|
||
return data
|
||
except Exception as e:
|
||
logger.warning(f"[TickFlowFetcher] 获取指数行情失败: {e}")
|
||
|
||
for fetcher in self._fetchers:
|
||
if region == "cn" and fetcher.name == "TickFlowFetcher":
|
||
continue
|
||
try:
|
||
data = fetcher.get_main_indices(region=region)
|
||
if data:
|
||
logger.info(f"[{fetcher.name}] 获取指数行情成功")
|
||
return data
|
||
except Exception as e:
|
||
logger.warning(f"[{fetcher.name}] 获取指数行情失败: {e}")
|
||
continue
|
||
return []
|
||
|
||
def get_market_stats(self, *, purpose: str = "unspecified") -> Dict[str, Any]:
|
||
"""获取市场涨跌统计（自动切换数据源）"""
|
||
logger.info("[MarketStats] component=market_stats action=start purpose=%s", purpose)
|
||
tickflow_fetcher = self._get_tickflow_fetcher()
|
||
if tickflow_fetcher is not None:
|
||
started_at = time.monotonic()
|
||
try:
|
||
data = tickflow_fetcher.get_market_stats()
|
||
elapsed = time.monotonic() - started_at
|
||
if data:
|
||
logger.info(
|
||
"[MarketStats] component=market_stats action=provider_success "
|
||
"purpose=%s provider=TickFlowFetcher elapsed=%.2fs",
|
||
purpose,
|
||
elapsed,
|
||
)
|
||
return data
|
||
logger.info(
|
||
"[MarketStats] component=market_stats action=provider_empty "
|
||
"purpose=%s provider=TickFlowFetcher elapsed=%.2fs",
|
||
purpose,
|
||
elapsed,
|
||
)
|
||
except Exception as e:
|
||
elapsed = time.monotonic() - started_at
|
||
logger.warning(
|
||
"[MarketStats] component=market_stats action=provider_failed "
|
||
"purpose=%s provider=TickFlowFetcher elapsed=%.2fs error=%s",
|
||
purpose,
|
||
elapsed,
|
||
e,
|
||
)
|
||
|
||
for fetcher in self._fetchers:
|
||
if fetcher.name == "TickFlowFetcher":
|
||
continue
|
||
started_at = time.monotonic()
|
||
try:
|
||
data = fetcher.get_market_stats()
|
||
elapsed = time.monotonic() - started_at
|
||
if data:
|
||
logger.info(
|
||
"[MarketStats] component=market_stats action=provider_success "
|
||
"purpose=%s provider=%s elapsed=%.2fs",
|
||
purpose,
|
||
fetcher.name,
|
||
elapsed,
|
||
)
|
||
return data
|
||
logger.info(
|
||
"[MarketStats] component=market_stats action=provider_empty "
|
||
"purpose=%s provider=%s elapsed=%.2fs",
|
||
purpose,
|
||
fetcher.name,
|
||
elapsed,
|
||
)
|
||
except Exception as e:
|
||
elapsed = time.monotonic() - started_at
|
||
logger.warning(
|
||
"[MarketStats] component=market_stats action=provider_failed "
|
||
"purpose=%s provider=%s elapsed=%.2fs error=%s",
|
||
purpose,
|
||
fetcher.name,
|
||
elapsed,
|
||
e,
|
||
)
|
||
continue
|
||
logger.warning("[MarketStats] component=market_stats action=complete status=empty purpose=%s", purpose)
|
||
return {}
|
||
|
||
def _run_with_timeout(
|
||
self,
|
||
task: Callable[[], Any],
|
||
timeout_seconds: float,
|
||
task_name: str,
|
||
) -> Tuple[Optional[Any], Optional[str], int]:
|
||
"""
|
||
Execute a task in a short-lived thread and enforce a timeout.
|
||
|
||
Returns:
|
||
(result, error, duration_ms)
|
||
"""
|
||
start = time.time()
|
||
timeout_value = max(0.0, timeout_seconds)
|
||
if timeout_value <= 0:
|
||
return None, f"{task_name} timeout", 0
|
||
result_holder: Dict[str, Any] = {}
|
||
error_holder: Dict[str, Exception] = {}
|
||
|
||
if not self._fundamental_timeout_slots.acquire(blocking=False):
|
||
return None, f"{task_name} timeout worker pool exhausted", int(timeout_value * 1000)
|
||
|
||
def runner() -> None:
|
||
try:
|
||
result_holder["value"] = task()
|
||
except Exception as exc:
|
||
error_holder["value"] = exc
|
||
finally:
|
||
try:
|
||
self._fundamental_timeout_slots.release()
|
||
except ValueError:
|
||
pass
|
||
|
||
worker = Thread(target=runner, daemon=True, name=f"fundamental-{task_name}")
|
||
try:
|
||
worker.start()
|
||
except Exception as exc:
|
||
try:
|
||
self._fundamental_timeout_slots.release()
|
||
except ValueError:
|
||
pass
|
||
return None, str(exc), int((time.time() - start) * 1000)
|
||
worker.join(timeout=timeout_value)
|
||
if worker.is_alive():
|
||
return None, f"{task_name} timeout", int(timeout_value * 1000)
|
||
if "value" in error_holder:
|
||
return None, str(error_holder["value"]), int((time.time() - start) * 1000)
|
||
return result_holder.get("value"), None, int((time.time() - start) * 1000)
|
||
|
||
def _run_with_retry(
|
||
self,
|
||
task: Callable[[], Any],
|
||
timeout_seconds: float,
|
||
task_name: str,
|
||
) -> Tuple[Optional[Any], Optional[str], int]:
|
||
"""
|
||
Execute a task with bounded budget and best-effort retries.
|
||
|
||
Returns:
|
||
(result, error, total_duration_ms)
|
||
"""
|
||
config = self._get_fundamental_config()
|
||
attempts = max(1, int(config.fundamental_retry_max))
|
||
remaining_seconds = max(0.0, float(timeout_seconds))
|
||
total_cost_ms = 0
|
||
last_error: Optional[str] = None
|
||
|
||
for _ in range(attempts):
|
||
if remaining_seconds <= 0:
|
||
break
|
||
result, err, cost_ms = self._run_with_timeout(task, remaining_seconds, task_name)
|
||
total_cost_ms += cost_ms
|
||
remaining_seconds = max(0.0, remaining_seconds - cost_ms / 1000)
|
||
if err is None:
|
||
return result, None, total_cost_ms
|
||
last_error = err
|
||
if remaining_seconds <= 0:
|
||
break
|
||
|
||
return None, last_error, total_cost_ms
|
||
|
||
def _get_fundamental_config(self):
|
||
from src.config import get_config
|
||
return get_config()
|
||
|
||
@staticmethod
|
||
def _normalize_source_chain(
|
||
entries: Any,
|
||
provider: str,
|
||
result: str,
|
||
duration_ms: int,
|
||
) -> List[Dict[str, Any]]:
|
||
"""Normalize free-form source chain entries to structured dict list."""
|
||
if entries is None:
|
||
return [{"provider": provider, "result": result, "duration_ms": duration_ms}]
|
||
|
||
normalized: List[Dict[str, Any]] = []
|
||
if not isinstance(entries, (list, tuple)):
|
||
entries = [entries]
|
||
|
||
for item in entries:
|
||
if isinstance(item, dict):
|
||
normalized.append({
|
||
"provider": str(item.get("provider") or provider),
|
||
"result": str(item.get("result") or result),
|
||
"duration_ms": int(item.get("duration_ms", duration_ms)),
|
||
})
|
||
continue
|
||
|
||
if item is None:
|
||
continue
|
||
|
||
provider_name = str(item)
|
||
normalized.append({
|
||
"provider": provider_name,
|
||
"result": result,
|
||
"duration_ms": duration_ms,
|
||
})
|
||
|
||
if not normalized:
|
||
return [{"provider": provider, "result": result, "duration_ms": duration_ms}]
|
||
|
||
return normalized
|
||
|
||
@staticmethod
|
||
def _block_status(payload: Dict[str, Any], available: bool = True) -> str:
|
||
if not available:
|
||
return "not_supported"
|
||
if not payload:
|
||
return "partial"
|
||
return "ok"
|
||
|
||
@staticmethod
|
||
def _build_fundamental_block(
|
||
status: str,
|
||
payload: Optional[Dict[str, Any]] = None,
|
||
source_chain: Optional[List[Dict[str, Any]]] = None,
|
||
errors: Optional[List[str]] = None,
|
||
) -> Dict[str, Any]:
|
||
return {
|
||
"status": status,
|
||
"coverage": {"status": status},
|
||
"source_chain": source_chain or [],
|
||
"errors": errors or [],
|
||
"data": payload or {},
|
||
}
|
||
|
||
@staticmethod
|
||
def _has_meaningful_payload(payload: Any) -> bool:
|
||
if payload is None:
|
||
return False
|
||
if isinstance(payload, str):
|
||
normalized = payload.strip().lower()
|
||
return normalized not in ("", "-", "nan", "none", "null", "n/a", "na")
|
||
if isinstance(payload, dict):
|
||
return any(DataFetcherManager._has_meaningful_payload(v) for v in payload.values())
|
||
if isinstance(payload, pd.DataFrame):
|
||
if payload.empty:
|
||
return False
|
||
return any(
|
||
DataFetcherManager._has_meaningful_payload(v)
|
||
for v in payload.to_numpy().flat
|
||
)
|
||
if isinstance(payload, (pd.Series, pd.Index)):
|
||
return any(DataFetcherManager._has_meaningful_payload(v) for v in payload.tolist())
|
||
if isinstance(payload, np.ndarray):
|
||
if payload.ndim == 0:
|
||
payload = payload.item()
|
||
else:
|
||
return any(
|
||
DataFetcherManager._has_meaningful_payload(v)
|
||
for v in payload.flat
|
||
)
|
||
if isinstance(payload, (list, tuple, set)):
|
||
return any(DataFetcherManager._has_meaningful_payload(v) for v in payload)
|
||
if DataFetcherManager._try_scalar_isna(payload, "fundamental_payload") is True:
|
||
return False
|
||
return True
|
||
|
||
@staticmethod
|
||
def _dividend_contract_has_values(payload: Any) -> bool:
|
||
"""Check whether the dividend block satisfies the repo contract.
|
||
|
||
Downstream consumers read ttm_cash_dividend_per_share /
|
||
ttm_dividend_yield_pct and events[].cash_dividend_per_share /
|
||
ex_dividend_date / event_date. Raw provider events that only carry
|
||
provider-native keys (statement/ex_date/record_date) do not satisfy
|
||
the contract, so they must be treated as missing.
|
||
"""
|
||
if not isinstance(payload, dict):
|
||
return DataFetcherManager._has_meaningful_payload(payload)
|
||
# The repo contract consumes ttm_cash_dividend_per_share and
|
||
# ttm_dividend_yield_pct as a pair. A block with TTM cash but no
|
||
# yield (e.g. the extra realtime price snapshot failed or returned
|
||
# no price) is still missing a consumed field, so it must count as
|
||
# a gap and be supplemented instead of being treated as complete.
|
||
if DataFetcherManager._has_meaningful_payload(payload.get("ttm_cash_dividend_per_share")):
|
||
return DataFetcherManager._has_meaningful_payload(payload.get("ttm_dividend_yield_pct"))
|
||
for key in ("ttm_cash_dividend_per_share", "ttm_dividend_yield_pct"):
|
||
if DataFetcherManager._has_meaningful_payload(payload.get(key)):
|
||
return True
|
||
events = payload.get("events")
|
||
if isinstance(events, list):
|
||
for event in events:
|
||
if not isinstance(event, dict):
|
||
continue
|
||
if DataFetcherManager._has_meaningful_payload(
|
||
event.get("cash_dividend_per_share")
|
||
) or DataFetcherManager._has_meaningful_payload(
|
||
event.get("ex_dividend_date") or event.get("event_date")
|
||
):
|
||
return True
|
||
return False
|
||
|
||
@staticmethod
|
||
def _earnings_block_has_values(payload: Any) -> bool:
|
||
"""Field-level check for the earnings block.
|
||
|
||
A truthy dict with only metadata (report_date/period/currency) is a
|
||
shell, not usable earnings. Require a core numeric field (revenue /
|
||
net_profit_parent / basic_eps / gross_profit) or a populated dividend
|
||
section before treating the block as usable.
|
||
"""
|
||
if not isinstance(payload, dict):
|
||
return DataFetcherManager._has_meaningful_payload(payload)
|
||
report = payload.get("financial_report")
|
||
if isinstance(report, dict):
|
||
for key in ("revenue", "net_profit_parent", "basic_eps", "gross_profit"):
|
||
if DataFetcherManager._has_meaningful_payload(report.get(key)):
|
||
return True
|
||
dividend = payload.get("dividend")
|
||
if DataFetcherManager._has_meaningful_payload(dividend):
|
||
return True
|
||
# Fall back to the generic check for other earnings sub-blocks.
|
||
for key in ("financial_reports", "indicators"):
|
||
if key in payload and DataFetcherManager._has_meaningful_payload(payload.get(key)):
|
||
return True
|
||
return False
|
||
|
||
@staticmethod
|
||
def _infer_block_status(payload: Any, fallback_status: str) -> str:
|
||
if DataFetcherManager._has_meaningful_payload(payload):
|
||
return "ok"
|
||
if fallback_status in ("failed", "partial", "not_supported"):
|
||
return fallback_status
|
||
return "partial"
|
||
|
||
@staticmethod
|
||
def _should_cache_fundamental_context(context: Any) -> bool:
|
||
if not isinstance(context, dict):
|
||
return False
|
||
status = str(context.get("status", "")).strip().lower()
|
||
if status == "ok":
|
||
return True
|
||
if status == "failed":
|
||
return False
|
||
for block in (
|
||
"valuation",
|
||
"growth",
|
||
"earnings",
|
||
"institution",
|
||
"capital_flow",
|
||
"dragon_tiger",
|
||
"boards",
|
||
):
|
||
payload = context.get(block, {})
|
||
if isinstance(payload, dict) and DataFetcherManager._has_meaningful_payload(payload.get("data")):
|
||
return True
|
||
return False
|
||
|
||
def _build_market_not_supported(self, market: str, reason: str) -> Dict[str, Any]:
|
||
blocks = {
|
||
"valuation": self._build_fundamental_block(
|
||
"partial" if market == "etf" else "not_supported",
|
||
{},
|
||
[{"provider": "fundamental_pipeline", "result": "not_supported", "duration_ms": 0}],
|
||
[reason],
|
||
),
|
||
"growth": self._build_fundamental_block(
|
||
"not_supported",
|
||
{},
|
||
[{"provider": "fundamental_pipeline", "result": "not_supported", "duration_ms": 0}],
|
||
[reason],
|
||
),
|
||
"earnings": self._build_fundamental_block(
|
||
"not_supported",
|
||
{},
|
||
[{"provider": "fundamental_pipeline", "result": "not_supported", "duration_ms": 0}],
|
||
[reason],
|
||
),
|
||
"institution": self._build_fundamental_block(
|
||
"not_supported",
|
||
{},
|
||
[{"provider": "fundamental_pipeline", "result": "not_supported", "duration_ms": 0}],
|
||
[reason],
|
||
),
|
||
"capital_flow": self._build_fundamental_block(
|
||
"not_supported",
|
||
{},
|
||
[{"provider": "fundamental_pipeline", "result": "not_supported", "duration_ms": 0}],
|
||
[reason],
|
||
),
|
||
"dragon_tiger": self._build_fundamental_block(
|
||
"not_supported",
|
||
{},
|
||
[{"provider": "fundamental_pipeline", "result": "not_supported", "duration_ms": 0}],
|
||
[reason],
|
||
),
|
||
"boards": self._build_fundamental_block(
|
||
"not_supported",
|
||
{},
|
||
[{"provider": "fundamental_pipeline", "result": "not_supported", "duration_ms": 0}],
|
||
[reason],
|
||
),
|
||
}
|
||
return {
|
||
"market": market,
|
||
"status": "partial" if market == "etf" else "not_supported",
|
||
"coverage": {
|
||
block: blocks[block]["status"] for block in blocks
|
||
},
|
||
"source_chain": [{"provider": "fundamental_pipeline", "result": "not_supported", "duration_ms": 0}],
|
||
"errors": [reason],
|
||
**blocks,
|
||
}
|
||
|
||
def _fetch_offshore_fundamental_bundle(
|
||
self,
|
||
stock_code: str,
|
||
market: str,
|
||
bundle_timeout: float,
|
||
) -> Tuple[Dict[str, Any], Optional[str], int, str]:
|
||
"""Fetch the fundamental bundle for offshore markets.
|
||
|
||
For HK with a configured Futu OpenD endpoint, try the Futu fundamental
|
||
adapter first (company profile, statements, dividends/splits, capital
|
||
flow, boards). When Futu succeeds only partially (e.g. statements
|
||
failed but static info worked), fetch the yfinance bundle as well and
|
||
merge the missing blocks so existing HK growth/earnings capability is
|
||
never silently downgraded. Returns (payload, error, duration_ms,
|
||
provider_name).
|
||
"""
|
||
from src.config import get_config
|
||
|
||
config = get_config()
|
||
|
||
def _use_yfinance() -> Tuple[Dict[str, Any], Optional[str], int, str]:
|
||
payload, err, ms = self._run_with_retry(
|
||
lambda: self._yfinance_fundamental_adapter.get_fundamental_bundle(stock_code),
|
||
bundle_timeout,
|
||
"fundamental_bundle_yfinance",
|
||
)
|
||
return payload or {}, err, ms, "fundamental_bundle_yfinance"
|
||
|
||
def _field_gaps(payload: Dict[str, Any]) -> List[str]:
|
||
"""List core growth/earnings fields that are missing or value-less.
|
||
|
||
A field is a gap when it has no usable value, so the yfinance
|
||
bundle should be consulted to fill it regardless of block-level
|
||
truthiness.
|
||
"""
|
||
gaps: List[str] = []
|
||
growth = payload.get("growth")
|
||
if isinstance(growth, dict):
|
||
for field in ("revenue_yoy", "net_profit_yoy", "gross_margin"):
|
||
if not self._has_meaningful_payload(growth.get(field)):
|
||
gaps.append(f"growth.{field}")
|
||
earnings = payload.get("earnings")
|
||
report = earnings.get("financial_report") if isinstance(earnings, dict) else None
|
||
if isinstance(report, dict):
|
||
for field in ("revenue", "net_profit_parent", "basic_eps", "gross_profit"):
|
||
if not self._has_meaningful_payload(report.get(field)):
|
||
gaps.append(f"earnings.financial_report.{field}")
|
||
# Dividend: the repo contract consumes ttm_* fields and/or
|
||
# events[].cash_dividend_per_share / ex_dividend_date. Raw OpenD
|
||
# events (statement/ex_date/record_date) without normalization do
|
||
# not satisfy it, so treat the block as a gap unless the contract
|
||
# fields carry usable values.
|
||
dividend = earnings.get("dividend") if isinstance(earnings, dict) else None
|
||
if not DataFetcherManager._dividend_contract_has_values(dividend):
|
||
gaps.append("earnings.dividend")
|
||
return gaps
|
||
|
||
def _merge_bundles(
|
||
futu_payload: Dict[str, Any],
|
||
yfinance_payload: Dict[str, Any],
|
||
futu_ms: int,
|
||
yfinance_ms: int,
|
||
) -> Tuple[Dict[str, Any], Optional[str], int, str]:
|
||
"""Field-level merge: keep Futu values, fill gaps from yfinance."""
|
||
merged: Dict[str, Any] = dict(futu_payload)
|
||
|
||
# growth: field-level fill.
|
||
futu_growth = futu_payload.get("growth")
|
||
yf_growth = yfinance_payload.get("growth")
|
||
if isinstance(yf_growth, dict):
|
||
growth = dict(futu_growth) if isinstance(futu_growth, dict) else {}
|
||
for field, value in yf_growth.items():
|
||
if not self._has_meaningful_payload(growth.get(field)):
|
||
growth[field] = value
|
||
if any(self._has_meaningful_payload(v) for v in growth.values()):
|
||
merged["growth"] = growth
|
||
|
||
# earnings: financial_report field-level fill, dividend block-level.
|
||
futu_earnings = futu_payload.get("earnings")
|
||
yf_earnings = yfinance_payload.get("earnings")
|
||
if isinstance(yf_earnings, dict):
|
||
earnings = dict(futu_earnings) if isinstance(futu_earnings, dict) else {}
|
||
futu_report = earnings.get("financial_report")
|
||
yf_report = yf_earnings.get("financial_report")
|
||
if isinstance(yf_report, dict):
|
||
report = dict(futu_report) if isinstance(futu_report, dict) else {}
|
||
for field, value in yf_report.items():
|
||
if not self._has_meaningful_payload(report.get(field)):
|
||
report[field] = value
|
||
if any(self._has_meaningful_payload(v) for v in report.values()):
|
||
earnings["financial_report"] = report
|
||
if not DataFetcherManager._dividend_contract_has_values(
|
||
earnings.get("dividend")
|
||
) and DataFetcherManager._dividend_contract_has_values(
|
||
yf_earnings.get("dividend")
|
||
):
|
||
earnings["dividend"] = yf_earnings.get("dividend")
|
||
if any(
|
||
DataFetcherManager._earnings_block_has_values(earnings.get(key))
|
||
or self._has_meaningful_payload(earnings.get(key))
|
||
for key in ("financial_report", "financial_reports", "dividend", "indicators")
|
||
):
|
||
merged["earnings"] = earnings
|
||
|
||
# Other blocks stay block-level (Futu wins, yfinance fills absent blocks).
|
||
for key in ("institution", "capital_flow", "belong_boards"):
|
||
if not self._has_meaningful_payload(merged.get(key)) and self._has_meaningful_payload(
|
||
yfinance_payload.get(key)
|
||
):
|
||
merged[key] = yfinance_payload.get(key)
|
||
|
||
merged["source_chain"] = list(
|
||
futu_payload.get("source_chain", [])
|
||
) + list(yfinance_payload.get("source_chain", []))
|
||
merged["errors"] = list(futu_payload.get("errors", [])) + list(
|
||
yfinance_payload.get("errors", [])
|
||
)
|
||
has_content = any(
|
||
merged.get(key)
|
||
for key in ("growth", "earnings", "institution", "capital_flow", "belong_boards")
|
||
)
|
||
merged["status"] = "partial" if has_content else "not_supported"
|
||
return merged, None, futu_ms + yfinance_ms, "fundamental_bundle_futu"
|
||
|
||
try:
|
||
from data_provider.futu_fetcher import FutuFetcher
|
||
from data_provider.futu_fundamental_adapter import FutuFundamentalAdapter
|
||
except Exception as exc: # noqa: BLE001 - fail open to yfinance
|
||
logger.warning("[futu-fundamental] import failed, using yfinance: %s", exc)
|
||
return _use_yfinance()
|
||
|
||
if market != "hk" or not FutuFetcher.has_configured_endpoint():
|
||
return _use_yfinance()
|
||
|
||
futu_fetcher = getattr(self, "_futu_fundamental_fetcher", None)
|
||
if futu_fetcher is None:
|
||
try:
|
||
futu_fetcher = FutuFetcher()
|
||
self._futu_fundamental_fetcher = futu_fetcher
|
||
except Exception as exc: # noqa: BLE001 - fail open to yfinance
|
||
logger.warning("[futu-fundamental] fetcher init failed, using yfinance: %s", exc)
|
||
return _use_yfinance()
|
||
|
||
adapter = FutuFundamentalAdapter(futu_fetcher)
|
||
futu_payload, futu_err, futu_ms = self._run_with_retry(
|
||
lambda: adapter.get_fundamental_bundle(stock_code),
|
||
bundle_timeout,
|
||
"fundamental_bundle_futu",
|
||
)
|
||
if futu_err is None and isinstance(futu_payload, dict):
|
||
has_content = any(
|
||
futu_payload.get(key)
|
||
for key in ("growth", "earnings", "institution", "capital_flow", "belong_boards")
|
||
)
|
||
if has_content:
|
||
# Futu partial success: keep the blocks it returned but do not
|
||
# silently drop any core growth/earnings field that yfinance
|
||
# could still provide. Decide by field gaps (not block-level
|
||
# truthiness) so partial Futu results are supplemented.
|
||
gaps = _field_gaps(futu_payload)
|
||
remaining_timeout = max(bundle_timeout - futu_ms / 1000.0, 0.0)
|
||
if gaps and remaining_timeout > 0:
|
||
yfinance_payload, yfinance_err, yfinance_ms = self._run_with_retry(
|
||
lambda: self._yfinance_fundamental_adapter.get_fundamental_bundle(stock_code),
|
||
remaining_timeout,
|
||
"fundamental_bundle_yfinance",
|
||
)
|
||
if yfinance_err is None and isinstance(yfinance_payload, dict):
|
||
return _merge_bundles(futu_payload, yfinance_payload, futu_ms, yfinance_ms)
|
||
logger.warning(
|
||
"[futu-fundamental] %s yfinance supplement failed (%s); keeping partial Futu bundle",
|
||
stock_code,
|
||
yfinance_err,
|
||
)
|
||
return futu_payload, None, futu_ms, "fundamental_bundle_futu"
|
||
logger.info(
|
||
"[futu-fundamental] %s bundle empty (status=%s), falling back to yfinance",
|
||
stock_code,
|
||
futu_payload.get("status"),
|
||
)
|
||
return _use_yfinance()
|
||
|
||
def _build_offshore_fundamental_context(
|
||
self,
|
||
stock_code: str,
|
||
market: str,
|
||
budget_seconds: Optional[float] = None,
|
||
) -> Dict[str, Any]:
|
||
"""HK/US fundamental aggregation via yfinance.
|
||
|
||
Mirrors :meth:`get_fundamental_context` but skips A-share-specific
|
||
blocks (capital_flow, dragon_tiger, sector rankings). belong_boards is
|
||
sourced from yfinance ``info.sector`` / ``info.industry``.
|
||
|
||
Cache, retry and fail-open semantics intentionally match the CN path so
|
||
upstream callers see the same shape regardless of market.
|
||
"""
|
||
from src.config import get_config
|
||
|
||
config = get_config()
|
||
stage_timeout = float(
|
||
budget_seconds if budget_seconds is not None else config.fundamental_stage_timeout_seconds
|
||
)
|
||
stage_timeout = max(0.0, stage_timeout)
|
||
fetch_timeout = float(config.fundamental_fetch_timeout_seconds)
|
||
fetch_timeout = max(0.0, fetch_timeout)
|
||
|
||
cache_ttl = int(config.fundamental_cache_ttl_seconds)
|
||
cache_max_entries = max(0, int(getattr(config, "fundamental_cache_max_entries", 256)))
|
||
cache_key = self._get_fundamental_cache_key(stock_code, stage_timeout)
|
||
if cache_ttl > 0:
|
||
self._prune_fundamental_cache(cache_ttl, cache_max_entries)
|
||
with self._fundamental_cache_lock:
|
||
cache_item = self._fundamental_cache.get(cache_key)
|
||
if cache_item:
|
||
age = time.time() - float(cache_item.get("ts", 0))
|
||
if age <= cache_ttl:
|
||
return cache_item.get("context", {})
|
||
|
||
result_ctx: Dict[str, Any] = {
|
||
"market": market,
|
||
"provider": "yfinance",
|
||
"as_of": datetime.now(timezone.utc).isoformat(),
|
||
"data_quality": "unavailable",
|
||
"missing_fields": [],
|
||
"valuation": {},
|
||
"growth": {},
|
||
"earnings": {},
|
||
"institution": {},
|
||
"capital_flow": {},
|
||
"dragon_tiger": {},
|
||
"boards": {},
|
||
"belong_boards": [],
|
||
"coverage": {},
|
||
"source_chain": [],
|
||
"errors": [],
|
||
}
|
||
start_ts = time.time()
|
||
|
||
# Valuation: reuse realtime quote payload — yfinance returns pe/pb in the
|
||
# same shape as AkShare, so the existing block formatter still works.
|
||
valuation_timeout = min(fetch_timeout, stage_timeout) if stage_timeout > 0 else 0
|
||
if valuation_timeout > 0:
|
||
quote_payload, valuation_err, valuation_ms = self._run_with_retry(
|
||
lambda: self.get_realtime_quote(stock_code),
|
||
valuation_timeout,
|
||
"fundamental_valuation",
|
||
)
|
||
else:
|
||
quote_payload, valuation_err, valuation_ms = None, "fundamental stage timeout", 0
|
||
valuation_payload = {
|
||
"pe_ratio": getattr(quote_payload, "pe_ratio", None) if quote_payload else None,
|
||
"pb_ratio": getattr(quote_payload, "pb_ratio", None) if quote_payload else None,
|
||
"total_mv": getattr(quote_payload, "total_mv", None) if quote_payload else None,
|
||
"circ_mv": getattr(quote_payload, "circ_mv", None) if quote_payload else None,
|
||
}
|
||
valuation_status = self._infer_block_status(
|
||
valuation_payload,
|
||
"partial" if quote_payload is not None else "not_supported",
|
||
)
|
||
if valuation_status == "partial" and valuation_err and not self._has_meaningful_payload(valuation_payload):
|
||
valuation_status = "failed"
|
||
result_ctx["valuation"] = self._build_fundamental_block(
|
||
valuation_status,
|
||
valuation_payload,
|
||
self._normalize_source_chain(
|
||
[{"provider": "realtime_quote", "result": valuation_status, "duration_ms": valuation_ms}],
|
||
"realtime_quote",
|
||
valuation_status,
|
||
valuation_ms,
|
||
),
|
||
[valuation_err] if valuation_err else [],
|
||
)
|
||
|
||
# Fundamental bundle via Futu (HK only, when OpenD is configured), then
|
||
# fall back to yfinance for the same payload shape.
|
||
bundle_timeout = min(fetch_timeout, max(stage_timeout - (time.time() - start_ts), 0.0))
|
||
if bundle_timeout <= 0:
|
||
bundle_payload, bundle_err, bundle_ms = {}, "fundamental stage timeout", 0
|
||
bundle_provider = "fundamental_bundle_yfinance"
|
||
else:
|
||
bundle_payload, bundle_err, bundle_ms, bundle_provider = (
|
||
self._fetch_offshore_fundamental_bundle(
|
||
stock_code,
|
||
market,
|
||
bundle_timeout,
|
||
)
|
||
)
|
||
if not isinstance(bundle_payload, dict):
|
||
bundle_payload = {}
|
||
|
||
bundle_chain = self._normalize_source_chain(
|
||
bundle_payload.get("source_chain", []),
|
||
bundle_provider,
|
||
str(bundle_payload.get("status", "not_supported")),
|
||
bundle_ms,
|
||
)
|
||
adapter_errors = list(bundle_payload.get("errors", []))
|
||
if bundle_err:
|
||
adapter_errors.append(bundle_err)
|
||
|
||
growth_payload = bundle_payload.get("growth", {}) if isinstance(bundle_payload.get("growth"), dict) else {}
|
||
earnings_payload = bundle_payload.get("earnings", {}) if isinstance(bundle_payload.get("earnings"), dict) else {}
|
||
belong_boards = bundle_payload.get("belong_boards") if isinstance(bundle_payload.get("belong_boards"), list) else []
|
||
|
||
growth_status = self._infer_block_status(growth_payload, str(bundle_payload.get("status", "not_supported")))
|
||
earnings_status = self._infer_block_status(earnings_payload, str(bundle_payload.get("status", "not_supported")))
|
||
|
||
result_ctx["growth"] = self._build_fundamental_block(
|
||
growth_status,
|
||
growth_payload,
|
||
bundle_chain,
|
||
list(adapter_errors),
|
||
)
|
||
result_ctx["earnings"] = self._build_fundamental_block(
|
||
earnings_status,
|
||
earnings_payload,
|
||
bundle_chain,
|
||
list(adapter_errors),
|
||
)
|
||
|
||
# capital_flow / dragon_tiger / boards: Futu fills capital_flow and
|
||
# belong_boards for HK; everything else keeps not_supported (fail-open).
|
||
futu_capital_flow = (
|
||
bundle_payload.get("capital_flow")
|
||
if isinstance(bundle_payload.get("capital_flow"), dict) and bundle_payload.get("capital_flow")
|
||
else {}
|
||
)
|
||
if futu_capital_flow:
|
||
result_ctx["capital_flow"] = self._build_fundamental_block(
|
||
"ok" if futu_capital_flow.get("latest") or futu_capital_flow.get("rows") else "partial",
|
||
futu_capital_flow,
|
||
bundle_chain,
|
||
[],
|
||
)
|
||
else:
|
||
result_ctx["capital_flow"] = self._build_fundamental_block(
|
||
"not_supported",
|
||
{},
|
||
[{"provider": "fundamental_pipeline", "result": "not_supported", "duration_ms": 0}],
|
||
["not supported for this source"],
|
||
)
|
||
result_ctx["dragon_tiger"] = self._build_fundamental_block(
|
||
"not_supported",
|
||
{},
|
||
[{"provider": "fundamental_pipeline", "result": "not_supported", "duration_ms": 0}],
|
||
["not supported for offshore market"],
|
||
)
|
||
futu_boards = (
|
||
bundle_payload.get("belong_boards")
|
||
if isinstance(bundle_payload.get("belong_boards"), list) and bundle_payload.get("belong_boards")
|
||
else []
|
||
)
|
||
if futu_boards:
|
||
result_ctx["boards"] = self._build_fundamental_block(
|
||
"ok",
|
||
{"boards": futu_boards},
|
||
bundle_chain,
|
||
[],
|
||
)
|
||
else:
|
||
result_ctx["boards"] = self._build_fundamental_block(
|
||
"not_supported",
|
||
{},
|
||
[{"provider": "fundamental_pipeline", "result": "not_supported", "duration_ms": 0}],
|
||
["not supported for offshore market"],
|
||
)
|
||
|
||
# institution: tw (台股) has a free official 三大法人 (institutional net buy/sell)
|
||
# feed (TWSE T86 / TPEx OpenAPI); every other offshore market keeps not_supported.
|
||
# tw-only + strictly additive + fail-open: any error or no-data -> not_supported,
|
||
# which never interrupts the main analysis. Raw net figures only — no derived
|
||
# signal / score / schema (per the v2 scope confirmed on issue #1777).
|
||
tw_record = None
|
||
if market == "tw":
|
||
fetcher = getattr(self, "_tw_institutional_fetcher", None)
|
||
if fetcher is None:
|
||
# Wiring (import + construct) is a one-time op; a failure here is a
|
||
# programming / deploy bug, so log it LOUD (error). Still fail-open
|
||
# (never interrupt the main analysis — a hard requirement of #1777).
|
||
try:
|
||
from data_provider.tw_institutional_fetcher import TwInstitutionalFetcher
|
||
|
||
fetcher = TwInstitutionalFetcher()
|
||
self._tw_institutional_fetcher = fetcher
|
||
except Exception as exc: # noqa: BLE001 - wiring failure: loud but fail-open
|
||
logger.error("[tw-inst] fetcher init failed (wiring bug?) code=%s: %s", stock_code, exc)
|
||
fetcher = None
|
||
# fetch_timeout == 0 disables per-fetch fundamental fetches (same as valuation /
|
||
# bundle above, which gate on fetch_timeout); honour that for institution too so
|
||
# the FUNDAMENTAL_FETCH_TIMEOUT_SECONDS=0 config semantic is not bypassed.
|
||
if fetcher is not None and fetch_timeout > 0:
|
||
# The tw institution block is a WHOLE-MARKET download (~4-5s), far slower
|
||
# than the per-symbol quote/bundle fetches, and it is the LAST offshore
|
||
# block. When enabled, give it the full REMAINING stage budget rather than
|
||
# the ~3s per-fetch cap that starves it and makes the first/only stock of a
|
||
# run coin-flip between ok and not_supported. Bounded by the stage deadline
|
||
# via _run_with_retry, so it fails open (never blocks).
|
||
inst_timeout = max(stage_timeout - (time.time() - start_ts), 0.0)
|
||
if inst_timeout > 0:
|
||
tw_record, inst_err, _inst_ms = self._run_with_retry(
|
||
lambda: fetcher.get_institutional_net(stock_code),
|
||
inst_timeout,
|
||
"fundamental_tw_institution",
|
||
)
|
||
if inst_err:
|
||
logger.warning("[tw-inst] fetch failed/timeout code=%s: %s", stock_code, inst_err)
|
||
else:
|
||
tw_record = None
|
||
# status 'ok' only when the record carries all core net figures (a genuine 0 is
|
||
# kept — 0 is not None); None / missing core field / fetch failure -> not_supported.
|
||
_tw_core = ("foreign_net", "trust_net", "dealer_net", "total_net")
|
||
if tw_record is not None and all(tw_record.get(key) is not None for key in _tw_core):
|
||
institution_status = "ok"
|
||
result_ctx["institution"] = self._build_fundamental_block(
|
||
"ok",
|
||
{
|
||
"foreign_net": tw_record.get("foreign_net"),
|
||
"trust_net": tw_record.get("trust_net"),
|
||
"dealer_net": tw_record.get("dealer_net"),
|
||
"total_net": tw_record.get("total_net"),
|
||
"unit": tw_record.get("unit"),
|
||
"date": tw_record.get("date"),
|
||
"source": tw_record.get("source"),
|
||
},
|
||
[{"provider": tw_record.get("source", "tw-institutional"), "result": "ok", "duration_ms": 0}],
|
||
[],
|
||
)
|
||
else:
|
||
institution_status = "not_supported"
|
||
result_ctx["institution"] = self._build_fundamental_block(
|
||
"not_supported",
|
||
{},
|
||
[{"provider": "fundamental_pipeline", "result": "not_supported", "duration_ms": 0}],
|
||
["not supported for offshore market"],
|
||
)
|
||
|
||
result_ctx["belong_boards"] = belong_boards or futu_boards
|
||
|
||
capital_flow_status = result_ctx["capital_flow"].get("status", "not_supported")
|
||
boards_status = result_ctx["boards"].get("status", "not_supported")
|
||
block_statuses = {
|
||
"valuation": result_ctx["valuation"].get("status", "not_supported"),
|
||
"growth": growth_status,
|
||
"earnings": earnings_status,
|
||
"institution": institution_status,
|
||
"capital_flow": capital_flow_status,
|
||
"dragon_tiger": "not_supported",
|
||
"boards": boards_status,
|
||
}
|
||
result_ctx["coverage"] = block_statuses
|
||
for block in ("valuation", "growth", "earnings", "institution", "capital_flow", "dragon_tiger", "boards"):
|
||
result_ctx["errors"].extend(result_ctx[block].get("errors", []))
|
||
result_ctx["source_chain"].extend(result_ctx[block].get("source_chain", []))
|
||
|
||
active_statuses = {
|
||
"valuation": valuation_status,
|
||
"growth": growth_status,
|
||
"earnings": earnings_status,
|
||
"capital_flow": capital_flow_status,
|
||
"boards": boards_status,
|
||
}
|
||
# tw institution (when present) counts toward the OVERALL status so a report that
|
||
# only has 三大法人 data still surfaces fundamentals (consumers key off the top-level
|
||
# status). Futu capital_flow / boards count the same way when they are available.
|
||
status_values = list(active_statuses.values())
|
||
if institution_status == "ok":
|
||
status_values.append("ok")
|
||
if all(value == "not_supported" for value in status_values):
|
||
result_ctx["status"] = "not_supported"
|
||
result_ctx["data_quality"] = "unavailable"
|
||
elif "failed" in status_values or "partial" in status_values:
|
||
result_ctx["status"] = "partial"
|
||
result_ctx["data_quality"] = "partial"
|
||
else:
|
||
result_ctx["status"] = "ok"
|
||
result_ctx["data_quality"] = "ok"
|
||
result_ctx["missing_fields"] = [
|
||
block for block, status in active_statuses.items() if status != "ok"
|
||
]
|
||
|
||
result_ctx["elapsed_ms"] = int((time.time() - start_ts) * 1000)
|
||
if cache_ttl > 0 and self._should_cache_fundamental_context(result_ctx):
|
||
with self._fundamental_cache_lock:
|
||
self._fundamental_cache[cache_key] = {
|
||
"ts": time.time(),
|
||
"context": result_ctx,
|
||
}
|
||
self._prune_fundamental_cache(cache_ttl, cache_max_entries)
|
||
return result_ctx
|
||
|
||
def build_failed_fundamental_context(self, stock_code: str, reason: str) -> Dict[str, Any]:
|
||
"""Build a consistent failed-context payload for caller-side fallback."""
|
||
market = _market_tag(stock_code)
|
||
block_names = (
|
||
"valuation",
|
||
"growth",
|
||
"earnings",
|
||
"institution",
|
||
"capital_flow",
|
||
"dragon_tiger",
|
||
"boards",
|
||
)
|
||
blocks = {
|
||
block: self._build_fundamental_block(
|
||
"failed",
|
||
{},
|
||
[{"provider": "fundamental_pipeline", "result": "failed", "duration_ms": 0}],
|
||
[reason],
|
||
)
|
||
for block in block_names
|
||
}
|
||
return {
|
||
"market": market,
|
||
"status": "failed",
|
||
"coverage": {block: "failed" for block in block_names},
|
||
"source_chain": [{"provider": "fundamental_pipeline", "result": "failed", "duration_ms": 0}],
|
||
"errors": [reason],
|
||
**blocks,
|
||
}
|
||
|
||
def build_not_supported_fundamental_context(
|
||
self, stock_code: str, reason: str
|
||
) -> Dict[str, Any]:
|
||
"""Build a consistent not-supported payload without calling providers."""
|
||
return self._build_market_not_supported(_market_tag(stock_code), reason)
|
||
|
||
def get_fundamental_context(
|
||
self,
|
||
stock_code: str,
|
||
budget_seconds: Optional[float] = None
|
||
) -> Dict[str, Any]:
|
||
"""
|
||
Aggregate fundamental blocks with fail-open semantics.
|
||
"""
|
||
from src.config import get_config
|
||
|
||
config = get_config()
|
||
if not config.enable_fundamental_pipeline:
|
||
return self._build_market_not_supported(
|
||
market=_market_tag(stock_code),
|
||
reason="fundamental pipeline disabled",
|
||
)
|
||
|
||
stock_code = normalize_stock_code(stock_code)
|
||
market = _market_tag(stock_code)
|
||
is_etf = _is_etf_code(stock_code)
|
||
if market in {"us", "hk", "jp", "kr", "tw"}:
|
||
return self._build_offshore_fundamental_context(
|
||
stock_code,
|
||
market=market,
|
||
budget_seconds=budget_seconds,
|
||
)
|
||
|
||
stage_timeout = float(
|
||
budget_seconds if budget_seconds is not None else config.fundamental_stage_timeout_seconds
|
||
)
|
||
stage_timeout = max(0.0, stage_timeout)
|
||
fetch_timeout = float(config.fundamental_fetch_timeout_seconds)
|
||
fetch_timeout = max(0.0, fetch_timeout)
|
||
|
||
cache_ttl = int(config.fundamental_cache_ttl_seconds)
|
||
cache_max_entries = max(0, int(getattr(config, "fundamental_cache_max_entries", 256)))
|
||
cache_key = self._get_fundamental_cache_key(stock_code, stage_timeout)
|
||
if cache_ttl > 0:
|
||
self._prune_fundamental_cache(cache_ttl, cache_max_entries)
|
||
with self._fundamental_cache_lock:
|
||
cache_item = self._fundamental_cache.get(cache_key)
|
||
if cache_item:
|
||
age = time.time() - float(cache_item.get("ts", 0))
|
||
if age <= cache_ttl:
|
||
return cache_item.get("context", {})
|
||
|
||
remaining_seconds = stage_timeout
|
||
result_ctx: Dict[str, Any] = {
|
||
"market": market,
|
||
"valuation": {},
|
||
"growth": {},
|
||
"earnings": {},
|
||
"institution": {},
|
||
"capital_flow": {},
|
||
"dragon_tiger": {},
|
||
"boards": {},
|
||
"coverage": {},
|
||
"source_chain": [],
|
||
"errors": [],
|
||
}
|
||
|
||
start_ts = time.time()
|
||
|
||
def _consume_budget(consumed_ms: int) -> None:
|
||
nonlocal remaining_seconds
|
||
remaining_seconds = max(0.0, remaining_seconds - consumed_ms / 1000.0)
|
||
|
||
valuation_timeout = min(fetch_timeout, remaining_seconds)
|
||
if valuation_timeout > 0:
|
||
quote_payload, valuation_err, valuation_ms = self._run_with_retry(
|
||
lambda: self.get_realtime_quote(stock_code),
|
||
valuation_timeout,
|
||
"fundamental_valuation",
|
||
)
|
||
_consume_budget(valuation_ms)
|
||
else:
|
||
quote_payload, valuation_err, valuation_ms = None, "fundamental stage timeout", 0
|
||
|
||
valuation_payload = {
|
||
"pe_ratio": getattr(quote_payload, "pe_ratio", None) if quote_payload else None,
|
||
"pb_ratio": getattr(quote_payload, "pb_ratio", None) if quote_payload else None,
|
||
"total_mv": getattr(quote_payload, "total_mv", None) if quote_payload else None,
|
||
"circ_mv": getattr(quote_payload, "circ_mv", None) if quote_payload else None,
|
||
}
|
||
valuation_status = self._infer_block_status(
|
||
valuation_payload,
|
||
"partial" if quote_payload is not None else "not_supported",
|
||
)
|
||
if valuation_status == "partial" and valuation_err and not self._has_meaningful_payload(valuation_payload):
|
||
valuation_status = "failed"
|
||
result_ctx["valuation"] = self._build_fundamental_block(
|
||
valuation_status,
|
||
valuation_payload,
|
||
self._normalize_source_chain(
|
||
[{"provider": "realtime_quote", "result": valuation_status, "duration_ms": valuation_ms}],
|
||
"realtime_quote",
|
||
valuation_status,
|
||
valuation_ms,
|
||
),
|
||
[valuation_err] if valuation_err else [],
|
||
)
|
||
|
||
# growth / earnings / institution (one AkShare call)
|
||
if remaining_seconds <= 0:
|
||
bundle_status = "failed"
|
||
bundle_payload: Dict[str, Any] = {}
|
||
bundle_errors = ["fundamental stage timeout"]
|
||
bundle_ms = 0
|
||
else:
|
||
bundle_timeout = min(fetch_timeout, remaining_seconds)
|
||
bundle_payload, bundle_err_msg, bundle_ms = self._run_with_retry(
|
||
lambda: self._fundamental_adapter.get_fundamental_bundle(stock_code),
|
||
bundle_timeout,
|
||
"fundamental_bundle",
|
||
)
|
||
_consume_budget(bundle_ms)
|
||
if not isinstance(bundle_payload, dict):
|
||
bundle_status = "failed"
|
||
bundle_payload = {}
|
||
bundle_errors = ["fundamental_bundle failed"]
|
||
if bundle_err_msg:
|
||
bundle_errors.append(bundle_err_msg)
|
||
else:
|
||
bundle_status = str(bundle_payload.get("status", "not_supported"))
|
||
bundle_errors = [bundle_err_msg] if bundle_err_msg else []
|
||
|
||
bundle_chain = self._normalize_source_chain(
|
||
bundle_payload.get("source_chain", []),
|
||
"fundamental_bundle",
|
||
bundle_status,
|
||
bundle_ms,
|
||
) if isinstance(bundle_payload, dict) else self._normalize_source_chain(
|
||
None,
|
||
"fundamental_bundle",
|
||
bundle_status,
|
||
bundle_ms,
|
||
)
|
||
growth_payload = bundle_payload.get("growth", {}) if isinstance(bundle_payload, dict) else {}
|
||
earnings_payload = bundle_payload.get("earnings", {}) if isinstance(bundle_payload, dict) else {}
|
||
institution_payload = bundle_payload.get("institution", {}) if isinstance(bundle_payload, dict) else {}
|
||
if not isinstance(growth_payload, dict):
|
||
growth_payload = {}
|
||
else:
|
||
growth_payload = dict(growth_payload)
|
||
if not isinstance(earnings_payload, dict):
|
||
earnings_payload = {}
|
||
else:
|
||
earnings_payload = dict(earnings_payload)
|
||
if not isinstance(institution_payload, dict):
|
||
institution_payload = {}
|
||
else:
|
||
institution_payload = dict(institution_payload)
|
||
|
||
# Derive TTM dividend yield from already-fetched quote price; avoid extra quote calls.
|
||
earnings_extra_errors: List[str] = []
|
||
dividend_payload = earnings_payload.get("dividend")
|
||
if isinstance(dividend_payload, dict):
|
||
dividend_payload = dict(dividend_payload)
|
||
ttm_cash_raw = dividend_payload.get("ttm_cash_dividend_per_share")
|
||
ttm_cash = None
|
||
if ttm_cash_raw is not None:
|
||
try:
|
||
ttm_cash = float(ttm_cash_raw)
|
||
except (TypeError, ValueError):
|
||
earnings_extra_errors.append("invalid_ttm_cash_dividend_per_share")
|
||
if isinstance(quote_payload, dict):
|
||
latest_price_raw = quote_payload.get("price")
|
||
else:
|
||
latest_price_raw = getattr(quote_payload, "price", None) if quote_payload else None
|
||
latest_price = None
|
||
if latest_price_raw is not None:
|
||
try:
|
||
latest_price = float(latest_price_raw)
|
||
except (TypeError, ValueError):
|
||
latest_price = None
|
||
ttm_yield = None
|
||
if ttm_cash is not None:
|
||
if latest_price is not None and latest_price > 0:
|
||
ttm_yield = round(ttm_cash / latest_price * 100.0, 4)
|
||
else:
|
||
earnings_extra_errors.append("invalid_price_for_ttm_dividend_yield")
|
||
|
||
dividend_payload["ttm_dividend_yield_pct"] = ttm_yield
|
||
if ttm_yield is not None:
|
||
dividend_payload["yield_formula"] = "ttm_cash_dividend_per_share / latest_price * 100"
|
||
earnings_payload["dividend"] = dividend_payload
|
||
|
||
adapter_errors = list(bundle_payload.get("errors", [])) if isinstance(bundle_payload, dict) else []
|
||
adapter_errors.extend(bundle_errors)
|
||
growth_errors = list(adapter_errors)
|
||
earnings_errors = list(adapter_errors)
|
||
earnings_errors.extend(earnings_extra_errors)
|
||
institution_errors = list(adapter_errors)
|
||
|
||
growth_status = self._infer_block_status(growth_payload, bundle_status)
|
||
earnings_status = self._infer_block_status(earnings_payload, bundle_status)
|
||
institution_status = self._infer_block_status(institution_payload, bundle_status)
|
||
|
||
result_ctx["growth"] = self._build_fundamental_block(
|
||
growth_status,
|
||
growth_payload,
|
||
bundle_chain,
|
||
growth_errors,
|
||
)
|
||
result_ctx["earnings"] = self._build_fundamental_block(
|
||
earnings_status,
|
||
earnings_payload,
|
||
bundle_chain,
|
||
earnings_errors,
|
||
)
|
||
result_ctx["institution"] = self._build_fundamental_block(
|
||
institution_status,
|
||
institution_payload,
|
||
bundle_chain,
|
||
institution_errors,
|
||
)
|
||
|
||
# capital flow
|
||
if is_etf:
|
||
result_ctx["capital_flow"] = self._build_fundamental_block(
|
||
"not_supported",
|
||
{},
|
||
[{"provider": "fundamental_pipeline", "result": "not_supported", "duration_ms": 0}],
|
||
["etf not fully supported"],
|
||
)
|
||
result_ctx["dragon_tiger"] = self._build_fundamental_block(
|
||
"not_supported",
|
||
{},
|
||
[{"provider": "fundamental_pipeline", "result": "not_supported", "duration_ms": 0}],
|
||
["etf not fully supported"],
|
||
)
|
||
result_ctx["boards"] = self._build_fundamental_block(
|
||
"not_supported",
|
||
{},
|
||
[{"provider": "fundamental_pipeline", "result": "not_supported", "duration_ms": 0}],
|
||
["etf not fully supported"],
|
||
)
|
||
result_ctx["status"] = "partial"
|
||
else:
|
||
capital_flow_budget = min(fetch_timeout, remaining_seconds)
|
||
capital_flow_start = time.time()
|
||
result_ctx["capital_flow"] = self.get_capital_flow_context(
|
||
stock_code,
|
||
budget_seconds=capital_flow_budget,
|
||
)
|
||
_consume_budget(int((time.time() - capital_flow_start) * 1000))
|
||
|
||
dragon_tiger_budget = min(fetch_timeout, remaining_seconds)
|
||
dragon_tiger_start = time.time()
|
||
result_ctx["dragon_tiger"] = self.get_dragon_tiger_context(
|
||
stock_code,
|
||
budget_seconds=dragon_tiger_budget,
|
||
)
|
||
_consume_budget(int((time.time() - dragon_tiger_start) * 1000))
|
||
|
||
result_ctx["boards"] = self.get_board_context(
|
||
stock_code,
|
||
budget_seconds=min(fetch_timeout, remaining_seconds),
|
||
)
|
||
|
||
block_statuses = {
|
||
"valuation": result_ctx["valuation"].get("status", "not_supported"),
|
||
"growth": result_ctx["growth"].get("status", "not_supported"),
|
||
"earnings": result_ctx["earnings"].get("status", "not_supported"),
|
||
"institution": result_ctx["institution"].get("status", "not_supported"),
|
||
"capital_flow": result_ctx["capital_flow"].get("status", "not_supported"),
|
||
"dragon_tiger": result_ctx["dragon_tiger"].get("status", "not_supported"),
|
||
"boards": result_ctx["boards"].get("status", "not_supported"),
|
||
}
|
||
result_ctx["coverage"] = block_statuses
|
||
for block in (
|
||
"valuation",
|
||
"growth",
|
||
"earnings",
|
||
"institution",
|
||
"capital_flow",
|
||
"dragon_tiger",
|
||
"boards",
|
||
):
|
||
result_ctx["errors"].extend(result_ctx[block].get("errors", []))
|
||
result_ctx["source_chain"].extend(result_ctx[block].get("source_chain", []))
|
||
|
||
if is_etf:
|
||
# Keep ETF downgrade semantics for overall status even when valuation is available.
|
||
result_ctx["status"] = (
|
||
"not_supported" if all(value == "not_supported" for value in block_statuses.values()) else "partial"
|
||
)
|
||
elif all(value == "not_supported" for value in block_statuses.values()):
|
||
result_ctx["status"] = "not_supported"
|
||
elif "failed" in block_statuses.values() or "partial" in block_statuses.values():
|
||
result_ctx["status"] = "partial"
|
||
else:
|
||
result_ctx["status"] = "ok"
|
||
|
||
result_ctx["elapsed_ms"] = int((time.time() - start_ts) * 1000)
|
||
if cache_ttl > 0 and self._should_cache_fundamental_context(result_ctx):
|
||
with self._fundamental_cache_lock:
|
||
self._fundamental_cache[cache_key] = {
|
||
"ts": time.time(),
|
||
"context": result_ctx,
|
||
}
|
||
self._prune_fundamental_cache(cache_ttl, cache_max_entries)
|
||
return result_ctx
|
||
|
||
def get_capital_flow_context(self, stock_code: str, budget_seconds: Optional[float] = None) -> Dict[str, Any]:
|
||
"""资金流向块（fail-open）。"""
|
||
from src.config import get_config
|
||
|
||
config = get_config()
|
||
stock_code = normalize_stock_code(stock_code)
|
||
timeout = float(budget_seconds if budget_seconds is not None else config.fundamental_fetch_timeout_seconds)
|
||
if _market_tag(stock_code) != "cn" or _is_etf_code(stock_code):
|
||
return self._build_fundamental_block(
|
||
"not_supported",
|
||
{},
|
||
[{"provider": "fundamental_pipeline", "result": "not_supported", "duration_ms": 0}],
|
||
["not supported"],
|
||
)
|
||
|
||
if timeout <= 0:
|
||
return self._build_fundamental_block(
|
||
"failed",
|
||
{},
|
||
[{"provider": "fundamental_pipeline", "result": "failed", "duration_ms": 0}],
|
||
["fundamental stage timeout"],
|
||
)
|
||
payload, err, cost_ms = self._run_with_retry(
|
||
lambda: self._fundamental_adapter.get_capital_flow(stock_code),
|
||
timeout,
|
||
"capital_flow",
|
||
)
|
||
if not isinstance(payload, dict):
|
||
return self._build_fundamental_block(
|
||
"failed",
|
||
{},
|
||
[{"provider": "fundamental_pipeline", "result": "failed", "duration_ms": cost_ms}],
|
||
[err or "capital_flow failed"],
|
||
)
|
||
|
||
stock_flow = payload.get("stock_flow") or {}
|
||
sector_rankings = payload.get("sector_rankings") or {}
|
||
has_stock_flow = False
|
||
if isinstance(stock_flow, dict):
|
||
has_stock_flow = any(v is not None for v in stock_flow.values())
|
||
has_sector_rankings = bool(sector_rankings.get("top")) or bool(sector_rankings.get("bottom"))
|
||
adapter_status = str(payload.get("status", "not_supported"))
|
||
if has_stock_flow or has_sector_rankings:
|
||
capital_flow_status = "ok"
|
||
elif adapter_status == "not_supported":
|
||
capital_flow_status = "not_supported"
|
||
else:
|
||
capital_flow_status = "partial"
|
||
|
||
return self._build_fundamental_block(
|
||
capital_flow_status,
|
||
{
|
||
"stock_flow": payload.get("stock_flow", {}),
|
||
"sector_rankings": payload.get("sector_rankings", {}),
|
||
},
|
||
self._normalize_source_chain(
|
||
payload.get("source_chain", []),
|
||
"capital_flow",
|
||
capital_flow_status,
|
||
cost_ms,
|
||
),
|
||
list(payload.get("errors", [])) + ([err] if err else []),
|
||
)
|
||
|
||
def get_dragon_tiger_context(self, stock_code: str, budget_seconds: Optional[float] = None) -> Dict[str, Any]:
|
||
"""龙虎榜块（fail-open）。"""
|
||
from src.config import get_config
|
||
|
||
config = get_config()
|
||
stock_code = normalize_stock_code(stock_code)
|
||
timeout = float(budget_seconds if budget_seconds is not None else config.fundamental_fetch_timeout_seconds)
|
||
if _market_tag(stock_code) != "cn" or _is_etf_code(stock_code):
|
||
return self._build_fundamental_block(
|
||
"not_supported",
|
||
{},
|
||
[{"provider": "fundamental_pipeline", "result": "not_supported", "duration_ms": 0}],
|
||
["not supported"],
|
||
)
|
||
|
||
if timeout <= 0:
|
||
return self._build_fundamental_block(
|
||
"failed",
|
||
{},
|
||
[{"provider": "fundamental_pipeline", "result": "failed", "duration_ms": 0}],
|
||
["fundamental stage timeout"],
|
||
)
|
||
payload, err, cost_ms = self._run_with_retry(
|
||
lambda: self._fundamental_adapter.get_dragon_tiger_flag(stock_code),
|
||
timeout,
|
||
"dragon_tiger",
|
||
)
|
||
if not isinstance(payload, dict):
|
||
return self._build_fundamental_block(
|
||
"failed",
|
||
{},
|
||
[{"provider": "fundamental_pipeline", "result": "failed", "duration_ms": cost_ms}],
|
||
[err or "dragon_tiger failed"],
|
||
)
|
||
return self._build_fundamental_block(
|
||
(payload.get("status") if isinstance(payload.get("status"), str) else "partial"),
|
||
{
|
||
"is_on_list": bool(payload.get("is_on_list", False)),
|
||
"recent_count": int(payload.get("recent_count", 0)),
|
||
"latest_date": payload.get("latest_date"),
|
||
},
|
||
self._normalize_source_chain(
|
||
payload.get("source_chain", []),
|
||
"dragon_tiger",
|
||
str(payload.get("status", "ok")),
|
||
cost_ms,
|
||
),
|
||
list(payload.get("errors", [])) + ([err] if err else []),
|
||
)
|
||
|
||
def get_board_context(self, stock_code: str, budget_seconds: Optional[float] = None) -> Dict[str, Any]:
|
||
"""板块榜单块（fail-open）。"""
|
||
from src.config import get_config
|
||
|
||
config = get_config()
|
||
stock_code = normalize_stock_code(stock_code)
|
||
timeout = float(budget_seconds if budget_seconds is not None else config.fundamental_fetch_timeout_seconds)
|
||
if _market_tag(stock_code) != "cn" or _is_etf_code(stock_code):
|
||
return self._build_fundamental_block(
|
||
"not_supported",
|
||
{},
|
||
[{"provider": "fundamental_pipeline", "result": "not_supported", "duration_ms": 0}],
|
||
["not supported"],
|
||
)
|
||
|
||
if timeout <= 0:
|
||
return self._build_fundamental_block(
|
||
"failed",
|
||
{},
|
||
[{"provider": "fundamental_pipeline", "result": "failed", "duration_ms": 0}],
|
||
["fundamental stage timeout"],
|
||
)
|
||
|
||
def task() -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]], List[Dict[str, Any]], str]:
|
||
return self._get_sector_rankings_with_meta(5)
|
||
|
||
rankings, err, cost_ms = self._run_with_retry(task, timeout, "boards")
|
||
if isinstance(rankings, tuple) and len(rankings) == 4:
|
||
top, bottom, chain, chain_error = rankings
|
||
if chain_error and not err:
|
||
err = chain_error
|
||
if not top and not bottom:
|
||
return self._build_fundamental_block(
|
||
"failed",
|
||
{},
|
||
chain if chain else [{"provider": "sector_rankings", "result": "failed", "duration_ms": cost_ms}],
|
||
[err or "boards empty from all sources"],
|
||
)
|
||
board_status = "ok" if top and bottom else "partial"
|
||
return self._build_fundamental_block(
|
||
board_status,
|
||
{"top": top or [], "bottom": bottom or []},
|
||
chain if chain else self._normalize_source_chain(
|
||
["sector_rankings"],
|
||
"boards",
|
||
board_status,
|
||
cost_ms,
|
||
),
|
||
[err] if err else [],
|
||
)
|
||
|
||
return self._build_fundamental_block(
|
||
"failed",
|
||
{},
|
||
[{"provider": "sector_rankings", "result": "failed", "duration_ms": cost_ms}],
|
||
[err or "boards failed"],
|
||
)
|
||
|
||
def _get_sector_rankings_with_meta(
|
||
self,
|
||
n: int = 5,
|
||
) -> Tuple[List[Dict], List[Dict], List[Dict[str, Any]], str]:
|
||
"""Get sector rankings with ordered fallback chain metadata."""
|
||
source_chain: List[Dict[str, Any]] = []
|
||
last_error = ""
|
||
|
||
# 直接遍历管理器已经按 priority 排好序的数据源列表
|
||
for fetcher in self._fetchers:
|
||
if not hasattr(fetcher, 'get_sector_rankings'):
|
||
continue
|
||
|
||
start = time.time()
|
||
try:
|
||
data = fetcher.get_sector_rankings(n)
|
||
duration_ms = int((time.time() - start) * 1000)
|
||
if data and data[0] is not None and data[1] is not None:
|
||
source_chain.append(
|
||
{
|
||
"provider": fetcher.name,
|
||
"result": "ok",
|
||
"duration_ms": duration_ms,
|
||
}
|
||
)
|
||
logger.info(f"[{fetcher.name}] 获取板块排行成功")
|
||
return data[0], data[1], source_chain, ""
|
||
|
||
last_error = f"{fetcher.name}返回空结果"
|
||
source_chain.append(
|
||
{
|
||
"provider": fetcher.name,
|
||
"result": "empty",
|
||
"duration_ms": duration_ms,
|
||
"error": last_error,
|
||
}
|
||
)
|
||
except Exception as e:
|
||
error_type, error_reason = summarize_exception(e)
|
||
last_error = f"{fetcher.name} ({error_type}) {error_reason}"
|
||
duration_ms = int((time.time() - start) * 1000)
|
||
source_chain.append(
|
||
{
|
||
"provider": fetcher.name,
|
||
"result": "failed",
|
||
"duration_ms": duration_ms,
|
||
"error": error_reason,
|
||
}
|
||
)
|
||
logger.warning(f"[{fetcher.name}] 获取板块排行失败: {error_reason}")
|
||
|
||
return [], [], source_chain, last_error
|
||
|
||
def get_sector_rankings(self, n: int = 5) -> Tuple[List[Dict], List[Dict]]:
|
||
"""获取板块涨跌榜（自动切换数据源）"""
|
||
# 按需求固定回退顺序：Akshare(EM) -> Akshare(Sina) -> Tushare -> Efinance
|
||
top, bottom, _, last_error = self._get_sector_rankings_with_meta(n)
|
||
if top or bottom:
|
||
return top, bottom
|
||
logger.warning(f"[板块排行] 所有数据源均失败，最终错误: {last_error}")
|
||
return [], []
|
||
|
||
@staticmethod
|
||
def _copy_ranking_rows(rows: List[Dict]) -> List[Dict]:
|
||
return [dict(row) if isinstance(row, dict) else row for row in rows or []]
|
||
|
||
@classmethod
|
||
def clear_concept_rankings_cache_for_tests(cls) -> None:
|
||
with cls._concept_rankings_cache_lock:
|
||
cls._concept_rankings_cache.clear()
|
||
|
||
def get_concept_rankings(self, n: int = 5) -> Tuple[List[Dict], List[Dict]]:
|
||
"""获取概念/题材涨跌榜（自动切换数据源）。"""
|
||
try:
|
||
normalized_n = int(n)
|
||
except (TypeError, ValueError):
|
||
normalized_n = 5
|
||
if normalized_n <= 0:
|
||
normalized_n = 5
|
||
|
||
last_error = ""
|
||
now = time.monotonic()
|
||
|
||
with self.__class__._concept_rankings_cache_lock:
|
||
cached = self.__class__._concept_rankings_cache.get(normalized_n)
|
||
if cached and cached[0] > now:
|
||
logger.debug("[概念排行] 命中共享缓存 n=%s", normalized_n)
|
||
return self._copy_ranking_rows(cached[1]), self._copy_ranking_rows(cached[2])
|
||
|
||
top: List[Dict] = []
|
||
bottom: List[Dict] = []
|
||
for fetcher in self._get_fetchers_snapshot():
|
||
try:
|
||
data = fetcher.get_concept_rankings(normalized_n)
|
||
if data and (data[0] or data[1]):
|
||
top = data[0] or []
|
||
bottom = data[1] or []
|
||
logger.info(f"[{fetcher.name}] 获取概念排行成功")
|
||
break
|
||
last_error = f"{fetcher.name}返回空结果"
|
||
except Exception as e:
|
||
error_type, error_reason = summarize_exception(e)
|
||
last_error = f"{fetcher.name} ({error_type}) {error_reason}"
|
||
logger.warning(f"[{fetcher.name}] 获取概念排行失败: {error_reason}")
|
||
|
||
if not top and not bottom and last_error:
|
||
logger.warning(f"[概念排行] 所有数据源均失败，最终错误: {last_error}")
|
||
|
||
ttl = (
|
||
self.__class__._CONCEPT_RANKINGS_CACHE_TTL_SECONDS
|
||
if top or bottom
|
||
else self.__class__._CONCEPT_RANKINGS_EMPTY_CACHE_TTL_SECONDS
|
||
)
|
||
cached_top = self._copy_ranking_rows(top)
|
||
cached_bottom = self._copy_ranking_rows(bottom)
|
||
self.__class__._concept_rankings_cache[normalized_n] = (
|
||
time.monotonic() + ttl,
|
||
cached_top,
|
||
cached_bottom,
|
||
)
|
||
return self._copy_ranking_rows(cached_top), self._copy_ranking_rows(cached_bottom)
|
||
|
||
def get_hot_stocks(self, n: int = 10) -> List[Dict[str, Any]]:
|
||
"""获取市场人气股榜（自动切换数据源）。"""
|
||
last_error = ""
|
||
for fetcher in self._fetchers:
|
||
try:
|
||
data = fetcher.get_hot_stocks(n)
|
||
if data:
|
||
logger.info(f"[{fetcher.name}] 获取人气股成功")
|
||
return data[:n]
|
||
last_error = f"{fetcher.name}返回空结果"
|
||
except Exception as e:
|
||
error_type, error_reason = summarize_exception(e)
|
||
last_error = f"{fetcher.name} ({error_type}) {error_reason}"
|
||
logger.warning(f"[{fetcher.name}] 获取人气股失败: {error_reason}")
|
||
if last_error:
|
||
logger.warning(f"[人气股] 所有数据源均失败，最终错误: {last_error}")
|
||
return []
|
||
|
||
def get_limit_up_pool(
|
||
self,
|
||
date: Optional[str] = None,
|
||
n: int = 20,
|
||
) -> List[Dict[str, Any]]:
|
||
"""获取涨停池与连板梯队（自动切换数据源）。"""
|
||
last_error = ""
|
||
for fetcher in self._fetchers:
|
||
try:
|
||
data = fetcher.get_limit_up_pool(date=date, n=n)
|
||
if data:
|
||
logger.info(f"[{fetcher.name}] 获取涨停池成功")
|
||
return data[:n]
|
||
last_error = f"{fetcher.name}返回空结果"
|
||
except Exception as e:
|
||
error_type, error_reason = summarize_exception(e)
|
||
last_error = f"{fetcher.name} ({error_type}) {error_reason}"
|
||
logger.warning(f"[{fetcher.name}] 获取涨停池失败: {error_reason}")
|
||
if last_error:
|
||
logger.warning(f"[涨停池] 所有数据源均失败，最终错误: {last_error}")
|
||
return []
|