mirror of
https://github.com/ZhuLinsen/daily_stock_analysis
synced 2026-09-20 10:53:33 +08:00
feat: add skill opinion outcome performance statistics (#2119)
* feat: add skill opinion outcome evaluation core * feat: add skill opinion outcome evaluation core * fix: unify expected-start resolution * changelog * fix: validate persisted daily start sessions * fix: preserve legacy local backtest windows * feat: add skill opinion outcome statistics * docs: clarify outcome statistics stage boundary * fix: clarify backtest-only legacy start fallback * fix: prevent pending outcome retry starvation * fix: preserve explicit backtest start contract * fix: rotate failed outcome candidates
This commit is contained in:
ObVious55
@@ -29,6 +29,23 @@ class SkillOpinionOutcomeCandidate:
|
||||
existing_outcome: Optional[SkillOpinionOutcomeRecord]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SkillOpinionPerformanceBucket:
|
||||
"""Raw persisted counts for one skill, horizon, and engine version."""
|
||||
|
||||
skill_id: str
|
||||
horizon: str
|
||||
engine_version: str
|
||||
total: int
|
||||
pending: int
|
||||
evaluated: int
|
||||
observational: int
|
||||
unable: int
|
||||
hit: int
|
||||
miss: int
|
||||
avg_directional_return_pct: Optional[float]
|
||||
|
||||
|
||||
class SkillOpinionOutcomeRepository:
|
||||
"""Read candidates and persist outcomes through the shared write guard."""
|
||||
|
||||
@@ -87,10 +104,6 @@ class SkillOpinionOutcomeRepository:
|
||||
.outerjoin(SkillOpinionOutcomeRecord, join_condition)
|
||||
.where(and_(*conditions))
|
||||
.order_by(
|
||||
case(
|
||||
(SkillOpinionOutcomeRecord.id.is_(None), 0),
|
||||
else_=1,
|
||||
),
|
||||
func.coalesce(
|
||||
SkillOpinionOutcomeRecord.updated_at,
|
||||
SkillOpinionSampleRecord.created_at,
|
||||
@@ -112,10 +125,10 @@ class SkillOpinionOutcomeRepository:
|
||||
horizon_rank = {horizon: index for index, horizon in enumerate(horizons)}
|
||||
candidates.sort(
|
||||
key=lambda item: (
|
||||
item.existing_outcome is not None,
|
||||
self._candidate_time(item),
|
||||
int(item.sample.id),
|
||||
horizon_rank[item.horizon],
|
||||
item.existing_outcome is not None,
|
||||
)
|
||||
)
|
||||
return candidates[:limit]
|
||||
@@ -188,6 +201,81 @@ class SkillOpinionOutcomeRepository:
|
||||
.limit(1)
|
||||
).scalar_one_or_none()
|
||||
|
||||
def list_performance_buckets(
|
||||
self,
|
||||
*,
|
||||
engine_version: str,
|
||||
skill_id: Optional[str] = None,
|
||||
horizons: Optional[Sequence[str]] = None,
|
||||
) -> List[SkillOpinionPerformanceBucket]:
|
||||
"""Aggregate persisted outcome facts without applying sample policy."""
|
||||
|
||||
status = SkillOpinionOutcomeRecord.eval_status
|
||||
outcome = SkillOpinionOutcomeRecord.outcome
|
||||
conditions = [
|
||||
SkillOpinionOutcomeRecord.engine_version == engine_version
|
||||
]
|
||||
if skill_id is not None:
|
||||
conditions.append(SkillOpinionSampleRecord.skill_id == skill_id)
|
||||
if horizons is not None:
|
||||
conditions.append(
|
||||
SkillOpinionOutcomeRecord.horizon.in_(list(horizons))
|
||||
)
|
||||
with self.db.get_session() as session:
|
||||
rows = session.execute(
|
||||
select(
|
||||
SkillOpinionSampleRecord.skill_id,
|
||||
SkillOpinionOutcomeRecord.horizon,
|
||||
SkillOpinionOutcomeRecord.engine_version,
|
||||
func.count(SkillOpinionOutcomeRecord.id),
|
||||
func.sum(case((status == "pending", 1), else_=0)),
|
||||
func.sum(case((status == "evaluated", 1), else_=0)),
|
||||
func.sum(case((status == "observational", 1), else_=0)),
|
||||
func.sum(case((status == "unable", 1), else_=0)),
|
||||
func.sum(case((outcome == "hit", 1), else_=0)),
|
||||
func.sum(case((outcome == "miss", 1), else_=0)),
|
||||
func.avg(
|
||||
case(
|
||||
(
|
||||
status == "evaluated",
|
||||
SkillOpinionOutcomeRecord.directional_return_pct,
|
||||
),
|
||||
else_=None,
|
||||
)
|
||||
),
|
||||
)
|
||||
.join(
|
||||
SkillOpinionSampleRecord,
|
||||
SkillOpinionSampleRecord.id
|
||||
== SkillOpinionOutcomeRecord.skill_opinion_sample_id,
|
||||
)
|
||||
.where(and_(*conditions))
|
||||
.group_by(
|
||||
SkillOpinionSampleRecord.skill_id,
|
||||
SkillOpinionOutcomeRecord.horizon,
|
||||
SkillOpinionOutcomeRecord.engine_version,
|
||||
)
|
||||
).all()
|
||||
|
||||
return [
|
||||
SkillOpinionPerformanceBucket(
|
||||
skill_id=str(row[0]),
|
||||
horizon=str(row[1]),
|
||||
engine_version=str(row[2]),
|
||||
total=int(row[3] or 0),
|
||||
pending=int(row[4] or 0),
|
||||
evaluated=int(row[5] or 0),
|
||||
observational=int(row[6] or 0),
|
||||
unable=int(row[7] or 0),
|
||||
hit=int(row[8] or 0),
|
||||
miss=int(row[9] or 0),
|
||||
avg_directional_return_pct=(
|
||||
float(row[10]) if row[10] is not None else None
|
||||
),
|
||||
)
|
||||
for row in rows
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _candidate_time(candidate: SkillOpinionOutcomeCandidate) -> datetime:
|
||||
outcome = candidate.existing_outcome
|
||||
|
||||
@@ -135,10 +135,7 @@ class BacktestService:
|
||||
else []
|
||||
)
|
||||
expected_start_date = start_resolution.expected_start_date
|
||||
local_start_date = (
|
||||
expected_start_date
|
||||
or start_resolution.legacy_local_start_date
|
||||
)
|
||||
local_start_date = start_resolution.backtest_start_date
|
||||
daily_window = None
|
||||
if local_start_date is not None:
|
||||
daily_window = resolve_stock_daily_window(
|
||||
|
||||
@@ -109,6 +109,16 @@ class SkillOpinionOutcomeService:
|
||||
"error_type": type(exc).__name__,
|
||||
}
|
||||
errors.append(error)
|
||||
try:
|
||||
self._record_retry_attempt(candidate)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Failed to advance Skill opinion outcome retry marker: "
|
||||
"sample_id=%s horizon=%s",
|
||||
error["sample_id"],
|
||||
error["horizon"],
|
||||
exc_info=True,
|
||||
)
|
||||
logger.warning(
|
||||
"Skill opinion outcome evaluation deferred after transient failure: "
|
||||
"sample_id=%s horizon=%s error_type=%s",
|
||||
@@ -130,6 +140,21 @@ class SkillOpinionOutcomeService:
|
||||
"engine_version": SKILL_OPINION_OUTCOME_ENGINE_VERSION,
|
||||
}
|
||||
|
||||
def _record_retry_attempt(
|
||||
self,
|
||||
candidate: SkillOpinionOutcomeCandidate,
|
||||
) -> None:
|
||||
self.repo.persist_outcome(
|
||||
{
|
||||
"skill_opinion_sample_id": int(candidate.sample.id),
|
||||
"horizon": candidate.horizon,
|
||||
"engine_version": SKILL_OPINION_OUTCOME_ENGINE_VERSION,
|
||||
"eval_status": "pending",
|
||||
"outcome": None,
|
||||
"direction_correct": None,
|
||||
}
|
||||
)
|
||||
|
||||
def _evaluate_candidate(
|
||||
self,
|
||||
candidate: SkillOpinionOutcomeCandidate,
|
||||
|
||||
153
src/services/skill_opinion_performance_service.py
Normal file
153
src/services/skill_opinion_performance_service.py
Normal file
@@ -0,0 +1,153 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Read-only statistics over persisted individual SkillAgent outcomes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Optional, Sequence
|
||||
|
||||
from src.core.skill_opinion_outcome_evaluator import (
|
||||
SUPPORTED_SKILL_OUTCOME_HORIZONS,
|
||||
)
|
||||
from src.repositories.skill_opinion_outcome_repo import (
|
||||
SkillOpinionOutcomeRepository,
|
||||
SkillOpinionPerformanceBucket,
|
||||
)
|
||||
from src.services.skill_opinion_outcome_service import (
|
||||
SKILL_OPINION_OUTCOME_ENGINE_VERSION,
|
||||
)
|
||||
from src.storage import DatabaseManager
|
||||
|
||||
|
||||
MIN_SKILL_OUTCOME_SAMPLE_SIZE = 30
|
||||
|
||||
|
||||
class SkillOpinionPerformanceService:
|
||||
"""Apply sample-sufficiency policy to raw Outcome aggregates."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
repo: Optional[SkillOpinionOutcomeRepository] = None,
|
||||
db_manager: Optional[DatabaseManager] = None,
|
||||
):
|
||||
self.repo = repo or SkillOpinionOutcomeRepository(db_manager)
|
||||
|
||||
def get_stats(
|
||||
self,
|
||||
*,
|
||||
skill_id: Optional[str] = None,
|
||||
horizons: Optional[Sequence[str]] = None,
|
||||
engine_version: str = SKILL_OPINION_OUTCOME_ENGINE_VERSION,
|
||||
) -> Dict[str, Any]:
|
||||
"""Return low-sensitive statistics for the current engine version."""
|
||||
|
||||
skill_id_norm = (
|
||||
self._required_text(skill_id, "skill_id")
|
||||
if skill_id is not None
|
||||
else None
|
||||
)
|
||||
horizons_norm = self._normalize_horizons(horizons)
|
||||
engine_version_norm = self._required_text(
|
||||
engine_version,
|
||||
"engine_version",
|
||||
)
|
||||
buckets = self.repo.list_performance_buckets(
|
||||
engine_version=engine_version_norm,
|
||||
skill_id=skill_id_norm,
|
||||
horizons=horizons_norm,
|
||||
)
|
||||
horizon_rank = {
|
||||
horizon: index
|
||||
for index, horizon in enumerate(
|
||||
SUPPORTED_SKILL_OUTCOME_HORIZONS
|
||||
)
|
||||
}
|
||||
buckets.sort(
|
||||
key=lambda bucket: (
|
||||
-bucket.total,
|
||||
bucket.skill_id,
|
||||
horizon_rank[bucket.horizon],
|
||||
)
|
||||
)
|
||||
return {
|
||||
"engine_version": engine_version_norm,
|
||||
"minimum_evaluated_sample_size": MIN_SKILL_OUTCOME_SAMPLE_SIZE,
|
||||
"buckets": [self._serialize_bucket(bucket) for bucket in buckets],
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _serialize_bucket(
|
||||
bucket: SkillOpinionPerformanceBucket,
|
||||
) -> Dict[str, Any]:
|
||||
sample_sufficient = (
|
||||
bucket.evaluated >= MIN_SKILL_OUTCOME_SAMPLE_SIZE
|
||||
)
|
||||
direction_denominator = bucket.hit + bucket.miss
|
||||
terminal_denominator = (
|
||||
bucket.evaluated + bucket.observational + bucket.unable
|
||||
)
|
||||
return {
|
||||
"skill_id": bucket.skill_id,
|
||||
"horizon": bucket.horizon,
|
||||
"engine_version": bucket.engine_version,
|
||||
"total": bucket.total,
|
||||
"pending": bucket.pending,
|
||||
"evaluated": bucket.evaluated,
|
||||
"observational": bucket.observational,
|
||||
"unable": bucket.unable,
|
||||
"hit": bucket.hit,
|
||||
"miss": bucket.miss,
|
||||
"sample_sufficient": sample_sufficient,
|
||||
"sample_status": (
|
||||
"sufficient" if sample_sufficient else "observational"
|
||||
),
|
||||
"hit_rate_pct": (
|
||||
round(bucket.hit / direction_denominator * 100, 2)
|
||||
if sample_sufficient and direction_denominator
|
||||
else None
|
||||
),
|
||||
"miss_rate_pct": (
|
||||
round(bucket.miss / direction_denominator * 100, 2)
|
||||
if sample_sufficient and direction_denominator
|
||||
else None
|
||||
),
|
||||
"avg_directional_return_pct": (
|
||||
round(bucket.avg_directional_return_pct, 4)
|
||||
if sample_sufficient
|
||||
and bucket.avg_directional_return_pct is not None
|
||||
else None
|
||||
),
|
||||
"unable_rate_pct": (
|
||||
round(bucket.unable / terminal_denominator * 100, 2)
|
||||
if sample_sufficient and terminal_denominator
|
||||
else None
|
||||
),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _required_text(value: Any, field_name: str) -> str:
|
||||
text = str(value or "").strip()
|
||||
if not text:
|
||||
raise ValueError(f"{field_name} must not be blank")
|
||||
return text
|
||||
|
||||
@staticmethod
|
||||
def _normalize_horizons(
|
||||
values: Optional[Sequence[str]],
|
||||
) -> Optional[List[str]]:
|
||||
if values is None:
|
||||
return None
|
||||
if not values:
|
||||
raise ValueError("horizons must not be empty")
|
||||
|
||||
normalized: List[str] = []
|
||||
for value in values:
|
||||
horizon = str(value or "").strip()
|
||||
if horizon not in SUPPORTED_SKILL_OUTCOME_HORIZONS:
|
||||
raise ValueError(
|
||||
"horizon must be one of "
|
||||
+ ", ".join(SUPPORTED_SKILL_OUTCOME_HORIZONS)
|
||||
)
|
||||
if horizon not in normalized:
|
||||
normalized.append(horizon)
|
||||
return normalized
|
||||
@@ -25,7 +25,13 @@ class DailyStockStartResolution:
|
||||
identity: Optional[DailyStockIdentity]
|
||||
expected_start_date: Optional[date]
|
||||
failure_reason: Optional[str]
|
||||
legacy_local_start_date: Optional[date] = None
|
||||
legacy_backtest_start_date: Optional[date] = None
|
||||
|
||||
@property
|
||||
def backtest_start_date(self) -> Optional[date]:
|
||||
"""Return the authoritative start or Backtest-only legacy fallback."""
|
||||
|
||||
return self.expected_start_date or self.legacy_backtest_start_date
|
||||
|
||||
|
||||
def resolve_stock_daily_start(
|
||||
@@ -99,7 +105,7 @@ def resolve_stock_daily_start(
|
||||
identity,
|
||||
None,
|
||||
"invalid_effective_daily_bar_date",
|
||||
legacy_local_start_date=effective_date,
|
||||
legacy_backtest_start_date=effective_date,
|
||||
)
|
||||
return DailyStockStartResolution(identity, effective_date, None)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user