diff --git a/strix/core/runner.py b/strix/core/runner.py index 71c08742..b40a36b0 100644 --- a/strix/core/runner.py +++ b/strix/core/runner.py @@ -45,6 +45,7 @@ from strix.core.paths import run_dir_for, runtime_state_dir from strix.core.sessions import open_agent_session from strix.report.state import get_global_report_state from strix.runtime import session_manager +from strix.telemetry import set_scan_phase from strix.telemetry.logging import set_scan_id, setup_scan_logging from strix.tools.output_store import ( WORKSPACE_SPILL_DIR, @@ -116,6 +117,13 @@ def _record_mcp_connections(connections: list[ConnectedMcpServer]) -> None: report_state.record_mcp_connections([connection.name for connection in connections]) +def _note_exit_reason(reason: str) -> None: + """Record why the scan stopped so the end-of-scan beacon reports it.""" + report_state = get_global_report_state() + if report_state is not None and report_state.scan_ended_exit_reason is None: + report_state.scan_ended_exit_reason = reason + + def _persist_mcp_status(roster: list[dict[str, Any]]) -> None: """Write the run's non-secret MCP connection status roster to run.json. @@ -313,6 +321,7 @@ async def run_strix_scan( root_id = uuid.uuid4().hex[:8] logger.info("Bringing up sandbox session for scan %s", scan_id) + set_scan_phase("sandbox_init") bundle = await session_manager.create_or_reuse( scan_id, image=image, @@ -322,6 +331,7 @@ async def run_strix_scan( ) report("Waiting for the first model response") logger.info("Sandbox ready for scan %s", scan_id) + set_scan_phase("agent_setup") sandbox_session = bundle["session"] @@ -573,6 +583,7 @@ async def run_strix_scan( async with coordinator._lock: root_status = coordinator.statuses.get(root_id) + set_scan_phase("agent_loop") result = await run_agent_loop( agent=root_agent, initial_input=initial_input, @@ -610,6 +621,7 @@ async def run_strix_scan( return result # noqa: TRY300 except BudgetExceededError as exc: logger.info("Scan %s stopped: %s", scan_id, exc) + _note_exit_reason("budget_exceeded") if root_id is not None: with contextlib.suppress(Exception): await coordinator.set_status(root_id, "stopped") @@ -622,6 +634,7 @@ async def run_strix_scan( exc, scan_id, ) + _note_exit_reason("rate_limited") if root_id is not None: with contextlib.suppress(Exception): await coordinator.set_status(root_id, "stopped") diff --git a/strix/interface/environment.py b/strix/interface/environment.py index 49bdd82b..522067df 100644 --- a/strix/interface/environment.py +++ b/strix/interface/environment.py @@ -14,6 +14,7 @@ from strix.interface.utils import ( image_exists, process_pull_line, ) +from strix.telemetry import report_error logger = logging.getLogger(__name__) @@ -44,6 +45,7 @@ def validate_environment() -> None: f"[red]STRIX_LLM={settings.llm.model} uses your ChatGPT subscription, " "but you're not signed in.[/] Run [cyan]strix auth login chatgpt[/] first." ) + report_error("subscription_not_signed_in") sys.exit(1) logger.info("Environment OK (ChatGPT subscription)") return @@ -153,6 +155,7 @@ def validate_environment() -> None: console.print("\n") console.print(panel) console.print() + report_error("missing_required_config") sys.exit(1) logger.info( "Environment OK (optional missing: %s)", @@ -180,6 +183,7 @@ def check_docker_installed() -> None: padding=(1, 2), ) console.print("\n", panel, "\n") + report_error("docker_not_installed") sys.exit(1) logger.debug("Docker CLI present") @@ -227,6 +231,7 @@ def pull_docker_image() -> None: padding=(1, 2), ) console.print(panel, "\n") + report_error("image_pull_failed", e) sys.exit(1) logger.info("Docker image %s ready", image) diff --git a/strix/interface/main.py b/strix/interface/main.py index 5723ef43..c9bd5596 100644 --- a/strix/interface/main.py +++ b/strix/interface/main.py @@ -42,7 +42,7 @@ from strix.interface.utils import ( build_final_stats_text, ) from strix.llm.warmup import start_import_warmup, wait_for_import_warmup -from strix.telemetry import posthog, scarf +from strix.telemetry import posthog, report_error, scarf, set_scan_phase from strix.telemetry.logging import configure_dependency_logging @@ -401,15 +401,18 @@ def _bootstrap_scan(args: argparse.Namespace) -> None: happen inside the TUI so the interface paints immediately instead of waiting on a model round trip. """ + set_scan_phase("preflight") try: asyncio.run(warm_up_llm(show_model_warning=True)) except ModelConnectionError as exc: + report_error("model_connection_failed", exc) _print_model_connection_error(exc, exc.model_name) sys.exit(1) persist_current() try: prepare_run(args) except ValueError as e: + report_error("scan_preparation_failed", e) _print_error_panel("SCAN PREPARATION FAILED", str(e)) sys.exit(1) telemetry_start(args) @@ -484,18 +487,21 @@ def main() -> None: from strix.interface.cli import run_cli asyncio.run(run_cli(args)) + # Headless runs have no user to quit: the agent either finished + # (already beaconed as finished_by_tool) or stopped on its own. + exit_reason = "agent_stopped" else: asyncio.run(run_tui(args)) except InteractiveSetupUnavailableError as exc: exit_reason = "error" + report_error("interactive_setup_unavailable", exc) _print_error_panel("INTERACTIVE SETUP UNAVAILABLE", str(exc)) sys.exit(1) except KeyboardInterrupt: exit_reason = "interrupted" - except Exception: + except Exception as exc: exit_reason = "error" - posthog.error("unhandled_exception") - scarf.error("unhandled_exception") + report_error("unhandled_exception", exc) raise finally: report_state = get_global_report_state() diff --git a/strix/interface/tui/runtime.py b/strix/interface/tui/runtime.py index 728e7023..08fa0522 100644 --- a/strix/interface/tui/runtime.py +++ b/strix/interface/tui/runtime.py @@ -37,6 +37,7 @@ from strix.interface.tui.sidecar import ( ) from strix.interface.utils import read_workspace_files from strix.report.state import ReportState, set_global_report_state +from strix.telemetry import report_error, set_scan_phase from strix.utils.resource_paths import get_strix_resource_path @@ -138,6 +139,7 @@ class GoTuiRuntime: await self._preflight_model() except Exception as exc: logger.exception("Go TUI setup model preflight failed") + report_error("model_connection_failed", exc) raise RuntimeError(f"Model connection failed: {exc}") from exc async def _preflight_model(self) -> None: @@ -195,13 +197,21 @@ class GoTuiRuntime: launch so the interface appears immediately. """ model = (load_settings().llm.model or "").strip() + set_scan_phase("preflight") try: await preflight_model_connection(model) + except Exception as exc: + logger.exception("Go TUI scan preparation failed") + report_error("model_connection_failed", exc) + self.controller.fail_preparation(str(exc)) + return + try: persist_current() prepare_run(self.args) telemetry_start(self.args) except Exception as exc: logger.exception("Go TUI scan preparation failed") + report_error("scan_preparation_failed", exc) self.controller.fail_preparation(str(exc)) return self.controller.scan_state = "running" @@ -240,6 +250,9 @@ class GoTuiRuntime: self.controller.scan_state = "completed" if report_status == "completed" else "stopped" except Exception as exc: logger.exception("Go TUI scan failed") + report_error("unhandled_exception", exc) + if self.report_state is not None and self.report_state.scan_ended_exit_reason is None: + self.report_state.scan_ended_exit_reason = "error" self.scan_error = exc self.controller.error = str(exc) self.controller.scan_state = "failed" diff --git a/strix/interface/utils.py b/strix/interface/utils.py index faab1772..e051a185 100644 --- a/strix/interface/utils.py +++ b/strix/interface/utils.py @@ -19,6 +19,7 @@ from rich.panel import Panel from rich.text import Text from strix.config import load_settings +from strix.telemetry import report_error from strix.utils.api_spec import detect_spec_format @@ -1602,7 +1603,8 @@ def check_docker_connection() -> Any: try: return docker.from_env() - except DockerException: + except DockerException as exc: + report_error("docker_unavailable", exc) console = Console() error_text = Text() error_text.append("DOCKER NOT AVAILABLE", style="bold red") diff --git a/strix/telemetry/README.md b/strix/telemetry/README.md index 023eb932..3985093e 100644 --- a/strix/telemetry/README.md +++ b/strix/telemetry/README.md @@ -12,7 +12,7 @@ Privacy is our priority. All collected data is anonymized by default. Each sessi We collect only very **basic** usage data including: -**Session Errors:** Duration and error types (not messages or stack traces)\ +**Session Errors:** Duration, the failure category, the scan phase, and the exception class name (not messages or stack traces)\ **System Context:** OS type, architecture, Strix version\ **Scan Context:** Scan mode (quick/standard/deep), scan type (whitebox/blackbox)\ **Model Usage:** Which LLM model is being used and whether it runs via an API key or a model subscription (not prompts or responses)\ diff --git a/strix/telemetry/__init__.py b/strix/telemetry/__init__.py index b9ca597e..30cc3b21 100644 --- a/strix/telemetry/__init__.py +++ b/strix/telemetry/__init__.py @@ -1,7 +1,19 @@ from . import posthog, scarf +from ._common import set_scan_phase + + +def report_error(error_type: str, exc: BaseException | None = None) -> None: + """Beacon a failure category, plus the exception class when one is given. + + Only class names travel: never the message, arguments, or traceback. + """ + posthog.error(error_type, exc) + scarf.error(error_type, exc) __all__ = [ "posthog", + "report_error", "scarf", + "set_scan_phase", ] diff --git a/strix/telemetry/_common.py b/strix/telemetry/_common.py index 7923a506..f7f2866c 100644 --- a/strix/telemetry/_common.py +++ b/strix/telemetry/_common.py @@ -5,7 +5,7 @@ import platform import sys from importlib.metadata import PackageNotFoundError, version from pathlib import Path -from typing import Any +from typing import Any, cast from uuid import uuid4 @@ -54,3 +54,43 @@ def base_props() -> dict[str, Any]: "python": f"{sys.version_info.major}.{sys.version_info.minor}", "strix_version": get_version(), } + + +# Coarse stage of the current run, attached to ``error`` beacons so a failure +# can be placed without a message or trace. Process-local, like the rest of the +# CLI telemetry: one process runs one scan. +_scan_phase = "startup" + + +def set_scan_phase(phase: str) -> None: + global _scan_phase # noqa: PLW0603 + _scan_phase = phase + + +def get_scan_phase() -> str: + return _scan_phase + + +def _exception_name(exc: BaseException) -> str: + cls = type(exc) + package = cls.__module__.split(".")[0] + return cls.__name__ if package == "builtins" else f"{package}.{cls.__name__}" + + +def _unwrap_group(exc: BaseException) -> BaseException: + if not isinstance(exc, BaseExceptionGroup): + return exc + group = cast("BaseExceptionGroup[BaseException]", exc) + return group.exceptions[0] if group.exceptions else group + + +def exception_props(exc: BaseException) -> dict[str, str]: + """Class names only. Messages, arguments, and tracebacks never leave the machine.""" + exc = _unwrap_group(exc) + props = {"exception_type": _exception_name(exc)} + cause = exc.__cause__ + if cause is None and not exc.__suppress_context__: + cause = exc.__context__ + if cause is not None: + props["exception_cause"] = _exception_name(cause) + return props diff --git a/strix/telemetry/posthog.py b/strix/telemetry/posthog.py index a85a843b..72927fbc 100644 --- a/strix/telemetry/posthog.py +++ b/strix/telemetry/posthog.py @@ -9,6 +9,8 @@ from strix.telemetry._common import ( SEND_TIMEOUT, SESSION_ID, base_props, + exception_props, + get_scan_phase, get_version, is_first_run, ) @@ -178,6 +180,12 @@ def viewer_agent_steered() -> None: _send("viewer_agent_steered", {**base_props()}) -def error(error_type: str) -> None: - props = {**base_props(), "error_type": error_type} +def error(error_type: str, exc: BaseException | None = None) -> None: + props: dict[str, Any] = { + **base_props(), + "error_type": error_type, + "phase": get_scan_phase(), + } + if exc is not None: + props.update(exception_props(exc)) _send("error", props) diff --git a/strix/telemetry/scarf.py b/strix/telemetry/scarf.py index fbaea998..c8161642 100644 --- a/strix/telemetry/scarf.py +++ b/strix/telemetry/scarf.py @@ -12,6 +12,8 @@ from strix.telemetry._common import ( SEND_TIMEOUT, SESSION_ID, base_props, + exception_props, + get_scan_phase, get_version, is_first_run, ) @@ -135,10 +137,13 @@ def end(report_state: ReportState, exit_reason: str = "completed") -> None: ) -def error(error_type: str) -> None: +def error(error_type: str, exc: BaseException | None = None) -> None: props: dict[str, Any] = { **base_props(), "session": SESSION_ID, "error_type": error_type, + "phase": get_scan_phase(), } + if exc is not None: + props.update(exception_props(exc)) _send("error", props) diff --git a/tests/test_telemetry_errors.py b/tests/test_telemetry_errors.py new file mode 100644 index 00000000..5e3b7c14 --- /dev/null +++ b/tests/test_telemetry_errors.py @@ -0,0 +1,125 @@ +"""Error beacons carry a category, phase, and exception class — never a message.""" + +from __future__ import annotations + +from typing import Any + +import pytest +import requests + +from strix.report.state import ReportState +from strix.telemetry import posthog, report_error, scarf, set_scan_phase +from strix.telemetry._common import exception_props + + +PRIVATE_MESSAGE = "private message that must stay on the machine" + + +def _capture(sent: list[dict[str, Any]], event: str, props: dict[str, Any]) -> bool: + sent.append({"event": event, **props}) + return True + + +def test_exception_props_uses_bare_name_for_builtins() -> None: + assert exception_props(ValueError(PRIVATE_MESSAGE)) == {"exception_type": "ValueError"} + + +def test_exception_props_prefixes_third_party_top_level_package() -> None: + props = exception_props(requests.exceptions.ConnectTimeout(PRIVATE_MESSAGE)) + assert props == {"exception_type": "requests.ConnectTimeout"} + + +def _chained(cause: BaseException | None, *, explicit: bool) -> RuntimeError: + exc = RuntimeError("wrapped") + if explicit: + exc.__cause__ = cause + exc.__suppress_context__ = True + else: + exc.__context__ = cause + return exc + + +def test_exception_props_reports_explicit_cause() -> None: + props = exception_props(_chained(ConnectionError(PRIVATE_MESSAGE), explicit=True)) + assert props == {"exception_type": "RuntimeError", "exception_cause": "ConnectionError"} + + +def test_exception_props_reports_implicit_context() -> None: + props = exception_props(_chained(KeyError("k"), explicit=False)) + assert props["exception_cause"] == "KeyError" + + +def test_exception_props_ignores_suppressed_context() -> None: + exc = _chained(None, explicit=True) + exc.__context__ = KeyError("k") + assert exception_props(exc) == {"exception_type": "RuntimeError"} + + +def test_exception_props_unwraps_exception_group() -> None: + group = ExceptionGroup("tasks", [TimeoutError("t"), ValueError("v")]) + assert exception_props(group) == {"exception_type": "TimeoutError"} + + +@pytest.mark.parametrize("telemetry", [posthog, scarf]) +def test_error_event_carries_phase_and_class_but_no_message( + telemetry: Any, + monkeypatch: pytest.MonkeyPatch, +) -> None: + sent: list[dict[str, Any]] = [] + monkeypatch.setattr(telemetry, "_send", lambda event, props: _capture(sent, event, props)) + set_scan_phase("sandbox_init") + + telemetry.error("scan_failed", RuntimeError(PRIVATE_MESSAGE)) + + assert len(sent) == 1 + event = sent[0] + assert event["event"] == "error" + assert event["error_type"] == "scan_failed" + assert event["phase"] == "sandbox_init" + assert event["exception_type"] == "RuntimeError" + assert PRIVATE_MESSAGE not in repr(event) + + +@pytest.mark.parametrize("telemetry", [posthog, scarf]) +def test_error_event_without_exception_omits_exception_fields( + telemetry: Any, + monkeypatch: pytest.MonkeyPatch, +) -> None: + sent: list[dict[str, Any]] = [] + monkeypatch.setattr(telemetry, "_send", lambda event, props: _capture(sent, event, props)) + set_scan_phase("startup") + + telemetry.error("docker_not_installed") + + assert sent[0]["error_type"] == "docker_not_installed" + assert sent[0]["phase"] == "startup" + assert "exception_type" not in sent[0] + assert "exception_cause" not in sent[0] + + +def test_report_error_fans_out_to_both_backends(monkeypatch: pytest.MonkeyPatch) -> None: + sent: list[dict[str, Any]] = [] + monkeypatch.setattr(posthog, "_send", lambda event, props: _capture(sent, event, props)) + monkeypatch.setattr(scarf, "_send", lambda event, props: _capture(sent, event, props)) + + report_error("model_connection_failed", TimeoutError(PRIVATE_MESSAGE)) + + assert len(sent) == 2 + assert {e["error_type"] for e in sent} == {"model_connection_failed"} + assert {e["exception_type"] for e in sent} == {"TimeoutError"} + + +@pytest.mark.parametrize("telemetry", [posthog, scarf]) +def test_scan_ended_prefers_recorded_exit_reason( + telemetry: Any, + monkeypatch: pytest.MonkeyPatch, +) -> None: + state = ReportState() + state.scan_ended_exit_reason = "budget_exceeded" + sent: list[dict[str, Any]] = [] + monkeypatch.setattr(telemetry, "_send", lambda event, props: _capture(sent, event, props)) + + telemetry.end(state, exit_reason="user_exit") + + assert sent[0]["event"] == "scan_ended" + assert sent[0]["exit_reason"] == "budget_exceeded"