[ { "productId": "betterstack", "storyId": "agent-acks-escalates-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Better Stack offers a documented Uptime API (token-authenticated, verified live via runtime probe) and an MCP server for querying telemetry/logs/metrics, plus a Slack-native AI SRE agent that can investigate and act on incidents, giving agents multiple non-dashboard paths to interact with incidents. However, evidence shows acknowledge/escalate/resolve actions are documented primarily via Slack, phone-tap, and dashboard UI (docs-3, docs-11, docs-19, docs-38), not explicitly via the MCP or REST API, so full end-to-end incident lifecycle control (create+ack+escalate+resolve) purely through API/MCP without any dashboard/Slack UI step is not directly documented. missing for 10: explicit API/MCP endpoints or docs for acknowledging/escalating/resolving incidents programmatically, and confirmation that MCP tool calls (not just Slack chat) can perform these actions end-to-end.", "evidenceIds": [ "betterstack-docs-1", "betterstack-docs-22", "betterstack-docs-23", "betterstack-docs-32", "betterstack-docs-3", "betterstack-docs-15", "betterstack-docs-45", "betterstack-probe-rt-1" ] }, { "productId": "betterstack", "storyId": "agentic-agent-docs", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "A direct llms.txt probe returned 404, but Better Stack does serve a machine-readable docs.md endpoint (HTTP 200) that could be pointed at by an agent as an alternative agent-oriented doc source. There's no dedicated llms.txt or explicit agent-first documentation framework beyond this. Missing for 10: an actual llms.txt file, explicit agent-oriented documentation index, or first-party guidance recommending agents consume a specific machine-readable doc format.", "evidenceIds": [ "betterstack-probe-1", "betterstack-probe-2" ] }, { "productId": "betterstack", "storyId": "agentic-ai-insights", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Better Stack's AI SRE feature explicitly offers natural-language querying of logs/charts, AI-generated post-mortems, AI-based incident fix suggestions with linked GitHub PRs, and AI-driven Linear ticket suggestions — directly matching the story of getting AI-generated insights from data. Missing for 10: independent/hands-on verification of these AI features' quality, and detail on what data sources power the natural-language insight generation beyond marketing copy.", "evidenceIds": [ "betterstack-docs-10", "betterstack-docs-9", "betterstack-docs-8", "betterstack-docs-33", "betterstack-docs-54", "betterstack-docs-23" ] }, { "productId": "betterstack", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Better Stack's monitors, CRON/serverless job tracking, and escalation policies run continuously and autonomously in the background (docs-6, docs-16, docs-37, docs-39), and the AI SRE agent can autonomously investigate incidents, draft post-mortems, and open PRs/Linear tickets (docs-8, docs-9, docs-23, docs-33, docs-54). However, several AI SRE features appear to be invoked on-demand (e.g., tagging @betterstack in Slack, docs-55) rather than fully self-initiating, and there's no independent corroboration of true unattended autonomous operation. Missing for 10: independent/hands-on verification of unattended autonomous triggering, and clearer documentation of scheduled/agentic workflow configuration beyond monitoring and incident response.", "evidenceIds": [ "betterstack-docs-6", "betterstack-docs-16", "betterstack-docs-8", "betterstack-docs-9", "betterstack-docs-23", "betterstack-docs-33", "betterstack-docs-54", "betterstack-docs-55", "betterstack-docs-39" ] }, { "productId": "betterstack", "storyId": "agentic-builtin-assistant", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Better Stack ships a built-in 'AI SRE' agent that investigates incidents, answers natural-language queries about logs/charts, drafts post-mortems, opens GitHub PRs and Linear tickets, and can be invoked directly from Slack/Teams by tagging @betterstack — this is a genuine in-product AI assistant users can delegate SRE tasks to, not just an external MCP integration. Missing for 10: independent/hands-on verification of the AI SRE's actual task outcomes and reliability, since all evidence is vendor-authored marketing/docs.", "evidenceIds": [ "betterstack-docs-23", "betterstack-docs-10", "betterstack-docs-32", "betterstack-docs-55", "betterstack-docs-8", "betterstack-docs-9", "betterstack-docs-33", "betterstack-docs-54" ] }, { "productId": "betterstack", "storyId": "agentic-headless", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Better Stack exposes a documented, token-authenticated REST API (confirmed live via runtime probe returning 401 on unauthenticated request) that could be scripted in CI for monitor/incident management, and it offers an MCP server for LLM workflow integration. However, there is no evidence of a CLI tool, headless automation examples, CI/CD pipeline integration guides, or non-interactive batch usage patterns specifically for automation contexts. missing for 10: CLI or SDK for scripting, documented CI/CD integration examples, headless/non-interactive automation workflows beyond the generic API and MCP mentions.", "evidenceIds": [ "betterstack-docs-15", "betterstack-docs-45", "betterstack-docs-51", "betterstack-probe-rt-1", "betterstack-docs-41", "betterstack-docs-1" ] }, { "productId": "betterstack", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Better Stack ships its own MCP server so other agents (e.g. Claude Code) can call its tools, but there is no evidence that Better Stack itself can consume/plug in external MCP servers to use their tools — the AI SRE agent is described only as accessible via Slack, Teams, or LLM workflows, not as an MCP client that ingests other servers' tools.", "evidenceIds": [ "betterstack-docs-1", "betterstack-docs-22", "betterstack-docs-32", "betterstack-docs-41", "betterstack-docs-42" ] }, { "productId": "betterstack", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Better Stack advertises a first-party MCP server that integrates logs, metrics, traces, and telemetry into LLM workflows, and explicitly mentions usage in Claude Code via MCP. Missing for 10: independent/hands-on corroboration of the MCP server's setup or reliability, and no dedicated docs page detailing MCP server configuration/auth beyond marketing claims.", "evidenceIds": [ "betterstack-docs-1", "betterstack-docs-22", "betterstack-docs-41", "betterstack-docs-42", "betterstack-docs-32" ] }, { "productId": "betterstack", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Better Stack's AI SRE explicitly supports natural-language interaction: users can ask for logs/charts in plain language, tag @betterstack in Slack, or use the MCP server to query telemetry from LLM workflows like Claude Code. This is a genuine natural-language operation surface, not just chat-adjacent automation. Missing for 10: independent/hands-on verification of natural-language query accuracy, and broader coverage beyond logs/charts (e.g., natural-language incident management or monitor configuration).", "evidenceIds": [ "betterstack-docs-10", "betterstack-docs-22", "betterstack-docs-23", "betterstack-docs-32", "betterstack-docs-55", "betterstack-docs-1" ] }, { "productId": "betterstack", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers a REST API, MCP server, and Slack/Teams integrations, but there is no mention of an official CLI tool anywhere in the docs or probes.", "evidenceIds": [] }, { "productId": "betterstack", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Better Stack documents a public API (global and team-scoped tokens, Bearer auth) with a getting-started guide, and a runtime probe confirms the Uptime API endpoint is live and correctly auth-gated, matching docs. It also offers an MCP server for LLM integration, directly supporting AI-native/agentic use. Missing for 10: a full OpenAPI/swagger spec discoverable at standard paths (probe found 404s) and broader API coverage documentation beyond Uptime.", "evidenceIds": [ "betterstack-docs-15", "betterstack-docs-45", "betterstack-docs-51", "betterstack-probe-rt-1", "betterstack-docs-41", "betterstack-docs-22", "betterstack-probe-3" ] }, { "productId": "betterstack", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Better Stack's API docs describe only two token tiers — a 'global' token valid across all teams and resources, and a team-scoped Uptime token — which offers coarse scoping but no evidence of fine-grained, per-agent or read/write-limited least-privilege credentials tailored for AI agents. Missing for 10: documented scopes/permissions (read-only, resource-specific), agent-specific credential issuance, expiring/rotatable tokens, and any MCP-server-specific auth scoping despite the product advertising an MCP server for LLM workflows.", "evidenceIds": [ "betterstack-docs-15", "betterstack-docs-51", "betterstack-docs-45", "betterstack-docs-41" ] }, { "productId": "betterstack", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows a documented REST API with bearer-token auth and an MCP server for LLM integration, but there is no mention of official client SDKs (e.g., Python, Node, Go libraries) for developers to build against.", "evidenceIds": [ "betterstack-docs-15", "betterstack-docs-45", "betterstack-probe-rt-1", "betterstack-probe-3" ] }, { "productId": "betterstack", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Better Stack’s evidence pack covers Slack/Teams/email/SMS alerting, an API with bearer-token auth, and an MCP server for LLM workflows, but nowhere mentions webhooks as a subscription mechanism for events (incidents, monitor status changes, etc.). Since webhook support is a standard, plausible capability for a monitoring/incident platform, its absence from all docs and probes means this axis is unmet rather than inapplicable.", "evidenceIds": [ "betterstack-docs-15", "betterstack-docs-45", "betterstack-docs-51", "betterstack-probe-rt-1", "betterstack-docs-48" ] }, { "productId": "betterstack", "storyId": "ai-drafted-postmortems", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Better Stack explicitly claims an automated post-mortem feature 'based on the incident timeline and Slack' as part of its AI SRE offering, directly matching the story's core ask. However, the evidence is marketing copy only — no detail on contributing-factors analysis, follow-up generation, or human-review workflow, and no independent/hands-on confirmation of output quality. Missing for 10: detailed documentation of postmortem structure (contributing factors, follow-ups), independent verification of the feature working, and explicit human-review/approval workflow.", "evidenceIds": [ "betterstack-docs-9", "betterstack-docs-23", "betterstack-docs-32" ] }, { "productId": "betterstack", "storyId": "ai-incident-summaries", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Better Stack's AI SRE agent investigates incidents via Slack and produces an automated post-mortem based on the incident timeline and Slack conversation, and can answer natural-language queries for logs/charts — but this is largely post-incident summarization plus Q&A, not live status update drafting or call-note scribing during the incident itself. Missing for 10: evidence of real-time drafted stakeholder updates while the incident is live, and any call/voice transcription or scribing feature.", "evidenceIds": [ "betterstack-docs-9", "betterstack-docs-23", "betterstack-docs-10", "betterstack-docs-55", "betterstack-docs-32" ] }, { "productId": "betterstack", "storyId": "ai-root-cause-investigation", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Better Stack's AI SRE agent explicitly investigates incidents using logs, metrics, traces, errors, and web events (docs-23), can be invoked proactively in Slack/Teams, and generates automated post-mortems from incident timelines plus AI-suggested fixes/PRs and tickets (docs-8, docs-9, docs-33, docs-54), matching the 'correlating changes, telemetry, and past incidents' framing. Missing for 10: independent/hands-on verification that it proactively investigates before a human asks (evidence shows it's invoked via tagging/chat, not fully autonomous), and no detail on how 'similar past incidents' correlation works technically.", "evidenceIds": [ "betterstack-docs-23", "betterstack-docs-9", "betterstack-docs-8", "betterstack-docs-33", "betterstack-docs-54", "betterstack-docs-32", "betterstack-docs-55" ] }, { "productId": "betterstack", "storyId": "alert-dedup-grouping", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers escalation policies, on-call scheduling, and notification cascades, but there is no mention of alert deduplication or incident grouping logic. In fact, docs-19/docs-53 describe '30 incidents created at the same time' with bulk one-tap acknowledgment as the mitigation — implying duplicates still spawn separate incidents rather than being merged into a single paged incident. No evidence of true dedup/grouping into one incident exists.", "evidenceIds": [ "betterstack-docs-19", "betterstack-docs-53", "betterstack-docs-39", "betterstack-docs-52", "betterstack-docs-20" ] }, { "productId": "betterstack", "storyId": "alert-ingest-routing", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Better Stack documents ingestion from third-party monitoring/observability sources (Datadog, New Relic, Grafana, Prometheus, Zabbix, cloud providers) plus its own monitors, and provides escalation policies that route alerts to on-call teams based on custom rules (time, team availability, incident origin), with schedules, overrides, and multi-channel notification confirmed by docs and a live API probe. missing for 10: independent/third-party confirmation of routing accuracy at scale and more detail on condition-based routing logic beyond escalation policy docs.", "evidenceIds": [ "betterstack-docs-13", "betterstack-docs-20", "betterstack-docs-27", "betterstack-docs-28", "betterstack-docs-39", "betterstack-docs-40", "betterstack-docs-12", "betterstack-docs-46", "betterstack-probe-rt-1" ] }, { "productId": "betterstack", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Better Stack has documented REST APIs (Uptime API with bearer token auth) but there is no evidence of an interactive API reference with runnable/try-it examples — OpenAPI/Swagger endpoint probes all returned 404, and no docs mention an interactive console or code sandbox.", "evidenceIds": [ "betterstack-docs-15", "betterstack-docs-45", "betterstack-docs-51", "betterstack-probe-3" ] }, { "productId": "betterstack", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Better Stack has a documented REST API (Uptime API with bearer-token auth) confirmed live via runtime probe, so a machine-readable spec is a fair expectation, but explicit probes for OpenAPI/Swagger files at all standard paths returned 404 and no docs mention an OpenAPI spec or downloadable schema.", "evidenceIds": [ "betterstack-probe-3", "betterstack-docs-15", "betterstack-docs-45", "betterstack-probe-rt-1" ] }, { "productId": "betterstack", "storyId": "api-sandbox", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Better Stack is an observability/incident-management/status-page platform, not a coding agent or dev tool with a sandbox-vs-production testing mode for AI-native usage; no evidence pack item discusses sandbox environments for testing separate from production data. This is a category mismatch rather than a missing feature.", "evidenceIds": [] }, { "productId": "betterstack", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack shows Better Stack has a documented, auth-gated Uptime API (bearer token, live endpoint) but contains no mention of API versioning scheme or any deprecation policy. Since this is a fair question for a SaaS product with a public API, absence of evidence means the story is not delivered.", "evidenceIds": [ "betterstack-docs-15", "betterstack-docs-45", "betterstack-docs-51", "betterstack-probe-rt-1", "betterstack-probe-3" ] }, { "productId": "betterstack", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Better Stack exposes a REST API (bearer-token auth, global tokens) and an MCP server for AI workflows, but no evidence describes batch/bulk endpoints or MCP tools that let an AI agent act on many items (monitors, incidents, schedules) in a single call. The only 'bulk' mentions (docs-19, docs-53) refer to a human tapping to acknowledge multiple simultaneous incidents on a phone, not AI-driven bulk automation. Missing for 10: documented bulk-create/update/delete API endpoints, MCP tool support for multi-item operations, or any example of an AI agent performing bulk actions.", "evidenceIds": [ "betterstack-docs-15", "betterstack-docs-45", "betterstack-docs-19", "betterstack-docs-53", "betterstack-docs-41" ] }, { "productId": "betterstack", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Better Stack provides configurable escalation policies and monitors that automatically trigger incidents/alerts based on defined conditions (time, team availability, incident origin), plus AI-SRE actions like auto-creating Linear tickets or GitHub PRs on exceptions/downtime — a real rule-based automation layer tied to events. However, this is scoped to incident/monitoring events rather than a general-purpose custom rule engine for arbitrary triggers/actions across the platform. Missing for 10: evidence of a broader custom automation/rule-builder (e.g., webhooks-to-arbitrary-action mapping) beyond incident escalation and the specific AI-SRE actions listed, and independent verification of these automated actions working in practice.", "evidenceIds": [ "betterstack-docs-20", "betterstack-docs-28", "betterstack-docs-37", "betterstack-docs-40", "betterstack-docs-33", "betterstack-docs-54", "betterstack-docs-8", "betterstack-docs-9" ] }, { "productId": "betterstack", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Better Stack only monitors external CRON jobs and serverless workers for failures (docs-6, docs-16) and manages recurring on-call schedules for incident escalation (docs-12, docs-24, docs-46) — none of this is a mechanism for an AI-native user to define or schedule their own recurring automated jobs/workflows within the platform. No evidence of a workflow/job scheduling engine, automation triggers, or recurring task creation exists in the pack.", "evidenceIds": [ "betterstack-docs-6", "betterstack-docs-16", "betterstack-docs-12", "betterstack-docs-24" ] }, { "productId": "betterstack", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Better Stack is an observability/incident-management/status-page platform; 'automations' here refer to monitors, escalation policies, and alerting configs, not AI-agent workflow automations with version history, diff/review, and rollback semantics. There is no evidence of any versioning/review/rollback feature for configurations or AI-driven automations, and this axis is a category mismatch for the product's role rather than an unmet applicable capability.", "evidenceIds": [] }, { "productId": "betterstack", "storyId": "calendar-sync-oncall", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Better Stack documents calendar integration for on-call schedules, stating it can 'Integrate with Google & Outlook calendar' and that schedules can be kept in 'a shared Google calendar' instead of the built-in interface, supporting the core need to see on-call shifts in a personal calendar. However, there's no explicit mention of a subscribable iCal/ICS feed URL, sync frequency, or Outlook-specific feed mechanics. Missing for 10: explicit iCal/ICS feed link details, documentation of sync behavior/real-time updates, and independent/hands-on confirmation of the calendar sync working reliably.", "evidenceIds": [ "betterstack-docs-25", "betterstack-docs-49" ] }, { "productId": "betterstack", "storyId": "declare-incident-chat", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Better Stack documents creating, acknowledging, and resolving incidents directly from Slack, plus a Slack-native AI SRE agent invoked by tagging @betterstack, and escalation policies define on-call roles/order. Teams support is mentioned only for alert delivery, not for declaring/running incidents, and there's no explicit evidence of automatic incident-channel creation or full role/update automation in Teams. missing for 10: explicit incident channel auto-creation, Teams-side full incident declare/manage parity, independent hands-on confirmation.", "evidenceIds": [ "betterstack-docs-3", "betterstack-docs-48", "betterstack-docs-55", "betterstack-docs-32", "betterstack-docs-29", "betterstack-docs-13", "betterstack-docs-40" ] }, { "productId": "betterstack", "storyId": "escalation-policies", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Docs explicitly describe escalation policies as custom rulesets defining who is alerted and in what order, continuing until acknowledged, with real examples (on-call person notified first, escalates to whole team after 3 minutes if unacknowledged), configurable complex rules based on time/team availability/incident origin, multi-region schedules, and phone/press-1 acknowledgment flows. Missing for 10: no independent/hands-on third-party corroboration of escalation behavior beyond vendor docs.", "evidenceIds": [ "betterstack-docs-13", "betterstack-docs-20", "betterstack-docs-28", "betterstack-docs-39", "betterstack-docs-52", "betterstack-docs-11", "betterstack-docs-46" ] }, { "productId": "betterstack", "storyId": "followup-tracking", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Better Stack's AI SRE creates Linear tickets and GitHub PRs from incidents/downtime, and generates automated post-mortems from the incident timeline, which supports the 'sync to issue tracker' half of the story. However, there's no evidence of a mechanism to track these follow-up items to completion (e.g., status sync back from Linear/GitHub, completion dashboards, or reporting on outstanding action items across incidents).\n\nmissing for 10: tracking/closure verification of follow-up action items, broader issue-tracker support beyond Linear/GitHub, reporting on completion rates across incidents.", "evidenceIds": [ "betterstack-docs-33", "betterstack-docs-54", "betterstack-docs-8", "betterstack-docs-9" ] }, { "productId": "betterstack", "storyId": "incident-timeline", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Better Stack automatically builds an incident timeline that includes alerts, on-call actions (ack/escalate), and Slack chat, and uses it to generate an automated post-mortem (docs-3, docs-9, docs-38, docs-23). However, there is no evidence describing the ability to manually edit or annotate the timeline after the fact. missing for 10: explicit editing/annotation UI or API for the incident timeline, evidence of manual timeline entry additions, independent confirmation of timeline accuracy/completeness.", "evidenceIds": [ "betterstack-docs-9", "betterstack-docs-3", "betterstack-docs-38", "betterstack-docs-23", "betterstack-docs-39" ] }, { "productId": "betterstack", "storyId": "internal-private-status", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Better Stack's status page docs explicitly support restricting page access to select people via password protection, and marketing docs list SSO, password, and IP protection as security options for status pages — directly matching private/internal status page access control for engineering leaders. missing for 10: independent/hands-on corroboration beyond vendor docs, and explicit detail on segmenting audiences (e.g., separate customer-only vs employee-only pages) rather than a single access-gated page.", "evidenceIds": [ "betterstack-docs-36", "betterstack-docs-47", "betterstack-docs-26", "betterstack-docs-4" ] }, { "productId": "betterstack", "storyId": "mobile-oncall-app", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Evidence shows on-call alerting via phone call (press 1 to acknowledge), push notifications, and single-tap acknowledgment of multiple incidents, implying a mobile app exists for at least acknowledging incidents. However, there is no explicit documentation of escalating or resolving incidents from the mobile app, only acknowledgment is described. Missing for 10: explicit mobile app screenshots/docs showing escalate and resolve actions, confirmation of full incident management (not just ack) from the phone interface, and any independent user reports of the mobile app experience at odd hours.", "evidenceIds": [ "betterstack-docs-11", "betterstack-docs-19", "betterstack-docs-29", "betterstack-docs-53", "betterstack-docs-39" ] }, { "productId": "betterstack", "storyId": "multichannel-paging", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Better Stack documents per-channel alerting (voice call, SMS, email, Slack, Teams, push) as part of its all-you-can-alert pricing, backed by escalation policy docs, phone-call acknowledgment (press 1), and Slack/Teams incident notifications, giving on-call engineers multi-channel reachability with configurable escalation rules. Missing for 10: no explicit documentation of independently configurable per-channel notification *rules* (e.g., choosing SMS vs push for specific severities) beyond escalation policy ordering, and no third-party/independent corroboration of reliability.", "evidenceIds": [ "betterstack-docs-29", "betterstack-docs-11", "betterstack-docs-13", "betterstack-docs-20", "betterstack-docs-39", "betterstack-docs-52", "betterstack-docs-28" ] }, { "productId": "betterstack", "storyId": "observability-integrations", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Better Stack explicitly claims integration with Datadog, Grafana, Prometheus, AWS/CloudWatch, and mentions a Datadog migration path, plus a general MCP server for connecting telemetry data into LLM workflows. However, evidence is limited to marketing one-liners on the status-page/homepage listing tool names (e.g. 'Plug in the services you already use including Datadog, New Relic, Grafana, Prometheus') with no dedicated per-integration setup docs, no explicit mention of Sentry or CloudWatch by name, and no walkthroughs showing configuration steps for each. Missing for 10: named Sentry/CloudWatch integration docs, dedicated setup guides per integration, and independent/hands-on confirmation these integrations work as advertised.", "evidenceIds": [ "betterstack-docs-27", "betterstack-docs-7", "betterstack-docs-43", "betterstack-docs-41", "betterstack-docs-42" ] }, { "productId": "betterstack", "storyId": "on-call-schedules", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Better Stack's docs show clear on-call scheduling primitives: multiple time-zone-based schedules for 24/7 coverage (betterstack-docs-12/46), standard rotation templates to pre-fill on-call calendars (betterstack-docs-24), overrides and cover requests (betterstack-docs-17/18), calendar integration (betterstack-docs-25/49), and escalation policies defining alert order (betterstack-docs-13/20/40/39/52). However, there is no explicit mention of 'layers' (multi-layer scheduling) or 'round-robin' rotation logic as distinct configurable concepts—only generic 'standard rotation schedules' are cited. Missing for 10: explicit layered scheduling support, explicit round-robin rotation mechanics, and independent/hands-on verification of complex rotation setups.", "evidenceIds": [ "betterstack-docs-12", "betterstack-docs-46", "betterstack-docs-24", "betterstack-docs-17", "betterstack-docs-18", "betterstack-docs-25", "betterstack-docs-49", "betterstack-docs-13", "betterstack-docs-20", "betterstack-docs-39", "betterstack-docs-52" ] }, { "productId": "betterstack", "storyId": "openness-api-parity", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Better Stack documents an Uptime API with global and team-scoped tokens explicitly described as usable 'for managing anything within Better Stack,' and a runtime probe confirms the API is live and properly auth-gated. However, there's no OpenAPI/swagger spec, no llms.txt, and no explicit evidence that incident management, status pages, or AI-SRE features are fully API-accessible beyond Uptime resources. Missing for 10: comprehensive API reference/OpenAPI spec, explicit API parity claims for non-Uptime features (incident management, status pages, on-call scheduling), and independent verification of full UI-to-API parity.", "evidenceIds": [ "betterstack-docs-15", "betterstack-docs-45", "betterstack-docs-51", "betterstack-probe-rt-1", "betterstack-probe-3", "betterstack-probe-1" ] }, { "productId": "betterstack", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence shows Better Stack offers APIs for managing monitors/incidents (e.g., Uptime API with bearer tokens) but nothing about bulk exporting logs, metrics, traces, or incident history in open formats, nor any stated data-portability policy for users who want to leave the platform. No docs, blog, or community evidence address full data export or open-format portability.", "evidenceIds": [ "betterstack-docs-15", "betterstack-docs-45", "betterstack-docs-51", "betterstack-probe-rt-1" ] }, { "productId": "betterstack", "storyId": "openness-open-license", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Better Stack is a closed-source SaaS observability/incident management platform; there is no evidence of an open-source license for its core product source. Reading source under an open license is a category mismatch for this type of hosted commercial SaaS, so the axis does not apply.", "evidenceIds": [] }, { "productId": "betterstack", "storyId": "openness-self-host", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Better Stack is a hosted SaaS observability/incident-management platform with no evidence of a self-hostable core product; self-hosting is not a plausible axis for this cloud service category, so this is a category mismatch rather than a missing feature.", "evidenceIds": [] }, { "productId": "betterstack", "storyId": "overrides-swaps", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Better Stack explicitly supports one-click 'Request cover' and one-click override creation, plus escalation policies that continue notifying until acknowledgment, all without admin intervention to rebuild schedules. Shift swap isn't explicitly named, but overrides and cover requests directly address the core story need for self-service schedule changes. Missing for 10: explicit documentation of a distinct 'shift swap' workflow and independent/hands-on verification beyond vendor docs.", "evidenceIds": [ "betterstack-docs-2", "betterstack-docs-17", "betterstack-docs-18", "betterstack-docs-13", "betterstack-docs-39", "betterstack-docs-24" ] }, { "productId": "betterstack", "storyId": "postmortem-workflow", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Better Stack documents an AI-generated postmortem drawn from the incident timeline and Slack discussion (betterstack-docs-9), showing drafting-from-timeline exists, but there is no evidence of a template library, a structured review step, or a publication/sharing workflow for postmortems. missing for 10: postmortem templates, review/approval workflow, publication mechanism.", "evidenceIds": [ "betterstack-docs-9" ] }, { "productId": "betterstack", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions data residency, region selection, or storage location controls for Better Stack; the evidence covers monitoring, incident management, and AI-SRE features only. This is an applicable axis for a SaaS platform storing customer telemetry data, so absence of evidence yields 'none'.", "evidenceIds": [] }, { "productId": "betterstack", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack contains no mention of AI training data opt-out, data usage policies for model training, or any privacy controls specific to preventing AI training on customer data. Better Stack's evidence is focused on monitoring, incident management, and AI SRE features, with no documentation addressing this specific privacy-posture axis.", "evidenceIds": [] }, { "productId": "betterstack", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers uptime monitoring, incident management, status pages, and AI SRE features but contains no documentation about data retention policies, configurable retention periods, or data deletion controls for logs/metrics/telemetry data. No mention of GDPR/data deletion tooling, retention settings, or export/delete APIs specific to user data. missing for 10: retention period configuration, data deletion/export controls, documentation on how long logs/metrics/traces are stored or how to purge them.", "evidenceIds": [] }, { "productId": "betterstack", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack addresses telemetry opt-out or usage-tracking controls for Better Stack itself; all citations describe monitoring/incident/AI-SRE features rather than privacy or telemetry settings.", "evidenceIds": [] }, { "productId": "betterstack", "storyId": "public-status-page", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Better Stack's status page product explicitly supports custom subdomains/branding, custom CSS, email subscriber notifications, password protection/SSO for private pages, and is driven from monitor/incident state (HTTP monitors auto-create incidents reflected on the page). This covers hosting, custom domain, subscriber notifications, and incident-driven state directly per docs. Missing for 10: independent/hands-on corroboration beyond vendor docs and no explicit mention of automatic custom SSL/domain verification flow details.", "evidenceIds": [ "betterstack-docs-4", "betterstack-docs-14", "betterstack-docs-21", "betterstack-docs-26", "betterstack-docs-36", "betterstack-docs-47", "betterstack-docs-37", "betterstack-docs-44" ] }, { "productId": "betterstack", "storyId": "reliability-analytics", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers incident creation, acknowledgment, escalation policies, on-call scheduling, and status pages, but contains no mention of MTTA/MTTR trend reporting, incident load analytics, or on-call health dashboards that would let a leader track improvement over time. This is a fair capability to expect from an incident/on-call platform, but no evidence shows it exists. Missing for 10: MTTA/MTTR trend metrics, incident load/volume analytics, on-call health/burnout reporting, any historical dashboard or exportable reliability report.", "evidenceIds": [] }, { "productId": "betterstack", "storyId": "roles-tasks-response", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Better Stack's incident management docs cover escalation policies, on-call scheduling, acknowledgment, and Slack/Teams integration, but nothing describes assigning structured incident roles (commander, comms lead) or task checklists for coordinated response. Escalation policies define notification order, not role-based incident command structure.", "evidenceIds": [ "betterstack-docs-13", "betterstack-docs-20", "betterstack-docs-28", "betterstack-docs-39" ] }, { "productId": "betterstack", "storyId": "runbook-automation", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Better Stack's AI SRE agent automates several incident-response actions (investigating incidents, generating post-mortems, creating Linear tickets, suggesting GitHub PR fixes) and escalation policies define automated notification chains, but there is no evidence of a structured 'runbook' object that attaches to incidents with discrete steps like auto-creating channels or assigning tasks. missing for 10: explicit runbook/playbook feature, automatic channel creation, task assignment automation, diagnostic step execution tied to a runbook template.", "evidenceIds": [ "betterstack-docs-23", "betterstack-docs-33", "betterstack-docs-8", "betterstack-docs-9", "betterstack-docs-20", "betterstack-docs-39" ] }, { "productId": "betterstack", "storyId": "stakeholder-comms", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Better Stack's status pages let subscribers receive e-mail updates without contacting the team directly, and private/password-protected status pages could be used to give internal stakeholders a dedicated, non-public view of incident status — a plausible mechanism for 'structured updates without joining the war room.' However, the evidence frames this feature around customers, not internal stakeholders, and there's no dedicated internal-stakeholder digest, dashboard, or subscription flow distinct from the public status-page mechanism. Missing for 10: explicit internal-stakeholder subscription workflow, structured incident-update format (vs generic subscribe), and evidence the private status page is actually used for this purpose rather than external customers.", "evidenceIds": [ "betterstack-docs-21", "betterstack-docs-36", "betterstack-docs-47", "betterstack-docs-9" ] }, { "productId": "betterstack", "storyId": "workflow-automation-engine", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Better Stack ships condition-based automation tied to incident events: escalation policies that alert people in order until acknowledged (docs-13, docs-20, docs-28, docs-39/52), plus AI SRE automations that auto-create Linear tickets, suggest PRs, and generate post-mortems from the incident timeline (docs-33/54, docs-8, docs-9). This covers much of the 'toil automation' story but is a set of pre-built triggers rather than a general-purpose condition/workflow engine for arbitrary field changes or custom reminders. Missing for 10: a documented generic rules/workflow builder letting SREs define custom conditions (e.g., field-change triggers, custom reminders) beyond escalation and the fixed AI-SRE actions.", "evidenceIds": [ "betterstack-docs-13", "betterstack-docs-20", "betterstack-docs-28", "betterstack-docs-39", "betterstack-docs-52", "betterstack-docs-33", "betterstack-docs-54", "betterstack-docs-8", "betterstack-docs-9" ] }, { "productId": "betterstack", "storyId": "workflow-tool-integrations", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Slack integration is well documented (create/ack/resolve incidents, alerts, AI SRE tagging via docs-3, docs-38, docs-48, docs-55), Teams appears only as an alert channel (docs-29, docs-32), and GitHub integration is one-directional (auto-generated PR with fix suggestion, docs-8) rather than bidirectional state sync. Jira and Zoom are not mentioned anywhere in the evidence pack; Linear (not Jira) is the ticketing tool referenced (docs-33, docs-54). missing for 10: any Jira integration, any Zoom integration, two-way (not just outbound) GitHub state sync, and deeper Teams bidirectional actions beyond alerting.", "evidenceIds": [ "betterstack-docs-3", "betterstack-docs-38", "betterstack-docs-48", "betterstack-docs-55", "betterstack-docs-29", "betterstack-docs-32", "betterstack-docs-8", "betterstack-docs-33", "betterstack-docs-54" ] }, { "productId": "firehydrant", "storyId": "agent-acks-escalates-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "FireHydrant ships a documented REST API described as able to do 'anything you can do on the FireHydrant UI' (docs-21, docs-48) and has a confirmed official MCP server that completes a full keyless handshake and advertises tools/prompts capabilities (docs-49, probe-3, probe-rt-1), enabling agent-driven incident workflows without a dashboard. However, the evidence never enumerates the specific MCP tool set (e.g., explicit 'acknowledge alert', 'escalate', 'resolve incident' endpoints/tools), so full end-to-end coverage of acknowledge→escalate→resolve purely via API/MCP is inferred rather than directly documented. Missing for 10: explicit MCP tool list showing acknowledge/escalate/resolve actions, and a hands-on example of an agent completing a full incident lifecycle via API/MCP.", "evidenceIds": [ "firehydrant-docs-21", "firehydrant-docs-48", "firehydrant-docs-49", "firehydrant-probe-3", "firehydrant-probe-rt-1", "firehydrant-docs-4", "firehydrant-docs-5" ] }, { "productId": "firehydrant", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "A probe confirms FireHydrant hosts a live llms.txt at docs.firehydrant.com/llms.txt returning HTTP 200 with structured documentation content, directly satisfying the ability to point an agent at agent-oriented docs. This is corroborated by an official MCP server that also exposes tool schemas for agent consumption. Missing for 10: no independent third-party report of an agent actually consuming llms.txt successfully, and no explicit docs page describing llms.txt as an intentional agent-facing feature.", "evidenceIds": [ "firehydrant-probe-1", "firehydrant-probe-3", "firehydrant-probe-rt-1" ] }, { "productId": "firehydrant", "storyId": "agentic-ai-insights", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "FireHydrant ships multiple AI-generated insight features inside the product: AI-generated incident summaries created automatically on declaration, AI-drafted retrospectives that produce timelines/impacts/lessons-learned, and a Copilot that proactively compares ongoing incidents to past ones and suggests actions in Command Center/Slack. Missing for 10: independent/hands-on validation of output quality and no detail on the underlying model or accuracy benchmarks.", "evidenceIds": [ "firehydrant-docs-11", "firehydrant-docs-13", "firehydrant-docs-30", "firehydrant-docs-35", "firehydrant-docs-45", "firehydrant-docs-52" ] }, { "productId": "firehydrant", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "FireHydrant's Runbooks are explicitly described as an automation engine that runs workflows of steps/actions automatically based on conditions (e.g., default runbook auto-attaches to every incident, auto-assigns roles, auto-creates Slack channels/Jira tickets), and Signals/Alert Rules auto-route events into alerts and incidents without human intervention — this is genuine autonomous background automation. However, these are reactive/triggered automations tied to incident events rather than fully agentic, self-initiating background processes, and there's no evidence of scheduled/proactive AI-driven automation beyond incident-triggered workflows. Missing for 10: evidence of autonomous scheduling or proactive (non-incident-triggered) automation, independent/hands-on confirmation that Runbooks run reliably unattended, and detail on how failures/edge-cases in autonomous execution are handled.", "evidenceIds": [ "firehydrant-docs-43", "firehydrant-docs-28", "firehydrant-docs-7", "firehydrant-docs-41", "firehydrant-docs-5", "firehydrant-docs-42", "firehydrant-docs-2", "firehydrant-docs-24" ] }, { "productId": "firehydrant", "storyId": "agentic-builtin-assistant", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "FireHydrant ships several AI-powered features—AI incident summaries, AI-drafted retrospectives ('Draft with AI'), and a 'Copilot' that proactively suggests related incidents in Command Center/Slack—which act as an embedded AI assistant for specific incident-management tasks. However, these are narrow, feature-specific AI actions (summarization, drafting, suggestions) rather than a general-purpose assistant to which a user can freely delegate open-ended tasks. Missing for 10: evidence of a conversational/chat-based assistant interface, ability to delegate arbitrary multi-step tasks beyond the predefined AI features, and independent user reports confirming assistant reliability.", "evidenceIds": [ "firehydrant-docs-11", "firehydrant-docs-13", "firehydrant-docs-30", "firehydrant-docs-35", "firehydrant-docs-52" ] }, { "productId": "firehydrant", "storyId": "agentic-headless", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "FireHydrant exposes a full REST API with token-based auth for configuring integrations, incidents, and webhooks, plus an official MCP server that supports headless/CI-style automation. However, this is an incident-management platform, not a CI-runnable tool itself — there's no evidence of a CLI, Docker image, or documented CI pipeline usage pattern. Missing for 10: dedicated CLI for automation, explicit CI/CD pipeline examples or integrations (e.g., GitHub Actions), and independent confirmation of headless API/MCP use in production automation workflows.", "evidenceIds": [ "firehydrant-docs-21", "firehydrant-docs-48", "firehydrant-docs-33", "firehydrant-docs-49", "firehydrant-probe-rt-1" ] }, { "productId": "firehydrant", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows FireHydrant ships an official MCP server so external AI agents can call FireHydrant's tools (docs-49, probe-3, probe-rt-1), but this is the reverse (server) role — there is no evidence FireHydrant itself can plug in third-party MCP servers to consume their tools within its own AI features (Copilot, AI summaries, retrospectives). Missing for 10: any documentation of FireHydrant acting as an MCP client, configuration for adding external MCP servers, or AI features that invoke external MCP tools.", "evidenceIds": [ "firehydrant-docs-49", "firehydrant-probe-3", "firehydrant-probe-rt-1" ] }, { "productId": "firehydrant", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "FireHydrant maintains an official MCP server (firehydrant/firehydrant-mcp) confirmed via docs reference and a runtime probe showing a successful stdio initialize handshake advertising tools/prompts capabilities. Missing for 10: independent third-party adoption reports or richer documentation of tool coverage beyond the handshake confirmation.", "evidenceIds": [ "firehydrant-docs-49", "firehydrant-probe-3", "firehydrant-probe-rt-1" ] }, { "productId": "firehydrant", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "FireHydrant ships an official MCP server (confirmed via live handshake exposing tools and prompts) that would let AI agents issue natural-language-driven commands against the platform, and Slack integration (/fh) offers some command-style interaction, but there's no evidence of a native in-app NL/chat interface for direct human natural-language operation—AI features documented are mostly summarization/drafting rather than command execution. Missing for 10: documented list of MCP tool commands, evidence of a native conversational command interface within the FireHydrant UI itself, and independent hands-on confirmation of NL command execution.", "evidenceIds": [ "firehydrant-docs-49", "firehydrant-probe-rt-1", "firehydrant-docs-8", "firehydrant-docs-30" ] }, { "productId": "firehydrant", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item references an official FireHydrant CLI tool; only an API (docs-18/21/48), an MCP server (docs-49, probe-3), and web/Slack integrations are documented. The axis is plausible for a platform like this, but absence of any CLI documentation means it is not evidenced.", "evidenceIds": [] }, { "productId": "firehydrant", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "FireHydrant documents a full public REST API with token-based key auth ('anything you can do on the FireHydrant UI'), and API key management docs describe permission scoping, plus an official MCP server exists and was verified to run a full handshake, further supporting programmatic/agentic access. missing for 10: a publicly discoverable OpenAPI/swagger spec (all probed paths 404'd) and independent third-party corroboration of API robustness.", "evidenceIds": [ "firehydrant-docs-21", "firehydrant-docs-18", "firehydrant-docs-33", "firehydrant-docs-48", "firehydrant-probe-2", "firehydrant-probe-3", "firehydrant-probe-rt-1" ] }, { "productId": "firehydrant", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "FireHydrant's docs describe API keys but explicitly state they grant Owner permissions by default with no mention of scoped or least-privilege roles for API keys or agent credentials (firehydrant-docs-33, firehydrant-docs-18, firehydrant-docs-48). No evidence of a mechanism to issue restricted-scope tokens specifically for AI agents.", "evidenceIds": [ "firehydrant-docs-33", "firehydrant-docs-18", "firehydrant-docs-48" ] }, { "productId": "firehydrant", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows a general REST API (docs-21, docs-48, docs-18, docs-33) and an official MCP server (docs-49), but there is no mention of official client SDKs (e.g., Python, JS, Go libraries) for developers to build against, and probes for an OpenAPI spec (firehydrant-probe-2) all returned 404s, suggesting no generated SDK tooling. Missing for 10: any documented official SDK/library in a specific language, package registry listings, or SDK usage examples.", "evidenceIds": [ "firehydrant-docs-21", "firehydrant-docs-48", "firehydrant-docs-33", "firehydrant-probe-2" ] }, { "productId": "firehydrant", "storyId": "agentic-webhooks", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The only evidence is a single line noting the FireHydrant API can be used to \"set up webhooks--anything you can do on the FireHydrant UI,\" implying webhook subscription is possible, but there's no dedicated documentation on webhook event types, payload structure, or subscription management shown in the pack. Missing for 10: dedicated webhooks documentation, list of subscribable event types, payload schema, and any hands-on/independent confirmation that webhook subscriptions work as described.", "evidenceIds": [ "firehydrant-docs-21" ] }, { "productId": "firehydrant", "storyId": "ai-drafted-postmortems", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "FireHydrant's AI-Drafted Retrospectives feature explicitly generates a comprehensive draft from the incident record including timelines, impacts, contributing-factor-style analysis, and preliminary Lessons Learned, triggered via a 'Draft with AI' button per retrospective question, clearly positioned for human review/editing. This directly matches the story's ask for AI-drafted postmortems with timeline/contributing factors/follow-ups ready for human review. Missing for 10: no independent/hands-on verification of output quality or explicit mention of 'follow-ups' as a distinct drafted section (only 'Lessons Learned' and impacts are named).", "evidenceIds": [ "firehydrant-docs-13", "firehydrant-docs-35", "firehydrant-docs-45", "firehydrant-docs-9" ] }, { "productId": "firehydrant", "storyId": "ai-incident-summaries", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "FireHydrant offers AI-generated incident summaries at declaration, AI-drafted retrospectives, and Zoom chat capture during calls, which cover live summaries and scribed call notes; however 'drafted updates' (e.g., AI-written status page/stakeholder updates during the incident) are not evidenced, and coverage is Slack/Zoom-centric with no mention of other call platforms or fully autonomous update drafting during the incident lifecycle. missing for 10: evidence of AI-drafted live status/stakeholder updates during an active incident, support for scribing calls outside Zoom, and independent/hands-on validation of accuracy of these AI features.", "evidenceIds": [ "firehydrant-docs-11", "firehydrant-docs-12", "firehydrant-docs-29", "firehydrant-docs-30", "firehydrant-docs-35", "firehydrant-docs-52" ] }, { "productId": "firehydrant", "storyId": "ai-root-cause-investigation", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "FireHydrant's AI Copilot proactively compares ongoing incidents to similar past incidents and surfaces suggestions in Command Center/Slack (firehydrant-docs-30), and AI-generated summaries appear as soon as an incident is declared (firehydrant-docs-52), giving some 'before a human asks' correlation. However, there's no documented evidence of correlating code/config changes or live telemetry data as part of this investigative process — it's limited to incident-text similarity and summarization, not a full root-cause 'investigator' pulling from deploys/metrics. Missing for 10: evidence of correlating infrastructure/code changes, telemetry/metrics ingestion into the AI analysis, and an autonomous 'investigation' output beyond summaries and incident-similarity suggestions.", "evidenceIds": [ "firehydrant-docs-30", "firehydrant-docs-52", "firehydrant-docs-11", "firehydrant-docs-35" ] }, { "productId": "firehydrant", "storyId": "alert-dedup-grouping", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes Alert Rules turning events into Alerts and routing them via Escalation Policies/Schedules, and an Alerting Analytics page showing how many alerts turn into incidents, but nothing describes deduplication or grouping of duplicate/related alerts into a single incident so only one person is paged. Missing for 10: explicit alert deduplication/grouping logic, evidence of noise reduction, and any documentation of merging correlated alerts into one incident.", "evidenceIds": [ "firehydrant-docs-5", "firehydrant-docs-27", "firehydrant-docs-42", "firehydrant-docs-34" ] }, { "productId": "firehydrant", "storyId": "alert-ingest-routing", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "FireHydrant's Signals feature documents ingesting alerts from any monitoring tool via Alert Rules (with CEL-based conditions/backtesting), routing to teams through Escalation Policies and On-Call Schedules, with multi-platform notifications and a migrator from PagerDuty/Opsgenie confirming this is a documented, mature ingestion+routing pipeline. Missing for 10: independent/hands-on validation of routing accuracy at scale and broader detail on the full catalog of supported monitoring source integrations beyond generic claims.", "evidenceIds": [ "firehydrant-docs-5", "firehydrant-docs-42", "firehydrant-docs-4", "firehydrant-docs-34", "firehydrant-docs-6", "firehydrant-docs-26", "firehydrant-docs-40", "firehydrant-docs-22", "firehydrant-docs-24" ] }, { "productId": "firehydrant", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "FireHydrant documents a REST API and API keys but there's no evidence of an interactive API reference with runnable examples; a probe explicitly found no OpenAPI/Swagger spec exposed (all candidate paths 404), suggesting no interactive API explorer exists.", "evidenceIds": [ "firehydrant-docs-21", "firehydrant-docs-18", "firehydrant-probe-2" ] }, { "productId": "firehydrant", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "FireHydrant has an API reference (docs-21, docs-48) but explicit probes for standard OpenAPI spec locations (openapi.json, swagger.json, etc.) all returned 404, and no evidence of a downloadable machine-readable spec anywhere else in the pack.", "evidenceIds": [ "firehydrant-probe-2", "firehydrant-docs-21" ] }, { "productId": "firehydrant", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "FireHydrant is an incident-management platform; no evidence describes a sandbox/staging environment, test mode, or data isolation feature for testing workflows without touching production data. The MCP server evidence covers agent connectivity, not sandbox testing. Missing for 10: any mention of a sandbox/test environment, non-production API mode, or data segregation guarantees.", "evidenceIds": [] }, { "productId": "firehydrant", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "There is a documented REST API (docs-18, docs-21, docs-48) and even a probe attempt for an OpenAPI spec, but that probe returned 404s for all candidate OpenAPI/versioning endpoints, and no evidence anywhere mentions API versioning scheme or a deprecation policy. Missing for 10: any documentation of API version numbers, a changelog, or a stated deprecation/EOL policy for the API.", "evidenceIds": [ "firehydrant-docs-21", "firehydrant-docs-18", "firehydrant-probe-2" ] }, { "productId": "firehydrant", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "FireHydrant's evidence describes a general REST API and MCP server but nowhere documents batch/bulk endpoints or bulk-update workflows (e.g., bulk-close incidents, bulk-edit alerts/schedules) that an AI agent could invoke across many items at once. Absence of any bulk-operation feature or API pattern means this applicable automation-depth axis is unmet.", "evidenceIds": [ "firehydrant-docs-21", "firehydrant-docs-48", "firehydrant-docs-49" ] }, { "productId": "firehydrant", "storyId": "automation-rules-engine", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "FireHydrant's Signals Alert Rules use CEL queries to match events and automatically route them into Alerts via Escalation Policies/On-Call Schedules, with a backtest tool to validate rule matches, and Runbooks serve as a full automation engine that triggers workflow steps based on specified conditions. This directly satisfies rule-based, event-triggered automation across both alerting and incident workflows. missing for 10: independent/hands-on validation of rule complexity or edge cases, and clearer detail on conditional branching logic within Runbooks beyond default templates", "evidenceIds": [ "firehydrant-docs-5", "firehydrant-docs-42", "firehydrant-docs-6", "firehydrant-docs-26", "firehydrant-docs-43", "firehydrant-docs-24", "firehydrant-docs-34" ] }, { "productId": "firehydrant", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "FireHydrant's 'Schedules' feature (docs-3, 25, 39) is for on-call rotation staffing, not recurring automated job/workflow execution, and Runbooks (docs-43) are event/condition-triggered automation rather than time-based recurring jobs. No evidence shows a cron-like or interval-based scheduler for workflows.", "evidenceIds": [] }, { "productId": "firehydrant", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "FireHydrant's Runbooks are its automation engine, and audit logs track 'Runbook modifications' for audit purposes, but there is no documented capability to version, review-approve changes, or roll back a Runbook/automation to a prior state. missing for 10: version history UI for Runbooks, diff/review workflow before automation changes go live, and an explicit rollback/restore mechanism.", "evidenceIds": [ "firehydrant-docs-19", "firehydrant-docs-43", "firehydrant-docs-54" ] }, { "productId": "firehydrant", "storyId": "calendar-sync-oncall", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack documents FireHydrant's Signals on-call schedules, rotations, and escalation policies, but nowhere mentions an iCal/calendar feed or personal calendar subscription for on-call shifts. Since on-call calendar sync is a standard, plausible capability for an on-call scheduling product, its absence in the evidence means the axis applies but is unsupported.", "evidenceIds": [ "firehydrant-docs-3", "firehydrant-docs-25", "firehydrant-docs-39", "firehydrant-docs-34" ] }, { "productId": "firehydrant", "storyId": "declare-incident-chat", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs show strong Slack-native incident declaration: automatic incident Slack channels (docs-2), default Runbook assigning Commander role (docs-7, docs-28, docs-41), Slack slash-command control (docs-8), and AI-generated incident summaries posted to Slack usable by anyone in the workspace (docs-11, docs-12, docs-44, docs-52). However, all evidence is Slack-specific; there is no mention of Microsoft Teams support for declaring incidents, creating channels, or assigning roles, and 'multi-platform notifications' (docs-34) refers to alerting, not chat-based incident declaration. Missing for 10: explicit Teams incident-declaration support, Teams channel/role automation, and any hands-on/independent confirmation of the chat-first workflow.", "evidenceIds": [ "firehydrant-docs-2", "firehydrant-docs-7", "firehydrant-docs-8", "firehydrant-docs-11", "firehydrant-docs-12", "firehydrant-docs-28", "firehydrant-docs-41", "firehydrant-docs-44", "firehydrant-docs-52", "firehydrant-docs-34" ] }, { "productId": "firehydrant", "storyId": "escalation-policies", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "FireHydrant's Signals product explicitly ships escalation policies that route alert notifications to on-call team members or individuals, with direct user/Slack notification options (docs-4, docs-34, docs-40). However, the evidence never details the specific mechanics of multi-step delays, fallback responders, or repeated escalation rounds until acknowledgment that the story asks about.\n\nmissing for 10: explicit documentation of configurable delay timers between escalation steps, fallback/backup responder chains, and repeat-round behavior on non-acknowledgment.", "evidenceIds": [ "firehydrant-docs-4", "firehydrant-docs-34", "firehydrant-docs-40", "firehydrant-docs-25" ] }, { "productId": "firehydrant", "storyId": "followup-tracking", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "FireHydrant's Runbooks can automate creating Jira tickets and include an 'Add Task List' step, implying some ability to generate and sync follow-up items to an issue tracker, but the evidence never describes a dedicated action-item/follow-up tracking feature with completion status or reporting back from Jira. Missing for 10: explicit follow-up/action-item tracking UI, completion status sync from Jira back to FireHydrant, and any analytics on follow-up closure rates.", "evidenceIds": [ "firehydrant-docs-2", "firehydrant-docs-41", "firehydrant-docs-43", "firehydrant-docs-9" ] }, { "productId": "firehydrant", "storyId": "incident-timeline", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "FireHydrant automatically creates incidents from alerts, runs Runbooks/automation, and captures Zoom/Slack chat insights via AI (docs-24, docs-29, docs-30, docs-43), and AI-drafted retrospectives generate timelines and summaries (docs-35, docs-45) which implies some automatic timeline construction. However, there is no explicit documentation of a unified, editable incident timeline UI where SREs can annotate or edit captured events after the fact — the evidence shows summary/retrospective generation and audit logs (docs-19, docs-54) but not a dedicated timeline edit/annotate feature. Missing for 10: explicit documentation of a dedicated incident timeline view, evidence of manual editing/annotation of auto-captured timeline entries, and independent/hands-on confirmation that alerts+actions+chat are merged into one editable record.", "evidenceIds": [ "firehydrant-docs-24", "firehydrant-docs-29", "firehydrant-docs-30", "firehydrant-docs-35", "firehydrant-docs-45", "firehydrant-docs-19", "firehydrant-docs-54" ] }, { "productId": "firehydrant", "storyId": "internal-private-status", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "FireHydrant offers internal status pages (org-only) and external status pages, but its own docs state internal status pages 'do not require auth or a FireHydrant license to view' — indicating no granular access control layer for customer-specific vs employee-only audiences. There is no evidence of role-based or audience-segmented private status pages beyond the binary internal/external split.", "evidenceIds": [ "firehydrant-docs-15", "firehydrant-docs-47", "firehydrant-docs-16" ] }, { "productId": "firehydrant", "storyId": "mobile-oncall-app", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers on-call schedules, escalation policies, Slack/API integrations, and AI summaries, but contains no mention of a native mobile app for acknowledging, escalating, or resolving incidents. Mobile app support is a fair axis for an incident-management/on-call platform, but no evidence confirms FireHydrant ships one.", "evidenceIds": [] }, { "productId": "firehydrant", "storyId": "multichannel-paging", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "FireHydrant's Signals docs mention a 'complete alerting solution' with 'multi-platform notifications' and direct notification of 'specific users and Slack channels,' implying some multi-channel delivery, but no evidence enumerates push, SMS, phone call, or email specifically, nor describes per-channel notification rule configuration. missing for 10: explicit documentation of SMS/phone/email/push channel options, per-channel rule configuration UI, and any hands-on confirmation of channel delivery.", "evidenceIds": [ "firehydrant-docs-34", "firehydrant-docs-40", "firehydrant-docs-4" ] }, { "productId": "firehydrant", "storyId": "observability-integrations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only shows generic claims like alerting from 'any monitoring tool' or 'alerting and monitoring integrations' without naming Datadog, Grafana, Prometheus, CloudWatch, or Sentry, and no documented setup guides for any of these specific tools appear in the pack. Missing for 10: named integration pages/setup docs for Datadog, Grafana, Prometheus, CloudWatch, and Sentry.", "evidenceIds": [ "firehydrant-docs-1", "firehydrant-docs-24" ] }, { "productId": "firehydrant", "storyId": "on-call-schedules", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "FireHydrant's Signals product documents on-call Schedules built from Rotations tied to teams, plus Escalation Policies for notifying via schedules or individuals, and multi-platform notifications — the core building blocks an SRE needs. However, the evidence never explicitly mentions layered/multi-tier rotations, time zone handling, or round-robin coverage patterns by name, so it's unclear whether these specific nuanced scheduling patterns are supported as described. Missing for 10: explicit documentation of scheduling layers, time zone-aware rotations, and round-robin distribution logic, plus independent/hands-on confirmation of these specifics.", "evidenceIds": [ "firehydrant-docs-25", "firehydrant-docs-39", "firehydrant-docs-3", "firehydrant-docs-4", "firehydrant-docs-34", "firehydrant-docs-40" ] }, { "productId": "firehydrant", "storyId": "openness-api-parity", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "FireHydrant's own API docs explicitly state 'you can use the FireHydrant API to configure integrations, define incidents, and set up webhooks--anything you can do on the FireHydrant UI,' directly claiming full UI/API parity, and API keys default to Owner permissions enabling full account-level actions (docs-21, docs-33, docs-48). Missing for 10: independent/hands-on verification of parity, a discoverable public OpenAPI/swagger spec (probe found only 404s), and confirmation that newer UI features (e.g., AI retrospectives, analytics dashboards) are all exposed via API.", "evidenceIds": [ "firehydrant-docs-21", "firehydrant-docs-33", "firehydrant-docs-48", "firehydrant-probe-2" ] }, { "productId": "firehydrant", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "There is no evidence of a bulk data export feature or open-format export/portability tool for FireHydrant; the API (docs-18,21,48) allows programmatic access but no documented full-data export/leave capability is described.", "evidenceIds": [] }, { "productId": "firehydrant", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "FireHydrant is a closed, commercial SaaS incident-management platform; no evidence indicates its core product source is available under any open license. The GitHub repo found is only the separate MCP server client tool, not the product itself, so it doesn't establish open licensing of the platform.", "evidenceIds": [] }, { "productId": "firehydrant", "storyId": "openness-self-host", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "FireHydrant is a SaaS incident management platform with no evidence of a self-hostable core product; all evidence describes a hosted service with API, SSO, and MCP integrations rather than deployable software. Self-hosting is not a plausible axis for this SaaS-only offering, so this is a category mismatch rather than a missing feature.", "evidenceIds": [] }, { "productId": "firehydrant", "storyId": "overrides-swaps", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "FireHydrant's Signals product documents creating schedules, rotations, and escalation policies (docs-3, docs-25, docs-39, docs-34), but no evidence describes an on-call engineer independently taking an override, swapping a shift, or requesting coverage without admin involvement.", "evidenceIds": [ "firehydrant-docs-3", "firehydrant-docs-25", "firehydrant-docs-39", "firehydrant-docs-34" ] }, { "productId": "firehydrant", "storyId": "postmortem-workflow", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "FireHydrant's docs show a real postmortem workflow: standardized templates for post-incident reviews (firehydrant-docs-9), AI-drafted retrospectives generated from the incident's timeline (firehydrant-docs-13, firehydrant-docs-35, firehydrant-docs-45), and live collaborative review (multiple viewers, cursors) on retrospective documents (firehydrant-docs-10). However, there is no explicit documentation of a distinct 'publication' step (e.g., sharing/exporting the finished retrospective externally or to stakeholders) beyond internal collaboration. Missing for 10: explicit publish/export/sharing step for finished retrospectives, and independent/hands-on corroboration of the end-to-end workflow.", "evidenceIds": [ "firehydrant-docs-9", "firehydrant-docs-10", "firehydrant-docs-13", "firehydrant-docs-35", "firehydrant-docs-45" ] }, { "productId": "firehydrant", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack items mention data residency, regional storage options, or geographic data controls for FireHydrant; this is an applicable axis for a SaaS incident-management platform but no documentation addresses it.", "evidenceIds": [] }, { "productId": "firehydrant", "storyId": "privacy-no-training", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "FireHydrant's AI security practices docs mention tenant-level control over which AI features are enabled, implying some data-governance control, but there is no explicit statement that customer data is excluded from model training. Missing for 10: an explicit no-training-on-customer-data policy, opt-out mechanism specific to model training, and independent verification of this claim.", "evidenceIds": [ "firehydrant-docs-14" ] }, { "productId": "firehydrant", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack has one line about controlling whether AI is available per tenant/feature (firehydrant-docs-14), but nothing about data retention policies, deletion of incident data, AI-generated content, or logs. No documentation of retention periods, deletion controls, or data lifecycle management is present, so this AI-native privacy-posture story is unsupported.", "evidenceIds": [] }, { "productId": "firehydrant", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item discusses telemetry, usage tracking, or opt-out controls for FireHydrant itself (docs focus on AI feature toggles, SSO, audit logs, etc.). This is a fair question for a SaaS incident management platform, so absence of evidence yields 'none'. Missing for 10: any documentation of telemetry/usage-data collection practices, an opt-out mechanism, or privacy settings addressing tracking.", "evidenceIds": [] }, { "productId": "firehydrant", "storyId": "public-status-page", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm FireHydrant offers external, FireHydrant-hosted status pages (in addition to Atlassian-hosted) driven from incident state, distinct from internal ephemeral status pages. However, the evidence never mentions custom domain support or subscriber notification functionality for these external pages. Missing for 10: custom domain configuration, subscriber/notification mechanics, and any independent confirmation of hosted page setup.", "evidenceIds": [ "firehydrant-docs-16", "firehydrant-docs-31", "firehydrant-docs-37", "firehydrant-docs-47", "firehydrant-docs-53", "firehydrant-docs-36" ] }, { "productId": "firehydrant", "storyId": "reliability-analytics", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "FireHydrant's Analytics page explicitly provides MTTx (MTTA/MTTR) metrics with grouping/filtering by team, service, severity, etc., letting leaders see overall health and trends, and a separate Alerting Analytics page tracks alert-to-incident conversion and acknowledgment rates relevant to on-call health. Missing for 10: independent/hands-on validation of the analytics UI, and no explicit dedicated 'on-call health' dashboard beyond alerting analytics and schedules.", "evidenceIds": [ "firehydrant-docs-17", "firehydrant-docs-32", "firehydrant-docs-38", "firehydrant-docs-56", "firehydrant-docs-27" ] }, { "productId": "firehydrant", "storyId": "roles-tasks-response", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "FireHydrant's default Runbook explicitly assigns a Commander role and attaches a task list on incident creation, with Runbooks acting as a customizable automation/checklist engine tied to teams and product areas, and Signals/escalation policies route comms to the right responders. Missing for 10: no explicit documented 'comms lead' named role (only Commander is default-named) and no independent/hands-on case study confirming role-based coordination in live incidents.", "evidenceIds": [ "firehydrant-docs-7", "firehydrant-docs-41", "firehydrant-docs-28", "firehydrant-docs-43", "firehydrant-docs-51", "firehydrant-docs-8" ] }, { "productId": "firehydrant", "storyId": "runbook-automation", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs describe Runbooks as FireHydrant's automation engine that attach to incidents (default Runbook auto-attaches to every incident) and execute steps like assigning roles, creating Slack channels/Jira tickets, task lists, and running diagnostics-type actions, with triggers via Slack (/fh) and API. This directly matches the SRE story of runbooks attaching to incidents and auto-triggering steps. Missing for 10: independent/hands-on verification of runbook execution reliability and broader diagnostic-step examples beyond docs.", "evidenceIds": [ "firehydrant-docs-43", "firehydrant-docs-28", "firehydrant-docs-7", "firehydrant-docs-41", "firehydrant-docs-2", "firehydrant-docs-8", "firehydrant-docs-51" ] }, { "productId": "firehydrant", "storyId": "stakeholder-comms", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "FireHydrant's internal status pages give stakeholders a no-login, no-license view of structured incident summaries without joining the incident channel/war room, and external status pages extend this further, which directly serves the 'update stakeholders without joining' need. However, the evidence never describes a subscribe/notify mechanism (e.g., email/SMS alerts on status changes) for stakeholders, only that pages exist to view. Missing for 10: explicit subscription/notification feature for internal status page updates, and independent/hands-on validation of stakeholder usage.", "evidenceIds": [ "firehydrant-docs-15", "firehydrant-docs-16", "firehydrant-docs-36", "firehydrant-docs-46", "firehydrant-docs-47" ] }, { "productId": "firehydrant", "storyId": "workflow-automation-engine", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "FireHydrant's Runbooks are explicitly documented as a condition-based automation engine that defines workflows of steps/actions triggered by incident events (role assignment, task lists, Slack/Jira automation, reminders), with default runbooks auto-attached to every incident and manual/automatic triggering options. Alert Rules similarly use conditional matching (CEL queries) to route events into incidents/escalations, reinforcing the condition-based automation theme.\n\nmissing for 10: independent/hands-on validation of runbook conditional logic in production use, and detail on field-update/reminder step types beyond role assignment and task lists.", "evidenceIds": [ "firehydrant-docs-43", "firehydrant-docs-7", "firehydrant-docs-41", "firehydrant-docs-28", "firehydrant-docs-8", "firehydrant-docs-5", "firehydrant-docs-42", "firehydrant-docs-2" ] }, { "productId": "firehydrant", "storyId": "workflow-tool-integrations", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Docs show strong two-way integration evidence for Slack (incident channels, /fh runbook commands, AI summaries usable without FH account), Jira (auto-created tickets), and Zoom (chat insights, meeting bridges), plus a general API/webhook layer for building custom integrations. However, GitHub and Microsoft Teams are never explicitly named as integrations — only generic 'multi-platform notifications' is mentioned — so state-flow for those specific tools is unevidenced. Missing for 10: explicit GitHub integration details, explicit Teams integration details, and independent/hands-on confirmation of bidirectional sync beyond FireHydrant's own docs.", "evidenceIds": [ "firehydrant-docs-2", "firehydrant-docs-8", "firehydrant-docs-12", "firehydrant-docs-29", "firehydrant-docs-34", "firehydrant-docs-40", "firehydrant-docs-21" ] }, { "productId": "incident-io", "storyId": "agent-acks-escalates-api", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "incident.io ships both a documented REST API (full OpenAPI spec, api.incident.io) and an official hosted remote MCP server that lets AI assistants 'query incidents, analyze alerts, check who's on call, manage escalations, and run deep operational analysis' — covering escalation and much of incident lifecycle management without a dashboard. However, the MCP/API docs emphasize querying, escalating, and analysis; explicit end-to-end 'acknowledge' and 'resolve' actions via MCP tools are not directly named in the evidence, only implied by broader incident-management API coverage. Missing for 10: explicit documentation naming acknowledge/resolve as MCP tool actions, and independent hands-on confirmation that an agent can fully close the incident lifecycle via MCP alone.", "evidenceIds": [ "incident-io-docs-29", "incident-io-docs-40", "incident-io-docs-47", "incident-io-probe-rt-1", "incident-io-probe-rt-2", "incident-io-docs-20", "incident-io-docs-34" ] }, { "productId": "incident-io", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "A direct runtime probe confirms `https://docs.incident.io/llms.txt` returns HTTP 200 with a structured index of 452+ help-center pages as markdown, and a further probe shows a companion llms.txt index covering 550+ API-reference pages, confirming agent-oriented docs are genuinely published and machine-readable. Missing for 10: independent/community confirmation that agents actually consume this successfully, and no evidence of an agents.md or similar beyond the llms.txt files.", "evidenceIds": [ "incident-io-probe-1", "incident-io-probe-rt-2" ] }, { "productId": "incident-io", "storyId": "agentic-ai-insights", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "incident.io ships multiple native AI-insight features directly in-product: Investigations auto-generates root-cause hypotheses from alerts, code, telemetry and past incidents (docs-14,16,23,31,56,62,66), Suggestions/summaries auto-draft incident updates and follow-ups (docs-12,53,54), and Insights dashboards offer trend/period comparisons (docs-18,32,64). Missing for 10: independent/hands-on validation of insight quality and no customer testimonial specifically praising the AI insights (only general company adoption quotes exist).", "evidenceIds": [ "incident-io-docs-14", "incident-io-docs-16", "incident-io-docs-23", "incident-io-docs-31", "incident-io-docs-56", "incident-io-docs-62", "incident-io-docs-66", "incident-io-docs-12", "incident-io-docs-53", "incident-io-docs-54", "incident-io-docs-18", "incident-io-docs-32" ] }, { "productId": "incident-io", "storyId": "agentic-autonomous-automation", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "incident.io ships both a background-automation engine (Workflows, e.g. auto-publishing status pages on incident updates) and an explicitly autonomous AI agent (Investigations) that runs the moment an incident is declared, gathering evidence, forming hypotheses, and testing them without a human prompt — with dedicated trust-and-safety docs describing how it stays 'under your control' while operating autonomously. This directly matches the story of AI-native automations running unattended in the background.\n\nmissing for 10: independent/hands-on verification that Investigations or Workflows genuinely operate unattended in production (only vendor docs cited), and broader documentation of the general-purpose Workflows automation engine beyond the status-page example.", "evidenceIds": [ "incident-io-docs-14", "incident-io-docs-16", "incident-io-docs-17", "incident-io-docs-23", "incident-io-docs-31", "incident-io-docs-9", "incident-io-docs-63" ] }, { "productId": "incident-io", "storyId": "agentic-builtin-assistant", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "incident.io ships several built-in AI capabilities users can delegate to: Investigations (an agentic, purpose-built harness that gathers evidence, forms/tests hypotheses, and even runs an adversarial agent to pressure-test findings), AI-assisted postmortem drafting/redrafting, and Suggestions which auto-drafts incident updates, summaries and follow-up actions. These are first-party product features distinct from the separate MCP client-connection story. Missing for 10: independent/hands-on user reports validating real-world reliability of these AI features beyond first-party docs.", "evidenceIds": [ "incident-io-docs-14", "incident-io-docs-16", "incident-io-docs-23", "incident-io-docs-31", "incident-io-docs-6", "incident-io-docs-12", "incident-io-docs-54" ] }, { "productId": "incident-io", "storyId": "agentic-headless", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "incident.io exposes a full REST API (api.incident.io) with API-key auth and downloadable per-tag OpenAPI 3.0.3 specs, plus webhooks for event-driven integration and a Terraform provider for infra-as-code — all of which enable declaring/managing incidents and syncing data headlessly from CI/automation pipelines without a UI. Missing for 10: explicit CI/CD pipeline examples or official SDKs/CLI wrapper, and no independent hands-on report of running it in a CI job.", "evidenceIds": [ "incident-io-docs-20", "incident-io-docs-34", "incident-io-docs-1", "incident-io-docs-19", "incident-io-docs-33", "incident-io-probe-rt-2" ] }, { "productId": "incident-io", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "All MCP evidence describes incident.io publishing its own remote MCP server so external AI assistants can query incident.io's data — the reverse of what this story asks (the product itself consuming/plugging in third-party MCP servers to gain their tools). No evidence shows incident.io's AI features (e.g. Investigations) can connect out to external MCP servers to use their tools.", "evidenceIds": [ "incident-io-docs-10", "incident-io-docs-29", "incident-io-docs-40", "incident-io-docs-47", "incident-io-probe-3", "incident-io-probe-rt-1" ] }, { "productId": "incident-io", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "incident.io documents and hosts an official remote MCP server (docs.incident.io/ai/remote-mcp) allowing any MCP-compatible AI assistant to connect and query incidents, alerts, on-call, and analytics; a live runtime probe confirms the hosted mcp.incident.io/mcp endpoint is real and enforces OAuth auth, corroborating the docs beyond marketing claims. Missing for 10: independent third-party (non-vendor) hands-on report of successfully using the MCP server in production.", "evidenceIds": [ "incident-io-docs-10", "incident-io-docs-29", "incident-io-docs-40", "incident-io-docs-47", "incident-io-probe-3", "incident-io-probe-rt-1" ] }, { "productId": "incident-io", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "incident.io ships an official hosted MCP server that lets any AI assistant query incidents, analyze alerts, check on-call, manage escalations and run operational analysis via natural language, confirmed live by a runtime probe (OAuth-protected MCP endpoint) rather than just docs. This is complemented by AI Investigations (agentic root-cause analysis) and an 'update your status page using the agent' feature, showing natural-language/agentic control across incident workflows. Missing for 10: evidence of a native built-in chat/NL command interface inside the product itself (e.g., Slack bot parsing free-form commands) independent of MCP, and independent/hands-on user validation of MCP-driven command accuracy.", "evidenceIds": [ "incident-io-docs-10", "incident-io-docs-29", "incident-io-docs-40", "incident-io-docs-47", "incident-io-docs-67", "incident-io-probe-3", "incident-io-probe-rt-1" ] }, { "productId": "incident-io", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of an official CLI tool for incident.io; the product offers a REST API, OpenAPI specs, and MCP server, but no dedicated CLI is documented anywhere in the evidence pack. Missing for 10: any mention of an incident.io CLI binary/package, its installation, or command-line usage documentation.", "evidenceIds": [] }, { "productId": "incident-io", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "incident.io publishes a documented public REST API (api.incident.io) with API-key auth and downloadable OpenAPI specs (per-tag OpenAPI 3.0.3 JSON verified live), plus 550+ API-reference doc pages, enabling programmatic/agentic control of incidents, alerts, schedules, workflows etc. Missing for 10: a single consolidated OpenAPI root file (only per-tag specs found) and independent third-party developer testimonials on API robustness.", "evidenceIds": [ "incident-io-docs-20", "incident-io-docs-34", "incident-io-docs-46", "incident-io-probe-rt-2", "incident-io-probe-1" ] }, { "productId": "incident-io", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "incident.io's hosted MCP server (used by AI agents) requires OAuth with a documented protected-resource metadata endpoint, implying some token-scoping mechanism, and the general API requires an API key. However, there is no documentation of configurable least-privilege scopes/roles for API keys or MCP tokens (e.g. read-only vs. write, per-resource permissions) that a user could explicitly issue to an agent. missing for 10: explicit API-key permission/scope configuration UI or docs, role-based least-privilege key issuance, confirmation that MCP OAuth tokens can be scoped down per-agent.", "evidenceIds": [ "incident-io-probe-rt-1", "incident-io-docs-20", "incident-io-docs-46", "incident-io-docs-10" ] }, { "productId": "incident-io", "storyId": "agentic-sdks", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "incident.io exposes a full public REST API with an OpenAPI 3.0 spec (and per-tag machine-readable specs verified live) that developers can use to generate client libraries/SDKs, plus API-key auth docs, but there is no evidence of first-party, officially maintained SDK packages (e.g., published npm/pip/Go client libraries) distinct from the raw OpenAPI spec. missing for 10: explicit official SDK packages/libraries in named languages, versioned SDK release notes, or SDK-specific quickstart guides beyond 'generate your own client from OpenAPI'.", "evidenceIds": [ "incident-io-docs-20", "incident-io-docs-34", "incident-io-docs-46", "incident-io-probe-rt-2" ] }, { "productId": "incident-io", "storyId": "agentic-webhooks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "incident.io documents a native webhooks feature for subscribing to platform events (incident updates, alert/escalation/schedule changes) for external syncing and automation, directly matching the story. Missing for 10: example payload/schema documentation, webhook signing/security details, and independent/hands-on confirmation beyond first-party docs.", "evidenceIds": [ "incident-io-docs-19", "incident-io-docs-33", "incident-io-docs-45" ] }, { "productId": "incident-io", "storyId": "ai-drafted-postmortems", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs explicitly describe AI generating first drafts of postmortems, inline review suggestions, and redrafting sections (incident-io-docs-6), plus customizable timelines (docs-4/49), follow-ups tracked separately (docs-7/39/61), and AI summaries/suggestions drawing on incident activity (docs-11/12/53/54) — together covering timeline, contributing factors via investigations (docs-14/31), and follow-ups feeding into a human-reviewable draft. Missing for 10: no independent/hands-on account of the postmortem draft quality or explicit confirmation that 'contributing factors' section specifically is auto-populated, and no third-party review of the AI drafting feature itself.", "evidenceIds": [ "incident-io-docs-6", "incident-io-docs-4", "incident-io-docs-49", "incident-io-docs-7", "incident-io-docs-39", "incident-io-docs-61", "incident-io-docs-11", "incident-io-docs-12", "incident-io-docs-53", "incident-io-docs-54", "incident-io-docs-14", "incident-io-docs-31" ] }, { "productId": "incident-io", "storyId": "ai-incident-summaries", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "incident.io documents AI Suggestions that auto-draft updates, summaries, and follow-up actions from incident activity, AI-generated suggested summaries for responders, and Scribe which transcribes/scribes incident calls — directly matching the story's live summaries, drafted updates, and call scribing. Missing for 10: independent/hands-on evidence validating real-world accuracy or adoption of these AI drafting features, and no first-party detail on latency/quality of live summary generation during actual calls.", "evidenceIds": [ "incident-io-docs-11", "incident-io-docs-12", "incident-io-docs-13", "incident-io-docs-41", "incident-io-docs-53", "incident-io-docs-54", "incident-io-docs-55", "incident-io-docs-30" ] }, { "productId": "incident-io", "storyId": "ai-root-cause-investigation", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "incident.io's Investigations feature explicitly proactively gathers evidence the moment an incident is declared — correlating alert stack traces, similar past incidents, code changes, runbooks, and telemetry — and builds/tests hypotheses via a structured harness with adversarial pressure-testing before a human digs in. This directly matches the story's ask for AI-driven correlation of changes, telemetry, and past incidents ahead of human investigation. missing for 10: independent/hands-on validation of investigation accuracy and speed claims beyond vendor docs, and more detail on how 'similar past incidents' matching works technically.", "evidenceIds": [ "incident-io-docs-14", "incident-io-docs-16", "incident-io-docs-23", "incident-io-docs-31", "incident-io-docs-56", "incident-io-docs-62", "incident-io-docs-66" ] }, { "productId": "incident-io", "storyId": "alert-dedup-grouping", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs directly document deduplication by key (dedup key prevents duplicate alerts while one remains active) and alert grouping (related alerts grouped into one alert group for single triage/escalation/incident attach), which together deliver exactly the story of not paging fifty times for one issue. Missing for 10: independent/hands-on validation of grouping accuracy at scale and configurability details beyond docs (e.g., custom grouping rules) aren't detailed.", "evidenceIds": [ "incident-io-docs-27", "incident-io-docs-36", "incident-io-docs-52", "incident-io-docs-28", "incident-io-docs-35" ] }, { "productId": "incident-io", "storyId": "alert-ingest-routing", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "incident.io documents 40+ pre-built alert sources plus custom HTTP sources, and describes escalation paths with configurable rules by priority and working hours, schedules, alert grouping and deduplication — directly matching the SRE story of documented ingestion and condition-based routing. Missing for 10: independent/hands-on verification of the routing conditions in practice and a public catalog of all 40+ sources beyond the marketing claim.", "evidenceIds": [ "incident-io-docs-15", "incident-io-docs-21", "incident-io-docs-25", "incident-io-docs-27", "incident-io-docs-28", "incident-io-docs-35", "incident-io-docs-50", "incident-io-docs-59" ] }, { "productId": "incident-io", "storyId": "api-interactive-docs", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "incident.io documents a public API reference with an OpenAPI 3.0 spec (downloadable, and confirmed live per-tag specs at runtime), which is the foundation for an interactive reference, but no evidence describes a 'try it out' console or runnable code examples within the docs themselves. missing for 10: explicit interactive/try-it-out UI in the API reference, runnable code samples, confirmation of live request execution from the docs site.", "evidenceIds": [ "incident-io-docs-20", "incident-io-docs-34", "incident-io-probe-rt-2", "incident-io-probe-2" ] }, { "productId": "incident-io", "storyId": "api-machine-spec", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "The API reference docs explicitly offer to 'Download the full OpenAPI 3.0 spec' and a runtime probe confirms machine-readable, keylessly-downloadable per-tag OpenAPI 3.0.3 specs (e.g. incidents-v2.json) are live and parseable. Missing for 10: a single consolidated full-spec file was not directly verified (only per-tag files), and no independent third-party corroboration beyond the probe.", "evidenceIds": [ "incident-io-docs-20", "incident-io-docs-34", "incident-io-docs-46", "incident-io-probe-rt-2" ] }, { "productId": "incident-io", "storyId": "api-sandbox", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Docs mention a 'test' incident type that lets users 'practice without affecting production' (docs-37), which is a lightweight sandbox-like mechanism within the core incident workflow, but there's no evidence of a dedicated sandbox environment, test API keys, or staging tenant for AI/MCP integrations or API testing separate from production data. Missing for 10: dedicated sandbox/staging environment for API or MCP testing, documented test API keys or environment flags, and any AI-native testing workflow beyond the single 'test incident' type.", "evidenceIds": [ "incident-io-docs-37", "incident-io-docs-10", "incident-io-probe-3" ] }, { "productId": "incident-io", "storyId": "api-versioning-policy", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "incident.io publishes a documented REST API with downloadable OpenAPI 3.0 specs (including versioned tags like 'incidents-v2') and an API-reference site, suggesting some versioning discipline, but there is no explicit deprecation-policy documentation, changelog for API breaking changes, or SLAs for version support lifecycle. missing for 10: explicit deprecation policy docs, version sunset timelines, migration guides between API versions.", "evidenceIds": [ "incident-io-docs-20", "incident-io-docs-34", "incident-io-probe-rt-2", "incident-io-probe-2" ] }, { "productId": "incident-io", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows a REST API, webhooks, and an MCP server for querying incidents/alerts/on-call data, but nothing describes bulk/batch operations (e.g., updating many incidents, alerts, or schedules in one call) that an AI agent could invoke at scale. Alert grouping/deduplication reduce duplicate alerts automatically but are not user- or agent-driven bulk actions across arbitrary item sets.", "evidenceIds": [ "incident-io-docs-20", "incident-io-docs-29", "incident-io-docs-40", "incident-io-probe-rt-2", "incident-io-docs-35" ] }, { "productId": "incident-io", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "incident.io's Workflows feature lets users configure rule-based automatic actions on incident events (e.g. auto-publish status pages, escalation-path rules keyed on priority/working-hours, alert deduplication/grouping), and webhooks let external systems react to events too. However the evidence never documents a general-purpose rules/workflow builder (trigger types, condition logic, full action catalog) in depth — only a couple of narrow examples are given. missing for 10: comprehensive workflow-engine documentation showing arbitrary event triggers and action library, independent/hands-on verification of workflow automation breadth.", "evidenceIds": [ "incident-io-docs-9", "incident-io-docs-25", "incident-io-docs-50", "incident-io-docs-27", "incident-io-docs-28", "incident-io-docs-19", "incident-io-docs-45" ] }, { "productId": "incident-io", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "incident.io's Workflows feature (docs-9) is event-triggered (e.g., publish to status page when an incident updates) and on-call 'Schedules' (docs-24, docs-42) refer to rotation calendars, not recurring automation jobs. No evidence describes cron-style or time-based recurring workflow scheduling that an AI-native user could configure via API, MCP, or workflow builder.", "evidenceIds": [ "incident-io-docs-9", "incident-io-docs-24", "incident-io-docs-42" ] }, { "productId": "incident-io", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "incident.io ships Workflows/automation features and even an in-progress Terraform provider for infra-as-code, but nothing in the evidence describes version history, diff/review, or rollback capabilities specifically for automations/workflows — the Terraform mention only covers general infra config, not workflow versioning or revert. Missing for 10: workflow version history, change review/diff UI, and a rollback mechanism for automations.", "evidenceIds": [ "incident-io-docs-1" ] }, { "productId": "incident-io", "storyId": "calendar-sync-oncall", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The docs describe calendar feeds only for importing personal holidays/PTO into incident.io, not for exporting an engineer's on-call shifts as a subscribable feed to their personal calendar. No evidence pack item shows an outbound iCal/webcal feed of on-call schedules for personal calendar apps.", "evidenceIds": [ "incident-io-docs-26", "incident-io-docs-43", "incident-io-docs-51" ] }, { "productId": "incident-io", "storyId": "declare-incident-chat", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs explicitly confirm incidents can be declared in seconds directly from Slack or Microsoft Teams, and other docs show incident.io auto-generates a timeline and provides post-incident/update tooling. However, the evidence pack doesn't explicitly detail automatic Slack/Teams channel creation or role assignment during declaration. Missing for 10: explicit documentation of automatic channel provisioning and role assignment during declare flow, hands-on/independent confirmation of the chat-native flow.", "evidenceIds": [ "incident-io-docs-3", "incident-io-docs-48", "incident-io-docs-4", "incident-io-docs-37", "incident-io-docs-53" ] }, { "productId": "incident-io", "storyId": "escalation-policies", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm escalation paths with priority/working-hours rules, on-call schedules, and routing alerts to escalation paths ([incident-io-docs-25],[incident-io-docs-50],[incident-io-docs-15],[incident-io-docs-24]), which implies multi-step escalation. However, the evidence never explicitly describes the specific mechanics the story asks about — configurable delay timers between steps, fallback responder chains, or repeat/looping rounds until acknowledgment. Missing for 10: explicit documentation of delay-based escalation steps, fallback responder configuration, and repeat-round behavior on non-acknowledgment.", "evidenceIds": [ "incident-io-docs-25", "incident-io-docs-50", "incident-io-docs-15", "incident-io-docs-24", "incident-io-docs-42" ] }, { "productId": "incident-io", "storyId": "followup-tracking", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "incident.io documents clear follow-up action creation/tracking (docs-7, docs-39, docs-61) and org-wide postmortem completion tracking via list views and policies (docs-5), but explicit native syncing of follow-ups to an issue tracker (e.g., Jira/Linear) is not documented — the closest evidence is a generic webhooks note that mentions 'keeping track of follow-ups' as a possible use case (docs-33/docs-45) plus a general API for building custom integrations (docs-20, docs-34). Missing for 10: a named native issue-tracker integration (Jira/Linear/Asana) for follow-ups, documentation of bidirectional sync/status updates, and any customer proof point of this specific sync working.", "evidenceIds": [ "incident-io-docs-5", "incident-io-docs-7", "incident-io-docs-39", "incident-io-docs-61", "incident-io-docs-33", "incident-io-docs-45", "incident-io-docs-20" ] }, { "productId": "incident-io", "storyId": "incident-timeline", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs confirm automatic timeline generation from key events (alerts, actions, chat) with full manual editing/annotation support, including adding custom narrative or external-system events (incident-io-docs-4, docs-38, docs-49). Alert/on-call integration and chat-based incident declaration (Slack/Teams) further support that timeline events originate from these sources automatically. Missing for 10: independent hands-on verification of exactly which event types (alerts vs chat decisions) populate automatically versus require manual entry, and no third-party review confirming the editing UX in practice.", "evidenceIds": [ "incident-io-docs-4", "incident-io-docs-38", "incident-io-docs-49", "incident-io-docs-3", "incident-io-docs-48", "incident-io-docs-15" ] }, { "productId": "incident-io", "storyId": "internal-private-status", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs explicitly confirm both public and private Status Pages exist ('public and private Status Pages') and status-page automation/API features, but no evidence details how access control is scoped (e.g., customer-only vs employee-only audiences, SSO/auth mechanisms) for private pages. missing for 10: explicit documentation of audience-based access control, authentication method for private pages, and independent confirmation of enterprise-grade access segmentation.", "evidenceIds": [ "incident-io-docs-22", "incident-io-docs-57", "incident-io-docs-8" ] }, { "productId": "incident-io", "storyId": "mobile-oncall-app", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows incident.io operates through Slack/Teams, web dashboard, escalation paths, and on-call schedules, but contains no mention of a dedicated mobile app for acknowledging, escalating, or resolving incidents from a phone. Mobile app capability is a reasonable expectation for an on-call/incident management product, so the axis applies, but nothing in the evidence confirms it exists.", "evidenceIds": [ "incident-io-docs-15", "incident-io-docs-25", "incident-io-docs-3" ] }, { "productId": "incident-io", "storyId": "multichannel-paging", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence covers on-call schedules, escalation paths (priority/working-hours rules), and alert sources, but nothing in the pack describes per-engineer notification channel preferences (push, SMS, phone call, email) for how a page reaches an individual on-call responder. Escalation-path docs (docs-25/50) discuss routing rules to people/teams, not personal channel selection.", "evidenceIds": [ "incident-io-docs-15", "incident-io-docs-25", "incident-io-docs-50", "incident-io-docs-24", "incident-io-docs-42" ] }, { "productId": "incident-io", "storyId": "observability-integrations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only mentions generic claims like '40 alert sources ready to go' and 'Configure alerts from your observability tools' without naming or documenting setup for Datadog, Grafana, Prometheus, CloudWatch, or Sentry specifically. No integration-specific setup docs, screenshots, or named connectors for any of these five tools appear anywhere in the pack.", "evidenceIds": [ "incident-io-docs-21", "incident-io-docs-15", "incident-io-docs-59" ] }, { "productId": "incident-io", "storyId": "on-call-schedules", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm core on-call scheduling: creating schedules with rotations and choosing on-call members, escalation paths with priority/working-hours rules, and calendar-feed imports for PTO/time-off (docs-15, docs-24, docs-42, docs-25, docs-50, docs-26, docs-43, docs-60). However, the evidence never explicitly mentions 'layers' as a schedule concept, multi-timezone handling within schedules, or round-robin rotation logic by name. missing for 10: explicit documentation of layered schedules, timezone-aware rotation configuration, and round-robin coverage patterns.", "evidenceIds": [ "incident-io-docs-15", "incident-io-docs-24", "incident-io-docs-42", "incident-io-docs-25", "incident-io-docs-50", "incident-io-docs-26", "incident-io-docs-43", "incident-io-docs-60" ] }, { "productId": "incident-io", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "incident.io documents a public REST API (api.incident.io) with per-tag OpenAPI 3.0.3 specs covering incidents, alerts, on-call/schedules, status pages, and webhooks, plus an MCP server that lets AI tools query incidents, alerts, on-call and escalations — evidence of broad, structured API coverage across most core UI functions. However, there is no explicit vendor claim or independent confirmation of full UI/API parity, and newer AI-native features like Investigations, AI summaries/suggestions, and Scribe are described only as UI/product capabilities without documented API/programmatic equivalents. Missing for 10: an explicit parity statement or docs, API coverage confirmation for Investigations/AI-suggestion features, and independent/hands-on verification that every UI action has an API equivalent.", "evidenceIds": [ "incident-io-docs-20", "incident-io-docs-34", "incident-io-probe-rt-2", "incident-io-docs-8", "incident-io-docs-19", "incident-io-docs-40", "incident-io-probe-3" ] }, { "productId": "incident-io", "storyId": "openness-full-export", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "incident.io exposes a documented REST API (OpenAPI 3.0 spec, per-tag JSON specs) and webhooks that could be used to pull out incidents, alerts, schedules, etc., giving a technical path to extract data, but there is no explicit 'export all your data' feature, bulk-export tool, or documented open-format full-account export/offboarding process. missing for 10: a dedicated data-export/backup feature, documentation of export formats (CSV/JSON dumps), and any account-closure/data-portability guarantee.", "evidenceIds": [ "incident-io-docs-20", "incident-io-docs-34", "incident-io-docs-46", "incident-io-probe-rt-2", "incident-io-docs-19", "incident-io-docs-45" ] }, { "productId": "incident-io", "storyId": "openness-open-license", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "incident.io is a closed commercial SaaS incident-management product; there is no evidence of an open-source license or public source repository, and this is not the kind of product where source-code openness is a standard offering. Providing an API/OpenAPI spec or MCP server is not the same as source availability, so the axis is a category mismatch rather than a gap.", "evidenceIds": [] }, { "productId": "incident-io", "storyId": "openness-self-host", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "incident.io is a SaaS incident management platform with no evidence of a self-hostable core product; self-hosting is not an axis that applies to this hosted SaaS category.", "evidenceIds": [] }, { "productId": "incident-io", "storyId": "overrides-swaps", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Docs describe schedule creation, rotations, escalation paths, and importing calendar/PTO feeds, but none of the evidence mentions an on-call engineer being able to self-service an override, swap a shift, or request coverage without admin intervention. This is a fair axis for an on-call scheduling product, so absence of evidence yields 'none'.", "evidenceIds": [ "incident-io-docs-24", "incident-io-docs-25", "incident-io-docs-26", "incident-io-docs-42", "incident-io-docs-50", "incident-io-docs-51", "incident-io-docs-60" ] }, { "productId": "incident-io", "storyId": "postmortem-workflow", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Docs show clear postmortem workflow pieces: timeline customization and narrative capture (docs-4/38/49), AI-assisted first-draft generation with inline review suggestions and redrafting (docs-6), follow-up action tracking (docs-7/39/61), and org-wide completion tracking via post-incident flow and policies (docs-5). However, there's no explicit mention of a postmortem 'template' system or a distinct publication/sign-off step, so the full templates→draft→review→publish pipeline isn't fully evidenced. Missing for 10: explicit template library/selection for postmortems, and a documented publication/approval step beyond 'policies' and 'completion tracking'.", "evidenceIds": [ "incident-io-docs-4", "incident-io-docs-5", "incident-io-docs-6", "incident-io-docs-7", "incident-io-docs-38", "incident-io-docs-39", "incident-io-docs-49", "incident-io-docs-61" ] }, { "productId": "incident-io", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions data residency, regional storage options, or geographic data location controls anywhere in incident.io's docs, changelog, or API references; this is an applicable privacy-posture axis for a SaaS platform handling incident/customer data, but no proof of a data-residency feature exists.", "evidenceIds": [] }, { "productId": "incident-io", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses any data usage, AI training opt-out, or privacy policy commitments regarding customer/incident data being used to train AI models; the docs cover AI features (Investigations, Suggestions, Scribe, MCP) but never mention data-training exclusion or opt-out controls.", "evidenceIds": [] }, { "productId": "incident-io", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack addresses data retention policies, deletion controls, or data lifecycle management for AI-native users; docs cover incidents, on-call, AI investigations, MCP, and API but nothing about retention/deletion settings.", "evidenceIds": [] }, { "productId": "incident-io", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item addresses telemetry/usage-tracking opt-out settings or a privacy control for AI-native users; the docs cover incident management, on-call, investigations, and MCP but nothing about disabling analytics/tracking.", "evidenceIds": [] }, { "productId": "incident-io", "storyId": "public-status-page", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Docs confirm hosted public and private Status Pages, workflow-driven auto-publishing from incident state, an agent that can update status pages, and an API to automate status page incidents and maintenance windows. However, the evidence pack contains no explicit mention of custom domain support or subscriber notification features, which are key parts of the story. Missing for 10: explicit documentation of custom domain hosting for status pages, subscriber/email-notification functionality, and independent/hands-on verification of these features.", "evidenceIds": [ "incident-io-docs-8", "incident-io-docs-9", "incident-io-docs-22", "incident-io-docs-57", "incident-io-docs-67" ] }, { "productId": "incident-io", "storyId": "reliability-analytics", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "incident.io's Insights product provides period-over-period comparisons and dashboards (docs-18/32/64), and its case study claims 'strengthened reliability company-wide' (docs-44), but the evidence pack lacks concrete detail on MTTA/MTTR trend tracking specifically, on-call health metrics, or incident load trending as named leadership-facing analytics — only generic 'Insights' mentions with period comparison toggles are documented. missing for 10: explicit MTTA/MTTR metric definitions and dashboards, on-call health scorecards, incident load/volume trend reporting, and independent validation that leaders use these to measure improvement over time.", "evidenceIds": [ "incident-io-docs-18", "incident-io-docs-32", "incident-io-docs-64", "incident-io-docs-44" ] }, { "productId": "incident-io", "storyId": "roles-tasks-response", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers incident declaration, timelines, follow-ups, on-call scheduling, and AI investigations, but nowhere mentions assignable incident roles (e.g., commander, comms lead) or structured task checklists during response. Missing for 10: any documentation of role assignment features, role definitions, or checklist/runbook task tracking during active incidents.", "evidenceIds": [] }, { "productId": "incident-io", "storyId": "runbook-automation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence only mentions 'runbooks' in passing as an input source that investigations reference (docs-31), and describes a separate 'Workflows' automation feature that can auto-publish status pages (docs-9), but there is no documentation tying runbook steps to automatic actions like creating channels, assigning tasks, or running diagnostics as the story describes. missing for 10: dedicated runbook-step automation, evidence of channel-creation/task-assignment/diagnostic triggers, any docs page specifically on runbook execution.", "evidenceIds": [ "incident-io-docs-31", "incident-io-docs-9" ] }, { "productId": "incident-io", "storyId": "stakeholder-comms", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Private/public Status Pages and auto-publish Workflows let stakeholders get structured updates without joining the incident channel, and AI-generated 'suggested summaries' support drafting those updates. However, no explicit 'subscribe' mechanism for internal stakeholders (e.g., follow/opt-in to specific incidents) is documented, and evidence focuses more on customer-facing status pages than internal exec/stakeholder subscriptions. Missing for 10: explicit internal-stakeholder subscription/opt-in feature, evidence of granular update cadence or digest for non-responders, and independent confirmation of use for internal (not just customer) communications.", "evidenceIds": [ "incident-io-docs-9", "incident-io-docs-22", "incident-io-docs-57", "incident-io-docs-53", "incident-io-docs-54", "incident-io-docs-8" ] }, { "productId": "incident-io", "storyId": "workflow-automation-engine", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "incident.io documents Workflows that trigger automated actions on incident events (e.g. auto-publishing status page updates, follow-up creation, webhooks for event-driven automation), which supports condition-based automation of toil like updates and field changes. However, the evidence pack lacks a dedicated 'Workflows' engine page detailing conditions/triggers/actions configuration in depth — only auto-publishing and adjacent features are directly cited. Missing for 10: a first-party doc explicitly describing the workflow engine's condition-based trigger/action model (e.g. 'when X happens, do Y' rules beyond status-page publishing), evidence of reminders/field-change automation specifically, and independent/hands-on confirmation of workflow reliability.", "evidenceIds": [ "incident-io-docs-9", "incident-io-docs-7", "incident-io-docs-19", "incident-io-docs-33", "incident-io-docs-45" ] }, { "productId": "incident-io", "storyId": "workflow-tool-integrations", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs clearly show two-way integration with Slack and Microsoft Teams (declaring incidents, timeline updates) plus a generic webhook/API layer and 40+ alert-source integrations that could plumb in tools like Jira/GitHub/Zoom, and investigations pull in 'recent code changes' implying some code-host integration. However, no evidence explicitly names Jira, GitHub, or Zoom as first-class two-way integrations. Missing for 10: explicit Jira ticket sync, GitHub PR/commit linkage, Zoom call integration documentation, and independent confirmation these named tools sync state bidirectionally.", "evidenceIds": [ "incident-io-docs-3", "incident-io-docs-48", "incident-io-docs-19", "incident-io-docs-33", "incident-io-docs-45", "incident-io-docs-21", "incident-io-docs-31", "incident-io-docs-20" ] }, { "productId": "pagerduty", "storyId": "agent-acks-escalates-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "PagerDuty ships an official local and hosted remote MCP server (verified live via runtime probe requiring only API-token/OAuth auth, no dashboard) plus a documented REST/Events API that lets integrations trigger, acknowledge, and resolve incidents, with escalation policies automating assignment without human/dashboard intervention. Independent probes confirm both the MCP endpoint and Events API v2 are functioning, self-describing endpoints, corroborating vendor docs claims. Missing for 10: explicit API/MCP documentation of an 'escalate' action being callable directly (escalation is largely policy-driven rather than an explicit agent-triggered API call), and no independent hands-on account of an agent performing the full ack→escalate→resolve loop end to end.", "evidenceIds": [ "pagerduty-docs-1", "pagerduty-docs-14", "pagerduty-docs-34", "pagerduty-probe-2", "pagerduty-probe-rt-1", "pagerduty-docs-21", "pagerduty-docs-32", "pagerduty-docs-37", "pagerduty-docs-39" ] }, { "productId": "pagerduty", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "A live probe confirms PagerDuty serves an actual llms.txt at support.pagerduty.com/llms.txt (HTTP 200) with structured knowledge-base content, and PagerDuty separately documents an official MCP server (local and hosted, verified live via runtime probe) enabling agent-native integration. Missing for 10: llms.txt coverage of developer/API docs specifically (only support KB confirmed) and independent hands-on confirmation that agents parse it effectively.", "evidenceIds": [ "pagerduty-probe-1", "pagerduty-probe-2", "pagerduty-probe-rt-1", "pagerduty-docs-14", "pagerduty-docs-34" ] }, { "productId": "pagerduty", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "PagerDuty Advance and its components (Scribe Agent for post-incident reviews, SRE Agent, Intelligent Alert Grouping via ML, AIOps insights claiming 91% alert reduction) provide AI-generated insights and suggestions embedded in the product, going beyond mere automation. However, evidence is largely first-party marketing copy with no independent/hands-on validation of insight quality, and community feedback is silent on these newer AI features (mostly discussing core paging/escalation). Missing for 10: independent verification of AI-generated insight quality, concrete examples of suggestions surfaced in-product, and community corroboration of the AI/Advance features specifically.", "evidenceIds": [ "pagerduty-docs-11", "pagerduty-docs-26", "pagerduty-docs-27", "pagerduty-docs-23", "pagerduty-docs-38", "pagerduty-docs-18", "pagerduty-docs-4" ] }, { "productId": "pagerduty", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "PagerDuty provides multiple background-automation mechanisms that run without manual triggering: Incident Workflows auto-fire on conditions (e.g., all P1/P2 incidents), Escalation Policies auto-escalate, Event Orchestration auto-dedupes/suppresses alerts, and PagerDuty Advance/SRE Agent is explicitly marketed as an agentic AI that 'handles incidents and fixes' autonomously while a human 'steps in when it matters.' This directly matches the ask of autonomous background automation. Missing for 10: independent/hands-on validation that the SRE Agent or Advance features actually operate unattended in production (community evidence is about legacy escalation/paging behavior, not the newer agentic AI layer), and no user testimonials confirming trust in fully autonomous incident resolution.", "evidenceIds": [ "pagerduty-docs-8", "pagerduty-docs-22", "pagerduty-docs-26", "pagerduty-docs-27", "pagerduty-docs-11", "pagerduty-docs-43", "pagerduty-comm-1" ] }, { "productId": "pagerduty", "storyId": "agentic-builtin-assistant", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "PagerDuty Advance embeds agentic AI directly in the product — the SRE Agent handles incidents/fixes and the Scribe Agent assists Post-Incident Reviews, both described as built-in assistants a user can delegate tasks to. However, this is marketing/first-party documentation only, with no independent or hands-on confirmation of how well delegation actually works in practice. Missing for 10: independent/hands-on validation of the SRE Agent or Scribe Agent in real use, and clarity on availability/tier restrictions for PagerDuty Advance.", "evidenceIds": [ "pagerduty-docs-11", "pagerduty-docs-26", "pagerduty-docs-27", "pagerduty-docs-23", "pagerduty-docs-38" ] }, { "productId": "pagerduty", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "PagerDuty exposes a fully headless REST API, async Events API v2, webhooks, and Incident Workflows, all confirmed live via runtime probes (400/401 structured responses), enabling CI/automation pipelines to trigger, manage, and orchestrate incidents without any UI. missing for 10: no first-party CLI tool explicitly marketed for CI pipelines, and no independent case study of running PagerDuty automation fully headless in a CI system.", "evidenceIds": [ "pagerduty-docs-31", "pagerduty-docs-32", "pagerduty-docs-33", "pagerduty-docs-39", "pagerduty-docs-8", "pagerduty-probe-rt-2", "pagerduty-probe-rt-1" ] }, { "productId": "pagerduty", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "All MCP-related evidence describes PagerDuty publishing its own MCP server (server role) so external AI clients can call PagerDuty's tools (pagerduty-docs-1, pagerduty-docs-14, pagerduty-docs-34, pagerduty-probe-2, pagerduty-probe-rt-1) — there is no evidence that PagerDuty itself, or its agentic features like PagerDuty Advance/SRE Agent, can act as an MCP client and plug in external MCP servers to use their tools. Since PagerDuty is a SaaS platform with agentic AI capabilities, this client-side capability is a fair question, but no evidence supports it.", "evidenceIds": [ "pagerduty-docs-1", "pagerduty-docs-14", "pagerduty-docs-34", "pagerduty-probe-2", "pagerduty-probe-rt-1" ] }, { "productId": "pagerduty", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "PagerDuty ships an official documented MCP server (both local GitHub package and hosted remote endpoint at mcp.pagerduty.com), confirmed live via runtime probe returning expected OAuth/Bearer auth challenge, matching official docs. missing for 10: independent third-party hands-on review of the MCP server's tool coverage beyond vendor docs.", "evidenceIds": [ "pagerduty-docs-1", "pagerduty-docs-14", "pagerduty-docs-34", "pagerduty-probe-2", "pagerduty-probe-rt-1" ] }, { "productId": "pagerduty", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "PagerDuty ships an official local and hosted MCP server (verified live via probe returning correct auth challenge) that exposes PagerDuty's incident/on-call tools to any MCP-enabled client, letting AI-native users issue natural-language commands (e.g., via Slack, IDE, or LLM assistants) to trigger/ack/resolve incidents and manage on-call data. This is reinforced by PagerDuty Advance and SRE Agent messaging around agentic/generative AI operating the platform. Missing for 10: no first-party native chat/NL interface within the PagerDuty product itself (reliant on external MCP clients) and no independent hands-on account of a user issuing NL commands end-to-end.", "evidenceIds": [ "pagerduty-docs-1", "pagerduty-docs-14", "pagerduty-docs-34", "pagerduty-probe-2", "pagerduty-probe-rt-1", "pagerduty-docs-27", "pagerduty-docs-26" ] }, { "productId": "pagerduty", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack documents PagerDuty's REST API, Events API, webhooks, and an official MCP server, but no official CLI tool for AI-native workflows is mentioned anywhere in the docs or community sources.", "evidenceIds": [] }, { "productId": "pagerduty", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "PagerDuty publishes a well-documented REST API, Events API v2, V3 webhooks, and an official MCP server, all confirmed live via runtime probes (401 auth challenge on MCP endpoint, structured validation errors on Events API), demonstrating genuine programmatic access beyond marketing claims. missing for 10: independent third-party developer testimonials specifically praising API completeness/documentation quality beyond PagerDuty's own docs.", "evidenceIds": [ "pagerduty-docs-31", "pagerduty-docs-32", "pagerduty-docs-33", "pagerduty-docs-39", "pagerduty-docs-44", "pagerduty-probe-2", "pagerduty-probe-rt-1", "pagerduty-probe-rt-2", "pagerduty-docs-14" ] }, { "productId": "pagerduty", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "PagerDuty's official MCP server and REST API require API-token or OAuth authentication (pagerduty-probe-rt-1, pagerduty-docs-32/39/44), which implies some credential-based access control an agent could use, but the evidence never describes granular scopes, roles, or least-privilege permission tiers specifically for AI-agent credentials. Missing for 10: explicit documentation of scoped/role-based API keys or OAuth scopes for agent use, any read-only vs write-restricted key options, and confirmation these can be issued distinctly per-agent.", "evidenceIds": [ "pagerduty-probe-rt-1", "pagerduty-docs-32", "pagerduty-docs-39", "pagerduty-docs-44", "pagerduty-probe-2" ] }, { "productId": "pagerduty", "storyId": "agentic-sdks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "PagerDuty provides multiple official developer surfaces — REST API, Events API v2, v3 webhooks, and an official (and now hosted/remote, OAuth-protected) MCP server — all documented on developer.pagerduty.com and confirmed live via runtime probes, which is exactly the kind of programmatic surface an AI-native builder would target. Missing for 10: explicit named official client-library SDKs in multiple languages (e.g., Python/Node SDK packages) beyond the REST/Events APIs, and independent hands-on developer corroboration of SDK ergonomics.", "evidenceIds": [ "pagerduty-docs-32", "pagerduty-docs-31", "pagerduty-docs-33", "pagerduty-docs-34", "pagerduty-probe-2", "pagerduty-probe-rt-1", "pagerduty-probe-rt-2", "pagerduty-docs-39" ] }, { "productId": "pagerduty", "storyId": "agentic-webhooks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "PagerDuty's developer docs describe V3 webhooks as a first-class subscription mechanism for incident/priority/responder change events, and the Events API v2 (confirmed live via runtime probe) provides the complementary ingestion side, giving AI-native integrators a documented way to subscribe to and consume PagerDuty events. Missing for 10: independent/hands-on confirmation of webhook subscription setup and payload handling, and no direct linkage of webhooks specifically to AI-agent workflows.", "evidenceIds": [ "pagerduty-docs-33", "pagerduty-docs-40", "pagerduty-docs-31", "pagerduty-probe-rt-2" ] }, { "productId": "pagerduty", "storyId": "ai-drafted-postmortems", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "PagerDuty's Post-Incident Reviews feature explicitly integrates with the PagerDuty Advance Scribe Agent to auto-build chronological incident narratives, uncover systemic patterns, and coordinate preventative follow-up actions — matching the AI-drafted timeline/contributing-factors/follow-ups story closely. Missing for 10: explicit documentation of 'human review' framing/draft-approval workflow, explicit mention of 'contributing factors' as a distinct output field, and any independent/hands-on confirmation of the Scribe Agent's output quality.", "evidenceIds": [ "pagerduty-docs-9", "pagerduty-docs-23", "pagerduty-docs-36", "pagerduty-docs-38", "pagerduty-docs-11", "pagerduty-docs-26" ] }, { "productId": "pagerduty", "storyId": "ai-incident-summaries", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "PagerDuty Advance's generative AI explicitly summarizes incidents and the Scribe Agent builds chronological incident narratives from Slack/incident data, directly matching 'AI writes the incident as it happens' and 'scribed call notes' (pagerduty-docs-23, pagerduty-docs-26, pagerduty-docs-38). However, the Scribe Agent is documented as tied to Post-Incident Reviews (after resolution) rather than clearly live/real-time during the call, and there's no explicit evidence of AI auto-drafting status-page updates in real time. Missing for 10: clear documentation that summaries/notes are generated live during an active incident (not just post-incident), and evidence of AI-drafted stakeholder updates during the incident itself.", "evidenceIds": [ "pagerduty-docs-23", "pagerduty-docs-26", "pagerduty-docs-38", "pagerduty-docs-11", "pagerduty-docs-27" ] }, { "productId": "pagerduty", "storyId": "ai-root-cause-investigation", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "PagerDuty Advance and the SRE Agent are marketed as agentic AI that 'handles incidents and fixes' and Intelligent Alert Grouping/AIOps use ML to reduce noise, suggesting some proactive investigative capability, but the evidence never describes the agent correlating changes, telemetry, and similar past incidents into a probable-cause narrative before a human asks. Post-Incident Review's Scribe Agent builds narratives, but only after/during incident lifecycle, not as a proactive pre-emptive root-cause investigator. Missing for 10: concrete description of autonomous correlation of deploy/change events + telemetry + historical incident similarity, and evidence of it surfacing findings unprompted.", "evidenceIds": [ "pagerduty-docs-11", "pagerduty-docs-26", "pagerduty-docs-27", "pagerduty-docs-23", "pagerduty-docs-18", "pagerduty-docs-25" ] }, { "productId": "pagerduty", "storyId": "alert-dedup-grouping", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "PagerDuty documents native dedup via `dedup_key` on Events API integrations, Event Orchestration to automatically deduplicate/suppress alerts, and Intelligent Alert Grouping using ML to consolidate related alerts into a single actionable incident specifically to reduce alert noise for SREs; escalation policies then notify a single target at a time until acknowledged, matching 'one incident pages one human.' Runtime probe confirms the Events API is live and enforces routing/event fields, corroborating the dedup mechanism is real and functioning. Missing for 10: independent/hands-on evidence quantifying grouping accuracy or false-negative rates beyond vendor claims.", "evidenceIds": [ "pagerduty-docs-3", "pagerduty-docs-4", "pagerduty-docs-25", "pagerduty-docs-43", "pagerduty-docs-37", "pagerduty-probe-rt-2" ] }, { "productId": "pagerduty", "storyId": "alert-ingest-routing", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "PagerDuty documents Events API v2 as a reliable, self-describing ingestion endpoint for monitoring alerts (confirmed live via runtime probe), plus Event Orchestration for condition-based routing/deduplication and Escalation Policies to route to the right team/individual per configurable rules. This directly matches the story of documented ingestion sources routed by defined conditions to the correct team. missing for 10: independent hands-on validation of complex routing/orchestration setups (community evidence only covers general escalation behavior and unrelated API versioning friction, not routing-condition failures).", "evidenceIds": [ "pagerduty-docs-31", "pagerduty-docs-43", "pagerduty-docs-7", "pagerduty-docs-37", "pagerduty-docs-21", "pagerduty-probe-rt-2", "pagerduty-comm-1" ] }, { "productId": "pagerduty", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows PagerDuty publishes REST/Events API overviews and developer docs (e.g. pagerduty-docs-31/32/33/39), but nothing describes an interactive API reference with runnable/try-it examples akin to a Swagger/OpenAPI explorer or code sandbox. This is a fair axis for a product with a public developer API, so absence of evidence yields 'none' rather than 'na'.", "evidenceIds": [ "pagerduty-docs-31", "pagerduty-docs-32", "pagerduty-docs-39" ] }, { "productId": "pagerduty", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack documents PagerDuty's REST API, Events API v2, and webhooks overviews, but none of the citations mention an OpenAPI/Swagger spec or any downloadable machine-readable API definition file. Missing for 10: explicit OpenAPI/Swagger spec file or reference, developer portal spec download link, schema versioning info.", "evidenceIds": [] }, { "productId": "pagerduty", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item mentions a sandbox/test environment, staging account, or test mode for safely trying PagerDuty's API/MCP tooling without affecting production incidents, escalations, or notifications; the closest analogues (MCP server, REST/Events API docs) describe only live production interactions.", "evidenceIds": [] }, { "productId": "pagerduty", "storyId": "api-versioning-policy", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "PagerDuty's developer docs show clear API versioning (REST API, Events API v2, webhooks v3) with dedicated overview pages, and probes confirm live, self-describing endpoints — but no evidence pack item states an explicit deprecation policy or sunset timeline for older versions. A dated community report also describes real confusion around distinguishing v1 vs v2 event API keys and an unhelpful support response, undercutting confidence that versioning transitions are cleanly documented in practice. Missing for 10: an explicit published deprecation/sunset policy document, and independent confirmation that version transitions are smooth rather than confusing.", "evidenceIds": [ "pagerduty-docs-31", "pagerduty-docs-32", "pagerduty-docs-33", "pagerduty-docs-40", "pagerduty-comm-6" ] }, { "productId": "pagerduty", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "PagerDuty exposes a REST API and MCP server that could be scripted to act on many items, and it explicitly supports bulk CSV user import, but there is no documented dedicated bulk-operations endpoint (e.g., bulk acknowledge/resolve many incidents in one call) or MCP tool description for batch actions across incidents/services. Missing for 10: explicit bulk incident/alert operation API or MCP tool documentation, and any hands-on evidence of performing large-scale bulk actions via AI-native tooling.", "evidenceIds": [ "pagerduty-docs-20", "pagerduty-docs-32", "pagerduty-docs-39", "pagerduty-docs-44", "pagerduty-docs-34", "pagerduty-probe-rt-1" ] }, { "productId": "pagerduty", "storyId": "automation-rules-engine", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "PagerDuty's Incident Workflows and Event Orchestration explicitly support configurable 'if-this-then-that' triggers/actions on incidents and events (auto-trigger for P1/P2, dedup/suppress alerts, escalation policies notifying on-call chains), and this automation is exposed via Events API, REST API, webhooks, and an official MCP server for AI-agent integration. Community evidence corroborates escalation policies working reliably in practice (comm-1), though some report friction with configuration and API versioning (comm-4, comm-6). Missing for 10: independent hands-on validation specifically of Incident Workflow/Orchestration rule automation (vs. just escalation) and confirmation that AI agents can programmatically define these rules via MCP rather than just fetch/manage data.", "evidenceIds": [ "pagerduty-docs-8", "pagerduty-docs-22", "pagerduty-docs-28", "pagerduty-docs-42", "pagerduty-docs-43", "pagerduty-docs-37", "pagerduty-docs-31", "pagerduty-comm-1", "pagerduty-probe-rt-1", "pagerduty-probe-rt-2" ] }, { "productId": "pagerduty", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "PagerDuty's automation features (Incident Workflows, escalation policies) are event-triggered by incident conditions, not time-based recurring job/workflow scheduling; the on-call 'Schedules' evidence (pagerduty-docs-12) covers staff rotation, not scheduled automation jobs. No evidence shows a mechanism for an AI-native user to schedule recurring automated workflows via API or MCP.", "evidenceIds": [ "pagerduty-docs-8", "pagerduty-docs-12", "pagerduty-docs-22", "pagerduty-docs-28", "pagerduty-docs-35", "pagerduty-docs-42" ] }, { "productId": "pagerduty", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "PagerDuty documents building automations (Incident Workflows, Event Orchestration, escalation policies) but no evidence pack item describes versioning, change history, review workflows, or rollback capability for these automations. The axis clearly applies since PagerDuty offers configurable automation objects, but no supporting evidence exists for version control or rollback.", "evidenceIds": [] }, { "productId": "pagerduty", "storyId": "calendar-sync-oncall", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers schedules, escalation policies, and mobile app notifications but contains no mention of an iCal/calendar feed or personal calendar sync capability for on-call shifts.", "evidenceIds": [ "pagerduty-docs-12", "pagerduty-docs-7", "pagerduty-docs-5" ] }, { "productId": "pagerduty", "storyId": "declare-incident-chat", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "PagerDuty's own marketing states you can 'manage incidents and coordinate response from Slack' (pagerduty-docs-2), and Post-Incident Reviews integrate with Slack data (pagerduty-docs-23/38), indicating some Slack tie-in, but there is no documented capability for declaring an incident directly from a Slack/Teams command with automatic channel creation, role assignment, and status updates. Missing for 10: explicit ChatOps slash-command/incident-declaration workflow docs, Teams support, evidence of auto-created channels and role assignment from chat, and any independent confirmation of this specific flow.", "evidenceIds": [ "pagerduty-docs-2", "pagerduty-docs-23", "pagerduty-docs-38" ] }, { "productId": "pagerduty", "storyId": "escalation-policies", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs explicitly describe escalation policies that notify a single target until acknowledged, with multi-level steps connecting to schedules and users, and incidents escalating per the policy automatically; a first-hand community report confirms it works as designed in practice (notify app, then call after a delay if unacknowledged). missing for 10: no explicit documentation of repeating the full escalation cycle multiple times or granular fallback-responder configuration details beyond general escalation-level language.", "evidenceIds": [ "pagerduty-docs-7", "pagerduty-docs-37", "pagerduty-docs-13", "pagerduty-docs-12", "pagerduty-comm-1" ] }, { "productId": "pagerduty", "storyId": "followup-tracking", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Post-Incident Reviews explicitly state they 'coordinate preventative follow-up actions' and integrate with incident lifecycle data (docs-36, docs-38, docs-23), showing PagerDuty tracks follow-up items generated from postmortems. However, no evidence documents a native or documented sync of these action items into an external issue tracker (Jira, Linear, etc.) or shows tracking-to-completion status/reporting; REST API/webhooks (docs-32, docs-33) could technically enable such integration but this is not demonstrated for follow-up actions specifically. Missing for 10: explicit issue-tracker sync documentation for post-incident follow-up items, evidence of completion-status tracking/reporting on these actions.", "evidenceIds": [ "pagerduty-docs-36", "pagerduty-docs-38", "pagerduty-docs-23", "pagerduty-docs-32", "pagerduty-docs-33" ] }, { "productId": "pagerduty", "storyId": "incident-timeline", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "PagerDuty's Post-Incident Reviews integrate with the incident lifecycle and Slack data (via the Scribe Agent) to automatically build 'chronological incident narratives,' which is evidence of automated timeline capture of alerts/actions/chat, and PIRs function as editable review documents. However, there's no explicit evidence of a dedicated incident timeline UI feature, nor documentation confirming that responders can directly edit or annotate the captured timeline entries after the fact. Missing for 10: explicit incident timeline/log feature docs, confirmation of manual editing/annotation of timeline entries, and independent corroboration of automatic chat-decision capture accuracy.", "evidenceIds": [ "pagerduty-docs-23", "pagerduty-docs-36", "pagerduty-docs-38", "pagerduty-docs-9" ] }, { "productId": "pagerduty", "storyId": "internal-private-status", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "PagerDuty Status Pages are documented as a general capability for real-time operational updates, but no evidence describes private/internal-only pages, audience-specific access control, or separate customer vs employee-only views.", "evidenceIds": [ "pagerduty-docs-10", "pagerduty-docs-17", "pagerduty-docs-24", "pagerduty-docs-29" ] }, { "productId": "pagerduty", "storyId": "mobile-oncall-app", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "PagerDuty's mobile app docs confirm responding to incidents, viewing objects, and managing settings on the go, and community evidence confirms the app is used for acknowledging pages and is core to escalation flow. However, the evidence doesn't explicitly confirm resolve/escalate actions from the app specifically (only 'respond to incidents' generally) and one community comment notes engineers 'only use the app for ack'ing the page,' suggesting other actions may be underused or less central. missing for 10: explicit documentation/hands-on confirmation of resolving and manually escalating incidents (not just acknowledging) from the mobile app, and independent third-party mobile UX reviews.", "evidenceIds": [ "pagerduty-docs-5", "pagerduty-comm-1", "pagerduty-comm-7" ] }, { "productId": "pagerduty", "storyId": "multichannel-paging", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "PagerDuty's docs confirm on-call notification rules and readiness reports tied to responder notification configurations, mobile app for push, and escalation policies with schedules controlling who is notified; a hands-on account also confirms app-then-phone-call escalation in practice. However, no evidence explicitly enumerates SMS, phone call, and email as configurable per-channel notification rule options together, so full breadth of channel choice isn't directly documented in this pack. Missing for 10: explicit documentation naming SMS/email/push/phone as selectable channels with per-channel ordering, and independent confirmation of all four channel types working.", "evidenceIds": [ "pagerduty-docs-6", "pagerduty-docs-12", "pagerduty-docs-5", "pagerduty-docs-30", "pagerduty-comm-1" ] }, { "productId": "pagerduty", "storyId": "observability-integrations", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack documents generic integration mechanisms (Events API v2, REST API, webhooks, service integrations) but never names or links documented setup guides for the specific tools cited in the story—Datadog, Grafana, Prometheus, CloudWatch, or Sentry—so there is no evidence of first-party, per-tool documented integrations for this observability stack.", "evidenceIds": [ "pagerduty-docs-3", "pagerduty-docs-31", "pagerduty-docs-32", "pagerduty-docs-33", "pagerduty-docs-43" ] }, { "productId": "pagerduty", "storyId": "on-call-schedules", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs confirm schedules exist to map coverage needs (pagerduty-docs-12) and escalation policies connect services to users/schedules (pagerduty-docs-7/37), but no evidence explicitly documents layers, rotations, round-robin coverage, or time-zone handling within schedules, and community feedback flags 'clunky calendar overrides' (pagerduty-comm-4) as a friction point. Missing for 10: explicit documentation of schedule layers, rotation types, round-robin distribution, and time-zone-aware scheduling, plus hands-on confirmation these work smoothly.", "evidenceIds": [ "pagerduty-docs-12", "pagerduty-docs-7", "pagerduty-docs-37", "pagerduty-comm-4", "pagerduty-comm-1" ] }, { "productId": "pagerduty", "storyId": "openness-api-parity", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "PagerDuty documents a broad REST API for configuration/user/workflow management, an Events API v2, V3 webhooks, and an official MCP server (with a live hosted endpoint) letting AI clients trigger/manage incidents, confirming strong API-driven parity with UI actions. However, no evidence explicitly confirms 100% feature parity between API and UI, and community comments note at least one case (OIDC/Azure AD SSO config) requiring manual support intervention outside normal UI/API self-service, suggesting some settings aren't fully API/UI accessible. Missing for 10: an explicit parity statement or audit showing all UI functions are API-exposed, and independent verification beyond vendor docs.", "evidenceIds": [ "pagerduty-docs-32", "pagerduty-docs-39", "pagerduty-docs-44", "pagerduty-docs-31", "pagerduty-docs-33", "pagerduty-docs-34", "pagerduty-probe-2", "pagerduty-probe-rt-1", "pagerduty-probe-rt-2", "pagerduty-comm-4" ] }, { "productId": "pagerduty", "storyId": "openness-full-export", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "PagerDuty's REST API exposes configuration and incident data programmatically (docs-32, docs-39, docs-44), which a technically capable user could use to pull out their data, but there is no documented bulk 'export all my data' feature, data-portability tool, or GDPR-style export mechanism in open formats. Missing for 10: an explicit account-wide export/backup feature, documentation of supported open export formats (CSV/JSON dump), and confirmation that all data types (incidents, schedules, users, workflows) can be fully extracted and are portable to another system.", "evidenceIds": [ "pagerduty-docs-32", "pagerduty-docs-39", "pagerduty-docs-44", "pagerduty-docs-20" ] }, { "productId": "pagerduty", "storyId": "openness-open-license", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "PagerDuty is a closed-source SaaS incident management platform; its core product is not open-licensed source code. The MCP server client is a separate small open-source tool, but the product itself is not open-source, so 'reading the product's source' is a category error for this SaaS platform.", "evidenceIds": [] }, { "productId": "pagerduty", "storyId": "openness-self-host", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "PagerDuty is a SaaS incident-management platform with no self-hosted/on-prem deployment option evidenced anywhere; the product is inherently a cloud service, making self-hosting a category error rather than a missing feature.", "evidenceIds": [] }, { "productId": "pagerduty", "storyId": "overrides-swaps", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "PagerDuty's schedule docs describe mapping coverage needs (docs-12) and escalation policies tied to schedules (docs-7), and a community comment confirms that calendar overrides exist in the product (comm-4), but no evidence details a self-service flow for taking an override, swapping a shift, or requesting coverage without admin involvement — the same community comment actually calls overrides 'clunky,' undercutting a smooth self-service experience. Missing for 10: first-party docs on override/shift-swap UI or API, evidence of coverage-request workflow, and confirmation these actions require no admin intervention.", "evidenceIds": [ "pagerduty-docs-12", "pagerduty-docs-7", "pagerduty-comm-4" ] }, { "productId": "pagerduty", "storyId": "postmortem-workflow", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "PagerDuty's Post-Incident Reviews (formerly Jeli PIR) integrate with the incident lifecycle and Slack data, and the PagerDuty Advance Scribe Agent helps 'build chronological incident narratives' and 'coordinate preventative follow-up actions,' which covers timeline-based drafting and some learning workflow. However, there is no explicit documentation of templates, a distinct review/approval step, or a publication mechanism for postmortems. missing for 10: named postmortem templates, explicit review/approval workflow, publication/sharing step, and independent corroboration of the drafting experience.", "evidenceIds": [ "pagerduty-docs-9", "pagerduty-docs-16", "pagerduty-docs-23", "pagerduty-docs-36", "pagerduty-docs-38" ] }, { "productId": "pagerduty", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses data residency, regional data storage, or the ability to choose where account/data is hosted for PagerDuty. Missing for 10: any mention of regional data centers, EU/US data residency options, or configurable data-storage location.", "evidenceIds": [] }, { "productId": "pagerduty", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item addresses AI training data usage, opt-out controls, or any privacy policy statement about model training on customer data; PagerDuty Advance and MCP references describe AI features, not data-training privacy controls.", "evidenceIds": [] }, { "productId": "pagerduty", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers MCP server access, incident workflows, alert grouping, and AI features (PagerDuty Advance), but contains no documentation on data retention policies, deletion controls, or data lifecycle management for AI-native users. Missing for 10: any retention policy documentation, data deletion/export mechanisms, or configurable retention windows relevant to AI-ingested incident data.", "evidenceIds": [] }, { "productId": "pagerduty", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses telemetry/usage-tracking opt-out settings for PagerDuty or its AI features (MCP server, Advance); this is an applicable privacy-posture axis for an AI-native user but no documentation or community evidence confirms such a control exists. Missing for 10: any privacy policy or settings page describing telemetry collection and an opt-out mechanism.", "evidenceIds": [] }, { "productId": "pagerduty", "storyId": "public-status-page", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "PagerDuty ships a dedicated Status Pages product that is explicitly driven from incident state and lets teams communicate real-time operational updates without leaving PagerDuty (pagerduty-docs-10, pagerduty-docs-17, pagerduty-docs-24). However, the evidence never confirms custom-domain hosting or subscriber notification/opt-in mechanics for the status page. Missing for 10: explicit documentation of custom domain support, subscriber sign-up/notification delivery, and any independent/hands-on confirmation of these specific features.", "evidenceIds": [ "pagerduty-docs-10", "pagerduty-docs-17", "pagerduty-docs-24", "pagerduty-docs-29" ] }, { "productId": "pagerduty", "storyId": "reliability-analytics", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "PagerDuty documents On-Call Readiness Reports (responder notification health) and Post-Incident Reviews (systemic pattern uncovering), which touch parts of 'on-call health' and post-incident learning, but no evidence explicitly describes MTTA/MTTR trend dashboards or incident-load analytics over time for leadership reporting. Missing for 10: explicit MTTA/MTTR trend reporting, incident load/volume analytics dashboards, and any leadership-facing 'are we improving' trend view.", "evidenceIds": [ "pagerduty-docs-6", "pagerduty-docs-15", "pagerduty-docs-30", "pagerduty-docs-9", "pagerduty-docs-36" ] }, { "productId": "pagerduty", "storyId": "roles-tasks-response", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "PagerDuty's Incident Workflows and Escalation Policies show automated, rule-based assignment and orchestration of incident response tasks, which speaks to coordinated response, but no evidence explicitly describes named incident roles like 'commander' or 'comms lead' or human-facing task checklists for responders. missing for 10: explicit incident commander/communications lead role definitions, structured task checklists per role, and any hands-on validation that these roles function as intended under pressure.", "evidenceIds": [ "pagerduty-docs-8", "pagerduty-docs-22", "pagerduty-docs-28", "pagerduty-docs-37", "pagerduty-docs-42" ] }, { "productId": "pagerduty", "storyId": "runbook-automation", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "PagerDuty's Incident Workflows explicitly support automated, triggerable 'if-this-then-that' responses attached to incidents (e.g., auto-trigger for P1/P2 incidents), which functionally matches the runbook-automation story, and Advance/agentic features add AI-driven remediation. However, the evidence never uses the term 'runbook' nor cites concrete step examples like auto-creating chat channels or running diagnostics scripts as named actions. Missing for 10: explicit runbook terminology, named step types (channel creation, task assignment, diagnostics execution), and independent/hands-on confirmation of these specific triggered actions.", "evidenceIds": [ "pagerduty-docs-8", "pagerduty-docs-22", "pagerduty-docs-28", "pagerduty-docs-35", "pagerduty-docs-42", "pagerduty-docs-11", "pagerduty-docs-26" ] }, { "productId": "pagerduty", "storyId": "stakeholder-comms", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "PagerDuty Status Pages give stakeholders visual, real-time updates on incident status 'without context switching to other tools,' and Incident Workflows/Post-Incident Reviews can push structured summaries, all without requiring war-room participation. However, the evidence never explicitly confirms a subscription/notification mechanism for external stakeholders or a dedicated 'structured update' template feature. Missing for 10: explicit subscribe/follow functionality for status pages, dedicated stakeholder-update templates, and independent confirmation that non-responders actually use this instead of joining incident channels.", "evidenceIds": [ "pagerduty-docs-10", "pagerduty-docs-17", "pagerduty-docs-24", "pagerduty-docs-29", "pagerduty-docs-8", "pagerduty-docs-9" ] }, { "productId": "pagerduty", "storyId": "workflow-automation-engine", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "PagerDuty Incident Workflows are explicitly documented as a condition-based automation engine using 'if-this-then-that' logic that triggers automated actions (e.g., field changes, notifications, reminders) based on incident event conditions like P1/P2 severity. Escalation policies and event orchestration further automate incident-response toil (assignment, deduplication, escalation) without manual intervention. Missing for 10: independent/hands-on corroboration of Incident Workflows specifically (community evidence discusses escalation and paging generally but not the workflow engine), and detailed examples of field-change/reminder automation beyond marketing-style docs.", "evidenceIds": [ "pagerduty-docs-8", "pagerduty-docs-22", "pagerduty-docs-28", "pagerduty-docs-35", "pagerduty-docs-42", "pagerduty-docs-13", "pagerduty-docs-37" ] }, { "productId": "pagerduty", "storyId": "workflow-tool-integrations", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "PagerDuty documents strong two-way integration infrastructure (REST API, Events API, V3 webhooks, MCP server) and explicitly calls out Slack integration for incident response and Post-Incident Reviews pulling in Slack data, but the evidence pack never names Jira, Teams, Zoom, or GitHub as specific bidirectional integrations — only Slack is concretely evidenced. Missing for 10: explicit documented integrations/state-sync with Jira, Microsoft Teams, Zoom, and GitHub specifically, plus confirmation these flow both ways rather than just PagerDuty→tool notifications.", "evidenceIds": [ "pagerduty-docs-2", "pagerduty-docs-23", "pagerduty-docs-38", "pagerduty-docs-32", "pagerduty-docs-33", "pagerduty-docs-40", "pagerduty-docs-34" ] }, { "productId": "rootly", "storyId": "agent-acks-escalates-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Rootly documents a full incident-management API (create incidents, timeline, retrospectives) and a hosted MCP server that lets AI clients query incidents, on-call schedules, and 'take action' with a large set of write-enabled tools (135 write paths per runtime probe), which strongly implies end-to-end incident handling without a dashboard. However, no evidence explicitly names 'acknowledge', 'escalate', or 'resolve' as documented API/MCP actions, leaving the exact scope of automatable lifecycle actions unconfirmed. Missing for 10: explicit doc/example showing ack/escalate/resolve calls via API or MCP tool names, and independent confirmation of successful end-to-end agent-driven resolution.", "evidenceIds": [ "rootly-docs-11", "rootly-docs-22", "rootly-docs-44", "rootly-probe-rt-2", "rootly-probe-3" ] }, { "productId": "rootly", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Rootly serves a live llms.txt file (HTTP 200) with a structured summary of the product, directly confirming an AI-native agent can be pointed at agent-oriented docs; this complements its documented MCP server for agentic tool use. Missing for 10: no independent/community confirmation that agents successfully consume llms.txt in practice.", "evidenceIds": [ "rootly-probe-1", "rootly-docs-22", "rootly-probe-3" ] }, { "productId": "rootly", "storyId": "agentic-ai-insights", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Rootly ships AI Summaries that auto-generate incident retrospectives and summaries from timeline/Slack/transcript data, plus an in-app 'Ask Rootly AI' chat for querying incident data in natural language and MCP-based similar-incident detection (TF-IDF). These are first-party documented AI-insight features embedded directly in the product. Missing for 10: independent/hands-on verification of output quality and no community corroboration of these specific AI features (community comments focus on pricing/workflow, not AI insights).", "evidenceIds": [ "rootly-docs-20", "rootly-docs-21", "rootly-docs-23", "rootly-docs-30", "rootly-docs-48" ] }, { "productId": "rootly", "storyId": "agentic-autonomous-automation", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Rootly's Workflows are explicitly documented as an 'automation engine' that runs structured actions automatically based on trigger events and run conditions, with 152 available action types spanning Slack, Jira, on-call, AI, and incident ops — enabling background automations like auto-creating channels, notifying stakeholders, filing tickets, and starting retros without manual intervention. missing for 10: independent/hands-on evidence confirming autonomous background execution in practice, and clearer detail on trigger types (e.g., scheduled/webhook triggers) beyond docs summaries.", "evidenceIds": [ "rootly-docs-16", "rootly-docs-17", "rootly-docs-29", "rootly-docs-34", "rootly-docs-41", "rootly-docs-45" ] }, { "productId": "rootly", "storyId": "agentic-builtin-assistant", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Rootly ships a built-in AI assistant ('Rootly AI in Web') embedded in the incident page for natural-language Q&A and an automated AI Summaries feature (/rootly summary) that drafts incident data — evidence of a built-in assistant exists. However, the documented 'take action' capability (query incidents, run actions) is explicitly scoped to external MCP-compatible clients (Cursor, Claude Code, etc.), not the in-product assistant itself, so true task delegation within the product's own UI is unproven. Missing for 10: documented examples of the built-in web AI assistant executing actions/workflows (not just answering questions), and independent/hands-on confirmation of task delegation via the in-app assistant.", "evidenceIds": [ "rootly-docs-20", "rootly-docs-21", "rootly-docs-30", "rootly-docs-47", "rootly-docs-48", "rootly-docs-22" ] }, { "productId": "rootly", "storyId": "agentic-headless", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Rootly's REST API supports creating/managing incidents from CI/CD pipelines and external automation systems (rootly-docs-11, rootly-docs-24), and the API key/token flow enables headless, non-interactive use, including via the MCP server which can run keylessly until gated by an API token (rootly-probe-rt-2). However, there's no dedicated CLI tool, no CI-specific integration guide/examples (e.g., GitHub Actions snippet), and no evidence of a documented headless batch/automation mode beyond API calls. missing for 10: dedicated CLI for CI pipelines, explicit CI/CD integration examples or GitHub Actions templates, and independent confirmation of headless automation use in production CI environments.", "evidenceIds": [ "rootly-docs-11", "rootly-docs-24", "rootly-docs-49", "rootly-probe-rt-2", "rootly-probe-rt-1" ] }, { "productId": "rootly", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "All evidence describes Rootly exposing its own MCP server so external AI tools (Cursor, Claude Code, etc.) can consume Rootly's data/actions — this is the reverse direction of the story, which asks whether a user can plug external MCP servers INTO Rootly so its own AI features (e.g. Rootly AI chat, workflows) can use their tools. No evidence shows Rootly acting as an MCP client or supporting external MCP server configuration.", "evidenceIds": [ "rootly-docs-22", "rootly-docs-44", "rootly-probe-3", "rootly-probe-rt-1", "rootly-probe-rt-2" ] }, { "productId": "rootly", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Rootly ships an official documented MCP server (docs.rootly.com/integrations/mcp-server) exposing incident data/tools to MCP-compatible clients like Cursor, Windsurf, Claude Code, Gemini CLI, and this is corroborated by a live runtime probe showing the hosted remote MCP endpoint responding with proper OAuth flow and a PyPI-installable server package starting with 124 tools. Missing for 10: independent third-party hands-on review of the MCP server's tool quality beyond vendor docs/probes.", "evidenceIds": [ "rootly-docs-22", "rootly-docs-44", "rootly-probe-3", "rootly-probe-rt-1", "rootly-probe-rt-2" ] }, { "productId": "rootly", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Rootly ships an in-app AI chat ('Ask Rootly AI') for natural-language queries on incidents, a Slack slash command (/rootly summary) and, notably, an official MCP server that lets users query incidents, check on-call schedules, and take actions using natural language from AI-native environments like Cursor, Claude Code, and Gemini CLI. Runtime probes confirm the MCP server is live and functioning (401/auth-gated as expected), corroborating the documented capability.\n\nmissing for 10: independent/hands-on evidence of natural-language command quality/accuracy, broader third-party review of the AI chat or MCP NL experience beyond first-party docs.", "evidenceIds": [ "rootly-docs-21", "rootly-docs-22", "rootly-docs-30", "rootly-docs-44", "rootly-probe-3", "rootly-probe-rt-1", "rootly-probe-rt-2" ] }, { "productId": "rootly", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows an API, MCP server, and Slack slash-commands, but no official CLI tool is documented anywhere in the pack. missing for 10: any mention of a dedicated Rootly CLI, its install method, or command reference.", "evidenceIds": [] }, { "productId": "rootly", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Rootly documents a public REST API for creating/managing incidents programmatically (rootly-docs-11), an API key generation/auth flow with scoped roles (rootly-docs-24, rootly-docs-49), and a public status-page JSON API (rootly-docs-19), giving AI-native users multiple documented ways to drive the product programmatically; this is reinforced by a documented MCP server exposing incident data/actions as callable tools for AI agents (rootly-docs-44, rootly-probe-3, rootly-probe-rt-1/2). Missing for 10: a discoverable OpenAPI/swagger spec (probe found only 404s at standard OpenAPI paths) and independent third-party corroboration of API robustness beyond vendor docs.", "evidenceIds": [ "rootly-docs-11", "rootly-docs-24", "rootly-docs-49", "rootly-docs-19", "rootly-docs-44", "rootly-probe-3", "rootly-probe-rt-1", "rootly-probe-rt-2", "rootly-probe-2" ] }, { "productId": "rootly", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Rootly documents generating API keys (rootly-docs-24) that are assigned a role ('On-Call and Incident Response') at creation (rootly-docs-49), and the official MCP server requires a distinct ROOTLY_API_TOKEN credential for agent access (rootly-probe-rt-2). This shows some notion of scoped credentials for machine/agent use, but there's no evidence of fine-grained least-privilege controls (e.g., read-only vs write, per-resource/team scoping) — indeed the MCP server logs show 135 write-capable tools enabled by default, suggesting broad rather than minimal privilege. Missing for 10: documented ability to restrict API keys to specific resources/teams, read-only or write-limited key types, and independent confirmation that scoping actually limits agent capability in practice.", "evidenceIds": [ "rootly-docs-24", "rootly-docs-49", "rootly-probe-rt-2", "rootly-docs-44" ] }, { "productId": "rootly", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence shows Rootly offers a REST API with API-key auth (rootly-docs-11, rootly-docs-24, rootly-docs-49) and an MCP server for agentic tool use, but there is no mention anywhere of official client SDKs (e.g., Python/JS/Go libraries) for developers to build against — the OpenAPI spec probe even returned 404s (rootly-probe-2), suggesting no formal SDK generation pipeline is publicly documented.", "evidenceIds": [ "rootly-docs-11", "rootly-docs-24", "rootly-docs-49", "rootly-probe-2" ] }, { "productId": "rootly", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack never mentions webhook subscriptions for events (e.g., incident created/updated) that an AI agent could subscribe to; it only covers inbound alert ingestion, workflows, and the MCP server for pull-based querying, not outbound webhook event subscriptions. Missing for 10: any documentation of a webhook subscription/registration mechanism, event payload schema, or signing/verification for outbound webhooks.", "evidenceIds": [] }, { "productId": "rootly", "storyId": "ai-drafted-postmortems", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Rootly's retrospectives feature auto-drafts Summary, Impact, Root Cause, Mitigation, Resolution, and Curated Timeline sections directly from incident data, Slack channel, and bridge-call transcripts, with real-time collaborative editing for human review — directly matching the story's ask for AI-drafted postmortems ready for review. missing for 10: explicit mention of 'follow-ups/action items' as a drafted postmortem section (action items are referenced only in AI Summaries, not explicitly tied to the retrospective draft), and independent/hands-on corroboration beyond vendor docs.", "evidenceIds": [ "rootly-docs-13", "rootly-docs-27", "rootly-docs-14", "rootly-docs-15", "rootly-docs-28", "rootly-docs-42", "rootly-docs-12", "rootly-docs-20" ] }, { "productId": "rootly", "storyId": "ai-incident-summaries", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Rootly's AI Summaries feature explicitly drafts live summaries from incident metadata, alerts, timeline, Slack messages and meeting transcripts, triggerable via a slash command mid-incident and delivered privately to responders (rootly-docs-20, -30, -47, -48). Retrospective drafting further auto-generates Summary/Impact/Root Cause/Timeline sections from Slack channel and bridge-call transcripts (rootly-docs-13, -27), covering the 'scribed call notes' and 'drafted updates' parts of the story. Missing for 10: independent/hands-on verification of summary/transcript quality, and no dedicated real-time call-transcription product page distinct from retrospective drafting.", "evidenceIds": [ "rootly-docs-20", "rootly-docs-30", "rootly-docs-47", "rootly-docs-48", "rootly-docs-13", "rootly-docs-27" ] }, { "productId": "rootly", "storyId": "ai-root-cause-investigation", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Rootly's AI features (AI Summaries, Ask Rootly AI, and the MCP `find_related_incidents` tool using TF-IDF similarity) can surface similar past incidents and draft a Root Cause section from incident data, timeline, and transcripts, giving partial coverage of the 'similar past incidents' and root-cause narrative pieces. However, these are user-triggered (slash command, chat query, or explicit MCP tool call) rather than an autonomous investigator that proactively correlates code/infra changes and live telemetry before a human asks. Missing for 10: evidence of automatic/unprompted change-correlation, telemetry ingestion and analysis, and a proactive (not command-triggered) investigation workflow.", "evidenceIds": [ "rootly-docs-13", "rootly-docs-20", "rootly-docs-23", "rootly-docs-21", "rootly-docs-30" ] }, { "productId": "rootly", "storyId": "alert-dedup-grouping", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Rootly's Alert Deduplication feature explicitly collapses repeat events from the same monitor into a single open alert using a stable identifier so responders aren't paged twice, and Alert Routing provides a unified layer to direct alerts to the right team/escalation policy, combining to reduce duplicate paging. Escalation Policies then ensure one incident routes to the correct on-call human via multi-step targets. Missing for 10: independent/hands-on validation of dedup accuracy at scale and no community evidence specifically confirming the anti-noise outcome in production.", "evidenceIds": [ "rootly-docs-9", "rootly-docs-36", "rootly-docs-10", "rootly-docs-37", "rootly-docs-7", "rootly-docs-39" ] }, { "productId": "rootly", "storyId": "alert-ingest-routing", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Rootly explicitly documents Alert Routes to route incoming monitoring alerts to teams/services/escalation policies 'based on conditions you define' (rootly-docs-10), backed by a unified routing layer across alert sources (rootly-docs-37), deduplication to avoid double-paging (rootly-docs-9, rootly-docs-36), and escalation policies with multi-step targets/time delays/round robin (rootly-docs-7, rootly-docs-39). Migration docs (PagerDuty/Opsgenie) confirm ingestion from common monitoring/alerting tool ecosystems. Missing for 10: independent hands-on validation of the routing-conditions engine itself (community evidence covers general sentiment, not this specific feature).", "evidenceIds": [ "rootly-docs-10", "rootly-docs-37", "rootly-docs-9", "rootly-docs-36", "rootly-docs-7", "rootly-docs-39", "rootly-docs-3", "rootly-docs-4" ] }, { "productId": "rootly", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Rootly documents an API reference page and API-key generation (rootly-docs-24, rootly-docs-49) but there's no evidence of an interactive, runnable-example explorer — probes for standard OpenAPI/Swagger specs at docs.rootly.com all returned 404 (rootly-probe-2), and no docs mention a 'try it out' console or embedded runnable code samples.", "evidenceIds": [ "rootly-docs-24", "rootly-docs-49", "rootly-probe-2" ] }, { "productId": "rootly", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Rootly has a documented REST API and API-key system (rootly-docs-24, rootly-docs-49), but an explicit probe for a machine-readable OpenAPI/Swagger spec at all standard paths (openapi.json, swagger.json, etc.) returned 404 on every candidate (rootly-probe-2), and no other evidence pack item shows a downloadable spec file.", "evidenceIds": [ "rootly-probe-2", "rootly-docs-24", "rootly-docs-49" ] }, { "productId": "rootly", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence anywhere in the pack of a sandbox/staging environment, test mode, or data isolation feature for Rootly or its MCP server; in fact the MCP server evidence shows write-enabled tools (135 write paths) operating against live data with no mention of a non-production mode.", "evidenceIds": [] }, { "productId": "rootly", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "There is evidence of an API with key generation (rootly-docs-24, rootly-docs-49) and an incident-creation API (rootly-docs-11), but no documentation of API versioning, version headers, or a deprecation policy anywhere in the pack; probes show no OpenAPI spec discoverable (rootly-probe-2). Missing for 10: any mention of API version numbers, backward-compatibility guarantees, or a published deprecation/sunset policy.", "evidenceIds": [ "rootly-docs-24", "rootly-docs-49", "rootly-docs-11", "rootly-probe-2" ] }, { "productId": "rootly", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Rootly's docs describe workflows, an API, and an MCP server for automating individual incident/alert actions, but nothing describes bulk operations (e.g., batch-updating or resolving many incidents/alerts/schedules in one call). Workflow triggers and API creation are per-incident/event, not a documented multi-item batch operation feature.", "evidenceIds": [ "rootly-docs-16", "rootly-docs-17", "rootly-docs-11", "rootly-docs-44" ] }, { "productId": "rootly", "storyId": "automation-rules-engine", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Rootly's Workflows engine is explicitly documented as its automation engine, running structured actions automatically based on trigger events, run conditions, and actions, with 152 distinct workflow action types across Slack, Jira, on-call, AI, and incident operations. This directly satisfies rule-based automatic triggering on events, reinforced by alert routing rules and escalation policies that also apply condition-based automated behavior. Missing for 10: independent/hands-on verification of workflow rule configuration in practice and more detail on conditional logic complexity beyond docs summaries.", "evidenceIds": [ "rootly-docs-16", "rootly-docs-17", "rootly-docs-29", "rootly-docs-41", "rootly-docs-10", "rootly-docs-34" ] }, { "productId": "rootly", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Rootly's Workflows are described as an automation engine triggered by incident-related events and conditions (docs-16, docs-29, docs-41), but there is no evidence of a scheduling/cron-style trigger or recurring-job capability for workflows — 'schedules' in the docs refer only to on-call rotation schedules (docs-6, docs-26), not automation scheduling. Missing for 10: any documented scheduled/cron trigger type in the workflow trigger-events list, recurring workflow execution examples, or API/UI support for time-based recurring automation.", "evidenceIds": [ "rootly-docs-16", "rootly-docs-29", "rootly-docs-41", "rootly-docs-6", "rootly-docs-26" ] }, { "productId": "rootly", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Rootly's workflow automation engine is documented in detail (triggers, conditions, 152 action types) but nothing in the evidence pack describes version history, diff/review workflows, or rollback capability for automations. Missing for 10: any mention of workflow versioning, change review/approval process, or rollback/revert mechanism for automations.", "evidenceIds": [ "rootly-docs-16", "rootly-docs-17", "rootly-docs-29", "rootly-docs-41" ] }, { "productId": "rootly", "storyId": "calendar-sync-oncall", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers Rootly's on-call schedules, rotations, escalation policies, and even a mobile app, but nowhere mentions a calendar feed (iCal/ICS export) for syncing on-call shifts to a personal calendar. This is a standard capability for on-call scheduling tools and its absence here means no evidence supports it.", "evidenceIds": [ "rootly-docs-6", "rootly-docs-26", "rootly-docs-43" ] }, { "productId": "rootly", "storyId": "declare-incident-chat", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs explicitly describe declaring/running/resolving incidents from Slack, Google Chat, or Teams, with automated channel creation, escalation-policy/role assignment, and stakeholder updates (rootly-docs-1, rootly-docs-34, rootly-docs-45, rootly-docs-7, rootly-docs-39). No community evidence concretely contradicts the chat-declare workflow itself (general skepticism in rootly-comm-7 is about PD parity, not a functional failure). Missing for 10: independent hands-on confirmation of the actual Slack/Teams declare flow and any real-world walkthrough beyond vendor docs.", "evidenceIds": [ "rootly-docs-1", "rootly-docs-34", "rootly-docs-45", "rootly-docs-7", "rootly-docs-39", "rootly-docs-6" ] }, { "productId": "rootly", "storyId": "escalation-policies", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Rootly's docs explicitly describe escalation policies with multi-step targets, time delays, and round robin to ensure alerts get acknowledged, directly matching the story's requirements for delays, fallback responders, and repeat rounds until acknowledgment (rootly-docs-39, rootly-docs-7). Missing for 10: independent hands-on validation of escalation behavior in practice and more detail on repeat-round/fallback mechanics beyond the doc summary.", "evidenceIds": [ "rootly-docs-39", "rootly-docs-7", "rootly-docs-5" ] }, { "productId": "rootly", "storyId": "followup-tracking", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Rootly's workflow automation supports Jira actions and can 'file tickets' as part of incident response automation, and AI summaries reference 'action items' captured during incidents, implying some link to follow-up tracking and ticket sync. However, there is no explicit documentation of a dedicated action-item/follow-up tracking system with completion status or two-way sync guarantees to an issue tracker. Missing for 10: a dedicated action-items/follow-up-tracking feature description, explicit two-way sync/status update with Jira or other trackers, and any evidence (docs or community) that follow-ups are tracked to completion rather than just ticket creation.", "evidenceIds": [ "rootly-docs-17", "rootly-docs-34", "rootly-docs-20" ] }, { "productId": "rootly", "storyId": "incident-timeline", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs explicitly describe the incident timeline as the 'authoritative record of what happened during an incident,' auto-populated from alerts, actions, Slack, and bridge-call data, and retrospectives support real-time collaborative editing/annotation via a rich text editor with live incident data. This covers automatic capture (alerts/actions/chat) plus post-hoc editing/annotation as described in the story. Missing for 10: independent/hands-on verification of timeline auto-capture accuracy and explicit UI evidence of inline timeline editing (only retrospective editing is directly documented).", "evidenceIds": [ "rootly-docs-12", "rootly-docs-40", "rootly-docs-13", "rootly-docs-27", "rootly-docs-28", "rootly-docs-15", "rootly-docs-42" ] }, { "productId": "rootly", "storyId": "internal-private-status", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs explicitly state Rootly offers both public and private status pages, and describe status pages as communicating health info to both internal stakeholders and external customers, directly matching the story's core ask. However, there's no detail on granular access-control mechanisms (e.g., password protection, audience segmentation between customer-specific vs employee-only groups, SSO-gated access), so the depth of access control is unclear. missing for 10: documentation of specific access-control mechanisms (password/SSO gating, audience segmentation rules), independent/hands-on confirmation of private page functionality.", "evidenceIds": [ "rootly-docs-46", "rootly-docs-38", "rootly-docs-18", "rootly-docs-19" ] }, { "productId": "rootly", "storyId": "mobile-oncall-app", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Docs confirm a native iOS/Android mobile app exists for on-call use (rootly-docs-43), but there is no detail confirming that acknowledge, escalate, and resolve actions are all available from the app, nor any hands-on/community validation of the mobile experience. missing for 10: documented list of mobile app actions (ack/escalate/resolve), UI/UX detail, independent user reviews of the app's reliability at odd hours.", "evidenceIds": [ "rootly-docs-43" ] }, { "productId": "rootly", "storyId": "multichannel-paging", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Rootly docs confirm push-style alerts via its mobile app and phone-call paging via Live Call Routing, plus escalation policies with multi-step targets and delays, but there is no explicit documentation of SMS or email as notification channels, nor of user-configurable per-channel notification rules (e.g., 'push first, then SMS, then call'). missing for 10: SMS channel support, email channel support, explicit per-channel notification-rule configuration UI/API.", "evidenceIds": [ "rootly-docs-5", "rootly-docs-8", "rootly-docs-35", "rootly-docs-39", "rootly-docs-43" ] }, { "productId": "rootly", "storyId": "observability-integrations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack describes generic alert routing, deduplication, and a 'unified routing layer that works across all alert sources' but never names or documents specific setup for Datadog, Grafana, Prometheus, CloudWatch, or Sentry integrations. Without concrete first-party docs pointing to these specific observability tools, the story's requirement for named, documented integrations is unmet.", "evidenceIds": [ "rootly-docs-9", "rootly-docs-10", "rootly-docs-37" ] }, { "productId": "rootly", "storyId": "on-call-schedules", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Rootly docs explicitly describe schedules with rotations and layers, plus multi-step escalation policies with round-robin support, directly matching the story's requirements, and migration docs confirm bringing existing schedules from PagerDuty/Opsgenie without rebuild. Time zone support is implied by 'previews to keep coverage clear' but not explicitly named, and there's no independent/hands-on validation beyond vendor docs. Missing for 10: explicit documentation of time-zone handling in schedule configuration, and independent/community verification of rotation/layer behavior in practice.", "evidenceIds": [ "rootly-docs-6", "rootly-docs-26", "rootly-docs-39", "rootly-docs-2", "rootly-docs-32" ] }, { "productId": "rootly", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Rootly documents a REST API for creating/managing incidents, API keys with scoped roles, a status-page public API, and an MCP server exposing 124 tools including 135 write paths for incidents, on-call, and alerts — indicating broad but not explicitly confirmed full parity with the UI. No official OpenAPI spec was discoverable (404s across candidate paths), and there's no explicit vendor claim that 100% of UI functionality (e.g., retrospectives editing, workflow builder, status page configuration) is API-accessible. Missing for 10: a published complete OpenAPI/API reference, explicit UI-API parity statement, and independent confirmation of full functional coverage beyond incidents/on-call/alerts.", "evidenceIds": [ "rootly-docs-11", "rootly-docs-19", "rootly-docs-24", "rootly-docs-49", "rootly-docs-44", "rootly-probe-2", "rootly-probe-rt-2" ] }, { "productId": "rootly", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Rootly's docs describe an API for creating/reading incident data and migration guides for importing from Opsgenie/PagerDuty, but there is no documented bulk data-export feature, open-format export of schedules/incidents/retrospectives, or account-portability/'leave the platform' capability. The API (rootly-docs-11, rootly-docs-24) hints at programmatic access but is not evidenced as a full data-export mechanism.", "evidenceIds": [ "rootly-docs-11", "rootly-docs-24", "rootly-docs-3", "rootly-docs-4", "rootly-docs-19" ] }, { "productId": "rootly", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Rootly is a closed, commercial SaaS incident-management platform; no evidence indicates its core product source is published under an open license (only an MCP server client integration exists on GitHub, not the product itself). No open-source license, repo, or source availability for the core platform is mentioned anywhere in the evidence pack.", "evidenceIds": [] }, { "productId": "rootly", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Rootly is exclusively offered as a hosted SaaS incident management platform; no evidence anywhere in the pack mentions a self-hosted/on-premise deployment option, Docker images, or open-source core for self-hosting. The MCP server is a client-side integration, not evidence of self-hosting the core product itself.", "evidenceIds": [] }, { "productId": "rootly", "storyId": "overrides-swaps", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm schedules support overrides directly (rotations, layers, overrides, restrictions, previews) without needing a rebuild, which addresses the 'take an override' part of the story. However, there is no evidence of self-service shift-swap or coverage-request workflows for individual on-call engineers. missing for 10: explicit shift-swap capability, coverage-request workflow, and confirmation that engineers (not admins) can perform these actions directly.", "evidenceIds": [ "rootly-docs-26", "rootly-docs-6", "rootly-docs-2" ] }, { "productId": "rootly", "storyId": "postmortem-workflow", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Rootly's retrospectives docs show a real workflow: multiple selectable process templates routed by severity/type/team (rootly-docs-14), auto-drafted sections (Summary, Impact, Root Cause, Mitigation, Resolution, Curated Timeline) generated from the authoritative incident timeline/Slack/transcripts (rootly-docs-12, rootly-docs-13, rootly-docs-27, rootly-docs-40), collaborative real-time editing for review by multiple authors (rootly-docs-15, rootly-docs-28, rootly-docs-42), and publication via status pages/communication channels (rootly-docs-18, rootly-docs-38, rootly-docs-45). This covers templates → draft-from-timeline → review → publish end-to-end. Missing for 10: independent/hands-on confirmation of the postmortem workflow specifically (community evidence is about on-call/pricing, not retrospectives) and explicit detail on an approval/sign-off publication step.", "evidenceIds": [ "rootly-docs-12", "rootly-docs-13", "rootly-docs-14", "rootly-docs-15", "rootly-docs-27", "rootly-docs-28", "rootly-docs-42", "rootly-docs-38", "rootly-docs-45" ] }, { "productId": "rootly", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack addresses data residency, regional storage options, or compliance/data-location controls for Rootly. The docs cover incidents, on-call, workflows, retrospectives, and MCP integrations but never mention region selection or data residency guarantees. missing for 10: any mention of data residency/region selection, compliance certifications (e.g., SOC2/GDPR data localization), or configurable storage location.", "evidenceIds": [] }, { "productId": "rootly", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses AI training data usage, opt-out controls, or any privacy policy statement about model training; Rootly's docs focus on incident management, on-call, and AI features (summaries, MCP) but never mention data-training exclusion or opt-out mechanisms. missing for 10: any privacy policy or documentation stating customer data is excluded from AI model training, opt-out settings, or data processing agreements addressing this concern.", "evidenceIds": [] }, { "productId": "rootly", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack items discuss data retention policies, data deletion controls, or export/purge mechanisms for AI-native users; the docs cover incident management, on-call, workflows, MCP server, and AI summaries but nothing about controlling retention or deletion of data. Missing for 10: any documentation of retention settings, deletion/purge APIs, data export controls, or privacy configuration options.", "evidenceIds": [] }, { "productId": "rootly", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses telemetry/usage-tracking opt-out settings, privacy controls, or data collection disclosures for Rootly; the docs focus entirely on incident management, on-call, and AI features. Missing for 10: any privacy policy or settings documentation on telemetry collection, any opt-out mechanism, or usage-tracking disclosure.", "evidenceIds": [] }, { "productId": "rootly", "storyId": "public-status-page", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm hosted public status pages with custom domains (public JSON API served on the custom domain) and quick setup, distinguishing public vs private status pages for external customers. However, there is no explicit documentation of subscriber notification features (email/SMS/webhook subscriptions) or an explicit description of how incident state automatically drives status page updates beyond general platform integration. missing for 10: subscriber notification mechanics, explicit incident-to-status-page state sync documentation, independent/hands-on confirmation of custom domain setup.", "evidenceIds": [ "rootly-docs-18", "rootly-docs-19", "rootly-docs-38", "rootly-docs-46" ] }, { "productId": "rootly", "storyId": "reliability-analytics", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers incident response, on-call scheduling, retrospectives, workflows, and AI summaries, but contains no mention of MTTA/MTTR trend dashboards, incident-load analytics, or on-call health reporting features that an engineering leader could use to track improvement over time. This is a fair ask for an incident management platform, but no documentation, screenshots, or community evidence support it.", "evidenceIds": [] }, { "productId": "rootly", "storyId": "roles-tasks-response", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers incident timelines, workflows, escalation policies, and automated actions like channel creation and stakeholder updates, but nowhere explicitly describes assigning defined incident roles (e.g., commander, comms lead) or maintaining task checklists during an incident. missing for 10: explicit role assignment (commander/comms lead) documentation, task checklist feature, evidence of role-based coordination during response.", "evidenceIds": [ "rootly-docs-34", "rootly-docs-14", "rootly-docs-16" ] }, { "productId": "rootly", "storyId": "runbook-automation", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Rootly's 'Workflows' engine is described as an automation system that runs structured actions automatically based on trigger events, run conditions, and actions — including creating incident channels, filing tickets, and updating stakeholders (rootly-docs-16, rootly-docs-17, rootly-docs-29, rootly-docs-34, rootly-docs-45). This closely maps to runbook-style automation attached to incidents, but the evidence never uses the term 'runbook' as a distinct attachable object, and doesn't explicitly show task assignment or diagnostic-running steps as workflow actions. Missing for 10: explicit runbook/incident attachment terminology, evidence of task-assignment actions, evidence of diagnostic-running actions, and independent/hands-on corroboration of workflow automation actually firing.", "evidenceIds": [ "rootly-docs-16", "rootly-docs-17", "rootly-docs-29", "rootly-docs-34", "rootly-docs-45", "rootly-docs-41" ] }, { "productId": "rootly", "storyId": "stakeholder-comms", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Rootly's private status pages are explicitly designed to give internal stakeholders structured incident health updates without needing to join the incident channel/war room, backed by automation that 'updates stakeholders' as part of workflows, plus AI summaries that can be pushed to Slack/Teams for those following along. missing for 10: explicit documentation of a stakeholder subscription/notification mechanism (e.g., email/RSS subscribe button) and independent confirmation that non-responders actually use status pages this way.", "evidenceIds": [ "rootly-docs-38", "rootly-docs-46", "rootly-docs-18", "rootly-docs-19", "rootly-docs-34", "rootly-docs-20", "rootly-docs-47", "rootly-docs-48" ] }, { "productId": "rootly", "storyId": "workflow-automation-engine", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Rootly's Workflows engine is explicitly documented as a condition-based automation system with trigger events, run conditions, and actions (rootly-docs-16/29/41), used to update fields, send reminders, create channels, update stakeholders, file tickets, and start retros automatically on incident events (rootly-docs-34, rootly-docs-45). A 152-action reference (rootly-docs-17) shows breadth across Slack, Jira, on-call, and incident-ops actions, directly matching the 'updates, reminders, field changes' toil-automation story. Missing for 10: independent hands-on validation of the workflow engine specifically (community evidence only discusses general product value, not workflow conditions in depth) and no detailed walkthrough of condition-branching logic beyond docs.", "evidenceIds": [ "rootly-docs-16", "rootly-docs-17", "rootly-docs-29", "rootly-docs-34", "rootly-docs-41", "rootly-docs-45" ] }, { "productId": "rootly", "storyId": "workflow-tool-integrations", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs show strong two-way integration with Slack, Microsoft Teams, and Google Chat (declare/resolve incidents from chat, create incident channels, /rootly summary commands) and a workflow action reference explicitly grouped by categories including Jira, alongside bidirectional automation (tickets filed, channels created, stakeholders updated). However, GitHub is never named as an integration, and Zoom is only implied via generic 'bridge-call transcripts'/'meeting transcripts' rather than a documented explicit Zoom integration. Missing for 10: explicit GitHub integration/state-sync evidence, explicit named Zoom integration (beyond generic call/meeting transcript mentions), and independent/hands-on confirmation that Jira ticket state actually syncs bidirectionally.", "evidenceIds": [ "rootly-docs-1", "rootly-docs-17", "rootly-docs-34", "rootly-docs-45", "rootly-docs-13", "rootly-docs-30" ] } ]