[ { "productId": "flagsmith", "storyId": "ab-experiments", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Flagsmith's Experimentation feature explicitly supports serving multivariate (A/B/n) flag variations, collecting events into a managed data warehouse, and reading results via a built-in Bayesian statistics engine, directly matching the PM story of running A/B/MVT tests and seeing which variation wins on metrics. Missing for 10: independent/hands-on evidence of the experimentation UI in action (e.g., screenshots or third-party reports of actual win/loss determination), and detail on which metrics/goals can be tracked or how statistical significance is surfaced to a non-technical PM.", "evidenceIds": [ "flagsmith-docs-7", "flagsmith-docs-38", "flagsmith-docs-4", "flagsmith-docs-25", "flagsmith-docs-35" ] }, { "productId": "flagsmith", "storyId": "agent-experiment-analysis", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Flagsmith documents an Experimentation feature with a Bayesian statistics engine for reading A/B test results (flagsmith-docs-7/25/35/38), and separately an official MCP server plus Admin API giving agents programmatic access to flags/segments/environments (flagsmith-docs-17/29, probe-3, probe-rt-1). However, no evidence confirms the MCP/Admin API surface exposes the experiment analytics/results data (vs. just flag/segment CRUD), so an agent's ability to programmatically pull experiment outcomes (not just configs) for ship/rollback recommendations is unconfirmed. Missing for 10: documented API/MCP endpoint for reading experiment statistics results, and any example of an agent consuming that data to produce a recommendation.", "evidenceIds": [ "flagsmith-docs-7", "flagsmith-docs-17", "flagsmith-docs-29", "flagsmith-probe-3", "flagsmith-probe-rt-1", "flagsmith-docs-38", "flagsmith-docs-16" ] }, { "productId": "flagsmith", "storyId": "agent-toggles-flag-safely", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Flagsmith documents an official MCP server, CLI, and Admin API that let an AI agent create/read/toggle flags (flagsmith-docs-17/29, probe-3, probe-rt-1, docs-18/40), and separately documents a four-eyes Change Request workflow plus RBAC that can restrict who may modify production flags, similar to PR review (flagsmith-docs-10, flagsmith-docs-26). This gives both halves of the story: programmatic write access and an approval-gate mechanism. missing for 10: explicit documentation/hands-on proof that API/MCP/CLI-originated writes are automatically routed through Change Requests (rather than change requests being a dashboard-only workflow), and no independent verification that RBAC blocks an agent's direct API write to production.", "evidenceIds": [ "flagsmith-docs-17", "flagsmith-probe-3", "flagsmith-probe-rt-1", "flagsmith-docs-18", "flagsmith-docs-40", "flagsmith-docs-10", "flagsmith-docs-26", "flagsmith-docs-8" ] }, { "productId": "flagsmith", "storyId": "agentic-agent-docs", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Flagsmith has no llms.txt file (confirmed 404 via direct probe), but it does publish agent-oriented documentation — a dedicated docs page describing the MCP server 'for AI assistants and agents' to programmatically access the Admin API, plus a documented CLI. This is genuine agent-facing documentation, just not in the llms.txt convention. Missing for 10: an actual llms.txt or equivalent machine-readable agent doc index, and independent confirmation an agent can successfully consume the MCP docs page as a pointer target.", "evidenceIds": [ "flagsmith-probe-1", "flagsmith-docs-17", "flagsmith-probe-3", "flagsmith-docs-18", "flagsmith-probe-4" ] }, { "productId": "flagsmith", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Flagsmith provides an MCP server for external AI agents to call its Admin API (docs-17/29) and a 'release pipeline' described as an 'intelligent release assistant' (docs-37), but this is rule-based automation and API access, not evidence of the product itself generating AI insights or suggestions from data (e.g., anomaly detection, usage analysis, recommended flags/segments). No documented AI/ML-driven insight or suggestion feature is present in the evidence pack.", "evidenceIds": [ "flagsmith-docs-37", "flagsmith-docs-17", "flagsmith-docs-29" ] }, { "productId": "flagsmith", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Flagsmith documents concrete autonomous background automation: Scheduled Flags apply changes at a future time without manual intervention, and Release Pipelines define stages with triggers/actions that automatically progress rollouts to environments/audiences, generating change requests on promotion. The MCP server also lets AI agents programmatically drive these flows, and a runtime probe confirms the hosted MCP endpoint is live (though key-gated, un-exercised end-to-end). Missing for 10: independent/hands-on verification that Release Pipelines actually execute staged autonomous rollouts in practice, and a verified authenticated MCP-driven automation run rather than just a 401 probe.", "evidenceIds": [ "flagsmith-docs-5", "flagsmith-docs-24", "flagsmith-docs-6", "flagsmith-docs-34", "flagsmith-docs-37", "flagsmith-docs-17", "flagsmith-docs-29", "flagsmith-probe-3", "flagsmith-probe-rt-1" ] }, { "productId": "flagsmith", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Flagsmith exposes an MCP server so external AI agents can call its Admin API, and calls its automated release pipeline an 'intelligent release assistant,' but neither is a built-in AI assistant inside the Flagsmith product that a user can converse with or delegate tasks to — the MCP server is a server-side integration point for external agents, not a first-party in-app assistant.", "evidenceIds": [ "flagsmith-docs-17", "flagsmith-docs-29", "flagsmith-docs-37", "flagsmith-probe-3", "flagsmith-probe-rt-1" ] }, { "productId": "flagsmith", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Flagsmith ships an official CLI and Admin API that can create/evaluate flags entirely headlessly, and a probe confirms a full keyless self-hosted Docker create-and-evaluate loop with no UI involved, plus server-side SDKs for automated environments. missing for 10: explicit CI/CD pipeline examples (e.g., GitHub Actions) and independent third-party CI usage reports.", "evidenceIds": [ "flagsmith-docs-18", "flagsmith-docs-30", "flagsmith-docs-40", "flagsmith-docs-13", "flagsmith-docs-27", "flagsmith-probe-4", "flagsmith-probe-rt-2" ] }, { "productId": "flagsmith", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Flagsmith is a feature-flag/SaaS platform, not an agent or AI host — it has no runtime that would consume external MCP servers' tools. The evidence instead shows Flagsmith ships an official MCP *server* (docs-17, docs-29, probe-3, probe-rt-1) so other AI agents can call Flagsmith's own API, which is the opposite role and belongs to a separate 'serves an MCP server' story, not this client-side axis.", "evidenceIds": [] }, { "productId": "flagsmith", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Flagsmith documents an official MCP Server that gives AI assistants/agents programmatic access to the Admin API via the Model Context Protocol, and a runtime probe confirms a live hosted MCP endpoint (mcp.flagsmith.com) responding to JSON-RPC initialize calls (key-gated). This is first-party, dedicated, and independently verified as functioning. Missing for 10: no independent/community hands-on report of actually using an agent through the MCP server in a real workflow.", "evidenceIds": [ "flagsmith-docs-17", "flagsmith-docs-29", "flagsmith-probe-3", "flagsmith-probe-rt-1" ] }, { "productId": "flagsmith", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Flagsmith ships an official hosted MCP server (mcp.flagsmith.com) exposing the Admin API to AI assistants/agents, confirmed live via a runtime probe (401 requiring auth, not a failure), which enables natural-language operation through an MCP-compatible AI client, plus a CLI for terminal-based flag management. However, this is agent-mediated (via MCP client) rather than native NLP in the product itself, and there's no hands-on evidence of a full natural-language command actually being executed end-to-end through an AI assistant. Missing for 10: a demonstrated end-to-end natural-language interaction transcript, and independent (non-vendor) corroboration of the MCP server working with a real AI agent.", "evidenceIds": [ "flagsmith-docs-17", "flagsmith-docs-29", "flagsmith-probe-3", "flagsmith-probe-rt-1", "flagsmith-docs-18", "flagsmith-probe-4" ] }, { "productId": "flagsmith", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Flagsmith documents an official CLI that manages flags, segments, features, projects and environments and can evaluate flags like an SDK, confirmed live at the documented URL. missing for 10: independent/hands-on community verification of the CLI's functionality and details on installation/versioning maturity.", "evidenceIds": [ "flagsmith-docs-18", "flagsmith-probe-4" ] }, { "productId": "flagsmith", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Flagsmith exposes a well-documented Admin API ('Anything you can do in the Flagsmith dashboard, you can also do via the Admin API'), a public CLI, and an official MCP server explicitly built for AI assistants/agents, with a runtime probe confirming the hosted MCP endpoint is live and key-gated and a self-hosted API create/evaluate roundtrip working keylessly. This directly satisfies driving the product via documented public API for an AI-native user. Missing for 10: a discoverable OpenAPI/swagger spec or llms.txt (both probed and returned 404), and independent/third-party corroboration of the MCP server's real-world agent usage beyond first-party docs.", "evidenceIds": [ "flagsmith-docs-17", "flagsmith-docs-29", "flagsmith-docs-30", "flagsmith-docs-40", "flagsmith-docs-18", "flagsmith-probe-3", "flagsmith-probe-4", "flagsmith-probe-rt-1", "flagsmith-probe-rt-2", "flagsmith-probe-1", "flagsmith-probe-2" ] }, { "productId": "flagsmith", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Flagsmith exposes an Admin API with RBAC for fine-grained access control, and its documented MCP server requires an API key to authenticate agent access (confirmed via a live 401 probe), showing credential-gated agent access. However, there is no explicit documentation of issuing scoped, least-privilege tokens specifically for agents (e.g., read-only or resource-limited keys) beyond general user-role RBAC. Missing for 10: explicit agent-specific scoped/short-lived API key issuance, documented token scopes (read/write/resource-limited), and independent confirmation of least-privilege enforcement for MCP/agent credentials.", "evidenceIds": [ "flagsmith-docs-11", "flagsmith-docs-26", "flagsmith-docs-16", "flagsmith-docs-17", "flagsmith-probe-rt-1", "flagsmith-probe-3" ] }, { "productId": "flagsmith", "storyId": "agentic-sdks", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Flagsmith advertises official SDKs across 15+ languages and frameworks (React, Next.js, .NET, Java, etc.) and positions itself for web, mobile, and server-side integration, giving AI-native developers concrete SDKs to build against. However the evidence pack lacks SDK-specific documentation depth (API references, code samples, changelogs) beyond a marketing blurb. Missing for 10: dedicated per-SDK docs/examples, versioning/maintenance evidence, and independent corroboration of SDK quality.", "evidenceIds": [ "flagsmith-gh-4", "flagsmith-docs-22" ] }, { "productId": "flagsmith", "storyId": "agentic-webhooks", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Flagsmith documents Audit Log Webhooks that let you stream events (flag/segment/identity changes) into your own infrastructure, which is a real webhook subscription mechanism. However, evidence only covers audit-log-triggered webhooks, not a broader event/webhook API for arbitrary flag or environment change subscriptions, and there's no independent corroboration of reliability. Missing for 10: documentation of a general-purpose webhook subscription API beyond audit logs, and any hands-on/independent verification of webhook delivery in production use.", "evidenceIds": [ "flagsmith-docs-9", "flagsmith-docs-39" ] }, { "productId": "flagsmith", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack explicitly shows probes for an OpenAPI/Swagger spec and llms.txt returning 404s, and no citation describes an interactive, runnable API reference (e.g., Swagger UI, Postman collection, or live code playground). Only static API overview docs and an Admin API description are mentioned, none of which are runnable/interactive.", "evidenceIds": [ "flagsmith-probe-2", "flagsmith-probe-1", "flagsmith-docs-15", "flagsmith-docs-40" ] }, { "productId": "flagsmith", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Flagsmith documents a full Admin API and CLI/MCP integrations, but explicit probes for a machine-readable spec (openapi.json, swagger.json, .well-known/openapi.json) all returned 404, and no evidence pack item points to a downloadable OpenAPI/Swagger spec.", "evidenceIds": [ "flagsmith-probe-2", "flagsmith-docs-40", "flagsmith-docs-16" ] }, { "productId": "flagsmith", "storyId": "api-sandbox", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Flagsmith supports multiple environments (dev/staging/prod) and fully self-hostable Docker setups that let users create, toggle, and evaluate flags without touching production, and a probe confirms a full local create-and-evaluate flag loop working offline; RBAC also lets teams restrict who can modify production. However, none of this is framed specifically as an 'AI-native sandbox' testing workflow, and the MCP/agent-facing docs don't call out sandbox vs. production isolation explicitly. Missing for 10: an explicit sandbox/staging environment guide oriented at AI agents, and confirmation that the MCP server itself supports pointing at a non-prod environment safely.", "evidenceIds": [ "flagsmith-gh-2", "flagsmith-docs-12", "flagsmith-docs-13", "flagsmith-docs-27", "flagsmith-docs-26", "flagsmith-probe-rt-2" ] }, { "productId": "flagsmith", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack documents Flagsmith's Admin API, SDKs, CLI, and MCP server, but nowhere mentions API version numbers, a versioning scheme, or a documented deprecation policy; probes even show openapi.json/spec endpoints returning 404, suggesting no discoverable API contract documentation. Missing for 10: any statement of API versioning strategy, deprecation timelines, or migration guides for breaking changes.", "evidenceIds": [ "flagsmith-docs-15", "flagsmith-docs-16", "flagsmith-docs-30", "flagsmith-probe-2" ] }, { "productId": "flagsmith", "storyId": "approvals-change-requests", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Flagsmith documents a native four-eyes Feature Change Request workflow (create, approve, publish) explicitly analogous to GitHub PRs, plus Release Pipelines that can auto-generate change requests when promoting flags to production, and RBAC to restrict who can modify production environments. This directly satisfies the governance requirement for approvals before production flag changes go live. Missing for 10: independent/hands-on verification of the approval gating in practice (only first-party docs cited).", "evidenceIds": [ "flagsmith-docs-10", "flagsmith-docs-34", "flagsmith-docs-37", "flagsmith-docs-26", "flagsmith-docs-11" ] }, { "productId": "flagsmith", "storyId": "audit-trail", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Flagsmith documents that every action in the admin application (flag, identity, segment changes) is tracked and logged, directly matching the who/what/when/value audit requirement, and offers webhook streaming to external systems for governance workflows. missing for 10: independent/hands-on confirmation of the audit log's exact field-level detail (e.g., explicit before/after value display) beyond vendor docs.", "evidenceIds": [ "flagsmith-docs-8", "flagsmith-docs-39", "flagsmith-docs-9" ] }, { "productId": "flagsmith", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Flagsmith's Admin API, CLI, and MCP server expose full CRUD over flags, segments, projects, environments and users programmatically, which lets an AI-native user script operations across many items in a loop (docs-16, docs-18, docs-17/29, probe-3/4). However, there is no documented explicit bulk/batch endpoint, bulk import/export tool, or multi-item transactional operation — only single-resource CRUD calls that must be iterated. Missing for 10: dedicated bulk/batch API endpoints, bulk import/export documentation, evidence of atomic multi-item operations.", "evidenceIds": [ "flagsmith-docs-16", "flagsmith-docs-18", "flagsmith-docs-17", "flagsmith-docs-29", "flagsmith-probe-3", "flagsmith-probe-4", "flagsmith-docs-40" ] }, { "productId": "flagsmith", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Release Pipelines let users define stages with 'triggers and actions' that automatically progress flags/rollouts to environments and audiences, and Scheduled Flags apply changes automatically at a specified time — both are rule-based automated actions. Audit Log Webhooks can stream internal events for downstream automation. However, evidence covers only time/stage-based and rollout triggers rather than generic arbitrary-event-triggered actions (e.g., inbound webhook conditions tied to external signals), and there's no independent/hands-on corroboration of the pipeline automation working as described. Missing for 10: broader event-source flexibility (e.g., external event ingestion triggering flag changes), and independent verification of release-pipeline automation in practice.", "evidenceIds": [ "flagsmith-docs-6", "flagsmith-docs-34", "flagsmith-docs-37", "flagsmith-docs-5", "flagsmith-docs-24", "flagsmith-docs-9" ] }, { "productId": "flagsmith", "storyId": "automation-scheduled-jobs", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Flagsmith supports 'Scheduled flags' to queue a one-time future flag change, and 'Release Pipelines' that automatically progress flags through stages with triggers/actions — both provide some automation depth, but neither is documented as a recurring/cron-style job scheduler that repeats on an interval. Missing for 10: explicit recurring/cron scheduling, repeatable automated workflow triggers beyond a single future timestamp, and any AI-agent-driven orchestration of repeating tasks.", "evidenceIds": [ "flagsmith-docs-5", "flagsmith-docs-24", "flagsmith-docs-6", "flagsmith-docs-34", "flagsmith-docs-37" ] }, { "productId": "flagsmith", "storyId": "automation-versioned-workflows", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Flagsmith provides a four-eyes Change Request workflow analogous to PR review (docs-10), full audit logs tracking every flag/segment/identity change over time (docs-8, docs-39), and Release Pipelines with automated stage-gated rollouts and auto-generated change requests (docs-6, docs-34, docs-37), plus instant toggle-off to revert behavior without redeploying (flagsmith-gh-1). Together these cover versioning, review, and effective rollback of flag-driven automations. missing for 10: an explicit named 'rollback to previous flag version' feature (evidence only shows audit trail + manual re-toggle, not one-click revert), and independent/hands-on confirmation of the change-request/pipeline workflow in practice.", "evidenceIds": [ "flagsmith-docs-8", "flagsmith-docs-10", "flagsmith-docs-6", "flagsmith-docs-34", "flagsmith-docs-37", "flagsmith-gh-1", "flagsmith-docs-39" ] }, { "productId": "flagsmith", "storyId": "create-toggle-flag-fast", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Docs and a hands-on runtime probe confirm the core flow: create a feature via the dashboard/API, name it, toggle it on/off per environment without deploying code, and evaluate it live via SDK/API within minutes — the probe even demonstrates a full keyless create-and-evaluate loop from scratch. Docker quickstart and clear UI steps (Create Feature, toggle) further support fast time-to-value.\n\nmissing for 10: no independent third-party timing benchmark or new-user onboarding video confirming the 'within minutes' claim beyond docs/probe.", "evidenceIds": [ "flagsmith-docs-31", "flagsmith-docs-20", "flagsmith-gh-1", "flagsmith-gh-3", "flagsmith-docs-13", "flagsmith-probe-rt-2" ] }, { "productId": "flagsmith", "storyId": "edge-evaluation", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Flagsmith ships an official Edge Proxy that can be self-hosted 'close to your servers' and queried by server-side SDKs (flagsmith-docs-14, flagsmith-docs-28), which addresses the general 'evaluate close to users' need, but this is a self-run relay service rather than a native CDN/edge-worker (e.g., Cloudflare Workers, Fastly Compute) integration. A community comment also notes Flagsmith's evaluation is 'primarily server-side' rather than fully local/in-memory, unlike some flagd-style hybrid approaches (flagsmith-comm-1), tempering claims of true edge-local evaluation. Missing for 10: a documented CDN/edge-worker runtime or SDK (e.g., Cloudflare/Fastly integration), multi-region edge network evidence, and independent latency benchmarks confirming edge-proximity gains.", "evidenceIds": [ "flagsmith-docs-14", "flagsmith-docs-28", "flagsmith-comm-1" ] }, { "productId": "flagsmith", "storyId": "experiment-stats-engine", "verdict": "partial", "quality": 6, "confidence": "low", "rationale": "Flagsmith explicitly documents an experimentation feature with a 'built-in Bayesian statistics engine' reading results from multivariate flags and warehoused events, giving product managers a named statistical method behind results. However, the docs give no detail on the underlying model, confidence intervals, sample-ratio checks, or frequentist alternative, and there is no independent/hands-on corroboration of the engine's correctness or variance-reduction techniques. Missing for 10: documentation of specific statistical methodology/assumptions, variance-reduction (e.g., CUPED) support, and third-party validation of the engine's results.", "evidenceIds": [ "flagsmith-docs-7", "flagsmith-docs-25", "flagsmith-docs-35", "flagsmith-docs-38" ] }, { "productId": "flagsmith", "storyId": "flag-environments", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Flagsmith's core model is organized around projects with multiple environments (dev/staging/prod), each with its own independent flag states and its own scoped Environment/API key that's safe to expose client-side, confirmed by docs and a live probe showing environment-scoped key evaluation working end to end. RBAC further lets teams restrict who can modify production environments specifically. missing for 10: no explicit doc snippet walking through creating multiple named environments side-by-side or promoting flag state between them, and no independent/community corroboration beyond vendor docs and the probe.", "evidenceIds": [ "flagsmith-docs-15", "flagsmith-docs-16", "flagsmith-docs-26", "flagsmith-probe-rt-2", "flagsmith-docs-22" ] }, { "productId": "flagsmith", "storyId": "guarded-rollouts-auto-rollback", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Flagsmith documents percentage rollouts, scheduled flags, and Release Pipelines with 'triggers and actions,' plus an Experimentation module with Bayesian stats — but none of the evidence describes automatic metric-based regression detection that triggers an automatic rollback of a release. The pipeline features found are about staged promotion and change-request automation, not closed-loop guarded rollouts tied to live health/error metrics.", "evidenceIds": [ "flagsmith-docs-6", "flagsmith-docs-34", "flagsmith-docs-37", "flagsmith-docs-38", "flagsmith-docs-21" ] }, { "productId": "flagsmith", "storyId": "individual-user-targeting", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Flagsmith explicitly supports overriding feature flags for individual users via 'identities', documented for internal testing, QA, and personalized experiences, plus segment-based targeting for group-based allowlists (e.g., beta testers). This is a well-documented core feature directly matching the story. Missing for 10: independent third-party hands-on confirmation of identity-level overrides specifically (probes verify flag creation/eval but not the identity-override workflow itself).", "evidenceIds": [ "flagsmith-docs-3", "flagsmith-docs-42", "flagsmith-docs-32", "flagsmith-docs-36" ] }, { "productId": "flagsmith", "storyId": "local-evaluation-latency", "verdict": "disputed", "quality": 3, "confidence": "medium", "rationale": "Flagsmith docs describe an Edge Proxy you can self-host to reduce latency (flagsmith-docs-14/28), but this is still a network hop to a proxy, not a local in-process cached rule evaluation. A knowledgeable community source explicitly states Flagsmith 'does primarily server-side eval' rather than in-memory local evaluation (flagsmith-comm-1), and the probed runtime evidence shows flag reads going through a live API call (flagsmith-probe-rt-2), directly contradicting the 'no network call per flag check' claim. missing for 10: first-party SDK docs describing local ruleset caching/polling with microsecond in-process evaluation, and hands-on confirmation that server SDKs avoid per-check network calls.", "evidenceIds": [ "flagsmith-docs-14", "flagsmith-docs-28", "flagsmith-comm-1", "flagsmith-probe-rt-2" ] }, { "productId": "flagsmith", "storyId": "metrics-warehouse-integration", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Flagsmith's experimentation feature explicitly funnels events into its own 'managed data warehouse' and evaluates them via a built-in Bayesian engine, the opposite of letting a PM define metrics from their own warehouse tables or ingested events — there is no evidence of BYO-warehouse or custom event-source metric definition.", "evidenceIds": [ "flagsmith-docs-7", "flagsmith-docs-35", "flagsmith-docs-38", "flagsmith-docs-25" ] }, { "productId": "flagsmith", "storyId": "multivariate-dynamic-config", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Flagsmith docs explicitly describe multivariate flags with percentage-weighted variants and remote config values, and core management docs show configurable flag fields (not just boolean toggles) plus experimentation using multivariate flags. This is corroborated by a self-hosted probe showing a real flag create/evaluate roundtrip. missing for 10: explicit doc confirmation of string/number/JSON config value types beyond variant weighting, and independent hands-on confirmation of non-boolean config value retrieval via SDK.", "evidenceIds": [ "flagsmith-docs-4", "flagsmith-docs-7", "flagsmith-docs-20", "flagsmith-docs-31", "flagsmith-docs-38", "flagsmith-probe-rt-2" ] }, { "productId": "flagsmith", "storyId": "openfeature-compatibility", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions OpenFeature support, an OpenFeature provider implementation, or any compatibility layer for the OpenFeature spec — Flagsmith is only shown with its own proprietary SDKs, API, and CLI. Missing for 10: any mention of an OpenFeature provider, spec compliance, or documented integration with the OpenFeature ecosystem.", "evidenceIds": [] }, { "productId": "flagsmith", "storyId": "openness-api-parity", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Flagsmith explicitly documents that 'Anything you can do in the Flagsmith dashboard, you can also do via the Admin API' (flagsmith-docs-30/40), and this is corroborated by a real probe showing a full create-project/environment/flag/evaluate roundtrip via the API (flagsmith-probe-rt-2). It further exposes this API to AI agents via an official MCP server and CLI (flagsmith-docs-17/18, flagsmith-probe-3/4, flagsmith-probe-rt-1), reinforcing API-UI parity for AI-native use. Missing for 10: a published OpenAPI/swagger spec was probed and returned 404 (flagsmith-probe-2), so machine-readable API completeness is not independently verifiable beyond the vendor's own claim.", "evidenceIds": [ "flagsmith-docs-30", "flagsmith-docs-40", "flagsmith-docs-17", "flagsmith-docs-18", "flagsmith-probe-3", "flagsmith-probe-4", "flagsmith-probe-rt-1", "flagsmith-probe-rt-2", "flagsmith-probe-2" ] }, { "productId": "flagsmith", "storyId": "openness-full-export", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Flagsmith's Admin API exposes full CRUD access to all projects, environments, flags, segments, and users (\"anything you can do in the dashboard, you can also do via the Admin API\"), and self-hosting via Docker gives full ownership of the underlying Postgres data store, together enabling a user to extract and own all their data outside the SaaS. However there is no documented one-click 'export all data' feature, no bulk export-to-JSON/CSV tool, and probes show no llms.txt or public OpenAPI spec is served, so a turnkey open-format export path is not explicitly evidenced. missing for 10: explicit bulk data-export feature/format, documented data-portability guarantee, and a working OpenAPI/machine-readable spec (docs.flagsmith.com/openapi.json returned 404).", "evidenceIds": [ "flagsmith-docs-30", "flagsmith-docs-40", "flagsmith-docs-12", "flagsmith-docs-13", "flagsmith-probe-rt-2", "flagsmith-probe-2" ] }, { "productId": "flagsmith", "storyId": "openness-open-license", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "The Flagsmith source code is publicly hosted on GitHub (repeatedly cited) and the project provides self-hosting docs, Docker images, and full platform control, all consistent with an openly licensed codebase. However, the evidence pack never explicitly names or links the license file/type, so the claim of a specific open license is inferred rather than directly documented. Missing for 10: explicit license identifier/file citation (e.g., BSD-3-Clause LICENSE), independent confirmation of license terms.", "evidenceIds": [ "flagsmith-gh-1", "flagsmith-gh-2", "flagsmith-gh-3", "flagsmith-docs-12", "flagsmith-docs-13" ] }, { "productId": "flagsmith", "storyId": "openness-self-host", "verdict": "full", "quality": 10, "confidence": "high", "rationale": "Flagsmith documents comprehensive self-hosting via Docker (full platform, own infrastructure, full control over data/security), and a hands-on probe confirms a complete keyless self-host roundtrip — running the official Docker image, creating a project/environment/flag, and successfully evaluating it via the API with no account or license required.", "evidenceIds": [ "flagsmith-docs-12", "flagsmith-docs-13", "flagsmith-docs-27", "flagsmith-probe-rt-2" ] }, { "productId": "flagsmith", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Flagsmith supports data residency indirectly via self-hosting (full control over data location, deployable in any infrastructure/region) and an Edge Proxy that can be run within a customer's own infrastructure, which lets an AI-native user choose where flag data lives. However, there's no documented SaaS-side region picker (e.g., dedicated EU/US managed cloud regions) — residency control is achieved only by fully self-hosting rather than a first-class 'choose region' setting. Missing for 10: explicit managed-cloud region selection options, documentation of data residency guarantees for the hosted SaaS offering.", "evidenceIds": [ "flagsmith-docs-12", "flagsmith-docs-13", "flagsmith-docs-14", "flagsmith-docs-27", "flagsmith-docs-28", "flagsmith-probe-rt-2" ] }, { "productId": "flagsmith", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Flagsmith is a feature-flag/experimentation platform, not an AI model or AI training data provider; there is no concept of 'training data opt-out' applicable to this product category. This axis is a category error for a feature-flagging tool.", "evidenceIds": [] }, { "productId": "flagsmith", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Flagsmith's Admin API and dashboard explicitly support creating, updating, and deleting projects, environments, flags, segments, and users, with a 'protected' flag option to guard against accidental deletion, and self-hosting gives full control over where data lives. However there is no explicit documentation of a data-retention policy, automated data expiry, or a GDPR-style 'delete my data' workflow beyond generic CRUD deletion of resources. Missing for 10: documented retention windows/expiry settings, formal data-deletion/right-to-erasure workflow, and audit-log retention controls.", "evidenceIds": [ "flagsmith-docs-16", "flagsmith-docs-43", "flagsmith-docs-12", "flagsmith-docs-8" ] }, { "productId": "flagsmith", "storyId": "privacy-telemetry-optout", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Flagsmith is a feature-flag/experimentation platform, not an AI agent or CLI/telemetry-emitting client tool; the story concerns opting out of an AI-native tool's own usage telemetry, which is not an applicable axis for this product category. Self-hosting evidence addresses data control generally but not a telemetry opt-out mechanism specifically.", "evidenceIds": [] }, { "productId": "flagsmith", "storyId": "progressive-percentage-rollouts", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Flagsmith docs explicitly document percentage-based rollouts via a % Split segment rule (1-100%) that can be adjusted without redeploying, and this is corroborated by a runtime probe showing a live flag create/evaluate loop. Consistent bucketing is implied by the segment-based percentage split mechanism used across SDKs/environments, though the docs don't explicitly describe the hashing/bucketing algorithm in detail. missing for 10: explicit documentation of the deterministic bucketing/hashing algorithm ensuring consistent user assignment across ramp changes.", "evidenceIds": [ "flagsmith-docs-21", "flagsmith-docs-23", "flagsmith-docs-33", "flagsmith-probe-rt-2" ] }, { "productId": "flagsmith", "storyId": "rbac-scoped-access", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Flagsmith documents fine-grained RBAC to restrict who can modify resources (e.g., 'only allow certain users to modify your production environments'), plus a four-eyes change-request workflow for approvals, and clearly distinguishes scoped API tokens (public non-secret client-side environment keys vs. privileged Admin API keys for CRUD operations). SSO/SAML and 2FA further support access governance. Missing for 10: no explicit documentation of scoped/granular API token permissions (e.g., per-project or read-only server tokens) beyond the client/server key distinction, and no independent/community validation of RBAC granularity in practice.", "evidenceIds": [ "flagsmith-docs-11", "flagsmith-docs-26", "flagsmith-docs-10", "flagsmith-docs-16", "flagsmith-docs-15", "flagsmith-docs-19", "flagsmith-docs-41" ] }, { "productId": "flagsmith", "storyId": "realtime-flag-updates", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack describes flag targeting, rollouts, and toggling but contains no documentation of streaming updates, real-time push, or specific polling intervals that would substantiate seconds-level propagation to SDKs; a community comment even notes Flagsmith does 'primarily server-side eval' rather than confirming fast client propagation. Missing for 10: documented real-time/streaming SDK update mechanism, polling interval specs, and evidence of kill-switch propagation latency.", "evidenceIds": [ "flagsmith-comm-1" ] }, { "productId": "flagsmith", "storyId": "relay-edge-proxy", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Flagsmith documents a self-hosted Edge Proxy service that runs the Flagsmith Engine within your own infrastructure and is connected to directly by server-side SDKs, keeping evaluation traffic local and resilient to vendor outages. This directly matches the platform-engineer story of running a relay/edge proxy for network-local flag serving. Missing for 10: independent hands-on validation of failover behavior when the vendor API is unreachable, and more detail on sync/refresh mechanics between proxy and main API.", "evidenceIds": [ "flagsmith-docs-14", "flagsmith-docs-28", "flagsmith-docs-12" ] }, { "productId": "flagsmith", "storyId": "scheduled-releases", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Flagsmith's docs explicitly describe a Scheduled Flags feature that lets you queue changes to feature flags and have them automatically applied at a specified future time, removing the need for manual intervention at that moment — directly matching the story. This is further complemented by Release Pipelines, which can automatically stage/progress rollouts with triggers, reinforcing scheduled/automated release workflows. missing for 10: independent hands-on confirmation of the scheduling UI/behavior beyond first-party docs.", "evidenceIds": [ "flagsmith-docs-5", "flagsmith-docs-24", "flagsmith-docs-6", "flagsmith-docs-37" ] }, { "productId": "flagsmith", "storyId": "sdk-language-breadth", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Flagsmith explicitly markets support for 15+ languages/frameworks across backend, web, and mobile, and its architecture (environment keys, consistent flag/segment/targeting model) implies consistent behavior across SDKs. However, the evidence pack lacks any per-SDK documentation detail, SDK-to-SDK consistency testing, or independent confirmation of parity across specific SDKs, and community commentary raises questions about server-side vs local evaluation consistency (flagd vs local evaluation) that isn't directly addressed. missing for 10: per-platform SDK documentation snippets, evidence of consistent flag evaluation logic/tests across SDKs, independent/hands-on confirmation of behavior parity across backend/web/mobile.", "evidenceIds": [ "flagsmith-gh-4", "flagsmith-docs-22", "flagsmith-comm-1" ] }, { "productId": "flagsmith", "storyId": "self-host-open-source", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Flagsmith's docs explicitly describe self-hosting the complete platform for full data/security control, with a Docker-based setup guide, and this was independently verified via a real keyless self-host probe that created an org/project/environment/flag and successfully evaluated it against the local instance. missing for 10: no independent third-party audit of production-scale self-hosted deployments or HA/scaling guidance beyond the basic Docker quickstart.", "evidenceIds": [ "flagsmith-docs-12", "flagsmith-docs-13", "flagsmith-docs-27", "flagsmith-probe-rt-2" ] }, { "productId": "flagsmith", "storyId": "stale-flag-cleanup", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers targeting, rollouts, experimentation, audit logs, RBAC, and change requests, but contains no mention of stale-flag detection, code reference scanning, or flag lifecycle/cleanup tooling that would help identify and remove unused flags from a codebase. This is a fair capability to expect from a mature flag-management platform (competitors ship code-reference scanners), so absence of evidence means 'none' rather than 'na'.", "evidenceIds": [] }, { "productId": "flagsmith", "storyId": "targeting-rules-segments", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Docs clearly describe trait-based targeting (geography, subscription, device, version), reusable segments defined by trait rules, per-identity overrides, percentage-split rollouts within segments, and multivariate flags — all core to attribute-based targeting and segment-driven variation delivery, corroborated by a runtime probe confirming a real flag create/evaluate loop. Missing for 10: independent (non-vendor) third-party validation of segment/attribute targeting behavior at scale beyond the single self-host probe.", "evidenceIds": [ "flagsmith-docs-1", "flagsmith-docs-32", "flagsmith-docs-36", "flagsmith-docs-42", "flagsmith-docs-21", "flagsmith-docs-4", "flagsmith-probe-rt-2" ] }, { "productId": "flagsmith", "storyId": "transparent-pricing-limits", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack only references the pricing page URL for isolated feature call-outs (SAML/SSO, 2FA) but contains no evidence about actual pricing tiers, seat costs, MAU/event/request-based pricing, or how usage drives cost. missing for 10: published pricing tiers, cost drivers (seats/MAUs/events/requests), any breakdown of what determines plan cost.", "evidenceIds": [ "flagsmith-docs-19", "flagsmith-docs-41" ] }, { "productId": "growthbook", "storyId": "ab-experiments", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "GrowthBook natively supports A/B and multivariate experiments attached to feature flags, with statistical engines (Bayesian/Frequentist, CUPED, Sequential, Bandits), templates for goal/secondary/guardrail metrics, and a results view showing 'Chance to Win' per variation to determine winners. Community reports corroborate quick setup and real-world use of feature flags plus A/B testing, though some note a gap between flags and full analytics requiring warehouse/metrics setup. missing for 10: independent hands-on validation of multivariate (>2 arm) experiment results and winner-declaration workflow specifically, beyond docs and community anecdotes.", "evidenceIds": [ "growthbook-docs-6", "growthbook-docs-7", "growthbook-docs-8", "growthbook-docs-9", "growthbook-docs-24", "growthbook-docs-29", "growthbook-docs-33", "growthbook-gh-2", "growthbook-comm-6", "growthbook-comm-12" ] }, { "productId": "growthbook", "storyId": "agent-experiment-analysis", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "GrowthBook exposes a full REST API and an official MCP server that makes authenticated calls to that API, explicitly loading 'Agent Skills (workflows and guardrails)' so AI tools can query feature/experiment data — confirmed working end-to-end in a runtime probe. Experiment results expose statistical decision signals like 'Chance to Win' (>95% highlighted as a clear winner) that directly support ship/rollback recommendations, and metrics/fact-table docs show how results are computed from stored data for programmatic reading. Missing for 10: a concrete worked example of an agent actually generating a ship/rollback summary via MCP, and independent (non-vendor) confirmation of recommendation quality.", "evidenceIds": [ "growthbook-docs-12", "growthbook-docs-13", "growthbook-docs-21", "growthbook-docs-24", "growthbook-docs-30", "growthbook-probe-rt-1", "growthbook-probe-3" ] }, { "productId": "growthbook", "storyId": "agent-toggles-flag-safely", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "GrowthBook documents an official MCP server that makes authenticated REST calls (growthbook-docs-13/21, growthbook-probe-3/rt-1) and a full REST API (growthbook-docs-12), and separately documents 'Draft feature revisions, request review... require approval before publishing' as a Pro+ approval-workflow feature (growthbook-docs-5, growthbook-docs-16/37). However, the runtime probe (growthbook-probe-rt-2) shows a flag created and immediately live via the API with no review step, and approval workflows appear gated behind a paid plan rather than proven to force-block an AI agent's writes by default. Missing for 10: a documented/verified CLI channel, and hands-on evidence that MCP/API writes are actually intercepted and blocked pending human approval rather than merely available as an opt-in paid feature.", "evidenceIds": [ "growthbook-docs-13", "growthbook-docs-21", "growthbook-probe-3", "growthbook-probe-rt-1", "growthbook-docs-5", "growthbook-docs-16", "growthbook-docs-37", "growthbook-docs-12", "growthbook-probe-rt-2" ] }, { "productId": "growthbook", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "A probe confirms llms.txt is live at docs.growthbook.io/llms.txt returning HTTP 200 with a structured index of 180 documentation pages, exactly matching the agent-oriented docs pattern; GrowthBook also documents an official MCP server for agent integration, reinforcing agent-friendliness. Missing for 10: no independent third-party confirmation of an agent successfully consuming llms.txt end-to-end beyond the probe.", "evidenceIds": [ "growthbook-probe-1", "growthbook-docs-13", "growthbook-docs-21", "growthbook-probe-3" ] }, { "productId": "growthbook", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "GrowthBook ships an official MCP server that lets AI tools like Cursor, VS Code, and Claude connect and make authenticated calls to pull experiment/feature data and get GrowthBook 'Agent Skills' workflows/guardrails, which can produce AI-generated insights and suggestions — this was verified working in a runtime probe. However, this capability lives entirely outside the product (in an external AI client) rather than as a native in-app AI insights feature; GrowthBook's own results UI only shows statistical outputs like 'Chance to Win', not LLM-generated narrative insights. Missing for 10: a native in-product AI insights/summarization feature surfaced in the GrowthBook UI itself, and evidence of automated suggestion-generation (e.g., 'this metric moved because X') without requiring an external MCP client.", "evidenceIds": [ "growthbook-docs-13", "growthbook-docs-21", "growthbook-probe-rt-1", "growthbook-docs-24" ] }, { "productId": "growthbook", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "GrowthBook has some automated background behaviors (automatic stale flag detection, ramp schedules, and 'safe rollouts' that presumably auto-guard rollouts), and its MCP server lets AI tools make authenticated calls to the product, but these are on-demand agent actions or passive detection features, not evidence of an AI agent configuring a persistent job that runs autonomously in the background on a schedule/trigger without human involvement. Missing for 10: a documented scheduler/trigger system for AI-configured autonomous jobs, evidence of MCP-driven automations persisting and executing unattended, and independent confirmation of any autonomous background workflow.", "evidenceIds": [ "growthbook-docs-4", "growthbook-docs-22", "growthbook-docs-16", "growthbook-docs-37", "growthbook-docs-15", "growthbook-docs-21", "growthbook-probe-rt-1" ] }, { "productId": "growthbook", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "GrowthBook documents an MCP server that lets external AI tools (Cursor, VS Code, Claude) connect to and query GrowthBook — this is the opposite direction (GrowthBook as a data/tool source for external assistants), not a built-in AI assistant living inside the GrowthBook product itself. No evidence anywhere in the pack shows an in-app chat/assistant feature for delegating tasks within GrowthBook's UI.", "evidenceIds": [ "growthbook-docs-13", "growthbook-docs-21", "growthbook-probe-3", "growthbook-probe-rt-1" ] }, { "productId": "growthbook", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "GrowthBook supports headless/CI usage via self-hosted Docker Compose deployment, a full REST API, SDK Connection Endpoints for programmatic flag evaluation, and inline server-side experiments requiring no third-party requests — all scriptable without a UI. A runtime probe confirms a real keyless self-host create-and-evaluate loop via API/docker, demonstrating genuine automation capability. missing for 10: no explicit CI/CD pipeline example or GitHub Actions integration doc, and no evidence of a dedicated CLI tool for automation workflows.", "evidenceIds": [ "growthbook-docs-10", "growthbook-docs-12", "growthbook-docs-19", "growthbook-docs-20", "growthbook-docs-35", "growthbook-probe-rt-2" ] }, { "productId": "growthbook", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "medium", "rationale": "GrowthBook is a feature-flagging/experimentation platform, not an AI agent or assistant with a tool-use surface; the evidence shows only the reverse integration — GrowthBook publishes an MCP server so external AI tools (Cursor, Claude, VS Code) can call GrowthBook's API — not that GrowthBook itself can consume/plug in external MCP servers as a client to use their tools.", "evidenceIds": [ "growthbook-docs-13", "growthbook-docs-21", "growthbook-probe-3", "growthbook-probe-rt-1" ] }, { "productId": "growthbook", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "GrowthBook documents an official MCP server that bridges AI tools (Cursor, VS Code, Claude) to GrowthBook via authenticated REST calls with Agent Skills/guardrails, and a runtime probe confirms the official npm package (@growthbook/mcp) completes a full stdio initialize handshake exposing read/write meta-tools. This is strong first-party plus hands-on evidence of a working official MCP server. Missing for 10: broader independent/community corroboration beyond the single runtime probe.", "evidenceIds": [ "growthbook-docs-13", "growthbook-docs-21", "growthbook-probe-3", "growthbook-probe-rt-1" ] }, { "productId": "growthbook", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "GrowthBook ships an official MCP server that bridges AI tools like Cursor, VS Code, and Claude to GrowthBook, loading 'Agent Skills' (workflows/guardrails) and making authenticated REST calls so users can operate flags/experiments via natural language in an AI client. A runtime probe confirms the official npm MCP server completes a full keyless handshake exposing read/write meta-tools covering the whole API, corroborating the docs' claims beyond marketing copy. Missing for 10: an end-to-end example of an actual natural-language command executing a real write action (only the handshake/tool-discovery was verified, not a full command-to-result flow) and independent (non-vendor) hands-on confirmation.", "evidenceIds": [ "growthbook-docs-13", "growthbook-docs-21", "growthbook-probe-3", "growthbook-probe-rt-1" ] }, { "productId": "growthbook", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack documents SDKs, REST API, self-host Docker, and an official MCP server, but there is no mention anywhere of an official standalone CLI tool for GrowthBook. Since a platform like GrowthBook could plausibly ship a CLI, the axis applies, but with no supporting evidence the verdict is none.", "evidenceIds": [] }, { "productId": "growthbook", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "GrowthBook documents a full REST API for interacting with the application (growthbook-docs-12), including a readonly SDK Connection Endpoint (growthbook-docs-20) and unauthenticated public endpoints (growthbook-docs-41), and a runtime probe confirms an actual API-driven create-and-evaluate loop (creating org, feature flag, SDK connection, and reading it back) worked end-to-end (growthbook-probe-rt-2). Missing for 10: a discoverable OpenAPI/swagger spec file (all candidate paths 404'd per growthbook-probe-2), which would make the API more machine-discoverable for agentic tooling.", "evidenceIds": [ "growthbook-docs-12", "growthbook-docs-20", "growthbook-docs-41", "growthbook-probe-rt-2", "growthbook-probe-2" ] }, { "productId": "growthbook", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "GrowthBook documents a read-only SDK Connection Endpoint that exposes only a subset of feature-flag data (least-privilege by design) and a full REST API, plus an 'Advanced access control' / 'Advanced permissioning' feature at higher pricing tiers, and the MCP server makes authenticated REST calls on behalf of an agent. However there is no explicit documentation of an admin flow for issuing a scoped, least-privilege API credential specifically for an AI agent (e.g., role-restricted API keys, token scoping UI, or agent-specific key generation). Missing for 10: documented API-key scoping/role assignment workflow, explicit agent-credential issuance guide, and evidence of granular per-tool permission limits for MCP/agent tokens beyond generic 'advanced access control' tier feature.", "evidenceIds": [ "growthbook-docs-20", "growthbook-docs-41", "growthbook-docs-12", "growthbook-docs-16", "growthbook-docs-37", "growthbook-docs-21", "growthbook-probe-rt-1" ] }, { "productId": "growthbook", "storyId": "agentic-sdks", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "GrowthBook ships 24 official language-specific SDKs (React, Python, Android, iOS, etc.) with documented quickstart code, local evaluation, targeting, and inline experiments, and this is corroborated by community hands-on reports of fast SDK integration. missing for 10: independent benchmarking across all 24 SDKs and more recent third-party validation beyond a single HN thread.", "evidenceIds": [ "growthbook-gh-1", "growthbook-docs-1", "growthbook-docs-17", "growthbook-docs-32", "growthbook-docs-19", "growthbook-comm-2", "growthbook-comm-4" ] }, { "productId": "growthbook", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers SDKs, REST API, MCP integration, self-hosting, and experimentation features but contains no mention of webhooks or event subscription mechanisms anywhere, so there is no evidence GrowthBook supports this capability.", "evidenceIds": [] }, { "productId": "growthbook", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "GrowthBook documents a REST API (growthbook-docs-12) but there is no evidence of an interactive, runnable API reference (e.g., Swagger/OpenAPI try-it console) — a direct probe for openapi/swagger specs returned 404 on all candidate paths, and no docs mention runnable code samples in an API explorer.", "evidenceIds": [ "growthbook-docs-12", "growthbook-probe-2" ] }, { "productId": "growthbook", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "While GrowthBook documents a full REST API (growthbook-docs-12) and an SDK Connection Endpoint (growthbook-docs-20), the evidence pack shows an explicit probe for OpenAPI/Swagger spec files at expected paths that all returned 404, and no other evidence surfaces a downloadable OpenAPI or machine-readable spec.", "evidenceIds": [ "growthbook-docs-12", "growthbook-probe-2" ] }, { "productId": "growthbook", "storyId": "api-sandbox", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "GrowthBook supports separate environments (dev/staging/production) for feature flags, self-hosting via Docker Compose for a fully isolated instance, and a verified runtime probe confirms a keyless self-host roundtrip creating and evaluating flags without any production account/license — enabling true sandbox testing. missing for 10: no explicit vendor guidance on a dedicated 'sandbox mode' distinct from staging environments, and no independent (non-vendor) confirmation of environment data isolation guarantees.", "evidenceIds": [ "growthbook-docs-3", "growthbook-docs-10", "growthbook-docs-35", "growthbook-probe-rt-2" ] }, { "productId": "growthbook", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "GrowthBook documents a REST API and SDK Connection Endpoint, but no evidence pack item describes API versioning scheme or a documented deprecation policy; the openapi/swagger probe returned 404s, suggesting no formal API spec is even discoverable. Missing for 10: any mention of API version numbers, backward-compatibility guarantees, or a deprecation/sunset policy.", "evidenceIds": [ "growthbook-docs-12", "growthbook-docs-20", "growthbook-probe-2" ] }, { "productId": "growthbook", "storyId": "approvals-change-requests", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "GrowthBook's docs explicitly describe drafting feature revisions, requesting review, resolving merge conflicts, and requiring approval before publishing changes (growthbook-docs-5), and this is packaged as a named 'Approval workflows' feature on paid tiers (growthbook-docs-16, growthbook-docs-37). This directly matches the story of gating production flag changes behind approvals. Missing for 10: independent/hands-on confirmation of the approval workflow in practice (only vendor docs cited) and detail on how approvals interact with environment-specific promotion.", "evidenceIds": [ "growthbook-docs-5", "growthbook-docs-16", "growthbook-docs-37", "growthbook-docs-3" ] }, { "productId": "growthbook", "storyId": "audit-trail", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Pricing docs confirm GrowthBook ships an audit log feature, with 'Exportable audit logs' listed as a Pro-tier capability, implying built-in tracking of changes for governance purposes. However, there is no first-party documentation detailing exactly what is captured (who/when/old-new value) or how the audit trail is presented/queried, and the capability is gated behind a paid plan rather than universally available. Missing for 10: dedicated audit-log documentation describing captured fields, retention, and access/export mechanics, plus confirmation it's available outside paid tiers or independent corroboration of its behavior.", "evidenceIds": [ "growthbook-docs-16", "growthbook-docs-37", "growthbook-docs-44" ] }, { "productId": "growthbook", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "GrowthBook exposes a full REST API (growthbook-docs-12) and an official MCP server that bridges AI tools to authenticated REST read/write calls across the whole API (growthbook-docs-21, growthbook-probe-rt-1), which could let an AI agent script operations across many flags/experiments programmatically. However, there is no documented bulk-specific endpoint, batch-update tool, or example of performing many-item operations in one call. Missing for 10: explicit bulk-create/bulk-update API endpoints, documented batch operation examples, and evidence of the MCP server or API actually handling many items in a single request rather than one-at-a-time calls.", "evidenceIds": [ "growthbook-docs-12", "growthbook-docs-21", "growthbook-probe-rt-1" ] }, { "productId": "growthbook", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "GrowthBook lets users define feature-flag 'rules' that automatically apply based on targeting conditions, scheduling, and ramping (docs-22, docs-28, docs-2), and has automated stale-flag detection (docs-4) — these are rule-based automations tied to feature evaluation. However, this falls short of a general 'on event X, trigger action Y' automation engine; there's no evidence of webhook-driven arbitrary actions, external event ingestion, or an automation/workflow layer beyond flag targeting and rollout scheduling. Missing for 10: generic event-trigger/webhook automation outside feature-flag context, evidence of arbitrary action execution on external events, and independent corroboration of rule automation depth.", "evidenceIds": [ "growthbook-docs-22", "growthbook-docs-28", "growthbook-docs-4", "growthbook-docs-2" ] }, { "productId": "growthbook", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only shows one-time scheduling of feature-flag rule start/stop dates and 'ramp schedules' for gradual rollouts (growthbook-docs-22, growthbook-docs-37) — nothing resembling recurring cron-like jobs or workflow automation that an AI-native user could schedule to run repeatedly. No mention of scheduled reports, recurring experiment analyses, or workflow automation triggers.", "evidenceIds": [ "growthbook-docs-22", "growthbook-docs-37", "growthbook-docs-16" ] }, { "productId": "growthbook", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "GrowthBook documents draft feature revisions, request-review, merge-conflict resolution, and required approval before publishing (docs-5, docs-22), and lists approval workflows plus exportable audit logs as paid-tier features (docs-37), which together enable versioning and review of flag/automation changes. However, no explicit rollback mechanism is documented and there's no independent/hands-on corroboration of the revision-history or rollback UI; a community comment even suggests config-as-code/git-based versioning isn't natively supported (growthbook-comm-11). Missing for 10: explicit rollback documentation, hands-on verification of revision history UI, and clarity on git-based config versioning.", "evidenceIds": [ "growthbook-docs-5", "growthbook-docs-22", "growthbook-docs-37", "growthbook-comm-11" ] }, { "productId": "growthbook", "storyId": "create-toggle-flag-fast", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "GrowthBook's quickstart docs show flag creation in a few lines of code with SDK hooks like useFeatureIsOn, and a runtime probe confirms a real keyless create-and-evaluate loop (create org, create flag, mint SDK connection, get correct value back) with no signup friction. Independent community testimony corroborates fast setup, with one user reporting a working flag in about 15 minutes and under 10 lines of code, and environments docs confirm dev/staging/prod toggling. missing for 10: a first-party or independent account of toggling a flag live specifically in a production environment end-to-end.", "evidenceIds": [ "growthbook-docs-1", "growthbook-docs-17", "growthbook-docs-3", "growthbook-comm-2", "growthbook-probe-rt-2" ] }, { "productId": "growthbook", "storyId": "edge-evaluation", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "GrowthBook documents a dedicated Cloudflare Workers edge SDK that evaluates feature flags and Visual Editor experiments at the edge to cut flicker before HTML ships, plus a self-hostable Proxy layer that caches SDK payloads near users and keeps serving flags if the API is down. Missing for 10: independent hands-on evidence specifically of the edge/Cloudflare SDK in production (community evidence covers general SDK/self-host use but not edge-worker deployments) and broader edge-platform coverage beyond Cloudflare.", "evidenceIds": [ "growthbook-docs-14", "growthbook-docs-11", "growthbook-docs-20" ] }, { "productId": "growthbook", "storyId": "experiment-stats-engine", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "GrowthBook documents both Bayesian and Frequentist statistics engines with explicit UI outputs like 'Chance to Win', and its GitHub/community materials detail variance-reduction and rigor features (CUPED, Sequential testing, Post-Stratification, SRM checks) plus rationale for choosing Bayesian to mitigate peeking/Type I error. This is corroborated by both official docs and first-party community explanation, giving a PM strong documented trust in the underlying stats methodology; missing for 10: independent third-party statistical audit or academic validation of the engine's correctness.", "evidenceIds": [ "growthbook-docs-7", "growthbook-docs-24", "growthbook-docs-30", "growthbook-gh-2", "growthbook-gh-6", "growthbook-comm-12" ] }, { "productId": "growthbook", "storyId": "flag-environments", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs explicitly describe dedicated environments (dev/staging/production) with independently toggled flag states [growthbook-docs-3], and the SDK Connection Endpoint provides scoped, readonly keys per environment for SDKs [growthbook-docs-20], with runtime probes confirming a real create-flag/scoped-key/evaluate loop [growthbook-probe-rt-2]. Custom environments are also called out as a paid-tier feature, confirming the capability exists in-product [growthbook-docs-16, growthbook-docs-37]. Missing for 10: independent/community verification specifically of environment-scoped SDK key isolation (evidence is docs/probe only, not third-party confirmation) and more detail on default-environment limits on lower tiers.", "evidenceIds": [ "growthbook-docs-3", "growthbook-docs-20", "growthbook-docs-16", "growthbook-docs-37", "growthbook-probe-rt-2" ] }, { "productId": "growthbook", "storyId": "guarded-rollouts-auto-rollback", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "GrowthBook's pricing pages list a named 'Safe rollouts' feature (Pro+ tier) alongside guardrail-metric support in experiment templates, implying automated regression detection and rollback, but the evidence pack contains no docs explaining how the automatic rollback mechanism actually works (thresholds, trigger logic, rollback speed) nor any hands-on confirmation it fires reliably in production. Missing for 10: detailed docs/description of the Safe Rollouts mechanism and guardrail-triggered auto-rollback, plus independent or hands-on evidence it works as claimed.", "evidenceIds": [ "growthbook-docs-15", "growthbook-docs-31", "growthbook-docs-36", "growthbook-docs-43", "growthbook-docs-8" ] }, { "productId": "growthbook", "storyId": "individual-user-targeting", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "GrowthBook's rule/targeting system lets you write conditions against arbitrary user attributes (e.g., user ID, email) to include or exclude specific users, which covers allowlisting beta testers or internal accounts (growthbook-docs-2, growthbook-docs-22, growthbook-docs-18, growthbook-docs-28). This is a first-class part of its rules engine (scope, ramp, schedule).\nmissing for 10: an explicit doc example/tutorial titled 'allowlist' or 'beta testers' walkthrough, and independent hands-on confirmation of exclusion-list behavior.", "evidenceIds": [ "growthbook-docs-2", "growthbook-docs-22", "growthbook-docs-18", "growthbook-docs-28" ] }, { "productId": "growthbook", "storyId": "local-evaluation-latency", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "GrowthBook's core architecture is a cacheable JSON ruleset that SDKs evaluate locally, confirmed by docs (SDK Connection Endpoint providing readonly flag data for local assignment, in-memory attribute evaluation) and corroborated by a community founder comment describing exactly this model plus a runtime probe showing the keyless payload endpoint serving flag data for local SDK evaluation. Missing for 10: explicit published latency/microsecond benchmarks or independent third-party performance testing.", "evidenceIds": [ "growthbook-docs-20", "growthbook-docs-18", "growthbook-comm-6", "growthbook-probe-rt-2", "growthbook-docs-11" ] }, { "productId": "growthbook", "storyId": "metrics-warehouse-integration", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "GrowthBook explicitly lets teams define Fact Tables and metrics computed directly from their own warehouse (BigQuery, Snowflake, Databricks, etc.) rather than relying on a proprietary metric store, and independent community commentary confirms this warehouse-native design as a key differentiator from black-box tools like Optimizely. missing for 10: hands-on PM-level walkthrough of defining a metric from ingested event data (vs. warehouse tables) and any independent report specifically validating metric-definition UX.", "evidenceIds": [ "growthbook-docs-9", "growthbook-gh-3", "growthbook-gh-7", "growthbook-comm-5", "growthbook-comm-8" ] }, { "productId": "growthbook", "storyId": "multivariate-dynamic-config", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "GrowthBook's SDK docs explicitly show non-boolean flag values via useFeatureValue (string example 'button-color') alongside useFeatureIsOn for booleans, and its targeting/rules docs describe rules that 'control which values a user sees' beyond simple on/off, plus a real runtime probe confirms a flag payload with a typed defaultValue served via the API. Missing for 10: explicit documented example of numeric and JSON-typed flag values (only string is shown in the quickstart snippet), and no independent hands-on confirmation of JSON config usage.", "evidenceIds": [ "growthbook-docs-32", "growthbook-docs-22", "growthbook-docs-28", "growthbook-probe-rt-2", "growthbook-gh-1" ] }, { "productId": "growthbook", "storyId": "openfeature-compatibility", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack describes GrowthBook's own 24 SDKs, REST API, and proxy, but contains no mention of OpenFeature or an OpenFeature provider integration anywhere. Since this axis (avoiding vendor lock-in via OpenFeature) plausibly applies to a flagging platform, the absence of any supporting evidence yields 'none' rather than 'na'.", "evidenceIds": [ "growthbook-gh-1", "growthbook-docs-1", "growthbook-docs-40" ] }, { "productId": "growthbook", "storyId": "openness-api-parity", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "GrowthBook documents a 'full REST API for interacting with the application' (growthbook-docs-12) and a runtime probe confirms it: a self-hosted instance created an org, user, and feature flag purely via API calls and served it through the SDK payload endpoint (growthbook-probe-rt-2), and the official MCP server exposes authenticated read/write meta-tools covering the whole REST API (growthbook-docs-21, growthbook-probe-rt-1). Missing for 10: an explicit enumerated API-vs-UI parity list (e.g. confirmation that experiment approval workflows, visual editor changes, and dashboard configuration are all API-exposed, not just feature flags) and independent third-party corroboration of API completeness.", "evidenceIds": [ "growthbook-docs-12", "growthbook-docs-21", "growthbook-probe-rt-1", "growthbook-probe-rt-2", "growthbook-docs-20" ] }, { "productId": "growthbook", "storyId": "openness-full-export", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "GrowthBook is open-source and self-hostable with its data stored in a standard MongoDB instance, and it exposes a full REST API for programmatic access to feature flags, experiments, and metrics, which together give a technically-inclined user a path to extract all their data without vendor lock-in. However, there is no explicit documented 'export all data' or bulk-export feature/wizard, and 'exportable audit logs' is gated behind an Enterprise tier rather than being a general open-export capability. Missing for 10: an explicit documented full-data export/backup tool or format, confirmation that experiment results/metrics history export in an open format, and evidence this works outside self-hosting for cloud customers.", "evidenceIds": [ "growthbook-docs-12", "growthbook-docs-10", "growthbook-docs-35", "growthbook-docs-37", "growthbook-comm-5" ] }, { "productId": "growthbook", "storyId": "openness-open-license", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Community evidence explicitly states GrowthBook is open source with an optional cloud SaaS offering, and a public GitHub repository (github.com/growthbook/growthbook) is cited, supporting that source is readable. However, no evidence pack item specifies the actual license type (e.g., MIT/Apache) or licensing terms, so the 'open license' claim isn't directly documented. Missing for 10: explicit license identification/text, first-party docs confirming license terms, independent confirmation of license permissiveness.", "evidenceIds": [ "growthbook-comm-5", "growthbook-gh-1", "growthbook-docs-10" ] }, { "productId": "growthbook", "storyId": "openness-self-host", "verdict": "full", "quality": 10, "confidence": "high", "rationale": "GrowthBook is documented and independently confirmed (via runtime probe) to self-host fully via Docker Compose including MongoDB, API, and front-end, with a real keyless create-and-evaluate flag loop working end-to-end; community reviews also corroborate long-term self-hosted usage.", "evidenceIds": [ "growthbook-docs-10", "growthbook-docs-35", "growthbook-probe-rt-2", "growthbook-comm-1" ] }, { "productId": "growthbook", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "GrowthBook can be fully self-hosted via Docker Compose (own MongoDB, API, and front-end), which lets an AI-native user control exactly where all data lives, and a probe confirms a working self-hosted flag create/evaluate loop with no third-party data storage. However, there is no documented cloud-hosted region/residency selection (e.g., EU vs US SaaS options) for users who don't want to self-host. Missing for 10: explicit cloud multi-region hosting options, formal data-residency/compliance certifications, and documentation letting SaaS customers pick a storage region without self-hosting.", "evidenceIds": [ "growthbook-docs-10", "growthbook-docs-35", "growthbook-probe-rt-2", "growthbook-docs-18" ] }, { "productId": "growthbook", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence addresses whether GrowthBook uses customer data to train AI models or offers an opt-out; while self-hosting (growthbook-docs-10) keeps data on-prem, there is no explicit privacy statement about AI training data usage.", "evidenceIds": [] }, { "productId": "growthbook", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "GrowthBook's privacy-by-design targeting (attribute values never sent to GrowthBook, kept in-memory in the SDK) and the option to self-host on your own infrastructure/MongoDB give users indirect control over what data is retained and how it's deleted, since they own the database. However, there is no explicit documentation of a data retention policy, a deletion API/endpoint, or GDPR-style 'delete my data' tooling in the evidence pack. Missing for 10: explicit retention-period settings, a documented deletion/erasure API or workflow, and any compliance certification (e.g., GDPR/CCPA) statements.", "evidenceIds": [ "growthbook-docs-18", "growthbook-docs-10", "growthbook-docs-35" ] }, { "productId": "growthbook", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack shows GrowthBook is self-hostable and that targeting-attribute values are never sent to the vendor, but there is no documentation of any product telemetry/usage-tracking opt-out setting or flag for the self-hosted or cloud application itself. Missing for 10: any explicit telemetry/usage-analytics toggle, docs on what data is phoned home from self-hosted instances, or a documented opt-out mechanism.", "evidenceIds": [] }, { "productId": "growthbook", "storyId": "progressive-percentage-rollouts", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "GrowthBook's rules docs explicitly cover 'rolling out gradually' and describe how rules can 'scope, ramp, and schedule' percentage rollouts, with experiment/flag assignment based on a hashing attribute for consistent bucketing, and SDKs evaluate the cached JSON payload client-side with no redeploy needed. Pricing tiers also list dedicated 'Ramp schedules' as a named capability, confirming this is a first-class shipped feature, and probe evidence confirms a live flag create/evaluate loop via the SDK payload endpoint. missing for 10: independent/hands-on confirmation of an actual 1%→100% ramp in practice, and detail on the specific hashing algorithm ensuring bucket stability across ramp changes", "evidenceIds": [ "growthbook-docs-22", "growthbook-docs-33", "growthbook-docs-38", "growthbook-docs-16", "growthbook-docs-37", "growthbook-docs-25", "growthbook-probe-rt-2" ] }, { "productId": "growthbook", "storyId": "rbac-scoped-access", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Pricing page confirms GrowthBook has role/permission features (\"Advanced permissioning\", \"Advanced access control\", SSO & SCIM) gated behind paid tiers, implying RBAC exists, but there is no docs page detailing how roles map to flag-level permissions or how API tokens can be scoped to restrict flag changes; the API docs even note the SDK connection endpoint is public/unauthenticated, which is a different, more limited access model. Missing for 10: a dedicated permissions/roles doc, explicit description of scoped/restricted API tokens for flag mutation, and independent confirmation of enforcement.", "evidenceIds": [ "growthbook-docs-15", "growthbook-docs-16", "growthbook-docs-36", "growthbook-docs-37", "growthbook-docs-12", "growthbook-docs-41" ] }, { "productId": "growthbook", "storyId": "realtime-flag-updates", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "GrowthBook's docs confirm a single cacheable feature-flag JSON payload that SDKs evaluate locally (growthbook-comm-6), a Proxy that caches payloads to reduce latency and keep serving flags if the API is down (growthbook-docs-11), and environment/rule toggles that could function as kill switches (growthbook-docs-3, growthbook-docs-22). However, no evidence in the pack documents actual streaming/SSE or fast-polling update latency in seconds, nor confirms a tested kill-switch effect at runtime. Missing for 10: explicit documentation of push/streaming update mechanism and propagation-time guarantees, and a hands-on/community test showing a kill switch immediately disabling a flag across connected SDKs.", "evidenceIds": [ "growthbook-docs-11", "growthbook-comm-6", "growthbook-docs-3", "growthbook-docs-22", "growthbook-docs-20" ] }, { "productId": "growthbook", "storyId": "relay-edge-proxy", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "GrowthBook explicitly documents a self-hostable Proxy that runs next to your app to cache SDK payloads, reduce latency, and keep serving flags if the API is down, directly matching the relay/edge proxy story, and this sits alongside full self-host Docker Compose deployment keeping SDK traffic in-network. Community evidence corroborates that evaluation happens locally within SDKs via a cacheable JSON payload, reinforcing offline resilience. Missing for 10: independent hands-on report specifically of the Proxy component (vs. general self-host) failing over during an outage.", "evidenceIds": [ "growthbook-docs-11", "growthbook-docs-10", "growthbook-docs-35", "growthbook-comm-6", "growthbook-docs-20" ] }, { "productId": "growthbook", "storyId": "scheduled-releases", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "GrowthBook explicitly documents that feature rules can be 'scoped, ramped, and scheduled,' and its pricing page lists 'Ramp schedules' as a distinct capability, confirming a future-time scheduling mechanism for flag rollouts. However, this appears gated to paid plans (Pro tier) and the evidence pack gives no detailed walkthrough of how scheduling is configured or any independent/hands-on confirmation it works as described. Missing for 10: detailed documentation of the scheduling UI/API, hands-on or community verification, and clarity on whether it applies broadly to all rule types.", "evidenceIds": [ "growthbook-docs-22", "growthbook-docs-16", "growthbook-docs-37" ] }, { "productId": "growthbook", "storyId": "sdk-language-breadth", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "GrowthBook offers 24 official SDKs spanning backend, web/React, and mobile (Android/iOS) per GitHub README, with docs showing consistent flag-evaluation patterns (useFeatureIsOn, useFeatureValue, targeting attributes evaluated locally) and community reports of quick multi-stack integration (Vue frontend in hours). An older 2021 community comment noted no native mobile SDKs yet, but this is superseded by the more recent GitHub claim explicitly listing Android/iOS SDKs, so it's not a live contradiction. Missing for 10: independent verification of identical targeting/rollout behavior across all 24 SDKs, and more recent hands-on confirmation of mobile SDK maturity/parity with backend/web SDKs.", "evidenceIds": [ "growthbook-gh-1", "growthbook-docs-1", "growthbook-docs-17", "growthbook-docs-32", "growthbook-docs-28", "growthbook-comm-4", "growthbook-comm-13" ] }, { "productId": "growthbook", "storyId": "self-host-open-source", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "GrowthBook documents a self-host distribution via Docker Compose (MongoDB, API, front-end) and this is corroborated by a hands-on probe showing a full keyless self-host boot, feature creation, and SDK payload evaluation working end-to-end on local infrastructure, plus community confirmation of production self-hosted usage. missing for 10: independent third-party audit of data residency guarantees beyond vendor docs/probe.", "evidenceIds": [ "growthbook-docs-10", "growthbook-docs-35", "growthbook-probe-rt-2", "growthbook-comm-1" ] }, { "productId": "growthbook", "storyId": "stale-flag-cleanup", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "GrowthBook's docs explicitly describe automatic stale flag detection ('GrowthBook can automatically detect these flags so you can clean them up') as a dedicated feature, directly addressing the story of finding stale flags for removal. This is corroborated by the product's broader flag-management infrastructure (environments, rules, revisions) but lacks detail on code-reference scanning across repos or independent/hands-on verification of the stale-detection feature itself. Missing for 10: evidence of code-reference/usage scanning across the codebase, and independent/community confirmation that stale detection works reliably in practice.", "evidenceIds": [ "growthbook-docs-4", "growthbook-docs-22", "growthbook-docs-3" ] }, { "productId": "growthbook", "storyId": "targeting-rules-segments", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "GrowthBook clearly supports attribute-based targeting rules (conditions evaluated against SDK-passed attributes, rule scoping/ramping/scheduling) as shown in growthbook-docs-2, -22, -28, and -18. However, the evidence pack never mentions a distinct 'segments' feature (reusable named audience definitions) that developers could apply across multiple flags/rules, so the reusable-segment half of the story is unevidenced. missing for 10: explicit segment/audience-reuse documentation, evidence of applying the same segment across multiple flags, independent confirmation of segment behavior.", "evidenceIds": [ "growthbook-docs-2", "growthbook-docs-22", "growthbook-docs-28", "growthbook-docs-18" ] }, { "productId": "growthbook", "storyId": "transparent-pricing-limits", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "GrowthBook publishes a tiered pricing page (Starter/Pro/Enterprise-like tiers) listing feature gates (visual editor, approval workflows, SSO/SCIM, etc.) and calls out unlimited flags/experiments/traffic and per-user caps like 'Up to 3 users', but none of the evidence shows usage-based cost drivers like MAUs, events, or API requests being priced or metered — the plans appear seat/feature-gated rather than usage-metered. missing for 10: explicit cost drivers tied to MAUs/events/requests, a pricing calculator or usage-based cost breakdown, and independent confirmation that published prices match real invoices.", "evidenceIds": [ "growthbook-docs-15", "growthbook-docs-16", "growthbook-docs-26", "growthbook-docs-31", "growthbook-docs-36", "growthbook-docs-37", "growthbook-docs-39", "growthbook-docs-44" ] }, { "productId": "launchdarkly", "storyId": "ab-experiments", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "LaunchDarkly's docs explicitly cover A/B Tests and Experiments, metric type selection, and Bayesian statistics for deciding a winning variation, directly matching the PM story of running experiments on flags and seeing which variation wins on metrics. Community evidence corroborates multivariate flags and rollout targeting as real, used features. Missing for 10: independent hands-on account of running a full experiment and reading results dashboards, and more detail on multivariate (not just binary) experiment support.", "evidenceIds": [ "launchdarkly-docs-27", "launchdarkly-docs-30", "launchdarkly-docs-8", "launchdarkly-docs-34", "launchdarkly-comm-9" ] }, { "productId": "launchdarkly", "storyId": "agent-experiment-analysis", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "LaunchDarkly documents an Experimentation feature with Bayesian statistical analysis, exposes API access tokens, an official CLI, and a hosted/local MCP server that an AI agent could use to read flag/experiment configuration data programmatically. However, no evidence explicitly confirms the MCP server or API exposes experiment results/metrics data (vs. just flag targeting) or that an agent has been shown reading such data to produce ship/rollback recommendations. Missing for 10: explicit documentation of an experiment-results API/MCP tool, and a hands-on example of an agent summarizing experiment outcomes or recommending ship/rollback decisions.", "evidenceIds": [ "launchdarkly-docs-30", "launchdarkly-docs-34", "launchdarkly-docs-37", "launchdarkly-docs-10", "launchdarkly-probe-4", "launchdarkly-probe-rt-2" ] }, { "productId": "launchdarkly", "storyId": "agent-toggles-flag-safely", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "LaunchDarkly documents an official CLI (ldcli) and both local and hosted MCP servers for programmatic flag creation/toggling, backed by API access tokens and RBAC roles, and runtime probes confirm the CLI and MCP handshakes actually work as documented. Separately, LaunchDarkly's Approvals feature lets teams require review/sign-off before flag changes go live, and Change History provides an audit trail, which together satisfy the 'force changes through approval workflows' requirement. Missing for 10: explicit documentation tying the Approvals workflow specifically to API/CLI/MCP-originated changes (vs. UI-only), and independent/hands-on confirmation that an agent's API write is actually blocked pending approval.", "evidenceIds": [ "launchdarkly-docs-6", "launchdarkly-docs-7", "launchdarkly-docs-10", "launchdarkly-docs-11", "launchdarkly-docs-12", "launchdarkly-docs-36", "launchdarkly-docs-37", "launchdarkly-probe-4", "launchdarkly-probe-5", "launchdarkly-probe-rt-1", "launchdarkly-probe-rt-2", "launchdarkly-probe-rt-3" ] }, { "productId": "launchdarkly", "storyId": "agentic-agent-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Direct probes show llms.txt and docs.md both return 404, and no evidence of any agent-oriented documentation endpoint; while LaunchDarkly ships an MCP server and CLI, these do not satisfy the specific 'llms.txt or agent-oriented docs' story.", "evidenceIds": [ "launchdarkly-probe-1", "launchdarkly-probe-2" ] }, { "productId": "launchdarkly", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence shows LaunchDarkly's AI-related features (AI Configs, 'AI Runs', LLM-as-judge evals) are about managing and evaluating AI/LLM application behavior via flags, not about the product itself surfacing AI-generated insights or suggestions from a user's flag/experiment/metrics data. No docs mention anomaly detection copilots, natural-language querying, or AI-generated recommendations inside the dashboard.", "evidenceIds": [ "launchdarkly-docs-15", "launchdarkly-docs-16", "launchdarkly-docs-5", "launchdarkly-docs-34" ] }, { "productId": "launchdarkly", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "LaunchDarkly supports autonomous background automations like scheduled flag changes and guarded rollouts that automatically monitor metrics and roll back without human intervention, plus webhooks that can trigger downstream automation, and its MCP server lets an AI agent configure these. But there's no evidence of a general-purpose agentic automation/orchestration capability beyond release-management rules. Missing for 10: evidence of AI agents autonomously creating/managing ongoing automations via MCP (only initial handshake is verified), broader agentic workflow orchestration beyond flag/rollout rules, and independent confirmation these autonomous rollbacks work reliably in practice.", "evidenceIds": [ "launchdarkly-docs-35", "launchdarkly-docs-5", "launchdarkly-docs-38", "launchdarkly-docs-10", "launchdarkly-probe-rt-2" ] }, { "productId": "launchdarkly", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "LaunchDarkly's evidence shows AI-related feature-flag tooling (AI Runs, LLM evaluation flags) and an MCP server that lets external AI agents call into LaunchDarkly as a tool provider, but there is no evidence of a built-in AI assistant inside the product itself that a user can delegate tasks to.", "evidenceIds": [ "launchdarkly-docs-15", "launchdarkly-docs-16", "launchdarkly-docs-10", "launchdarkly-docs-11" ] }, { "productId": "launchdarkly", "storyId": "agentic-headless", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "LaunchDarkly ships an official CLI (ldcli) confirmed to run keylessly in a runtime probe, plus API access tokens, SDKs, and webhooks that together support headless/CI automation of flag management and evaluation. Missing for 10: explicit CI/CD pipeline integration examples (e.g., GitHub Actions recipes) or documented non-interactive CLI auth flows for pipelines.", "evidenceIds": [ "launchdarkly-probe-rt-1", "launchdarkly-docs-12", "launchdarkly-docs-37", "launchdarkly-docs-38", "launchdarkly-docs-24" ] }, { "productId": "launchdarkly", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "LaunchDarkly is a feature-flag/experimentation SaaS platform, not an agentic assistant that consumes external tools; all evidence shows LaunchDarkly instead exposing its own functionality as an MCP server for other agents to call, which is the reverse of this client-side story.", "evidenceIds": [] }, { "productId": "launchdarkly", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "LaunchDarkly documents and ships both a local npm MCP server and a hosted MCP server endpoint, and runtime probes confirm the npm server completes a full stdio initialize handshake and the hosted endpoint is live and auth-gated as documented. Missing for 10: independent third-party (non-vendor) hands-on review of the MCP server's tool functionality beyond the handshake.", "evidenceIds": [ "launchdarkly-docs-10", "launchdarkly-docs-11", "launchdarkly-probe-4", "launchdarkly-probe-rt-2", "launchdarkly-probe-rt-3" ] }, { "productId": "launchdarkly", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "LaunchDarkly ships an official MCP server (local npm package and hosted endpoint) that exposes flag/targeting/rollout operations to AI agents, letting an AI-native user issue natural-language commands through an MCP-compatible client; runtime probes confirm the local server completes a full stdio handshake and the hosted endpoint is live and auth-gated as documented. Missing for 10: no evidence of actual authenticated tool-call transcripts showing natural-language commands successfully executing flag changes, and no independent/community corroboration of the MCP workflow in practice.", "evidenceIds": [ "launchdarkly-docs-10", "launchdarkly-docs-11", "launchdarkly-probe-4", "launchdarkly-probe-rt-2", "launchdarkly-probe-rt-3" ] }, { "productId": "launchdarkly", "storyId": "agentic-official-cli", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "LaunchDarkly ships an official CLI (`ldcli`), documented and verified via a hands-on runtime probe that installed via Homebrew and successfully ran `ldcli --version` keylessly, confirming it is real and functional. Missing for 10: independent third-party community commentary specifically evaluating the CLI's AI-native/agentic workflows.", "evidenceIds": [ "launchdarkly-docs-12", "launchdarkly-probe-5", "launchdarkly-probe-rt-1" ] }, { "productId": "launchdarkly", "storyId": "agentic-public-api", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "LaunchDarkly documents API access tokens for programmatic access (launchdarkly-docs-37), an official CLI (launchdarkly-docs-12/probe-5) and both local and hosted MCP servers that were verified live at runtime (launchdarkly-probe-rt-1/2/3), all of which wrap the public API for automated/agentic control. Missing for 10: a discoverable OpenAPI/swagger spec (probe-3 shows 404s at standard paths) and explicit REST API reference docs in the evidence pack.", "evidenceIds": [ "launchdarkly-docs-37", "launchdarkly-docs-12", "launchdarkly-probe-5", "launchdarkly-probe-rt-1", "launchdarkly-probe-rt-2", "launchdarkly-probe-rt-3", "launchdarkly-probe-3" ] }, { "productId": "launchdarkly", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "LaunchDarkly documents API access tokens and custom role assignment, which together let a user create scoped, least-privilege credentials for any client — and the MCP server explicitly requires an API key for tool calls (agent authenticates via this token), confirming the mechanism applies to AI-agent use. However, no docs specifically address agent-oriented token types, short-lived/ephemeral credentials, or per-tool scoping tailored to autonomous agents. missing for 10: agent-specific credential type or lifecycle, fine-grained per-tool/action scoping beyond general custom roles, independent verification that scoped tokens work end-to-end with the MCP server.", "evidenceIds": [ "launchdarkly-docs-37", "launchdarkly-docs-36", "launchdarkly-probe-rt-2", "launchdarkly-probe-rt-3", "launchdarkly-docs-10" ] }, { "productId": "launchdarkly", "storyId": "agentic-sdks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "LaunchDarkly documents 30 idiomatic SDKs, OpenFeature providers, and edge SDKs (e.g., Cloudflare) plus a CLI and MCP server for AI-native/agentic workflows, giving developers official building blocks across languages and AI tooling. missing for 10: independent hands-on developer reviews specifically praising SDK quality/documentation depth, and no evidence of AI-specific SDK features (e.g., LLM-context helpers) beyond general SDK/OpenFeature support.", "evidenceIds": [ "launchdarkly-docs-24", "launchdarkly-docs-33", "launchdarkly-docs-14", "launchdarkly-docs-12", "launchdarkly-docs-10", "launchdarkly-probe-rt-1", "launchdarkly-probe-rt-2" ] }, { "productId": "launchdarkly", "storyId": "agentic-webhooks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "LaunchDarkly documents a dedicated Webhooks feature under infrastructure integrations, allowing subscription to flag/project events for automation, which directly supports agentic/event-driven workflows. Missing for 10: independent hands-on confirmation of webhook payload structure/reliability and no evidence of fine-grained event filtering options.", "evidenceIds": [ "launchdarkly-docs-38" ] }, { "productId": "launchdarkly", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows only static docs pages and confirms no OpenAPI/swagger spec discoverable (probe-3 all 404s) and no llms.txt/docs.md; nothing indicates an interactive, runnable API reference/playground exists.", "evidenceIds": [ "launchdarkly-probe-3", "launchdarkly-probe-1", "launchdarkly-probe-2" ] }, { "productId": "launchdarkly", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows explicit probes for an OpenAPI/swagger spec at LaunchDarkly's standard candidate paths all returning 404, and no other citation surfaces a downloadable machine-readable API spec (only human-readable API access token docs are referenced). Despite LaunchDarkly having a public REST API in reality, this evidence pack contains no proof of an accessible OpenAPI spec.", "evidenceIds": [ "launchdarkly-probe-3", "launchdarkly-docs-37" ] }, { "productId": "launchdarkly", "storyId": "api-sandbox", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "LaunchDarkly's Environments feature explicitly separates test/staging from production, letting users evaluate and target flags in a non-production environment, and 'Testing changes to flag targeting' documents workflows to validate rule changes safely before they affect live traffic. Missing for 10: no explicit 'sandbox for AI-native/agent testing' framing, no independent/hands-on confirmation that environment data is fully isolated from production telemetry.", "evidenceIds": [ "launchdarkly-docs-9", "launchdarkly-docs-3", "launchdarkly-docs-6", "launchdarkly-docs-7" ] }, { "productId": "launchdarkly", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows API access tokens and CLI/MCP tooling but no documentation of API versioning scheme or a deprecation policy; OpenAPI probes returned 404s, providing no confirmation of a discoverable, versioned API spec.", "evidenceIds": [ "launchdarkly-docs-37", "launchdarkly-probe-3" ] }, { "productId": "launchdarkly", "storyId": "approvals-change-requests", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "LaunchDarkly documents a dedicated Approvals feature requiring sign-off before flag changes are applied, plus Scheduled changes and Change history for governance/audit trails around production flag modifications, backed by role-based access control to enforce who can approve. missing for 10: independent hands-on confirmation of the approval workflow in practice and details on customizable approval policies (e.g., required approver count, bypass rules).", "evidenceIds": [ "launchdarkly-docs-6", "launchdarkly-docs-7", "launchdarkly-docs-35", "launchdarkly-docs-36" ] }, { "productId": "launchdarkly", "storyId": "audit-trail", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "LaunchDarkly documents 'Change history' for flags, which tracks flag configuration changes, and community mentions of Slack integration for visibility into changes corroborates change tracking exists; however, no evidence explicitly confirms a full audit log capturing who/when/old-value/new-value in a governance-focused, exportable/immutable format. Missing for 10: explicit documentation of audit-log fields (actor, timestamp, before/after value), retention/export/SIEM integration details, and independent confirmation of completeness.", "evidenceIds": [ "launchdarkly-docs-7", "launchdarkly-comm-9" ] }, { "productId": "launchdarkly", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "LaunchDarkly exposes a CLI, REST API tokens, and an MCP server that could be scripted by an AI-native user to act on many flags/segments programmatically, but no documented UI or API feature explicitly named 'bulk operations' (e.g., bulk edit/bulk archive across flags) appears in the evidence pack. Missing for 10: explicit bulk-edit/bulk-update API endpoints or UI, documentation of batch operations across contexts/segments, and independent confirmation of bulk-scale usage.", "evidenceIds": [ "launchdarkly-docs-12", "launchdarkly-docs-37", "launchdarkly-probe-4", "launchdarkly-probe-5", "launchdarkly-probe-rt-1", "launchdarkly-probe-rt-2" ] }, { "productId": "launchdarkly", "storyId": "automation-rules-engine", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "LaunchDarkly ships rule-based targeting (custom rules, adaptive triggers), scheduled flag changes, and webhooks that let flag state changes or conditions automatically trigger downstream actions/integrations — directly matching the 'rules trigger actions on events' story. Missing for 10: independent/hands-on verification of adaptive-trigger and webhook automation behavior in practice, and clearer documentation of arbitrary event-driven (vs. flag-state) triggers.", "evidenceIds": [ "launchdarkly-docs-26", "launchdarkly-docs-29", "launchdarkly-docs-35", "launchdarkly-docs-38", "launchdarkly-docs-1" ] }, { "productId": "launchdarkly", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "LaunchDarkly documents one-time 'Scheduled flag changes' (docs-35) but there is no evidence of recurring/cron-style job or workflow scheduling; its CLI, API, and MCP server (docs-10/11/12) could be used by external automation but no first-party recurring scheduler is documented.", "evidenceIds": [ "launchdarkly-docs-35", "launchdarkly-docs-12", "launchdarkly-docs-10" ] }, { "productId": "launchdarkly", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "LaunchDarkly's flags (its closest analog to 'automations') have change history, approvals, scheduled changes, and rollback via targeting/rollout controls, which cover versioning and review workflows for flag changes. However, this is framed around feature flags/rollouts, not AI agent automations specifically, and there's no evidence of a dedicated automation-versioning UI, diff/review workflow for AI-agent-driven changes, or audit trail tied to AI-native usage. missing for 10: explicit versioning/rollback UI for AI-generated automations, review/approval workflow tied to AI agent changes, independent confirmation that change history supports full rollback of complex rule sets.", "evidenceIds": [ "launchdarkly-docs-6", "launchdarkly-docs-7", "launchdarkly-docs-35", "launchdarkly-docs-4", "launchdarkly-docs-5" ] }, { "productId": "launchdarkly", "storyId": "create-toggle-flag-fast", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "LaunchDarkly's docs explicitly cover creating flags, targeting rules, percentage rollouts, and environments—the core primitives needed to create and toggle a flag live in production—and community accounts confirm real-world use of rules-based targeting and rollout by percentage/attributes with minimal latency. However, missing for 10: a documented quickstart/time-to-first-flag walkthrough proving the 'minutes after signup' claim, and independent hands-on confirmation of onboarding speed (one community comment even calls the product generally 'a pain to use').", "evidenceIds": [ "launchdarkly-docs-19", "launchdarkly-docs-23", "launchdarkly-docs-9", "launchdarkly-docs-17", "launchdarkly-comm-9", "launchdarkly-comm-8", "launchdarkly-comm-10" ] }, { "productId": "launchdarkly", "storyId": "edge-evaluation", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "LaunchDarkly documents a dedicated Cloudflare edge SDK (launchdarkly-docs-14) for evaluating flags directly in CDN workers, plus a self-hosted Relay Proxy (launchdarkly-docs-13/32, corroborated by community usage in launchdarkly-comm-2) that lets platform engineers run an edge/relay layer close to users for low-latency evaluation. Missing for 10: independent hands-on benchmarks of edge SDK latency/behavior and broader coverage of other edge platforms beyond Cloudflare.", "evidenceIds": [ "launchdarkly-docs-14", "launchdarkly-docs-13", "launchdarkly-docs-32", "launchdarkly-comm-2" ] }, { "productId": "launchdarkly", "storyId": "experiment-stats-engine", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "LaunchDarkly documents a Bayesian statistics engine for experiment results and a dedicated experimentation/metrics framework, giving PMs a documented methodology behind results. However, the evidence pack shows no explicit mention of a frequentist alternative or variance-reduction techniques (e.g., CUPED), and there's no independent/hands-on corroboration of the engine's real-world reliability. missing for 10: frequentist option documentation, variance-reduction feature details, independent validation of statistical results.", "evidenceIds": [ "launchdarkly-docs-34", "launchdarkly-docs-30", "launchdarkly-docs-8" ] }, { "productId": "launchdarkly", "storyId": "flag-environments", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "LaunchDarkly's docs explicitly cover Environments as a first-class construct with independent flag states per environment, and API access tokens/roles docs indicate scoped SDK keys per environment; community evidence corroborates the SDK/streaming architecture supporting per-environment keys. Missing for 10: no independent hands-on confirmation of scoped SDK key isolation across environments beyond docs.", "evidenceIds": [ "launchdarkly-docs-9", "launchdarkly-docs-37", "launchdarkly-docs-36", "launchdarkly-comm-9" ] }, { "productId": "launchdarkly", "storyId": "guarded-rollouts-auto-rollback", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "LaunchDarkly's Guarded Rollouts feature explicitly monitors metrics during a progressive rollout and automatically rolls back on regression detection, directly matching the story, and is backed by supporting docs on metrics, progressive rollouts, and change history. However, the evidence pack is entirely first-party docs with no independent/hands-on confirmation that automatic rollback actually triggers correctly in practice, and no detail on metric types supported for regression detection or configuration limits. Missing for 10: independent/hands-on validation of guarded rollout auto-rollback behavior, detail on which metric types/thresholds trigger rollback, and confirmation of reliability at scale.", "evidenceIds": [ "launchdarkly-docs-5", "launchdarkly-docs-4", "launchdarkly-docs-8", "launchdarkly-docs-7" ] }, { "productId": "launchdarkly", "storyId": "individual-user-targeting", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "LaunchDarkly's docs explicitly cover 'Individual targeting' and 'Target with flags' allowing developers to include/exclude specific users by key, plus segments and custom rules for building allowlists (beta testers, internal accounts), corroborated by community mention of rules-based group-level targeting. Missing for 10: independent hands-on verification of individual-user allowlist workflow, dedicated example of internal-account exclusion in practice.", "evidenceIds": [ "launchdarkly-docs-1", "launchdarkly-docs-2", "launchdarkly-docs-31", "launchdarkly-docs-21", "launchdarkly-docs-29", "launchdarkly-comm-9" ] }, { "productId": "launchdarkly", "storyId": "local-evaluation-latency", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Community evidence confirms LaunchDarkly's core streaming architecture — SDKs download the full flag ruleset once and then stream updates, enabling local evaluation without a per-flag network call (launchdarkly-comm-2, launchdarkly-comm-9), and the Relay Proxy exists specifically to cache rulesets on-prem for resiliency (launchdarkly-docs-13, launchdarkly-docs-32). However there is no first-party SDK doc in this pack explicitly describing the local-evaluation/streaming-store mechanism or microsecond latency claims, and one community comment raises concern about network dependency per request (launchdarkly-comm-6), showing the story is not fully substantiated end-to-end. Missing for 10: dedicated SDK architecture docs describing the in-memory flag store/streaming model, explicit benchmark/latency numbers, and resolution of the community concern about request-time dependency.", "evidenceIds": [ "launchdarkly-comm-2", "launchdarkly-comm-9", "launchdarkly-docs-13", "launchdarkly-docs-32", "launchdarkly-comm-6" ] }, { "productId": "launchdarkly", "storyId": "metrics-warehouse-integration", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "LaunchDarkly's experimentation docs (metrics, Bayesian stats, A/B tests) describe metrics defined via LaunchDarkly's own event tracking and metric store; there is no evidence of connecting to a customer's data warehouse or ingesting external event tables to define metrics.", "evidenceIds": [ "launchdarkly-docs-8", "launchdarkly-docs-30", "launchdarkly-docs-34", "launchdarkly-docs-27" ] }, { "productId": "launchdarkly", "storyId": "multivariate-dynamic-config", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Community evidence explicitly confirms multivariate flags as a key LaunchDarkly benefit (launchdarkly-comm-9), and docs reference custom rules, targeting, and percentage rollouts that imply non-boolean variation support, but no evidence pack item directly documents string/number/JSON typed flag variations or dynamic config value usage. missing for 10: explicit docs on flag variation types (string/number/JSON), examples of dynamic config values, independent hands-on confirmation beyond the single HN mention.", "evidenceIds": [ "launchdarkly-comm-9", "launchdarkly-docs-1", "launchdarkly-docs-23", "launchdarkly-docs-29" ] }, { "productId": "launchdarkly", "storyId": "openfeature-compatibility", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "LaunchDarkly documents official OpenFeature providers (launchdarkly-docs-33) alongside its 30 idiomatic SDKs, letting platform engineers write against the vendor-neutral OpenFeature API instead of a proprietary SDK surface. missing for 10: independent hands-on confirmation of OpenFeature provider behavior/parity across languages and any community discussion validating the OpenFeature integration in practice.", "evidenceIds": [ "launchdarkly-docs-33", "launchdarkly-docs-24" ] }, { "productId": "launchdarkly", "storyId": "openness-api-parity", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "LaunchDarkly documents API access tokens for programmatic access, an official CLI (ldcli, verified installable and working) and both a local and hosted MCP server (verified live and functioning), indicating broad programmatic control over flags, targeting, and other resources. However, the evidence pack never explicitly states or demonstrates full parity between UI and API (e.g., no OpenAPI/swagger spec was discoverable, and no docs confirm every UI feature—like approvals, guarded rollouts, or experimentation setup—is API-exposed). Missing for 10: an explicit statement or reference doc confirming 100% UI/API feature parity, and evidence covering API support for approvals, guarded rollouts, and experimentation configuration specifically.", "evidenceIds": [ "launchdarkly-docs-37", "launchdarkly-docs-12", "launchdarkly-probe-4", "launchdarkly-probe-5", "launchdarkly-probe-rt-1", "launchdarkly-probe-rt-2", "launchdarkly-probe-rt-3", "launchdarkly-probe-3" ] }, { "productId": "launchdarkly", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "LaunchDarkly offers API access tokens and a CLI (ldcli) that could programmatically read flag/segment configurations, but no evidence pack item documents a dedicated bulk data-export feature, an open-format export tool, or a stated data-portability/exit policy for account data.", "evidenceIds": [ "launchdarkly-docs-37", "launchdarkly-docs-12" ] }, { "productId": "launchdarkly", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "LaunchDarkly is a closed-source SaaS platform; evidence shows only proprietary docs, SDKs (some open-source client SDKs and CLI/relay proxy tools exist per community notes, but the core product source is not open-licensed), with no indication the core platform source is available under an open license.", "evidenceIds": [ "launchdarkly-comm-2" ] }, { "productId": "launchdarkly", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "LaunchDarkly is a hosted SaaS platform; the only self-hostable component is the Relay Proxy, a caching/streaming layer that still depends on the LaunchDarkly SaaS control plane for flag configuration, not the core product itself. No evidence of a self-hostable core (dashboard, rules engine, data store) exists in the pack.", "evidenceIds": [ "launchdarkly-docs-13", "launchdarkly-docs-32", "launchdarkly-comm-2", "launchdarkly-comm-6" ] }, { "productId": "launchdarkly", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack items address data residency, region selection, or storage location controls for LaunchDarkly; the only tangential item is 'federal environments' docs, which is not about general data residency choice. missing for 10: documentation of selectable data regions/residency, EU/US hosting options, or any data localization controls.", "evidenceIds": [ "launchdarkly-docs-39" ] }, { "productId": "launchdarkly", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "LaunchDarkly is a feature-flag/experimentation platform, not an AI model provider or data-processing service that trains models on user data; 'prevent my data from being used to train AI models' is a category mismatch for this product type.", "evidenceIds": [] }, { "productId": "launchdarkly", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "LaunchDarkly is a feature-flag/experimentation platform; while it stores user/context data (contexts, segments), no evidence in the pack addresses data retention policies, deletion controls, or privacy/compliance tooling for AI-native users. missing for 10: data retention policy docs, deletion/erasure API or workflow, GDPR/CCPA data-subject-request handling, retention configuration options.", "evidenceIds": [] }, { "productId": "launchdarkly", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item addresses telemetry/usage-tracking opt-out for LaunchDarkly itself; the docs cover flagging, targeting, MCP/CLI tooling, and pricing but nothing about the product's own data collection practices or an opt-out mechanism.", "evidenceIds": [] }, { "productId": "launchdarkly", "storyId": "progressive-percentage-rollouts", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "LaunchDarkly's core product is percentage-based progressive rollouts with consistent bucketing via context attributes, explicitly documented (progressive rollouts, percentage rollouts, guarded rollouts) and corroborated by community users citing 'custom rollout by % of users or attributes' as a key benefit, all changed instantly without redeploy through the flag dashboard/API. Missing for 10: no independent hands-on verification of bucketing consistency across ramp changes or a documented technical explanation of the hashing algorithm.", "evidenceIds": [ "launchdarkly-docs-4", "launchdarkly-docs-5", "launchdarkly-docs-23", "launchdarkly-comm-9" ] }, { "productId": "launchdarkly", "storyId": "rbac-scoped-access", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "LaunchDarkly documents role-based access control ('Assigning roles to members') and scoped API access tokens ('API access tokens'), directly matching the story's request for roles, permissions, and scoped tokens to restrict who can change flags. Missing for 10: independent/hands-on verification of custom role granularity and confirmation of per-token flag/project scoping restrictions in practice, and no community corroboration of this specific governance feature.", "evidenceIds": [ "launchdarkly-docs-36", "launchdarkly-docs-37" ] }, { "productId": "launchdarkly", "storyId": "realtime-flag-updates", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "LaunchDarkly's SDK architecture is documented and independently corroborated as a streaming (SSE) model where clients download all flags then receive real-time updates, with an optional self-hosted Relay Proxy for resiliency — this is the mechanism that lets a flag toggle (kill switch) propagate quickly to connected SDKs. Community commentary explicitly calls out 'centralized feature flag service via SSE with minimal latency' as a key benefit, though some users note occasional API/SDK timeout issues that are latency-adjacent concerns rather than a contradiction of the streaming design. Missing for 10: first-party documentation with explicit propagation-time benchmarks (e.g. 'seconds'), explicit 'kill switch' terminology/case study, and hands-on confirmation of polling fallback speed.", "evidenceIds": [ "launchdarkly-comm-2", "launchdarkly-comm-9", "launchdarkly-docs-13", "launchdarkly-docs-32", "launchdarkly-comm-5" ] }, { "productId": "launchdarkly", "storyId": "relay-edge-proxy", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "LaunchDarkly documents the Relay Proxy as a self-hosted Go application that runs on customer infrastructure to serve flags and cache SDK connections, explicitly for resiliency, and this is independently corroborated by community commentary describing it as open source and used 'for extra resiliency' when LaunchDarkly's servers are unreachable, keeping SDK traffic within the customer's network. Missing for 10: independent hands-on verification of offline-serving behavior during vendor outage, and explicit first-party detail on network-isolation guarantees beyond general resiliency claims.", "evidenceIds": [ "launchdarkly-docs-13", "launchdarkly-docs-32", "launchdarkly-comm-2" ] }, { "productId": "launchdarkly", "storyId": "scheduled-releases", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "LaunchDarkly documents a dedicated 'Scheduled flag changes' feature allowing flag changes to be set to take effect at a future time, alongside related release tooling like progressive/guarded rollouts and change history for tracking scheduled edits. Missing for 10: independent hands-on confirmation or community corroboration of the scheduling workflow specifically (only vendor docs cited).", "evidenceIds": [ "launchdarkly-docs-35", "launchdarkly-docs-4", "launchdarkly-docs-5", "launchdarkly-docs-7" ] }, { "productId": "launchdarkly", "storyId": "sdk-language-breadth", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "LaunchDarkly documents '30 idiomatic SDKs' across languages/platforms, OpenFeature provider support, edge/Cloudflare SDKs, and a Relay Proxy for consistent flag delivery, and community evidence independently confirms a working mobile client SDK and consistent streaming/flag-download behavior across SDKs. Missing for 10: an explicit enumerated list of backend/web/mobile SDK names in the pack and independent hands-on cross-platform consistency testing (community notes mobile SDK's MAU-based pricing as a practical caveat, not a functional gap).", "evidenceIds": [ "launchdarkly-docs-24", "launchdarkly-docs-33", "launchdarkly-docs-13", "launchdarkly-docs-32", "launchdarkly-docs-14", "launchdarkly-comm-1", "launchdarkly-comm-2" ] }, { "productId": "launchdarkly", "storyId": "self-host-open-source", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "LaunchDarkly's evidence only describes an open-source Relay Proxy that caches/streams flag data from the LaunchDarkly cloud for resiliency and low latency, not a full self-hostable flag-management platform; the control plane, rules engine, and dashboard remain SaaS-hosted, and community comments explicitly flag this as a third-party critical-path dependency rather than a self-hosted deployment (launchdarkly-docs-13, launchdarkly-comm-2, launchdarkly-comm-6). No evidence shows an OSS distribution of the full platform keeping evaluation data entirely on customer infrastructure.", "evidenceIds": [ "launchdarkly-docs-13", "launchdarkly-docs-32", "launchdarkly-comm-2", "launchdarkly-comm-6" ] }, { "productId": "launchdarkly", "storyId": "stale-flag-cleanup", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "LaunchDarkly documents a dedicated 'Code References' feature that shows where flags are used in code, corroborated by independent community commentary ([launchdarkly-docs-22], [launchdarkly-comm-3]), which directly supports finding code references for stale flags. However, the evidence pack lacks explicit documentation of flag-status/stale-flag detection (e.g., 'last evaluated' tracking or automated stale-flag alerts) to complete the workflow. Missing for 10: dedicated stale-flag status/health dashboard docs, evidence of automated staleness alerts, and independent confirmation that code-reference removal workflows are reliably used to clean up flags in practice.", "evidenceIds": [ "launchdarkly-docs-22", "launchdarkly-comm-3" ] }, { "productId": "launchdarkly", "storyId": "targeting-rules-segments", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "LaunchDarkly's docs and community evidence confirm attribute-based targeting rules, contexts, individual targeting, custom rules, percentage rollouts, and reusable segments (including converting rules into segments), directly matching the story; community input corroborates rules-based targeting by group/attributes in production use. missing for 10: independent hands-on verification of segment reuse across multiple flags beyond docs claims.", "evidenceIds": [ "launchdarkly-docs-1", "launchdarkly-docs-2", "launchdarkly-docs-20", "launchdarkly-docs-21", "launchdarkly-docs-25", "launchdarkly-docs-29", "launchdarkly-docs-31", "launchdarkly-comm-9" ] }, { "productId": "launchdarkly", "storyId": "transparent-pricing-limits", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "LaunchDarkly's public pricing page confirms a usage-based model (unlimited seats, no per-seat pricing) and lists some usage components like AI Runs and session-replay/error caps, plus community confirmation that mobile MAUs factor into cost. However, the evidence pack lacks concrete published tiers/numbers for events or API requests, and community commentary calls the pricing 'incredibly over priced' and confusing to reason about ahead of committing.\n\nmissing for 10: detailed published rate tables for MAUs/events/requests, a cost calculator or worked pricing examples, and independent confirmation that the pricing page is fully transparent pre-sales.", "evidenceIds": [ "launchdarkly-docs-15", "launchdarkly-docs-16", "launchdarkly-docs-17", "launchdarkly-docs-18", "launchdarkly-docs-28", "launchdarkly-comm-1", "launchdarkly-comm-7" ] }, { "productId": "statsig", "storyId": "ab-experiments", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Statsig's docs directly describe running A/B/n and multivariate experiments with primary/secondary metrics via the Scorecard, plus Layers for managing multiple experiments and Autotune for identifying/shifting toward winning variants, and community feedback corroborates its statistical rigor versus competitors. Missing for 10: independent hands-on validation of the results/analysis UI itself (e.g., screenshots or third-party review of the 'winning variation' scorecard output) rather than only docs.", "evidenceIds": [ "statsig-docs-4", "statsig-docs-5", "statsig-docs-6", "statsig-docs-27", "statsig-docs-30", "statsig-comm-1", "statsig-comm-2" ] }, { "productId": "statsig", "storyId": "agent-experiment-analysis", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Statsig provides a Console API (CRUD for experiments/gates), an HTTP API for retrieving experiment/gate values, a CLI scriptable in CI, and an official MCP server explicitly documented to let AI tools like Claude Code/Cursor/Codex 'ask questions, explore experiments, and access your Statsig data' — directly enabling an AI agent to programmatically read experiment configs/results. Runtime probes confirm the MCP server and CLI are live and reachable. Missing for 10: explicit documented example of an agent using these APIs to produce a ship/rollback recommendation (only general data-access capability is evidenced, not the recommendation workflow itself).", "evidenceIds": [ "statsig-docs-11", "statsig-docs-10", "statsig-docs-13", "statsig-docs-9", "statsig-docs-31", "statsig-probe-rt-1", "statsig-probe-rt-2" ] }, { "productId": "statsig", "storyId": "agent-toggles-flag-safely", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Statsig documents multiple agent-friendly interfaces for creating/toggling flags — a CLI (also usable in CI/CD), a Console CRUD API, and an official MCP server reachable via OAuth — giving strong coverage of the 'documented APIs, CLIs, or MCP' half of the story. However, there is no evidence of an approval/review workflow that intercepts or blocks agent-initiated changes before they hit production; access-management docs mention SSO/SCIM for identity but not change-approval gating. Missing for 10: documented approval-workflow/change-review gating for flag mutations, evidence that CLI/API/MCP writes can be routed through a review step rather than applied directly.", "evidenceIds": [ "statsig-docs-9", "statsig-docs-11", "statsig-docs-13", "statsig-probe-rt-1", "statsig-probe-rt-2", "statsig-docs-12" ] }, { "productId": "statsig", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Statsig publishes a working llms.txt (probed live, HTTP 200) and offers a dedicated Docs MCP server that lets AI clients read documentation directly, confirmed via a full keyless initialize handshake at /api/mcp. This directly satisfies pointing an agent at agent-oriented docs. Missing for 10: independent third-party confirmation of agents actually using llms.txt in practice beyond the probe.", "evidenceIds": [ "statsig-probe-1", "statsig-docs-14", "statsig-docs-22", "statsig-probe-rt-3" ] }, { "productId": "statsig", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Statsig's MCP server explicitly lets AI clients (Claude Code, Cursor, Codex) 'ask questions, explore experiments, and access your Statsig data using AI,' which is a form of AI-generated insight retrieval, and this is confirmed live via a working OAuth-gated MCP endpoint. However this is an external-agent integration rather than a native in-product AI insights/suggestions feature (e.g., no evidence of built-in AI-generated experiment summaries, anomaly explanations, or suggestion widgets inside the Statsig console itself). Missing for 10: native in-app AI-generated insights/copilot UI, evidence of proactive AI suggestions surfaced directly in the console rather than via external MCP client.", "evidenceIds": [ "statsig-docs-13", "statsig-docs-28", "statsig-probe-rt-2", "statsig-probe-3" ] }, { "productId": "statsig", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Statsig supports background-running automations like Scheduled Rollouts (gradual gate rollout without manual intervention) and Autotune (autonomously shifts traffic toward winning variants), plus CLI integration into CI/CD pipelines for automated gate/experiment management. These are real autonomous background mechanisms, but they are experimentation/rollout automations rather than general-purpose AI-agent automations, and there's no evidence of scheduling/triggering arbitrary agentic workflows. Missing for 10: evidence of general-purpose scheduled/triggered agent workflows beyond rollout/autotune, and independent verification that these run reliably unattended in production.", "evidenceIds": [ "statsig-docs-3", "statsig-docs-6", "statsig-docs-27", "statsig-docs-31", "statsig-probe-rt-1" ] }, { "productId": "statsig", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers Statsig's MCP server, which lets external AI tools (Cursor, Claude Code, Codex) query Statsig data — the opposite direction of an in-product assistant that users delegate tasks to. No evidence of a built-in AI assistant embedded in the Statsig console itself.", "evidenceIds": [] }, { "productId": "statsig", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Statsig ships an official CLI (@statsig/siggy) that runs keylessly and is explicitly documented and probe-verified for CI/CD automation of Feature Gates and Experiments, plus a Console API/HTTP API for headless programmatic control. Runtime probes confirm the CLI executes via npx in a scriptable, non-interactive way suitable for automation pipelines. Missing for 10: no independent third-party CI pipeline case study or deeper docs on CLI command coverage beyond gates/experiments.", "evidenceIds": [ "statsig-docs-9", "statsig-docs-31", "statsig-docs-11", "statsig-docs-10", "statsig-probe-4", "statsig-probe-rt-1" ] }, { "productId": "statsig", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Statsig is a feature-flagging/experimentation SaaS platform, not an AI agent or assistant that consumes external tools; the evidence shows the reverse relationship—Statsig publishes its own MCP server so that AI tools (Codex, Cursor, Claude Code) can consume Statsig's data, not that Statsig itself plugs in and uses external MCP servers' tools. This client-side 'product acts as MCP consumer' story is a category error for this kind of product.", "evidenceIds": [ "statsig-docs-13", "statsig-probe-rt-2", "statsig-probe-rt-3" ] }, { "productId": "statsig", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Statsig ships an official MCP server integrating with agent tools like Codex, Cursor, and Claude Code for querying Statsig data, plus a separate docs MCP server; runtime probes confirm both endpoints are live, with the main MCP server enforcing OAuth and the docs server completing a full handshake. missing for 10: independent third-party (non-vendor) hands-on report of using the MCP server in a real agent workflow beyond docs/probes.", "evidenceIds": [ "statsig-docs-13", "statsig-docs-28", "statsig-docs-14", "statsig-probe-3", "statsig-probe-rt-2", "statsig-probe-rt-3" ] }, { "productId": "statsig", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Statsig ships a first-party MCP server (verified live via runtime probe) that lets AI clients like Claude Code and Cursor query experiments, gates, and data using natural language, which is the core mechanism for natural-language operation. However, this is an agentic/AI-integration layer bolted onto a console/CLI/API-driven product rather than a native NL command interface across the whole product surface. missing for 10: evidence of natural-language command support within the core Statsig console UI itself (not just via external MCP-connected agents), independent hands-on validation of NL command accuracy/reliability, and coverage of write-actions (creating/modifying gates or experiments) via natural language rather than read-only querying.", "evidenceIds": [ "statsig-docs-13", "statsig-docs-28", "statsig-probe-rt-2", "statsig-probe-3" ] }, { "productId": "statsig", "storyId": "agentic-official-cli", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Statsig ships an official CLI (Statsig CLI / @statsig/siggy) for managing Feature Gates, Experiments, and Dynamic Configs, documented for CI/CD automation, and a runtime probe confirms it executes keylessly via npx and is scriptable. missing for 10: no independent (non-Statsig) hands-on review of the CLI's full feature set beyond the version-check probe.", "evidenceIds": [ "statsig-docs-9", "statsig-docs-31", "statsig-probe-4", "statsig-probe-rt-1" ] }, { "productId": "statsig", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Statsig documents multiple public API surfaces (HTTP API for gate/experiment retrieval and event logging, full CRUD Console API, and CLI) that let developers/agents drive the product programmatically, and runtime probes confirm the CLI works keylessly and the MCP servers respond live. missing for 10: a discoverable OpenAPI/Swagger spec (probe found 404s on standard OpenAPI paths), which would give full machine-readable API discoverability.", "evidenceIds": [ "statsig-docs-10", "statsig-docs-11", "statsig-docs-9", "statsig-probe-rt-1", "statsig-probe-2" ] }, { "productId": "statsig", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Statsig documents general access-management/SSO/SCIM controls and shows its hosted MCP server requires an OAuth bearer token rather than being fully open (statsig-probe-rt-2), implying some authenticated, permissioned access surface an agent could use. However there is no explicit documentation of issuing scoped or least-privilege API keys/tokens specifically for agent use (e.g., read-only or project-scoped Console API keys). Missing for 10: explicit docs on creating scoped/limited-permission API keys, role-based key scoping for agents, and any guidance on least-privilege credential issuance for AI agents.", "evidenceIds": [ "statsig-docs-12", "statsig-docs-32", "statsig-probe-rt-2" ] }, { "productId": "statsig", "storyId": "agentic-sdks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Statsig's docs explicitly describe SDK architecture (client vs. server evaluation, on-device eval SDKs, deterministic evaluation across platforms) and even offer an HTTP API for cases without an SDK, confirming a first-party SDK ecosystem that AI-native developers can build against. Missing for 10: a concrete list/matrix of supported languages/platforms and independent hands-on confirmation of SDK usage beyond docs.", "evidenceIds": [ "statsig-docs-7", "statsig-docs-8", "statsig-docs-20", "statsig-docs-21", "statsig-docs-10" ] }, { "productId": "statsig", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack contains no mention of webhooks or event subscription mechanisms anywhere in Statsig's docs, community posts, or probes — only feature gates, experiments, CLI, MCP, and access management are covered. Webhooks are a plausible axis for a flagging/analytics platform, so absence of evidence yields none rather than na.", "evidenceIds": [] }, { "productId": "statsig", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Statsig documents an HTTP API and Console API but there is no evidence of an interactive API reference with runnable/try-it examples; explicit probes for OpenAPI/Swagger endpoints returned 404s, indicating no such interactive explorer exists.", "evidenceIds": [ "statsig-docs-10", "statsig-docs-11", "statsig-probe-2" ] }, { "productId": "statsig", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Statsig documents a Console API and HTTP API but a direct probe for OpenAPI/swagger specs at common paths (openapi.json, swagger.json, api/openapi.json, .well-known/openapi.json) all returned 404, and no docs reference a downloadable machine-readable spec.", "evidenceIds": [ "statsig-probe-2", "statsig-docs-11", "statsig-docs-10" ] }, { "productId": "statsig", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack documents feature gates, experiments, CLI, and MCP integrations, but nothing describes a distinct sandbox/staging environment or project isolated from production data for testing purposes — only production-oriented tools like emergency disable switches, gate overrides, and scheduled rollouts are mentioned. This is a fair question for a feature-flag/experimentation platform, but no evidence confirms a sandbox capability exists.", "evidenceIds": [ "statsig-docs-1", "statsig-docs-2", "statsig-docs-19", "statsig-docs-29", "statsig-docs-32" ] }, { "productId": "statsig", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Statsig documents an HTTP API and Console API but the evidence pack contains no mention of API versioning scheme or a documented deprecation policy; a probe for OpenAPI spec files (which would typically reveal version info) returned 404s across all candidate paths.", "evidenceIds": [ "statsig-docs-10", "statsig-docs-11", "statsig-probe-2" ] }, { "productId": "statsig", "storyId": "approvals-change-requests", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence describes an approval workflow, change request, or review gate before flag changes go live in production; docs mention scheduled rollouts, overrides, and audit-adjacent features like exposure logging but not a governance approval mechanism.", "evidenceIds": [] }, { "productId": "statsig", "storyId": "audit-trail", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers feature gates, experiments, SSO/SCIM, CLI, Console API, and MCP integrations, but nowhere mentions an audit log or change history capturing who changed a flag, when, or to what value. This is a standard governance capability for feature-flag platforms, so its complete absence from the docs pack means the axis applies but is unsupported.", "evidenceIds": [] }, { "productId": "statsig", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Statsig's Console API provides a CRUD API for managing gates, experiments, and configs programmatically, and the CLI (confirmed runnable via npx) can be scripted/integrated into CI/CD, enabling automation across many items rather than one-by-one UI clicks. However, no evidence explicitly documents a bulk/batch endpoint or bulk-edit feature, so bulk operations must be assembled by looping API/CLI calls rather than a native bulk primitive; missing for 10: explicit bulk/batch API endpoints, documented multi-item transactional updates, and independent confirmation of large-scale automation use.", "evidenceIds": [ "statsig-docs-11", "statsig-docs-9", "statsig-docs-31", "statsig-probe-rt-1" ] }, { "productId": "statsig", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Statsig supports rule-based automation such as targeting rules on Feature Gates, Scheduled Rollouts, and Autotune (which automatically shifts traffic toward a winning variant based on live metrics) — these are forms of automatic rule-triggered behavior, but they are evaluation/targeting rules and traffic-shift algorithms rather than a general 'on event X, trigger action Y' automation engine. Missing for 10: documented event-triggered workflow/webhook automation (e.g., alert-to-action pipelines), explicit event-based trigger definitions distinct from targeting/eval rules, and independent hands-on evidence of such automation working end-to-end.", "evidenceIds": [ "statsig-docs-3", "statsig-docs-6", "statsig-docs-25", "statsig-docs-27", "statsig-docs-29", "statsig-docs-18" ] }, { "productId": "statsig", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Statsig only documents one-time 'Scheduled Rollouts' for feature gates and CLI/CI-CD scripting of flag management (statsig-docs-3, statsig-docs-25, statsig-docs-31); there is no evidence of a recurring job/workflow scheduler (e.g., cron-like recurring automation) that an AI-native user could set up. missing for 10: any documented recurring/cron-style job scheduling, workflow orchestration, or automated recurring task execution beyond a single scheduled feature rollout.", "evidenceIds": [ "statsig-docs-3", "statsig-docs-25", "statsig-docs-31" ] }, { "productId": "statsig", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Statsig's feature gates support rollback-like controls (emergency disable switch, scheduled rollouts) and CLI/Console API can be scripted into CI/CD for programmatic management, implying some versioned control over flag state, but the evidence never documents an explicit audit/version history, diffing, or approval-review workflow for these 'automations'. missing for 10: explicit version history/audit log for gate or experiment changes, a review/approval workflow before changes go live, and a documented one-click rollback to a prior version (only an on/off disable switch is shown).", "evidenceIds": [ "statsig-docs-2", "statsig-docs-3", "statsig-docs-9", "statsig-docs-31", "statsig-docs-11", "statsig-docs-19" ] }, { "productId": "statsig", "storyId": "create-toggle-flag-fast", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs provide a first-feature tutorial creating a gate, targeting users, and checking it live via SDK within a browser console (statsig-docs-24), plus a free-tier signup with feature flags included (statsig-docs-17), console-based exposure monitoring and overrides (statsig-docs-19, statsig-docs-29), and community praise confirming Statsig ships fast, developer-friendly tooling (statsig-comm-1, statsig-comm-2). Missing for 10: independent hands-on account of actual time-to-first-flag from signup and no third-party benchmark of onboarding speed.", "evidenceIds": [ "statsig-docs-24", "statsig-docs-17", "statsig-docs-19", "statsig-docs-29", "statsig-comm-1", "statsig-comm-2" ] }, { "productId": "statsig", "storyId": "edge-evaluation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers Statsig's feature gates, on-device eval SDKs for client-side, server SDKs, and CLI/API surfaces, but nothing addresses evaluating flags specifically at CDN/edge workers or an edge/relay layer close to users. Missing for 10: any mention of Cloudflare Workers, edge runtime SDKs, or a proxy/relay layer for edge evaluation.", "evidenceIds": [ "statsig-docs-8", "statsig-docs-21", "statsig-docs-7" ] }, { "productId": "statsig", "storyId": "experiment-stats-engine", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Statsig documents a full experimentation suite with variance-reduction techniques (CUPED, stratified sampling, switchback tests) and hypothesis/metric scorecards, and community sentiment praises its 'stat rigor,' but the evidence never names or details whether the underlying engine is Bayesian, frequentist, or a documented hybrid methodology. missing for 10: explicit documentation of the statistical model (Bayesian vs frequentist), confidence-interval/p-value methodology details, and independent validation of statistical correctness beyond anecdotal praise.", "evidenceIds": [ "statsig-docs-16", "statsig-docs-5", "statsig-docs-15", "statsig-comm-1" ] }, { "productId": "statsig", "storyId": "flag-environments", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers feature gates, experiments, CLI, console API, and access management, but nowhere mentions environment tags (dev/staging/prod) or environment-scoped SDK keys — a core Statsig platform capability that isn't documented here. Missing for 10: explicit environment/project configuration docs, environment-scoped SDK key generation, and any mention of promoting flag state across environments.", "evidenceIds": [] }, { "productId": "statsig", "storyId": "guarded-rollouts-auto-rollback", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Statsig supports gradual/scheduled rollouts, feature-gate emergency kill switches, and metric-based experimentation with alerting for regressions (Infra Analytics 'set alerts to catch regressions'), giving the pieces needed to guard a rollout with metrics. However, none of the evidence shows an automated rollback action triggered by a detected metric regression — alerts and kill switches appear to require manual intervention rather than an automatic release rollback loop. Missing for 10: documented automatic rollback/kill-switch triggered by metric threshold breach, closed-loop guardrail-to-rollback automation, and independent confirmation this works in production.", "evidenceIds": [ "statsig-docs-2", "statsig-docs-3", "statsig-docs-18", "statsig-docs-23", "statsig-docs-25", "statsig-docs-5" ] }, { "productId": "statsig", "storyId": "individual-user-targeting", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs explicitly cover overrides for targeting/excluding specific users and a guided tutorial that targets internal testers/mobile platforms, directly matching the allowlist/beta-tester/internal-account use case. missing for 10: no independent hands-on confirmation of override UI behavior beyond first-party docs.", "evidenceIds": [ "statsig-docs-29", "statsig-docs-24", "statsig-docs-19" ] }, { "productId": "statsig", "storyId": "local-evaluation-latency", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm deterministic evaluation 'even across different platforms (client or server)' implying server SDKs replicate rule logic locally, and explicitly describe on-device eval SDKs keeping definitions in memory for fast evaluation — but this on-device architecture is documented only for client SDKs, not explicitly server SDKs. Missing for 10: explicit server-SDK docs on background polling/streaming to a local cache, no-network-call guarantee per check, and microsecond-latency benchmarks specific to server SDKs.", "evidenceIds": [ "statsig-docs-7", "statsig-docs-8", "statsig-docs-20", "statsig-docs-21" ] }, { "productId": "statsig", "storyId": "metrics-warehouse-integration", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Statsig Warehouse Native explicitly runs experiment analysis directly on the customer's own data warehouse rather than a black-box metric store, and Statsig's core event-ingestion pipeline lets PMs define metrics from ingested events with a full analysis toolkit (CUPED, stratified sampling, switchback tests). missing for 10: independent/hands-on corroboration of Warehouse Native in production and detail on the custom-metric-definition UI/workflow itself.", "evidenceIds": [ "statsig-docs-15", "statsig-docs-16", "statsig-docs-4", "statsig-docs-26" ] }, { "productId": "statsig", "storyId": "multivariate-dynamic-config", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Statsig docs reference Dynamic Configs alongside Feature Gates and Experiments (e.g., statsig-docs-9, statsig-docs-10, statsig-docs-11, statsig-docs-31 mention 'Dynamic Configs' as a managed entity distinct from booleans), and experiments/layers imply multivariate parameter delivery (statsig-docs-4, statsig-docs-26, statsig-docs-30). However, no evidence explicitly confirms support for strings/numbers/JSON value types or shows a dynamic config JSON payload example. missing for 10: explicit documentation or example showing dynamic config value types (string/number/JSON), SDK code sample retrieving a non-boolean config value.", "evidenceIds": [ "statsig-docs-9", "statsig-docs-10", "statsig-docs-11", "statsig-docs-31", "statsig-docs-30", "statsig-docs-4" ] }, { "productId": "statsig", "storyId": "openfeature-compatibility", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item mentions OpenFeature or a provider implementation for it; all SDK references are to Statsig's own proprietary SDKs, CLI, HTTP API, and Console API. This is a fair axis for a feature-flag/experimentation platform, so absence of evidence yields 'none'.", "evidenceIds": [] }, { "productId": "statsig", "storyId": "openness-api-parity", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Statsig documents a full Console API that is explicitly the CRUD API for 'performing the actions available on console.statsig.com without using the web UI,' plus a separate HTTP API for gate/experiment evaluation and event logging, and an official CLI wrapping these for CI/CD scripting — together covering feature gates, experiments, dynamic configs, and access management via API/CLI rather than only the UI. MCP servers and CLI were confirmed live/reachable in runtime probes, reinforcing that programmatic parity is real and operational, not just documented aspiration. Missing for 10: no independent third-party audit confirming 1:1 UI/API feature parity (e.g., some newer UI features like Autotune or Warehouse Native scorecards aren't explicitly confirmed as API-exposed), and no public OpenAPI spec was found (404s on standard paths).", "evidenceIds": [ "statsig-docs-11", "statsig-docs-10", "statsig-docs-9", "statsig-docs-31", "statsig-probe-rt-1", "statsig-probe-2" ] }, { "productId": "statsig", "storyId": "openness-full-export", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Statsig provides a Console API and CLI for programmatic access to configuration data (feature gates, experiments, dynamic configs), which supports exporting configuration/setup data in a scriptable way, but there is no evidence of a bulk data export feature for user-level event/analytics data in open/standard formats, nor documentation of data portability or account deletion/export workflows for users wanting to leave the platform. missing for 10: documented bulk export of analytics/event data in open formats (CSV/JSON/Parquet), explicit data portability or 'leave the platform' export tooling, evidence of exporting experiment results/metrics data, and any community confirmation of successful full data export.", "evidenceIds": [ "statsig-docs-11", "statsig-docs-9", "statsig-docs-31" ] }, { "productId": "statsig", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack indicates Statsig's core platform or source code is released under an open license; only docs, CLI, MCP servers, and SaaS features are mentioned, none of which speak to open-source licensing of the product's source.", "evidenceIds": [] }, { "productId": "statsig", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack mentions self-hosting or an on-prem/open-source deployment of the core Statsig platform; all documentation describes a hosted SaaS console, hosted MCP/API endpoints, and cloud-based warehouse-native analysis. Self-hosting is a fair ask for this category (competitors offer it), so absence of evidence means 'none' rather than 'na'.", "evidenceIds": [] }, { "productId": "statsig", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions data residency, regional storage options, or geographic data controls for Statsig's platform; the documentation excerpts cover feature flags, experiments, MCP, CLI, and access management but never address where data is stored or user choice of region.", "evidenceIds": [] }, { "productId": "statsig", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Statsig is a feature-flagging/experimentation platform, not an AI model provider or chatbot; the concept of preventing user data from being used to 'train AI models' does not apply to its product category.", "evidenceIds": [] }, { "productId": "statsig", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers feature flags, experimentation, MCP integrations, and access management (SSO/SCIM), but contains no mention of data retention policies, deletion controls, or GDPR/CCPA-style data lifecycle management. This is a fair axis for a SaaS analytics/experimentation platform handling user event data, so absence of evidence yields 'none' rather than 'na'.", "evidenceIds": [] }, { "productId": "statsig", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack addresses whether Statsig's own tools (CLI, SDKs, MCP server, docs site) collect telemetry/usage data or provide an opt-out mechanism; the pack only covers Statsig's core flagging/experimentation features, not its own instrumentation privacy controls.", "evidenceIds": [] }, { "productId": "statsig", "storyId": "progressive-percentage-rollouts", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Statsig docs explicitly describe scheduled/gradual rollouts to safely deploy new code without redeploying, and deterministic evaluation ensures consistent bucketing across evaluations for the same user given the same gate state. Percentage-based ramping (1%→100%) is a core documented Feature Gate/Scheduled Rollout capability, backed by consistent hashing via deterministic evaluation. missing for 10: explicit mention of percentage granularity/UI slider mechanics and independent hands-on confirmation of ramp behavior.", "evidenceIds": [ "statsig-docs-3", "statsig-docs-25", "statsig-docs-7", "statsig-docs-20", "statsig-docs-1" ] }, { "productId": "statsig", "storyId": "rbac-scoped-access", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Statsig docs reference enterprise access management with SSO/SCIM and a Console API for programmatic control, implying some access-control layer exists, but no evidence details role definitions, granular permissions, or scoped/limited API tokens for restricting who can change specific flags. Missing for 10: explicit RBAC documentation, permission granularity for flag edits, and scoped/read-vs-write API token support.", "evidenceIds": [ "statsig-docs-12", "statsig-docs-32", "statsig-docs-11" ] }, { "productId": "statsig", "storyId": "realtime-flag-updates", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Statsig's docs explicitly describe real-time toggling and an 'emergency disable switch' to immediately turn off a code branch in production, plus deterministic evaluation and on-device SDKs for fast client updates — directly supporting the kill-switch and fast-propagation story. However, there is no explicit documentation of streaming vs polling mechanics, propagation latency in seconds, or independent/hands-on verification of SDK update speed. Missing for 10: documented streaming/polling interval specifics, third-party benchmark or hands-on confirmation of sub-second/seconds-level propagation.", "evidenceIds": [ "statsig-docs-1", "statsig-docs-2", "statsig-docs-7", "statsig-docs-8" ] }, { "productId": "statsig", "storyId": "relay-edge-proxy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack describes a self-hostable relay/edge proxy for offline or in-network flag serving; documentation covers SDKs, HTTP API, CLI, and MCP but nothing about a proxy component that caches flags locally when Statsig's servers are unreachable.", "evidenceIds": [] }, { "productId": "statsig", "storyId": "scheduled-releases", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm Statsig supports 'Scheduled Rollout' for Feature Gates, letting you gradually deploy a feature over time rather than instantly ('Gradually deploy a feature over time by setting up a Feature Gate as a Scheduled Rollout', 'Schedule gradual feature rollouts to safely deploy new code'), which implies future-time scheduling of flag state changes. However, the pack lacks detail on setting an exact future timestamp for a release event, calendar/date-picker UI specifics, or independent confirmation of this exact workflow. Missing for 10: explicit documentation of specifying an exact date/time for a rollout step, screenshots or hands-on walkthrough, and third-party confirmation that scheduled changes fire reliably.", "evidenceIds": [ "statsig-docs-3", "statsig-docs-25" ] }, { "productId": "statsig", "storyId": "sdk-language-breadth", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Statsig documents deterministic evaluation guaranteeing identical results 'across different platforms (client or server)' and offers On-Device Eval client SDKs plus a JS SDK tutorial that also targets mobile platforms, implying a consistent cross-stack SDK model. However, the pack never enumerates specific backend/web/mobile SDK names or shows hands-on parity testing across a real multi-platform stack. Missing for 10: an explicit list of officially supported language/platform SDKs, and independent/hands-on evidence confirming identical flag behavior when used together across backend, web, and mobile in a real project.", "evidenceIds": [ "statsig-docs-7", "statsig-docs-20", "statsig-docs-8", "statsig-docs-21", "statsig-docs-24" ] }, { "productId": "statsig", "storyId": "self-host-open-source", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions an open-source self-hosted distribution of the full Statsig platform; all material describes the hosted SaaS console, SDKs, CLI, and MCP integrations that connect to Statsig's cloud API. Absence of evidence for this applicable axis means it is not delivered.", "evidenceIds": [] }, { "productId": "statsig", "storyId": "stale-flag-cleanup", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers feature gates, experiments, rollout scheduling, overrides, CLI, and MCP integrations, but nothing documents stale-flag detection, flag usage/code-reference scanning, or cleanup workflows. This is a reasonable axis for a feature-flag platform, but no capability is evidenced.", "evidenceIds": [] }, { "productId": "statsig", "storyId": "targeting-rules-segments", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm Feature Gates support attribute-based targeting (e.g., targeting mobile platforms and internal testers) and per-user overrides, indicating rule-based flag targeting is supported. However, the evidence pack never mentions a dedicated 'segments' or reusable audience-list feature, so reusable, cross-flag targeting groups are unevidenced. missing for 10: explicit documentation of reusable segment/audience definitions, evidence of segment reuse across multiple flags/experiments, and independent/hands-on confirmation of complex rule builders.", "evidenceIds": [ "statsig-docs-24", "statsig-docs-29", "statsig-docs-1" ] }, { "productId": "statsig", "storyId": "transparent-pricing-limits", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Docs mention a free tier tied to 'events' as a metered unit, giving a partial glimpse of cost drivers, but there is no evidence of a full published pricing page detailing seats, MAUs, or request-based costs across paid tiers. missing for 10: published pricing tiers/page, seat-based pricing, MAU-based pricing, request-based pricing, enterprise pricing details.", "evidenceIds": [ "statsig-docs-17" ] }, { "productId": "unleash", "storyId": "ab-experiments", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Unleash documents 'variants' explicitly for A/B and multivariate testing (unleash-docs-8) and has 'impact metrics' that can gate/automate rollout progression based on threshold metrics (unleash-docs-14, unleash-docs-22, unleash-docs-29), giving PMs some data-driven feedback loop on flag variations. However, there is no evidence of a dedicated experimentation dashboard, statistical significance/winner-declaration reporting, or goal-metric comparison across variants akin to a full A/B testing suite. Missing for 10: statistical significance engine, experiment results/winner dashboard, and metric-goal configuration UI tied directly to variant performance.", "evidenceIds": [ "unleash-docs-8", "unleash-docs-14", "unleash-docs-22", "unleash-docs-29" ] }, { "productId": "unleash", "storyId": "agent-experiment-analysis", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Unleash exposes a comprehensive Admin API for programmatic access to flags, variants, and impact metrics (thresholds, error-rate pausing) that an agent could poll to gauge rollout health, and an official MCP server lets LLM agents manage flags. However, evidence shows the MCP server is scoped to 'create and manage feature flags' rather than reading experiment results/outcomes or generating ship/rollback recommendations, and no docs describe an agent-facing results-summary or decision-recommendation workflow. missing for 10: explicit API/MCP support for reading experiment results/outcome data, and any documented agent workflow that summarizes outcomes or recommends ship/rollback decisions.", "evidenceIds": [ "unleash-docs-6", "unleash-docs-14", "unleash-docs-22", "unleash-docs-29", "unleash-docs-7", "unleash-probe-rt-2", "unleash-docs-8" ] }, { "productId": "unleash", "storyId": "agent-toggles-flag-safely", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Unleash documents an Admin API and MCP server for creating/managing flags (unleash-docs-6, unleash-docs-7, unleash-probe-2) and the runtime probe confirms real flag creation via the Admin API (unleash-probe-rt-1). Change requests (unleash-docs-11) and RBAC (unleash-docs-12) provide an approval-gating mechanism for environment changes, but this is an Enterprise-tier feature and evidence does not explicitly confirm it enforces approval on MCP/API-driven writes specifically (only that it applies to 'changes in an environment') nor is a documented CLI shown. Missing for 10: explicit CLI support, confirmation that change-request approval gates API/MCP-originated writes specifically, and independent (non-vendor) verification of the approval workflow blocking agent writes.", "evidenceIds": [ "unleash-docs-6", "unleash-docs-7", "unleash-probe-2", "unleash-probe-rt-1", "unleash-docs-11", "unleash-docs-12", "unleash-gh-3" ] }, { "productId": "unleash", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Unleash publishes a working llms.txt (HTTP 200, with explicit AI-agent instructions for fetching Markdown) plus a documented, MCP-reachable docs endpoint, giving agents structured, agent-oriented documentation entry points. Missing for 10: independent (non-vendor) confirmation that third-party agents actually consume these successfully in the wild.", "evidenceIds": [ "unleash-probe-1", "unleash-probe-rt-2", "unleash-docs-7" ] }, { "productId": "unleash", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Unleash's evidence covers feature flag management, impact metrics thresholds, and an MCP server that lets external LLM coding assistants create/manage flags — but nothing shows Unleash itself generating AI insights or suggestions from data inside the product (e.g., AI-driven rollout recommendations, anomaly analysis, or narrative summaries). The impact metrics feature is rule-based automation, not AI-generated insight, and the MCP server is for external agents to act on Unleash, not Unleash surfacing AI-generated analysis to users.", "evidenceIds": [ "unleash-docs-14", "unleash-docs-22", "unleash-docs-29", "unleash-docs-7", "unleash-probe-rt-2" ] }, { "productId": "unleash", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Unleash documents metrics-driven automation — progressing rollout milestones or triggering safeguards (pause/disable) automatically when thresholds are crossed — which is a real background automation running without manual intervention, and its MCP integration lets AI agents create/manage flags. However this automation is narrowly scoped to release/rollout progression rather than a general-purpose agentic automation framework, and there's no evidence of scheduling arbitrary autonomous agent tasks beyond flag rollout safeguards. Missing for 10: evidence of broader autonomous task scheduling beyond rollout thresholds, independent corroboration of the impact-metrics automation working in production, and confirmation the MCP server itself can trigger these automations autonomously rather than just via API calls.", "evidenceIds": [ "unleash-docs-13", "unleash-docs-14", "unleash-docs-22", "unleash-docs-29", "unleash-docs-7", "unleash-probe-rt-2" ] }, { "productId": "unleash", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Unleash exposes an MCP server so external LLM coding assistants can manage flags (unleash-docs-7, unleash-probe-2, unleash-probe-rt-2), but this is the reverse of the story — it makes Unleash a tool for outside agents, not a built-in assistant living inside the Unleash product that users delegate tasks to. No evidence of an in-product AI assistant/chat feature for creating rollouts, analyzing flags, etc.", "evidenceIds": [ "unleash-docs-7", "unleash-probe-2", "unleash-probe-rt-2" ] }, { "productId": "unleash", "storyId": "agentic-headless", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Unleash ships a self-hostable Docker image, a full Admin/Client API, and scoped API tokens designed for automation/SDKs rather than human UI use, and a hands-on probe confirms a complete keyless headless workflow (docker boot, flag creation via Admin API, evaluation via Client API) with no manual UI steps — exactly the CI/automation pattern the story asks for. missing for 10: no independent third-party CI pipeline example or official CI/CD docs page walking through pipeline integration.", "evidenceIds": [ "unleash-docs-4", "unleash-docs-6", "unleash-docs-23", "unleash-docs-15", "unleash-probe-rt-1" ] }, { "productId": "unleash", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Unleash is a feature-flag management platform, not an agent or AI assistant; the evidence only shows Unleash exposing its own functionality via an MCP server for external coding assistants to use (docs-7, probe-2), which is the reverse relationship (Unleash as server, not as client consuming other MCP tools). There is no notion of Unleash itself plugging in external MCP servers to gain new tool capabilities, so this client-side MCP axis is a category mismatch for this product type.", "evidenceIds": [ "unleash-docs-7", "unleash-probe-2", "unleash-probe-rt-2" ] }, { "productId": "unleash", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Unleash documents an official MCP server (@unleash/mcp) that lets LLM-powered coding assistants create and manage feature flags following best practices, and a probe confirms the docs MCP endpoint completes a full handshake. missing for 10: independent/hands-on third-party validation of the flag-management MCP server itself (only the docs-MCP handshake was runtime-verified, not the @unleash/mcp flag server), and no evidence of broader community adoption reports.", "evidenceIds": [ "unleash-docs-7", "unleash-probe-2", "unleash-probe-rt-2" ] }, { "productId": "unleash", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Unleash ships an official MCP server (@unleash/mcp) documented to let LLM-powered coding assistants create and manage feature flags following best practices, and a probe confirms the docs' MCP endpoint completes a handshake, corroborating the integration is real and agent-reachable. Missing for 10: no independent/hands-on demonstration of the flag-management MCP server itself in use (only the docs MCP endpoint was probed), and it requires local setup with a PAT rather than being zero-config.", "evidenceIds": [ "unleash-docs-7", "unleash-probe-2", "unleash-probe-rt-2" ] }, { "productId": "unleash", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack details SDKs, Admin API, MCP server, and Docker deployment, but no official CLI tool for Unleash is mentioned anywhere in the docs or GitHub materials.", "evidenceIds": [] }, { "productId": "unleash", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Unleash's Admin API is explicitly documented as providing comprehensive programmatic access to all resources, backed by scoped API tokens/keys, and this is corroborated by a hands-on probe showing a full keyless create-and-evaluate flag roundtrip via the Admin and Client APIs. This directly demonstrates AI-native drivability through a documented public API. Missing for 10: no independent third-party API client showcase beyond Unleash's own SDK/probe evidence.", "evidenceIds": [ "unleash-docs-6", "unleash-docs-15", "unleash-docs-23", "unleash-probe-rt-1" ] }, { "productId": "unleash", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Unleash supports scoped credentials generally: backend API tokens are scoped to specific projects and a single environment, frontend tokens are explicitly non-secret/least-privilege by design, and RBAC governs permissions; the official MCP flag-management server requires a PAT (a scoped credential) at startup, which is directly relevant to agent access. However, there's no explicit documentation of fine-grained, agent-specific credential scoping (e.g., read-only vs write scopes tailored for an AI agent) beyond project/environment granularity. Missing for 10: dedicated agent-credential scoping guide, granular permission scopes beyond project+environment, and independent confirmation of least-privilege PAT usage in agent workflows.", "evidenceIds": [ "unleash-docs-15", "unleash-docs-23", "unleash-docs-26", "unleash-docs-12", "unleash-probe-rt-2", "unleash-docs-7" ] }, { "productId": "unleash", "storyId": "agentic-sdks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Unleash documents official client/backend SDKs as the primary integration path (quickstart guide, API token docs distinguishing frontend/backend SDK scoping), which AI-native developers can build against directly. missing for 10: no independent/hands-on corroboration of SDK quality or breadth across languages, and no explicit SDK versioning/changelog evidence.", "evidenceIds": [ "unleash-docs-1", "unleash-docs-24", "unleash-docs-25", "unleash-docs-15", "unleash-docs-23", "unleash-docs-26" ] }, { "productId": "unleash", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack contains no mention of webhooks, event subscriptions, or push-based notification mechanisms anywhere in Unleash's docs, API overview, or integrations list — only SDKs, Admin/Client APIs, Edge caching, and MCP flag-management tools are documented.", "evidenceIds": [] }, { "productId": "unleash", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows Admin API docs and an MCP server for AI agents to manage flags, but nothing describes an interactive API reference (e.g., Swagger/OpenAPI explorer) with runnable/try-it-out examples. This is a fair capability to expect from an API-driven product, but no evidence supports it.", "evidenceIds": [] }, { "productId": "unleash", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack describes Unleash's Admin API as providing 'comprehensive programmatic access to all Unleash resources' but never mentions an OpenAPI/Swagger spec, schema file, or any downloadable machine-readable API definition. Missing for 10: explicit mention of an OpenAPI/Swagger document, a spec download endpoint, or documentation confirming machine-readable API schema availability.", "evidenceIds": [ "unleash-docs-6" ] }, { "productId": "unleash", "storyId": "api-sandbox", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Unleash's environments concept explicitly supports enabling flags in development/test without affecting production (unleash-docs-21, unleash-docs-10), and a hands-on probe confirms a full self-hosted Docker instance can be spun up locally with its own Postgres, tokens, and flag create/evaluate loop entirely separate from any production account (unleash-probe-rt-1). This gives an AI-native user a genuine sandbox path via self-hosting plus environment isolation. Missing for 10: no dedicated 'sandbox mode' or ephemeral test-environment feature purpose-built for AI agents, and no independent (non-vendor) corroboration of the environment-isolation guarantees in production use.", "evidenceIds": [ "unleash-docs-21", "unleash-docs-10", "unleash-docs-4", "unleash-probe-rt-1" ] }, { "productId": "unleash", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack documents the Admin API, tokens, and SDKs but contains no mention of API versioning scheme or a documented deprecation policy for Unleash's APIs. Missing for 10: any documentation of API version numbers, changelogs, or deprecation/sunset policy for endpoints or SDKs.", "evidenceIds": [ "unleash-docs-6", "unleash-docs-15", "unleash-docs-23" ] }, { "productId": "unleash", "storyId": "approvals-change-requests", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Unleash docs explicitly describe Change Requests as an approval workflow that requires an additional approval step before changes go live in an environment, and this is corroborated by GitHub listing change requests as a hosted/enterprise feature alongside RBAC and SSO. Missing for 10: independent hands-on verification of the approval workflow in practice and details on multi-stage/production-specific enforcement.", "evidenceIds": [ "unleash-docs-11", "unleash-gh-3", "unleash-docs-10", "unleash-docs-12" ] }, { "productId": "unleash", "storyId": "audit-trail", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers RBAC, change requests, environments, and API tokens, but contains no mention of an audit log recording who changed what flag, when, or to what value — the specific capability the story asks about is unevidenced.", "evidenceIds": [] }, { "productId": "unleash", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The Admin API is described as providing 'comprehensive programmatic access to all Unleash resources,' which implies scripting could achieve bulk create/update flows, but no evidence documents an explicit bulk-operations feature (e.g., batch enable/disable, bulk import/export, multi-select actions in UI) beyond a vague pricing-page mention of 'export/import feature flags for easy migration.' missing for 10: explicit bulk-update/bulk-tagging API endpoints, UI multi-select bulk actions, and any hands-on demonstration of operating on many items in one call.", "evidenceIds": [ "unleash-docs-6", "unleash-docs-16" ] }, { "productId": "unleash", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Unleash's impact-metrics feature lets users define threshold-based rules that automatically trigger actions — progressing rollouts, pausing on latency/error-rate breaches, or disabling environments — which is genuine event-triggered automation ([unleash-docs-14],[unleash-docs-22],[unleash-docs-29]). However this is scoped narrowly to rollout safeguards rather than a general-purpose rule/event engine for arbitrary actions across the system. Missing for 10: evidence of a broader rule-definition system covering non-rollout events/actions, and independent confirmation the automation works reliably in practice.", "evidenceIds": [ "unleash-docs-14", "unleash-docs-22", "unleash-docs-29", "unleash-docs-13" ] }, { "productId": "unleash", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Unleash supports one-time scheduled feature releases (specific date/time) and conditional automation of rollout milestones/pauses based on metrics, but there is no evidence of recurring/cron-style job scheduling or workflow automation that repeats on an interval.", "evidenceIds": [ "unleash-docs-3", "unleash-docs-20", "unleash-docs-22", "unleash-docs-29" ] }, { "productId": "unleash", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Unleash's 'automations' are feature-flag rollouts: change requests provide a review/approval step before changes go live (docs-11), and flags can be toggled or reverted instantly without redeploying (docs-18), giving a rollback-like mechanism, plus RBAC constrains who can change what (docs-12). However there is no evidence of an explicit version-history/audit trail or diff view for flag configuration changes, only approval gating and on/off toggling. Missing for 10: documented changelog/audit-log of flag config versions, explicit 'rollback to previous version' feature, and independent confirmation of change-request review workflow in practice.", "evidenceIds": [ "unleash-docs-11", "unleash-docs-18", "unleash-docs-12", "unleash-docs-13" ] }, { "productId": "unleash", "storyId": "create-toggle-flag-fast", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Docs show a clear quickstart flow: create a feature flag in the Admin UI, connect an SDK, and toggle it via activation strategies without redeploying — and a runtime probe confirms an actual create-flag-and-evaluate loop working end-to-end within minutes on a fresh self-hosted instance. Missing for 10: independent third-party account of a brand-new signup-to-production toggle timing (only a self-hosted probe, not hosted SaaS signup, is verified).", "evidenceIds": [ "unleash-docs-1", "unleash-docs-18", "unleash-docs-24", "unleash-probe-rt-1" ] }, { "productId": "unleash", "storyId": "edge-evaluation", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Unleash Edge is a documented purpose-built component (Rust-based caching/relay layer) that sits between SDKs and the primary server, described explicitly as reducing read load and improving scalability/performance/resilience for connected SDKs, functioning as a low-latency edge/relay evaluation layer close to users. This directly matches the story, though evidence doesn't explicitly show deployment as a CDN/edge-worker runtime (e.g., Cloudflare Workers) itself. missing for 10: explicit CDN-worker (e.g., Cloudflare Workers) deployment guide, independent performance/latency benchmarks for Edge in production.", "evidenceIds": [ "unleash-docs-5", "unleash-docs-28", "unleash-docs-23" ] }, { "productId": "unleash", "storyId": "experiment-stats-engine", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Unleash's docs mention 'impact metrics' for pausing/progressing rollouts based on thresholds, but there is no evidence of a documented statistics engine (Bayesian/frequentist), significance testing, confidence intervals, or variance-reduction techniques (e.g., CUPED) behind experiment/variant results. A Hacker News comment even notes Unleash's rule system is more limited than competitors, but this doesn't address the statistics engine specifically. missing for 10: any mention of a named statistical methodology, significance/power calculations, confidence intervals, or variance-reduction options for A/B test analysis.", "evidenceIds": [ "unleash-docs-14", "unleash-docs-22", "unleash-docs-29", "unleash-docs-8", "unleash-comm-1" ] }, { "productId": "unleash", "storyId": "flag-environments", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Unleash's docs explicitly describe environments as distinct stages (dev/staging/prod) with independent flag states per environment, and API tokens/keys are scoped to specific project(s) and a single environment, satisfying both independent state and scoped SDK keys. Missing for 10: independent third-party corroboration of multi-environment isolation beyond docs/probe.", "evidenceIds": [ "unleash-docs-10", "unleash-docs-21", "unleash-docs-15", "unleash-docs-23", "unleash-probe-rt-1" ] }, { "productId": "unleash", "storyId": "guarded-rollouts-auto-rollback", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Unleash's impact-metrics feature explicitly supports guarding rollouts with metrics, including automatic pause/rollback when error rates or latency cross thresholds (e.g., 'Automate milestone progression or trigger safeguards that pause rollouts or disable environments when metrics cross a threshold' and 'Pause automatically when request latency increases during a rollout'). This directly matches the story's requirement of metric-guarded rollout with automatic rollback. missing for 10: independent/hands-on verification of the automatic rollback behavior in production, and more detail on how 'rollback' (vs. pause/disable) is technically executed.", "evidenceIds": [ "unleash-docs-14", "unleash-docs-22", "unleash-docs-29", "unleash-docs-13" ] }, { "productId": "unleash", "storyId": "individual-user-targeting", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Unleash supports activation strategies and constraints (segments, custom activation strategies and targeting) that can target or exclude specific users/groups, and tokens/environments allow scoping test rollouts to internal accounts, but no evidence pack item explicitly documents a simple 'individual user allowlist/exclude by userId' UI feature or shows a hands-on example of listing specific beta tester IDs. missing for 10: explicit documentation/example of per-user allowlist or exclude-by-userId targeting UI, independent/hands-on confirmation of this exact workflow beyond generic 'custom activation strategies and targeting' marketing copy.", "evidenceIds": [ "unleash-docs-9", "unleash-docs-17", "unleash-docs-18", "unleash-docs-15", "unleash-docs-21" ] }, { "productId": "unleash", "storyId": "local-evaluation-latency", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Evidence shows Unleash SDKs communicate via API keys and that Edge exists as a caching layer to reduce read load from 'thousands of connected SDKs,' implying a poll-and-cache model, but no evidence explicitly describes local, in-process flag evaluation from a cached ruleset or confirms 'no network call per flag check' / microsecond latency. missing for 10: explicit SDK architecture docs describing local evaluation engine, cache refresh intervals, and no-network-per-check guarantee, plus independent benchmarks.", "evidenceIds": [ "unleash-docs-5", "unleash-docs-28", "unleash-docs-23" ] }, { "productId": "unleash", "storyId": "metrics-warehouse-integration", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "unleash", "storyId": "multivariate-dynamic-config", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Unleash's docs confirm a dedicated 'variants' concept explicitly used for multivariate/A-B testing, which supports serving multiple values per flag, but the evidence never explicitly documents the variant payload types (string, number, JSON) or shows an SDK example reading a non-boolean value at runtime. missing for 10: explicit documentation of variant payload types (string/number/json) and hands-on/SDK evidence of retrieving a non-boolean variant value.", "evidenceIds": [ "unleash-docs-8", "unleash-docs-13", "unleash-probe-rt-1" ] }, { "productId": "unleash", "storyId": "openfeature-compatibility", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack discusses Unleash's own SDKs, Admin API, Edge, and MCP integration but never mentions OpenFeature or any OpenFeature provider, so there is no evidence supporting vendor-neutral SDK usage via OpenFeature.", "evidenceIds": [ "unleash-docs-1", "unleash-docs-24", "unleash-docs-5", "unleash-docs-6" ] }, { "productId": "unleash", "storyId": "openness-api-parity", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "The Admin API is documented as providing 'comprehensive programmatic access to all Unleash resources', and a runtime probe confirms a real create-and-evaluate flag loop via the Admin/Client APIs with no UI involved, showing core flag management is fully API-driven. However, some UI-centric workflows (e.g., change request approval flows, RBAC/SSO configuration, impact-metrics dashboards) are not shown to have full API parity in the evidence. missing for 10: explicit documentation/proof that change-request approvals, RBAC/SSO setup, and impact-metrics automation can be fully performed via API rather than just referenced conceptually.", "evidenceIds": [ "unleash-docs-6", "unleash-probe-rt-1", "unleash-docs-11", "unleash-docs-12", "unleash-docs-22" ] }, { "productId": "unleash", "storyId": "openness-full-export", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Unleash is open source and self-hostable, and pricing docs mention 'export/import feature flags for easy migration,' plus the Admin API gives programmatic access to all resources, supporting data portability. However, there's no explicit documented full-data-export format spec, no evidence of exporting audit logs, metrics history, or user/RBAC data, and no dedicated 'leave and take everything' migration guide. Missing for 10: a documented comprehensive data-export/backup format covering all entities (not just flags), evidence of full account data portability, and independent confirmation that exports are truly complete and open-format.", "evidenceIds": [ "unleash-docs-16", "unleash-docs-6", "unleash-docs-4" ] }, { "productId": "unleash", "storyId": "openness-open-license", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "The GitHub repo (github.com/Unleash/unleash) is cited multiple times and docs explicitly distinguish 'Unleash Open Source' from 'Unleash Enterprise', indicating the core product's source is publicly available. However, the evidence pack never states the specific open-source license (e.g. Apache-2.0) or confirms the full extent of what's open vs. enterprise-gated. Missing for 10: explicit license name/file citation, clarity on which modules are open vs. proprietary enterprise-only code.", "evidenceIds": [ "unleash-gh-1", "unleash-gh-2", "unleash-gh-3", "unleash-docs-4" ] }, { "productId": "unleash", "storyId": "openness-self-host", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Unleash Open Source is explicitly documented as self-hostable via Docker (unleash-docs-4), and a runtime probe confirms a full keyless self-host roundtrip creating and evaluating a flag with no account or license (unleash-probe-rt-1), directly demonstrating self-hosting of the core product. missing for 10: independent third-party (non-vendor) write-up confirming production-grade self-hosted deployment at scale.", "evidenceIds": [ "unleash-docs-4", "unleash-probe-rt-1", "unleash-gh-1" ] }, { "productId": "unleash", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Unleash's self-hosted deployment (Docker, own environment) inherently lets users control where infrastructure and thus data resides, and this was verified in a runtime probe of a full self-host flag create/evaluate loop. However, there is no explicit documentation of region/residency options for the hosted Enterprise/Cloud offering (e.g., choosing EU/US data centers), so residency is only achievable via self-hosting rather than a first-class SaaS feature. Missing for 10: explicit region-selection or data-residency documentation for the managed/cloud product, and any compliance certifications tied to specific regions.", "evidenceIds": [ "unleash-docs-4", "unleash-probe-rt-1", "unleash-gh-3" ] }, { "productId": "unleash", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Unleash is a feature-flag/release-management platform, not a data/AI-training product; there is no concept of user data being used to train AI models within its product scope, making this axis a category error.", "evidenceIds": [] }, { "productId": "unleash", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers feature-flag mechanics, deployment, RBAC, and API tokens, but contains no documentation of data retention policies, deletion workflows, or GDPR-style data controls for flags, audit logs, or user data. Self-hosting is mentioned but that alone doesn't constitute a documented retention/deletion control feature.", "evidenceIds": [] }, { "productId": "unleash", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses telemetry or usage-tracking opt-out settings for Unleash itself (self-hosted or SaaS); nothing documents what data Unleash collects about its own usage or how to disable it. missing for 10: any mention of telemetry collection, opt-out flags/env vars, or privacy documentation regarding usage tracking.", "evidenceIds": [] }, { "productId": "unleash", "storyId": "progressive-percentage-rollouts", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Unleash's gradual rollout activation strategy is a documented first-class concept, explicitly supporting progressive percentage rollout with consistent bucketing (via stickiness) without redeploying, and impact-metrics docs show automated progression between percentage thresholds (e.g., 25% to 50%) based on health checks. Missing for 10: no independent hands-on evidence of the specific 1%→100% ramp UI/automation flow in practice beyond docs.", "evidenceIds": [ "unleash-docs-2", "unleash-docs-18", "unleash-docs-14", "unleash-docs-22", "unleash-docs-29" ] }, { "productId": "unleash", "storyId": "rbac-scoped-access", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Unleash documents RBAC at two levels (project and instance), scoped API tokens restricted to specific projects/environments (backend tokens) with non-secret frontend tokens, and change requests requiring approval before changes apply in an environment — together enabling platform engineers to restrict who can change which flags. GitHub evidence also confirms RBAC, SSO, and SCIM as enterprise features. missing for 10: independent/hands-on verification of granular permission enforcement and no first-party detail on custom fine-grained permission roles beyond RBAC docs.", "evidenceIds": [ "unleash-docs-12", "unleash-docs-15", "unleash-docs-26", "unleash-docs-11", "unleash-gh-3", "unleash-docs-23" ] }, { "productId": "unleash", "storyId": "realtime-flag-updates", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs confirm the core mechanic — flags can be enabled/disabled without redeploy (unleash-docs-18) and SDKs/Edge communicate via API keys with Edge scaling to thousands of connected SDKs (unleash-docs-5, unleash-docs-23, unleash-docs-28) — but there is no explicit documentation of streaming vs. polling propagation times, no stated 'seconds' latency figure, and no hands-on/independent test confirming a kill switch instantly disables a live SDK. missing for 10: explicit streaming/polling interval docs, measured propagation latency, independent or hands-on confirmation of kill-switch behavior in production SDKs.", "evidenceIds": [ "unleash-docs-18", "unleash-docs-5", "unleash-docs-23", "unleash-docs-28" ] }, { "productId": "unleash", "storyId": "relay-edge-proxy", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Unleash Edge (open source) and Unleash Enterprise Edge are documented as a lightweight caching/relay layer that sits between SDKs and the Unleash server, acting as a read replica that can serve thousands of SDKs and improve resilience — keeping SDK traffic local and flags served even if the vendor/backend is unreachable. Self-hosting via Docker is also documented, reinforcing the on-network deployment model. Missing for 10: independent/hands-on validation of Edge's offline-resilience behavior and more detail on failover semantics beyond docs claims.", "evidenceIds": [ "unleash-docs-5", "unleash-docs-28", "unleash-docs-4", "unleash-docs-23" ] }, { "productId": "unleash", "storyId": "scheduled-releases", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Unleash documents a dedicated how-to guide for scheduling feature releases at a specific date/time, framed explicitly for use cases like product launches, and this sits alongside broader release management, gradual rollout, and environment concepts that support scheduled changes. Missing for 10: independent/hands-on corroboration of the scheduling feature working in practice, and detail on granularity/timezone handling or recurring schedules.", "evidenceIds": [ "unleash-docs-3", "unleash-docs-20", "unleash-docs-13" ] }, { "productId": "unleash", "storyId": "sdk-language-breadth", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm Unleash's SDK model — distinguishing backend/server SDKs (secret tokens) from client-side SDKs for web/mobile (safe-to-expose frontend tokens) and instruct developers to 'use one of the client or backend SDKs to connect Unleash with your application.' However, the pack never lists concrete SDK names/languages (e.g., specific mobile or web SDKs) or documents consistent flag-evaluation behavior across them. Missing for 10: explicit enumeration of officially supported backend/web/mobile SDKs, evidence of consistent evaluation semantics across SDKs, and independent developer corroboration of cross-stack consistency.", "evidenceIds": [ "unleash-docs-24", "unleash-docs-26", "unleash-docs-23", "unleash-docs-15" ] }, { "productId": "unleash", "storyId": "self-host-open-source", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Docs explicitly describe deploying Unleash Open Source via Docker in your own environment, and a hands-on probe confirms a full keyless self-hosted flag create-and-evaluate loop using the official unleash-server image against Postgres with no external account, demonstrating evaluation data stays on the operator's infrastructure. missing for 10: no independent third-party production case study of long-term self-hosted operation at scale.", "evidenceIds": [ "unleash-docs-4", "unleash-probe-rt-1", "unleash-gh-1" ] }, { "productId": "unleash", "storyId": "stale-flag-cleanup", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers flag creation, rollout strategies, environments, RBAC, change requests, and impact metrics, but contains no mention of stale-flag detection, flag lifecycle status, or code-reference scanning tools that would help a platform engineer find and remove unused flags from source code.", "evidenceIds": [] }, { "productId": "unleash", "storyId": "targeting-rules-segments", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Unleash docs directly describe segments as reusable collections of strategy constraints, plus custom activation strategies/targeting and attribute-based rules for flag rollout (unleash-docs-9, unleash-docs-17, unleash-docs-18). This directly matches the story's request for attribute-based rules and reusable segments to target specific user variations. Missing for 10: independent/hands-on validation of complex multi-attribute rule evaluation and the HN comment noting relative rule-system limitations versus competitors introduces some doubt but is not a concrete contradiction.", "evidenceIds": [ "unleash-docs-9", "unleash-docs-17", "unleash-docs-18", "unleash-docs-8", "unleash-comm-1" ] }, { "productId": "unleash", "storyId": "transparent-pricing-limits", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The pricing page is referenced and shows some plan-gated features (export/import, custom strategies, SSO) but the evidence never shows actual published prices or explicit cost drivers like per-seat, MAU, event, or request-based pricing that a PM could use to estimate cost before committing. Missing for 10: explicit price figures, cost-driver metrics (seats/MAUs/events/requests), and any tier comparison table showing how usage translates to price.", "evidenceIds": [ "unleash-docs-16", "unleash-docs-17", "unleash-docs-30", "unleash-gh-3" ] } ]