[ { "productId": "amplitude", "storyId": "ab-experiments", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Amplitude's Feature Experiment product explicitly supports A/B, multivariate, and bandit testing with audience targeting, causal lift measurement, and visual editing to ship variants, plus Experiment APIs for programmatic control (docs-25, docs-41, docs-42, docs-13, docs-15, docs-28). Statistical significance/causal lift is referenced ('measure causal lift') but no deep documentation of the specific stats methodology or independent hands-on validation of experiment result reporting is present. Missing for 10: detailed documentation of significance testing methodology/confidence intervals, and independent/hands-on corroboration of the experiment results UI.", "evidenceIds": [ "amplitude-docs-25", "amplitude-docs-41", "amplitude-docs-42", "amplitude-docs-13", "amplitude-docs-15", "amplitude-docs-28", "amplitude-docs-35", "amplitude-docs-19" ] }, { "productId": "amplitude", "storyId": "agent-answers-growth-questions", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Amplitude ships an official MCP server that lets AI tools (Claude, Cursor, OpenAI, etc.) query behavioral data conversationally without SQL or dashboards, plus REST/Dashboard/Export APIs for programmatic access to live product data. Together these enable an agent to answer growth questions from live data via API or MCP. Missing for 10: independent hands-on verification of the MCP server's query accuracy/reliability and no community corroboration of agent-driven analysis in practice.", "evidenceIds": [ "amplitude-docs-11", "amplitude-docs-17", "amplitude-docs-34", "amplitude-docs-30", "amplitude-docs-38", "amplitude-probe-4", "amplitude-probe-3" ] }, { "productId": "amplitude", "storyId": "agent-creates-dashboard", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Amplitude has documented APIs (Dashboard REST API for reading chart data, event/property CRUD, cohort APIs) and an official MCP server that lets AI tools like Claude/Cursor query Amplitude data in plain language, which covers agent-driven analytics querying. However, evidence shows the Dashboard API is read-only (fetching JSON of existing dashboard graphs) and 'Create a dashboard' appears to be a UI action, not an API/MCP-exposed operation, so there's no documented end-to-end agent workflow that programmatically builds a new dashboard from scratch via API or MCP. Missing for 10: an API/MCP endpoint or example specifically for creating/configuring dashboards (not just querying data), and any hands-on/community proof of an agent successfully building a dashboard end-to-end.", "evidenceIds": [ "amplitude-docs-11", "amplitude-docs-17", "amplitude-docs-34", "amplitude-docs-30", "amplitude-docs-39", "amplitude-docs-38", "amplitude-probe-4" ] }, { "productId": "amplitude", "storyId": "agent-manages-feature-flags", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Amplitude's Experiment APIs let scripts/CI/tools create, target, and toggle flags (amplitude-docs-13, amplitude-docs-19, amplitude-docs-28, amplitude-docs-42), so an agent could drive flag lifecycle via API. However, the documented MCP server (amplitude-docs-11/17/34) is scoped to bringing analytics/behavioral context into AI tools for Q&A, not to flag creation/toggling, so the 'MCP server' half of the story is unsupported. Missing for 10: MCP-server tools for flag CRUD/targeting/toggling, explicit agent-oriented examples of calling the Experiment API, and any independent verification of this workflow.", "evidenceIds": [ "amplitude-docs-13", "amplitude-docs-19", "amplitude-docs-28", "amplitude-docs-42", "amplitude-docs-11", "amplitude-docs-34" ] }, { "productId": "amplitude", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Direct probe confirms a live llms.txt at amplitude.com/llms.txt describing itself as an AI-native platform, plus a docs.md machine-readable docs mirror and an OpenAPI spec, giving agents multiple structured entry points. missing for 10: independent third-party confirmation that agents actually consume these files effectively in practice.", "evidenceIds": [ "amplitude-probe-1", "amplitude-probe-2", "amplitude-probe-3" ] }, { "productId": "amplitude", "storyId": "agentic-ai-insights", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Amplitude AI (docs-10, docs-34) explicitly lets users ask plain-language questions and get expert-level answers/insights across their data, and the MCP server (docs-11/17, probe-4) surfaces this behavioral context directly inside AI tools. This directly matches the story of getting AI-generated insights inside the product. Missing for 10: independent/hands-on validation of the AI insight quality (only vendor docs present), and no detail on proactive suggestion generation vs. query-response only.", "evidenceIds": [ "amplitude-docs-10", "amplitude-docs-34", "amplitude-docs-11", "amplitude-docs-17", "amplitude-probe-4" ] }, { "productId": "amplitude", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Amplitude offers pieces that resemble autonomous background automation—flag rollout/rollback rules that react to data (amplitude-docs-19), automated in-product guides/surveys triggers (amplitude-docs-14), CI/script-driven experiment management (amplitude-docs-13), and an embeddable agent that 'completes tasks for users' (amplitude-docs-26)—but there is no dedicated, documented automation/workflow builder explicitly for AI-native users to configure and run persistent background automations. missing for 10: a first-party 'automations' or workflow engine with triggers/schedules, independent evidence of autonomous background execution, and confirmation the embedded agent runs unattended rather than on-demand.", "evidenceIds": [ "amplitude-docs-19", "amplitude-docs-14", "amplitude-docs-13", "amplitude-docs-26" ] }, { "productId": "amplitude", "storyId": "agentic-builtin-assistant", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Amplitude AI lets users ask plain-language questions and get answers across their data, and an embeddable agent can 'answer questions, guide flows, and complete tasks for users,' which shows some built-in delegatable AI assistant capability. However, evidence is limited to marketing docs snippets with no hands-on demonstration of task delegation scope, autonomy limits, or independent corroboration. missing for 10: independent/hands-on verification of the AI assistant actually completing delegated tasks, detail on task scope/autonomy, and confirmation this is available to the analytics-user persona (not just embedded end-user product).", "evidenceIds": [ "amplitude-docs-10", "amplitude-docs-26", "amplitude-docs-34" ] }, { "productId": "amplitude", "storyId": "agentic-headless", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Amplitude offers headless automation surfaces suitable for CI: HTTP V2 event ingestion API, Dashboard REST API, Experiment APIs for managing flags/experiments 'from CI, scripts, or your own tools', a CLI-based wizard, and an OpenAPI spec probe. These support scripted/automated use without a GUI, but there's no evidence of a full CLI for ongoing operations (only a setup wizard), no CI-specific recipes/examples, and no independent/hands-on confirmation that headless CI workflows work reliably. missing for 10: dedicated CLI beyond setup wizard, documented CI/CD pipeline examples, independent verification of headless automation in practice.", "evidenceIds": [ "amplitude-docs-2", "amplitude-docs-13", "amplitude-docs-27", "amplitude-docs-30", "amplitude-probe-3" ] }, { "productId": "amplitude", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "All evidence describes Amplitude publishing its own MCP server so external AI tools (Claude, Cursor, etc.) can pull Amplitude data — this is Amplitude acting as an MCP *server*, not as a client that plugs in and consumes external MCP servers' tools. There is no evidence Amplitude itself connects to or invokes third-party MCP servers to extend its own AI features.", "evidenceIds": [ "amplitude-docs-11", "amplitude-docs-17", "amplitude-docs-34", "amplitude-probe-4" ] }, { "productId": "amplitude", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Amplitude is not itself an agent but a SaaS analytics platform, so the axis applies; it documents an official MCP server at amplitude.com/mcp-server that brings Amplitude's behavioral data into AI tools like Claude, Cursor, OpenAI, and Lovable, corroborated by a probe confirming the page exists. Missing for 10: independent/hands-on third-party verification of the MCP server actually working in practice.", "evidenceIds": [ "amplitude-docs-11", "amplitude-docs-17", "amplitude-docs-34", "amplitude-probe-4" ] }, { "productId": "amplitude", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Amplitude AI lets users 'ask questions in plain language' and get answers across data, and the MCP server brings this natural-language querying into external AI tools (Claude, Cursor, etc.), directly enabling natural-language operation of the product's analytics capabilities. This is corroborated by dedicated docs pages and a probe confirming the MCP server exists. Missing for 10: independent/hands-on verification of natural-language command accuracy and breadth (e.g., can it also configure flags/dashboards via NL, not just query), and no community evidence testing this specific feature.", "evidenceIds": [ "amplitude-docs-10", "amplitude-docs-11", "amplitude-docs-17", "amplitude-docs-34", "amplitude-probe-4" ] }, { "productId": "amplitude", "storyId": "agentic-official-cli", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Amplitude ships a Wizard CLI that auto-instruments a codebase and gets a first dashboard from one terminal command (amplitude-docs-27, amplitude-docs-1), which counts as an official CLI, but this is scoped to onboarding/setup rather than a general-purpose AI-native CLI for ongoing agentic workflows. Missing for 10: documentation of a broader CLI with commands beyond initial instrumentation, evidence of AI-native/agentic use cases via the CLI, and independent hands-on corroboration of the CLI's capabilities.", "evidenceIds": [ "amplitude-docs-27", "amplitude-docs-1" ] }, { "productId": "amplitude", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Amplitude documents a broad public HTTP API surface (event ingestion, dashboard REST API, cohort management, event/property CRUD, experiment/feature-flag APIs) plus a confirmed OpenAPI spec and llms.txt/docs.md endpoints for machine consumption, indicating a well-documented, programmatically drivable API. missing for 10: independent third-party corroboration of API completeness/reliability and no evidence of official client libraries per language beyond SDK mentions.", "evidenceIds": [ "amplitude-docs-2", "amplitude-docs-12", "amplitude-docs-13", "amplitude-docs-29", "amplitude-docs-30", "amplitude-docs-38", "amplitude-probe-3", "amplitude-probe-1", "amplitude-probe-2" ] }, { "productId": "amplitude", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence shows API keys, an MCP server, and OpenAPI spec, but nothing about scoped/least-privilege credential issuance (e.g., role-based API tokens, granular permission scopes) for agent use is documented.", "evidenceIds": [] }, { "productId": "amplitude", "storyId": "agentic-sdks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Amplitude documents a full suite of official SDKs (web, mobile, server, game engines) sharing a unified identity/event model and API surface, backed by an installer wizard, HTTP V2 ingestion endpoint, and a published OpenAPI spec, all discoverable via llms.txt/docs.md for AI-native consumption. Missing for 10: independent/hands-on developer corroboration of SDK integration ease and no explicit AI-agent-specific SDK usage examples beyond the MCP server.", "evidenceIds": [ "amplitude-docs-3", "amplitude-docs-4", "amplitude-docs-16", "amplitude-docs-27", "amplitude-docs-35", "amplitude-docs-36", "amplitude-docs-37", "amplitude-docs-2", "amplitude-probe-1", "amplitude-probe-2", "amplitude-probe-3" ] }, { "productId": "amplitude", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of any webhook subscription mechanism for events; Amplitude's evidence covers SDKs, APIs for ingest/export, integrations, and an MCP server, but nothing describes outbound webhooks for event subscriptions. missing for 10: any webhook API/documentation, event subscription mechanism, or third-party confirmation of webhook support.", "evidenceIds": [] }, { "productId": "amplitude", "storyId": "analytics-query-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Amplitude documents a Dashboard REST API returning JSON query results, an Export API for raw event data, and other CRUD APIs (events, cohorts, taxonomy), with an OpenAPI spec confirmed reachable — covering both analytical queries and raw event export as the story requires. Missing for 10: independent/hands-on developer corroboration of the export/query API's reliability and rate limits beyond first-party docs.", "evidenceIds": [ "amplitude-docs-2", "amplitude-docs-12", "amplitude-docs-30", "amplitude-docs-38", "amplitude-probe-3" ] }, { "productId": "amplitude", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence confirms Amplitude has API documentation (HTTP V2 endpoint, Dashboard REST API, various CRUD endpoints) and even an openapi.json spec file, but nothing in the pack shows an interactive reference UI with runnable/try-it examples — no Swagger-style console, no code sandbox, no mention of 'try it' functionality.", "evidenceIds": [ "amplitude-docs-2", "amplitude-docs-30", "amplitude-docs-38", "amplitude-probe-3" ] }, { "productId": "amplitude", "storyId": "api-machine-spec", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "A direct probe confirms a machine-readable OpenAPI spec is available at https://amplitude.com/openapi.json (contains an 'openapi' key), and Amplitude also exposes llms.txt/docs.md machine-readable docs. Missing for 10: no independent/third-party corroboration of the spec's completeness or usage examples beyond the probe.", "evidenceIds": [ "amplitude-probe-3", "amplitude-probe-1", "amplitude-probe-2", "amplitude-docs-2" ] }, { "productId": "amplitude", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack mentions a sandbox, test project, or staging environment separate from production data; docs reference API keys, SDKs, and CI/deployment but nothing about isolating test data from production. Missing for 10: sandbox/test environment documentation, guidance on separating test vs production data, any staging API keys or environment flags.", "evidenceIds": [] }, { "productId": "amplitude", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Amplitude has REST/HTTP APIs (e.g., HTTP V2 endpoint, Dashboard REST API, Experiment APIs) and an OpenAPI spec, confirming APIs exist, but nothing in the pack documents API versioning conventions or a deprecation/sunset policy for these APIs. The axis applies to any API-driven platform, but no evidence supports the specific claim of a documented deprecation policy.", "evidenceIds": [ "amplitude-docs-2", "amplitude-docs-30", "amplitude-docs-13", "amplitude-probe-3" ] }, { "productId": "amplitude", "storyId": "autocapture-events", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Amplitude's docs claim built-in autocapture ('Auto-instrument page views, clicks, and sessions out of the box' and a Wizard CLI that 'instruments your codebase' in one command), directly matching the story of code-free automatic event capture. However, all support is first-party marketing/docs with no independent hands-on confirmation of the autocapture specifically, and community feedback (amplitude-comm-3) reports that real-world integration is 'very hard' and data is 'often wrong,' introducing real-world doubt about the ease claimed. Missing for 10: independent/hands-on verification that autocapture works as advertised without extra config, and resolution of the community-reported integration difficulty.", "evidenceIds": [ "amplitude-docs-4", "amplitude-docs-27", "amplitude-docs-1", "amplitude-comm-3" ] }, { "productId": "amplitude", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Amplitude's APIs support programmatic mass operations—batch event ingestion via HTTP V2 (docs-2), CRUD across event types/properties (docs-38), cohort list/export/upload (docs-29), and CI/script-driven flag/experiment management (docs-13)—which enable bulk-style automation for AI-native users. However there's no explicit documentation of a dedicated bulk-operations UI or API endpoint for acting on 'many items at once' as a single call. Missing for 10: explicit bulk-update/bulk-delete endpoints, documented batch size limits, and independent evidence of bulk workflows being used successfully.", "evidenceIds": [ "amplitude-docs-2", "amplitude-docs-38", "amplitude-docs-29", "amplitude-docs-13", "amplitude-docs-12" ] }, { "productId": "amplitude", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers analytics, experimentation, session replay, and AI-query features, but contains no documentation of a rules/automation engine that triggers actions automatically based on events (e.g., event-triggered webhooks, workflows, or conditional messaging). Feature flags/experiments and in-product guides are mentioned but only as manually configured campaigns, not as an automatic event-triggered rules system.", "evidenceIds": [ "amplitude-docs-14", "amplitude-docs-19", "amplitude-docs-28" ] }, { "productId": "amplitude", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Amplitude is an analytics/experimentation platform; nothing in the evidence pack references scheduling recurring jobs, workflows, or automated cron-like tasks. There's no mention of scheduled reports, recurring data exports, or workflow automation triggers.", "evidenceIds": [] }, { "productId": "amplitude", "storyId": "automation-versioned-workflows", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Amplitude's feature-flag/experiment system supports rollback-like control ('turn it off the moment something looks wrong') and can be managed via CI/scripts, but there is no evidence of a version history, change review, or audit/rollback log for automations. missing for 10: explicit versioning/change-history for flags or automations, a review/approval workflow, and documented rollback-to-prior-version mechanics.", "evidenceIds": [ "amplitude-docs-19", "amplitude-docs-13", "amplitude-docs-28" ] }, { "productId": "amplitude", "storyId": "cdp-destinations", "verdict": "disputed", "quality": 6, "confidence": "medium", "rationale": "Amplitude's integrations directory documents connections to CRM (Salesforce, HubSpot), reverse-ETL/streaming tools (Hightouch, RudderStack, Segment), and states it can integrate with any system, supporting onward event streaming to third-party destinations. However, a hands-on community report explicitly states 'they dont do a good job in sending data back to things like SFDC,' directly contradicting the claimed CRM-sync capability for at least one major destination. Missing for 10: independent corroboration of reliable webhook/ad-platform streaming, and resolution of the documented SFDC sync complaint.", "evidenceIds": [ "amplitude-docs-44", "amplitude-intdir-7", "amplitude-intdir-13", "amplitude-intdir-12", "amplitude-comm-3" ] }, { "productId": "amplitude", "storyId": "client-server-sdks", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs explicitly describe a unified SDK model spanting web, mobile, server, and game engines with a shared identity/event model (amplitude-docs-36, amplitude-docs-16, amplitude-docs-37), plus a direct HTTP V2 endpoint for backend/server-side event sending (amplitude-docs-2) and auto-instrumentation for web/mobile (amplitude-docs-4). This covers the core of the story across major platforms via official SDKs. Missing for 10: an explicit enumerated list of supported languages/platforms and independent (non-vendor) confirmation of SDK reliability/breadth beyond docs claims.", "evidenceIds": [ "amplitude-docs-2", "amplitude-docs-4", "amplitude-docs-16", "amplitude-docs-35", "amplitude-docs-36", "amplitude-docs-37" ] }, { "productId": "amplitude", "storyId": "cookieless-tracking", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Amplitude's docs mention server-side event sending (HTTP V2 endpoint) and a shared consent model with options to mask sensitive content and honor consent, which implies some capacity for privacy-conscious, non-cookie-dependent tracking, but there is no explicit claim about avoiding third-party cookies or reducing cookie-consent banners. missing for 10: explicit documentation of cookieless/first-party tracking, cookie policy details, or guidance on reducing consent-banner requirements.", "evidenceIds": [ "amplitude-docs-2", "amplitude-docs-16", "amplitude-docs-33" ] }, { "productId": "amplitude", "storyId": "dashboards-api", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The docs mention a Dashboard REST API that returns JSON data for existing dashboard graphs and a general 'Create a dashboard' doc, plus API endpoints for cohorts and event/property metadata, but none of these clearly describe full CRUD (create/update/delete) for dashboards or 'saved insights' via API—only read/export access is explicitly documented. Missing for 10: explicit API endpoints for creating/updating/deleting dashboards, a documented 'saved insights' API object, and independent confirmation that these operations work programmatically.", "evidenceIds": [ "amplitude-docs-30", "amplitude-docs-39", "amplitude-docs-29", "amplitude-docs-38" ] }, { "productId": "amplitude", "storyId": "dashboards-builder", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs confirm core dashboard-building capabilities: creating dashboards, charts/insights, funnels/segments, and organizing dashboards/notebooks into team-owned spaces for collaboration, which directly supports composing insights into shareable team dashboards. Missing for 10: explicit documentation of sharing/permissions mechanics (e.g., link sharing, access controls) and independent/hands-on confirmation of the dashboard-building workflow beyond first-party docs.", "evidenceIds": [ "amplitude-docs-18", "amplitude-docs-39", "amplitude-docs-40", "amplitude-docs-5", "amplitude-docs-30" ] }, { "productId": "amplitude", "storyId": "feature-flags-targeting", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Amplitude's Experiment/Feature Flags docs explicitly cover targeting by user properties, cohorts, and segments (docs-42), percentage-based ramping/rollouts (docs-19, docs-28), local evaluation for low latency (docs-3), and SDKs across web/mobile/server (docs-35), directly matching the story. missing for 10: independent/hands-on verification of flag targeting behavior beyond vendor docs, and no community evidence specifically confirming the flagging/rollout feature works as described.", "evidenceIds": [ "amplitude-docs-3", "amplitude-docs-19", "amplitude-docs-28", "amplitude-docs-35", "amplitude-docs-42", "amplitude-docs-13" ] }, { "productId": "amplitude", "storyId": "founder-self-serve-answers", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Amplitude's core product is explicitly built around funnels, segmentation, and self-serve answers ('Build funnels, segment users, answer why') plus an AI assistant that answers plain-language questions without SQL or a data team, directly matching the founder self-serve need. Community evidence corroborates ease of adoption across non-analyst roles, though one review notes implementation/data-quality difficulties. Missing for 10: independent hands-on validation specifically of funnel/drop-off analysis accuracy and ease for a non-analyst founder persona.", "evidenceIds": [ "amplitude-docs-5", "amplitude-docs-18", "amplitude-docs-10", "amplitude-docs-34", "amplitude-comm-1", "amplitude-comm-3" ] }, { "productId": "amplitude", "storyId": "funnel-analysis", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Docs explicitly reference building funnels, segmenting users, and answering 'why' via charts/dashboards, which is core to this story, but there is no direct documentation detail on breakdown/filter mechanics within a funnel, nor independent hands-on validation of funnel accuracy — one community comment even flags data accuracy concerns during implementation. missing for 10: explicit funnel filter/breakdown feature documentation, independent hands-on corroboration of funnel drop-off analysis working as described.", "evidenceIds": [ "amplitude-docs-5", "amplitude-docs-18", "amplitude-docs-6", "amplitude-comm-3" ] }, { "productId": "amplitude", "storyId": "group-account-analytics", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers user-level analytics, cohorts, funnels, session replay, and experimentation, but contains no mention of account-level or company/group-based analytics (e.g., grouping events by an 'account' or 'company' entity) that B2B PMs need. Since account-based analysis is a plausible and common analytics-platform capability, its absence here is 'none' rather than 'na'.", "evidenceIds": [ "amplitude-docs-5", "amplitude-docs-6", "amplitude-docs-18", "amplitude-docs-42" ] }, { "productId": "amplitude", "storyId": "historical-event-import", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Amplitude's HTTP V2 API for sending events from a server (amplitude-docs-2) and third-party ETL/integration connectors like Fivetran and RudderStack (amplitude-intdir-5, amplitude-intdir-12) suggest a path for importing external or historical event data, but there is no explicit documentation of a dedicated bulk/batch historical import tool or backfill process. Missing for 10: explicit bulk-import/backfill API documentation, guidance on importing from competitor tools (e.g., Mixpanel/Segment exports), and any hands-on confirmation of successful historical data migration.", "evidenceIds": [ "amplitude-docs-2", "amplitude-intdir-5", "amplitude-intdir-12", "amplitude-docs-12" ] }, { "productId": "amplitude", "storyId": "identity-resolution", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The only relevant evidence is a generic identity-model claim ('Every SDK shares the same identity, event, and consent model, so a user instrumented once is understood everywhere'), which gestures at cross-context identity resolution but never explicitly describes cross-device merging or converting anonymous pre-signup activity into an identified user profile. missing for 10: explicit documentation of user/device ID merge behavior, identify API or alias mechanism for anonymous-to-known user transition, and any independent confirmation of cross-device identity stitching.", "evidenceIds": [ "amplitude-docs-16" ] }, { "productId": "amplitude", "storyId": "metric-alerts", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers dashboards, charts, cohorts, and a Dashboard REST API, but contains no mention of subscribing to dashboards or receiving alerts when a metric moves abnormally (e.g., anomaly detection, alerting rules, or subscription/notification features). missing for 10: evidence of dashboard subscription feature, evidence of anomaly/alert detection on metrics, evidence of notification delivery (email/Slack) tied to metric changes.", "evidenceIds": [ "amplitude-docs-39", "amplitude-docs-40", "amplitude-docs-30" ] }, { "productId": "amplitude", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Amplitude exposes broad APIs (HTTP V2 event ingestion, Dashboard REST API, Experiment APIs for flags/experiments, Cohorts API, taxonomy CRUD, data export) that mirror many core UI workflows, and an official MCP server lets AI tools query behavioral data. However, several flagship UI capabilities—session replay viewing/analysis, the visual A/B test editor, in-product guides/surveys creation, dashboard/chart creation, and the AI 'ask in plain language' analyst—are documented only as UI features with no corresponding API endpoints in the evidence pack. Missing for 10: API parity for session replay, visual experiment editor, dashboard/notebook creation, and guides/surveys authoring; independent confirmation that API coverage matches UI feature-for-feature.", "evidenceIds": [ "amplitude-docs-2", "amplitude-docs-12", "amplitude-docs-13", "amplitude-docs-29", "amplitude-docs-30", "amplitude-docs-38", "amplitude-docs-8", "amplitude-docs-15", "amplitude-docs-39", "amplitude-probe-3", "amplitude-probe-4" ] }, { "productId": "amplitude", "storyId": "openness-full-export", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Amplitude provides data export mechanisms (event data export, Dashboard REST API returning JSON, cohort export/upload) via documented APIs, which support portability of raw data. However, there's no evidence of a comprehensive full-account export (e.g., all configuration, dashboards, experiments, session replays) in a single open-format bundle, nor any documented account-deletion/data-portability workflow for 'leaving' the platform. missing for 10: full-account export tooling covering dashboards/configs/experiments/session-replay data, an explicit data-portability/account-closure workflow, and independent verification that exported formats are truly open and complete.", "evidenceIds": [ "amplitude-docs-12", "amplitude-docs-29", "amplitude-docs-30" ] }, { "productId": "amplitude", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Amplitude is a closed-source, proprietary SaaS analytics platform; the evidence pack shows only some SDKs (e.g., Flutter) are open-source but no evidence the core product's source is open-licensed or readable. This axis applies to any product (a buyer could ask about source availability), but there's no evidence of an open license covering Amplitude's core codebase.", "evidenceIds": [ "amplitude-intdir-6" ] }, { "productId": "amplitude", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Amplitude is offered exclusively as a hosted SaaS analytics platform; no evidence of a self-hostable core product, open-source release, or on-prem deployment option appears anywhere in the docs, integrations, or community threads.", "evidenceIds": [] }, { "productId": "amplitude", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions data residency, region selection, EU/US hosting options, or any data storage location controls for Amplitude. This is a fair axis for an analytics SaaS platform handling user data, but absence of evidence means it cannot be credited as delivered. missing for 10: any mention of regional data hosting options, EU/US residency controls, or data location settings.", "evidenceIds": [] }, { "productId": "amplitude", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses AI-training opt-out or data-usage-for-AI-training controls; the pack covers analytics, session replay, feature flags, and an MCP server but nothing about preventing data from being used to train AI models.", "evidenceIds": [] }, { "productId": "amplitude", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Docs mention consent controls and content masking for session replay (amplitude-docs-33) and CRUD including delete for event/user properties via API (amplitude-docs-38), which touch on privacy controls, but there's no explicit documentation of a user-level data retention policy or GDPR-style deletion/export mechanism for end-user data. missing for 10: explicit data retention policy settings, a documented user data deletion/erasure API or workflow, and any independent confirmation that deletion requests are honored.", "evidenceIds": [ "amplitude-docs-33", "amplitude-docs-38" ] }, { "productId": "amplitude", "storyId": "privacy-telemetry-optout", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Amplitude documents a shared 'consent model' across SDKs and features to 'honor consent, and control what gets captured' for session replay, implying some opt-out/consent tooling exists in its data-collection stack, but there is no dedicated documentation about an AI-native user opting out of Amplitude's own telemetry or usage tracking of the product itself. missing for 10: explicit opt-out/DNT mechanism documentation, privacy policy specifics for AI-native/developer users, independent confirmation that consent controls actually block tracking.", "evidenceIds": [ "amplitude-docs-16", "amplitude-docs-33" ] }, { "productId": "amplitude", "storyId": "replay-linked-to-events", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Amplitude's session replay docs explicitly describe going from a chart, funnel, or support ticket to the exact session, catching errors and rage clicks across sessions, and tying errors to UX — directly matching the funnel-drop-off/error-to-replay jump story. This is well documented across multiple docs pages with clear first-party detail on the workflow. missing for 10: independent/hands-on third-party corroboration that the funnel-to-replay jump works smoothly in practice.", "evidenceIds": [ "amplitude-docs-31", "amplitude-docs-32", "amplitude-docs-21", "amplitude-docs-24", "amplitude-docs-43" ] }, { "productId": "amplitude", "storyId": "retention-cohorts", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Amplitude is core analytics product with explicit retention/behavioral-cohort features: funnels/segmentation ('Build funnels, segment users, answer why'), user cohort creation, and cohort management APIs (list/export/upload). This directly supports retention-over-time analysis sliced by behavioral cohorts, which is a first-class Amplitude use case. Missing for 10: no explicit named 'Retention chart' documentation excerpt and no independent/hands-on report specifically confirming retention-by-cohort workflow quality.", "evidenceIds": [ "amplitude-docs-5", "amplitude-docs-6", "amplitude-docs-18", "amplitude-docs-29", "amplitude-docs-39" ] }, { "productId": "amplitude", "storyId": "session-replay-capture", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Amplitude's session replay docs explicitly state you can watch real user sessions ([amplitude-docs-8],[amplitude-docs-21],[amplitude-docs-24]) and that sensitive content is masked by default with consent controls ([amplitude-docs-33]), directly matching the story. Missing for 10: independent/hands-on verification of default masking behavior and more detail on configurability/exceptions.", "evidenceIds": [ "amplitude-docs-8", "amplitude-docs-21", "amplitude-docs-24", "amplitude-docs-33", "amplitude-docs-31", "amplitude-docs-43" ] }, { "productId": "amplitude", "storyId": "sql-query-access", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Amplitude offers dashboards, charts, cohorts, and an AI natural-language query interface (which explicitly markets 'No SQL' as a benefit), plus REST/export APIs and warehouse integrations like BigQuery/Snowflake for data export — but nothing indicates an in-platform SQL or SQL-equivalent query editor over raw events. missing for 10: any documented SQL/notebook query interface inside Amplitude, evidence of a query language for ad-hoc raw event access, or hands-on confirmation of SQL support.", "evidenceIds": [ "amplitude-docs-10", "amplitude-docs-34", "amplitude-docs-12", "amplitude-docs-30", "amplitude-intdir-3", "amplitude-intdir-16" ] }, { "productId": "amplitude", "storyId": "user-paths", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Amplitude's docs confirm funnel building and segmentation ('Build funnels, segment users, answer why') and session replay lets PMs see exact user sessions tied to funnel drop-off points, which supports exploring behavior around key events. However, no explicit mention of a dedicated Path/Journey analysis feature (e.g., Amplitude's Pathfinder/User Paths chart type) appears in the evidence, so the story's specific 'paths before/after an event' capability is only indirectly covered via funnels + session replay. Missing for 10: explicit path/journey analysis chart documentation, independent user validation of path exploration accuracy.", "evidenceIds": [ "amplitude-docs-5", "amplitude-docs-18", "amplitude-docs-31", "amplitude-docs-21", "amplitude-docs-24" ] }, { "productId": "amplitude", "storyId": "warehouse-sync", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Amplitude documents an Export API for event data and lists first-party integrations with warehouses (BigQuery, Snowflake, Databricks) plus reverse-ETL tools (Hightouch, Fivetran, RudderStack) that support syncing events to/from a warehouse, satisfying the 'continuous export' half of the story. However, there's no evidence of an in-product warehouse query feature (e.g., querying warehouse tables directly within Amplitude's UI). Missing for 10: in-product warehouse querying capability, and independent/hands-on confirmation of the export-to-warehouse pipelines actually working reliably.", "evidenceIds": [ "amplitude-docs-12", "amplitude-intdir-3", "amplitude-intdir-4", "amplitude-intdir-16", "amplitude-intdir-7", "amplitude-intdir-5" ] }, { "productId": "mixpanel", "storyId": "ab-experiments", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Mixpanel documents a Feature Flags API for assigning variants to experiments/rollouts and marketing copy claims users can 'design, run, and analyze tests and rollouts' alongside analytics, indicating some experimentation capability. However, there is no explicit documentation of goal-metric configuration or statistical-significance calculation/reporting for A/B tests. missing for 10: explicit stat-sig reporting docs, goal-metric setup workflow, independent corroboration of experiment results UI.", "evidenceIds": [ "mixpanel-docs-13", "mixpanel-docs-36" ] }, { "productId": "mixpanel", "storyId": "agent-answers-growth-questions", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Mixpanel documents a hosted MCP server that lets AI assistants query events, funnels, flows, retention, and session replays via natural language (no dashboard navigation), plus a Query API for filtering/breaking down events, cohorts, and profiles, and a raw export/JQL API for deeper queries—together covering agent-driven growth Q&A over live product data. Missing for 10: independent/hands-on verification of MCP server reliability or accuracy, and no evidence of complex ad-hoc SQL-like warehouse querying via MCP beyond the documented report types.", "evidenceIds": [ "mixpanel-docs-10", "mixpanel-docs-19", "mixpanel-docs-26", "mixpanel-docs-32", "mixpanel-docs-39", "mixpanel-docs-43", "mixpanel-probe-3" ] }, { "productId": "mixpanel", "storyId": "agent-creates-dashboard", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Mixpanel provides a documented hosted MCP server explicitly with a Create-Dashboard action ('Create a new dashboard with text cards and reports') plus natural-language querying of events, funnels, flows, retention, session replays, complementing the Query/Export APIs an agent could also use end-to-end. This directly matches the story of an agent building a dashboard via MCP or API, though there's no independent hands-on report confirming a real agent successfully completed this end-to-end. missing for 10: independent/hands-on verification of an agent actually building a dashboard via MCP or API, and more detail on API-only dashboard creation outside MCP.", "evidenceIds": [ "mixpanel-docs-10", "mixpanel-docs-11", "mixpanel-docs-19", "mixpanel-docs-26", "mixpanel-docs-32", "mixpanel-probe-3", "mixpanel-docs-39", "mixpanel-docs-43" ] }, { "productId": "mixpanel", "storyId": "agent-manages-feature-flags", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Mixpanel documents a Feature Flags API for assigning users to variants for experiments, rollout, and releases (mixpanel-docs-36), and a separate hosted MCP server for natural-language querying and dashboard creation (mixpanel-docs-10/11/19). However, there is no evidence that flag creation or toggling is exposed via the MCP server, nor documentation confirming full CRUD (create/toggle) operations through the API beyond targeting/assignment. missing for 10: MCP tool list including feature-flag creation/toggle, explicit API endpoints for creating and toggling flags, and any hands-on confirmation of agent-driven flag management.", "evidenceIds": [ "mixpanel-docs-36", "mixpanel-docs-10", "mixpanel-docs-11", "mixpanel-docs-19" ] }, { "productId": "mixpanel", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "A direct probe confirms Mixpanel serves a working llms.txt at docs.mixpanel.com/llms.txt (HTTP 200) with a curated index of docs, and Mixpanel also documents an MCP server for agent-native data access. Missing for 10: no independent/community corroboration of an agent actually consuming llms.txt successfully.", "evidenceIds": [ "mixpanel-probe-1", "mixpanel-docs-10", "mixpanel-probe-3" ] }, { "productId": "mixpanel", "storyId": "agentic-ai-insights", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Mixpanel's own docs/marketing describe AI-driven insight surfacing ('Agents monitor your product continuously, surface insights before you think to ask' and example outputs like 'Signups dipped 19% last week'), AI trend-spotting across replays/heatmaps, AI summaries of replay playlists, and natural-language querying via its MCP server. These directly match the story of AI-generated insights/suggestions inside the product. Missing for 10: independent/hands-on validation of these AI insight features' accuracy or usefulness, and more concrete product detail beyond marketing copy.", "evidenceIds": [ "mixpanel-docs-44", "mixpanel-docs-15", "mixpanel-docs-20", "mixpanel-docs-21", "mixpanel-docs-27", "mixpanel-docs-19", "mixpanel-docs-26" ] }, { "productId": "mixpanel", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Mixpanel markets background 'Agents' that continuously monitor data and surface insights/recommendations autonomously (mixpanel-docs-44), suggesting some agentic automation capability, but there is no documentation on how a user actually configures or triggers these automations, what actions they can take, or independent confirmation they run unattended. missing for 10: setup/configuration docs for these agents, description of trigger conditions and autonomous actions, independent/hands-on evidence of them running in the background.", "evidenceIds": [ "mixpanel-docs-44" ] }, { "productId": "mixpanel", "storyId": "agentic-builtin-assistant", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Mixpanel's marketing copy references built-in AI features such as 'Agents' that 'monitor your product continuously... and tell your team exactly what to do next,' AI-generated replay summaries, and AI trend-spotting across replays/heatmaps, suggesting some in-product AI assistant capability. However, these are thin marketing snippets rather than detailed product docs describing how a user actually delegates multi-step tasks to an assistant, and the only well-documented AI integration (MCP server) is for external AI assistants to query Mixpanel data, not a built-in assistant for users. Missing for 10: detailed first-party documentation of the 'Agents' feature and how tasks are delegated/executed, concrete UI/workflow examples, and independent hands-on corroboration.", "evidenceIds": [ "mixpanel-docs-44", "mixpanel-docs-21", "mixpanel-docs-27", "mixpanel-docs-10" ] }, { "productId": "mixpanel", "storyId": "agentic-headless", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Mixpanel exposes REST-based APIs (Raw Event Export, Query API, Engage API, JQL) that can be scripted and run headlessly/in CI for data export and querying, and warehouse connectors support automated syncs. However, there's no evidence of an official CLI, SDK for automation pipelines, or documented CI/CD integration pattern — the product is primarily a UI-driven analytics platform with API access as a secondary capability. missing for 10: dedicated CLI tool, CI/CD pipeline examples or GitHub Actions integration, headless automation SDK, and any hands-on/community evidence of running Mixpanel in automated pipelines.", "evidenceIds": [ "mixpanel-docs-7", "mixpanel-docs-9", "mixpanel-docs-39", "mixpanel-docs-40", "mixpanel-docs-43", "mixpanel-docs-6", "mixpanel-docs-36" ] }, { "productId": "mixpanel", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only shows Mixpanel exposing its OWN hosted MCP server so external AI assistants can query Mixpanel data (mixpanel-docs-10, mixpanel-docs-19, mixpanel-probe-3) — this is the reverse direction of the story, which asks whether Mixpanel can consume/plug in external MCP servers to use their tools. No evidence exists of Mixpanel acting as an MCP client or supporting third-party MCP tool integration.", "evidenceIds": [ "mixpanel-docs-10", "mixpanel-docs-19", "mixpanel-probe-3" ] }, { "productId": "mixpanel", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Mixpanel documents a hosted, official MCP server that gives AI assistants direct access to Mixpanel data, exposing tools like natural-language querying of events/funnels/retention and dashboard creation, confirmed by both docs and a live probe of the docs page. missing for 10: independent/hands-on verification of the MCP server working in practice, and details on authentication/scopes beyond the docs.", "evidenceIds": [ "mixpanel-docs-10", "mixpanel-docs-11", "mixpanel-docs-19", "mixpanel-docs-26", "mixpanel-docs-32", "mixpanel-probe-3" ] }, { "productId": "mixpanel", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Mixpanel documents a hosted MCP server that lets AI assistants query events, funnels, flows, retention, and session replays using natural language without dashboard navigation, directly matching the story of operating the product via NL commands. This is first-party documentation without independent hands-on corroboration of the NL experience itself. Missing for 10: independent/community validation of the MCP natural-language workflow in practice, and detail on the breadth/reliability of commands beyond the listed examples.", "evidenceIds": [ "mixpanel-docs-10", "mixpanel-docs-19", "mixpanel-docs-26", "mixpanel-docs-32", "mixpanel-docs-11", "mixpanel-probe-3" ] }, { "productId": "mixpanel", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack documents Mixpanel's APIs, MCP server, JQL query language, and export methods, but contains no mention of an official command-line interface (CLI) tool for Mixpanel. A SaaS analytics platform could plausibly ship a CLI for automation/scripting, so the axis applies, but no evidence supports it.", "evidenceIds": [] }, { "productId": "mixpanel", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Mixpanel documents a comprehensive public REST-style API surface (Query API, Raw Event Export API, Engage API, Feature Flags API, JQL) plus a hosted MCP server that lets AI assistants query events, funnels, retention, and replays via natural language, giving AI-native users both programmatic and agentic access. missing for 10: no discoverable OpenAPI/Swagger spec (probe found only 404s) and no independent/hands-on corroboration of the MCP server or APIs actually working reliably.", "evidenceIds": [ "mixpanel-docs-7", "mixpanel-docs-9", "mixpanel-docs-39", "mixpanel-docs-43", "mixpanel-docs-36", "mixpanel-docs-37", "mixpanel-docs-10", "mixpanel-docs-19", "mixpanel-probe-2", "mixpanel-probe-3" ] }, { "productId": "mixpanel", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Mixpanel offers an MCP server and various APIs, but the evidence pack contains no mention of scoped/least-privilege API keys, service accounts, or permission-limited tokens for agent use — only generic API/export/query docs and generic security incident commentary unrelated to credential scoping.", "evidenceIds": [] }, { "productId": "mixpanel", "storyId": "agentic-sdks", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Evidence shows official APIs (Query API, Raw/Engage Export API, Feature Flags API) and a JQL scripting interface for building custom analytics, which support programmatic integration, but there is no explicit mention of official client SDKs (e.g., JS/Python/mobile libraries), language coverage, or code samples that an AI-native builder would need. Missing for 10: explicit SDK documentation/list of languages, code snippets, GitHub repos, and independent corroboration of SDK reliability.", "evidenceIds": [ "mixpanel-docs-39", "mixpanel-docs-43", "mixpanel-docs-36", "mixpanel-docs-37", "mixpanel-docs-40" ] }, { "productId": "mixpanel", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of Mixpanel offering webhooks to push/subscribe to events; the evidence only covers exports (raw export API, warehouse connectors, MCP server for querying data), which are pull-based, not event-driven webhook subscriptions.", "evidenceIds": [] }, { "productId": "mixpanel", "storyId": "analytics-query-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Mixpanel documents a Query API for analytical queries (filter/breakdown events, cohorts, activity streams) and a separate Raw Event Export API returning full JSONL event data, plus JQL for custom JS-based queries — directly matching the query/export API story. Missing for 10: no independent/hands-on developer corroboration of API reliability or rate limits beyond first-party docs.", "evidenceIds": [ "mixpanel-docs-39", "mixpanel-docs-43", "mixpanel-docs-7", "mixpanel-docs-9", "mixpanel-docs-17", "mixpanel-docs-37", "mixpanel-docs-40" ] }, { "productId": "mixpanel", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "While Mixpanel documents various APIs (Query API, Raw Export API, JQL), there is no evidence of an interactive API reference with runnable/try-it examples; a probe for a standard OpenAPI/Swagger spec returned 404s across all candidate paths, suggesting no interactive API console exists.", "evidenceIds": [ "mixpanel-probe-2", "mixpanel-docs-39", "mixpanel-docs-43" ] }, { "productId": "mixpanel", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Mixpanel has extensive REST/Query/Export APIs, but direct probes for a machine-readable OpenAPI/Swagger spec (openapi.json, swagger.json, etc.) all returned 404, and no documentation evidence points to a downloadable spec file—only human-readable reference docs are cited.", "evidenceIds": [ "mixpanel-probe-2", "mixpanel-docs-39", "mixpanel-docs-43" ] }, { "productId": "mixpanel", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item mentions a sandbox, test/dev project, or any mechanism to let an AI assistant query or write data without touching the production Mixpanel dataset — the MCP server docs (mixpanel-docs-10, mixpanel-docs-19) only describe querying live production data. Missing for 10: any documented sandbox/test project mode, data isolation for AI testing, or non-production environment support.", "evidenceIds": [ "mixpanel-docs-10", "mixpanel-docs-19", "mixpanel-probe-3" ] }, { "productId": "mixpanel", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item mentions API versioning, version numbers, or a documented deprecation/sunset policy for Mixpanel's APIs; the openapi.json probe returned 404s and no versioning docs were found. missing for 10: API version scheme documentation, deprecation/sunset policy, changelog or migration guides for breaking changes.", "evidenceIds": [ "mixpanel-probe-2" ] }, { "productId": "mixpanel", "storyId": "autocapture-events", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers Mixpanel's event model, funnels, retention, exports, warehouse connectors, and session replay, but nowhere documents an autocapture/autotrack feature that automatically records clicks, pageviews, and form interactions without custom instrumentation. Session Replay (mixpanel-docs-4/23/28/33) records video sessions but does not equate to automatic event-level tracking for analytics events.", "evidenceIds": [] }, { "productId": "mixpanel", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Mixpanel offers APIs (Raw Event Export, JQL, Query API) that allow programmatic, high-volume data operations (bulk export, custom scripted queries, bulk filtering/cohort retrieval), and its MCP server lets AI agents query events, funnels, retention, etc. via natural language. However, the MCP tool list only shows single-item actions like 'Create-Dashboard', with no documented bulk-operation tools (e.g., batch create/update/delete across many items) for an AI-native agent to invoke. missing for 10: explicit AI-agent-invocable bulk operation tools (batch create/update/delete), independent confirmation of bulk actions succeeding at scale via MCP or API in practice.", "evidenceIds": [ "mixpanel-docs-7", "mixpanel-docs-17", "mixpanel-docs-37", "mixpanel-docs-39", "mixpanel-docs-40", "mixpanel-docs-10", "mixpanel-docs-11", "mixpanel-docs-19" ] }, { "productId": "mixpanel", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Mixpanel's evidence covers analytics reporting, exports, session replay, and an MCP server for querying data with natural language, but there is no documented feature for defining rule-based triggers that automatically fire actions on events (e.g., alerts, webhooks, workflow automation). The closest mention, 'Agents monitor your product continuously... tell your team what to do next' (mixpanel-docs-44), describes passive AI insight surfacing, not user-defined event-triggered automation.", "evidenceIds": [ "mixpanel-docs-44", "mixpanel-docs-36", "mixpanel-docs-19" ] }, { "productId": "mixpanel", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Mixpanel is an analytics platform with export APIs, MCP server for queries, and dashboards, but there is no evidence of any capability to schedule recurring jobs or workflows (e.g., scheduled reports, cron-like automation, recurring data syncs beyond warehouse mirroring). No scheduling/automation feature is documented anywhere in the evidence pack.", "evidenceIds": [] }, { "productId": "mixpanel", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Mixpanel is a product analytics platform (events, funnels, retention, session replay, dashboards, MCP queries) rather than an automation/workflow tool with versionable automations; there is no concept of 'automations' to version, review, or roll back in the evidence pack, making this axis a category mismatch rather than a missing feature.", "evidenceIds": [] }, { "productId": "mixpanel", "storyId": "cdp-destinations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows only inbound warehouse syncing (Snowflake/BigQuery/etc. into Mixpanel) and pull-based export APIs (Raw Event Export, Query API, CSV), none of which describe pushing/streaming events out to CRM systems, ad platforms, or webhooks. The partner integrations directory lists third-party platforms (mParticle, Census, Vendo) but their descriptions frame them as bringing data INTO Mixpanel, not Mixpanel streaming events out to them.", "evidenceIds": [ "mixpanel-docs-6", "mixpanel-docs-7", "mixpanel-docs-24", "mixpanel-docs-25", "mixpanel-intdir-1", "mixpanel-intdir-2", "mixpanel-intdir-3" ] }, { "productId": "mixpanel", "storyId": "client-server-sdks", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack never mentions any specific SDKs (JavaScript, iOS, Android, Python, Java, etc.) for sending events from web, mobile, or backend apps — it only discusses tracking methods generically (warehouse connectors, event ingestion server rewrite, export APIs) without documenting an SDK catalog. Missing for 10: explicit mention of official web SDK, mobile SDKs (iOS/Android), backend/server SDKs, and any platform coverage list.", "evidenceIds": [ "mixpanel-comm-18", "mixpanel-docs-18", "mixpanel-intdir-5" ] }, { "productId": "mixpanel", "storyId": "cookieless-tracking", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack contains no documentation on cookieless tracking, first-party identity resolution, or reducing/avoiding cookie consent banners — Mixpanel's docs focus on events, funnels, retention, session replay, exports, and MCP, none of which address cookie usage or consent-banner avoidance. Missing for 10: any statement on cookieless tracking methods, first-party data collection design, or consent-banner impact.", "evidenceIds": [] }, { "productId": "mixpanel", "storyId": "dashboards-api", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Mixpanel's hosted MCP server exposes a 'Create-Dashboard' tool that can create dashboards with text cards and reports, and lets you query events/funnels/retention/etc. programmatically, but this is MCP-tool access for AI assistants, not a documented general-purpose REST API for full CRUD (create/update/delete) of dashboards and saved insights. The Query API (docs-39, docs-43) only supports reading/filtering data, not creating or managing dashboard objects. Missing for 10: explicit REST API endpoints for creating, updating, and deleting dashboards/saved insights outside the MCP context, and any evidence of programmatic dashboard sharing/permissions management.", "evidenceIds": [ "mixpanel-docs-10", "mixpanel-docs-11", "mixpanel-docs-19", "mixpanel-docs-39", "mixpanel-docs-43", "mixpanel-probe-3" ] }, { "productId": "mixpanel", "storyId": "dashboards-builder", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Mixpanel's marketing/docs reference dashboards as an output (e.g. MCP tool 'Create-Dashboard' with text cards and reports, and marketing copy about 'a dashboard you and your team can keep working from'), implying saved insights can be composed into shareable team dashboards. However there is no first-party documentation detailing the dashboard-building UI, sharing/permissions workflow, or team collaboration mechanics beyond these brief mentions. Missing for 10: dedicated dashboard-creation docs, sharing/permission controls, and independent/hands-on evidence of team usage.", "evidenceIds": [ "mixpanel-docs-11", "mixpanel-docs-15", "mixpanel-docs-22" ] }, { "productId": "mixpanel", "storyId": "feature-flags-targeting", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Mixpanel documents a Feature Flags API for assigning users to variants for experiments, rollouts, and releases, and separately supports cohort creation via user properties, plus a marketing page claiming tests/rollouts are done in the same place as analytics. However, there's no documentation showing explicit percentage-rollout controls or cohort/property-based targeting rules specifically applied to feature flags. missing for 10: detailed docs on percentage-based rollout configuration, explicit flag targeting rules by user property/cohort, and any independent/hands-on confirmation of the flagging workflow.", "evidenceIds": [ "mixpanel-docs-36", "mixpanel-docs-16", "mixpanel-docs-13", "mixpanel-docs-39" ] }, { "productId": "mixpanel", "storyId": "founder-self-serve-answers", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Mixpanel's core product is self-serve funnels ('measure conversions through a series of events') and drop-off analysis, with docs explicitly framing this as a no-code, click-based, self-serve capability ('interactive reports that let you query and visualize the results with just a few clicks', 'Discover critical drop-off points'), plus Session Replay to explore 'why customers drop off' — all designed for non-analyst self-service. Community evidence corroborates the self-serve UI power ('PMs spend hours in mixpanel quickly generating custom report after custom report') though also notes UI friction in an older Flow feature. Missing for 10: no first-party founder/non-technical-user case study specifically validating a founder (not a PM/analyst) independently diagnosing activation drop-off, and one community note flags usability complaints on a related flow visualization.", "evidenceIds": [ "mixpanel-docs-2", "mixpanel-docs-45", "mixpanel-docs-41", "mixpanel-docs-23", "mixpanel-comm-12", "mixpanel-comm-15", "mixpanel-comm-17" ] }, { "productId": "mixpanel", "storyId": "funnel-analysis", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Mixpanel's funnels report explicitly measures conversion through a series of events, properties enable filtering/cohorting, and the Query API supports breakdowns and filtering of events; marketing copy also touts discovering 'critical drop-off points.' Missing for 10: a hands-on community example specifically confirming breakdown usage inside funnels (only adjacent Flow feature has direct user testimonials) and independent verification of the breakdown UI itself.", "evidenceIds": [ "mixpanel-docs-2", "mixpanel-docs-16", "mixpanel-docs-39", "mixpanel-docs-45", "mixpanel-comm-15" ] }, { "productId": "mixpanel", "storyId": "group-account-analytics", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Mixpanel does support account/company-level (group) analytics for B2B use cases, but the evidence pack lacks first-party docs describing this feature directly — the only confirmation comes from a community report noting that group-level/account-level analytics 'useful for B2B SaaS' exists but is 'tucked away on one of Mixpanel's higher price plans,' limiting access for many customers. Missing for 10: first-party documentation of the Group Analytics feature, details on setup/limits, and independent hands-on validation beyond the pricing-gate complaint.", "evidenceIds": [ "mixpanel-comm-9" ] }, { "productId": "mixpanel", "storyId": "historical-event-import", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Mixpanel supports ingesting historical/bulk data via Warehouse Connectors (syncing from Snowflake, BigQuery, Databricks, Redshift, Postgres) and has a general import/ingestion API implied by its event JSON model, but the evidence pack lacks explicit documentation of a dedicated bulk 'Import API' for historical event backfill or migration from another analytics tool. missing for 10: explicit historical/bulk import API docs, guidance on importing from other analytics platforms (e.g., Amplitude/Segment migration), and confirmation of backdated event timestamp support.", "evidenceIds": [ "mixpanel-docs-6", "mixpanel-docs-24", "mixpanel-docs-34", "mixpanel-intdir-5", "mixpanel-intdir-6", "mixpanel-docs-18" ] }, { "productId": "mixpanel", "storyId": "identity-resolution", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack contains no mention of Mixpanel's identity resolution features (e.g., identify(), $merge, alias, distinct_id merging) for tying anonymous pre-signup activity to an identified user profile. While this is a well-known Mixpanel capability generally, none of the provided docs, community posts, or probes reference it, so per the evidence-only rule this must be judged as unsupported.", "evidenceIds": [] }, { "productId": "mixpanel", "storyId": "metric-alerts", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Evidence shows Mixpanel has AI 'Agents' that monitor products continuously and surface anomalies (e.g., 'Signups dipped 19% last week') and can 'tell your team exactly what to do next,' suggesting some proactive alerting capability, but there is no explicit documentation of a dashboard subscription mechanism or configurable alert thresholds for PMs. missing for 10: explicit dashboard subscription feature, alert configuration/thresholds, delivery channels (email/Slack) for alerts.", "evidenceIds": [ "mixpanel-docs-44", "mixpanel-docs-20", "mixpanel-docs-15" ] }, { "productId": "mixpanel", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Mixpanel exposes a broad API surface—Query API (mirrors report results), Raw Export/Engage APIs, Feature Flags API, JQL for custom queries, and an MCP server that can create dashboards and query via natural language—giving strong programmatic coverage of most analytics workflows (mixpanel-docs-39, mixpanel-docs-43, mixpanel-docs-36, mixpanel-docs-37, mixpanel-docs-10, mixpanel-docs-11). However, some UI-only capabilities are explicitly documented, e.g. Session Replay sampling is managed 'directly from the Mixpanel UI — no code changes or redeployment required' (mixpanel-docs-5, mixpanel-docs-29), indicating a gap rather than full UI/API parity. Missing for 10: explicit confirmation that dashboard configuration, session-replay settings, and all admin/UI-only controls are fully API-accessible, and independent verification of complete parity beyond vendor docs.", "evidenceIds": [ "mixpanel-docs-39", "mixpanel-docs-43", "mixpanel-docs-36", "mixpanel-docs-37", "mixpanel-docs-10", "mixpanel-docs-11", "mixpanel-docs-5", "mixpanel-docs-29" ] }, { "productId": "mixpanel", "storyId": "openness-full-export", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Mixpanel documents multiple open-format export paths — Raw Event Export API returning JSONL (mixpanel-docs-7,9,18,31), CSV export from the UI (mixpanel-docs-25,30), Engage API for user profiles (mixpanel-docs-8), and Query API for aggregated data (mixpanel-docs-39,43) — enabling a user to pull out event, profile, and query data in standard formats. Community commentary even praises Mixpanel's export policy as 'great' when discussing data ownership (mixpanel-comm-13), corroborating that full data export works in practice. Missing for 10: no explicit statement of exporting *all* config/dashboard/schema metadata, and no independent bulk-export walkthrough beyond forum anecdote.", "evidenceIds": [ "mixpanel-docs-7", "mixpanel-docs-9", "mixpanel-docs-18", "mixpanel-docs-25", "mixpanel-docs-30", "mixpanel-docs-8", "mixpanel-docs-39", "mixpanel-comm-13" ] }, { "productId": "mixpanel", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Mixpanel is closed-source SaaS; no evidence of any open-licensed source code repository or license grant is present in the evidence pack, only product docs, integrations, and community commentary unrelated to source licensing.", "evidenceIds": [] }, { "productId": "mixpanel", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Mixpanel is offered exclusively as a hosted SaaS product; no evidence pack item mentions a self-hosted or on-premise deployment option, source code availability, or Docker/Kubernetes install path. Absence of evidence for this applicable capability yields a 'none' verdict.", "evidenceIds": [] }, { "productId": "mixpanel", "storyId": "privacy-data-residency", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Mixpanel documents an EU Data Residency Program allowing customers to process and store personal data in Europe, directly addressing regional data residency choice [mixpanel-docs-38], backed by GDPR-oriented deletion/retrieval APIs [mixpanel-docs-12]. Missing for 10: no detail on which regions beyond EU are supported, no independent/hands-on confirmation of the residency mechanics or migration process, and no mention of per-project region selection UI.", "evidenceIds": [ "mixpanel-docs-38", "mixpanel-docs-12" ] }, { "productId": "mixpanel", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses AI-training opt-out or any policy preventing customer data from being used to train AI models; privacy docs only mention GDPR deletion/retrieval APIs and EU data residency, not AI-training exclusion.", "evidenceIds": [ "mixpanel-docs-12", "mixpanel-docs-38" ] }, { "productId": "mixpanel", "storyId": "privacy-retention-controls", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Mixpanel documents explicit deletion and retrieval APIs built to meet GDPR requirements, plus an EU Data Residency Program for regional data storage/processing, giving users concrete levers to control retention and deletion of their data ([mixpanel-docs-12], [mixpanel-docs-38]). Missing for 10: independent/hands-on verification of the deletion APIs actually working, and documentation of configurable retention windows or automated data-expiry settings beyond GDPR compliance language.", "evidenceIds": [ "mixpanel-docs-12", "mixpanel-docs-38" ] }, { "productId": "mixpanel", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence pack shows Mixpanel's GDPR deletion/retrieval APIs and EU data residency options for customer data stored in Mixpanel, but there is no mention of any mechanism for an AI-native user (e.g., of the MCP server or AI features) to opt out of telemetry/usage tracking collected about their own interactions with Mixpanel's tools.", "evidenceIds": [ "mixpanel-docs-12", "mixpanel-docs-38" ] }, { "productId": "mixpanel", "storyId": "replay-linked-to-events", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Mixpanel explicitly documents Session Replay as a way to 'follow up by exploring why customers drop off' from funnels, and playlists can be saved 'based on events or properties' for review, supporting drop-off-to-replay workflows. However, there is no explicit documentation of jumping from an error event specifically into replays, nor independent/hands-on confirmation of this workflow. Missing for 10: explicit error-event-to-replay linkage, independent user corroboration of the workflow.", "evidenceIds": [ "mixpanel-docs-4", "mixpanel-docs-23", "mixpanel-docs-27", "mixpanel-docs-45", "mixpanel-docs-21" ] }, { "productId": "mixpanel", "storyId": "retention-cohorts", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Mixpanel's Retention report explicitly assesses user engagement over time, and properties/cohorts allow filtering and grouping of users for behavioral segmentation, directly matching the story. Documentation confirms retention analysis, cohort creation via properties, and query/export APIs that support slicing by cohort. Missing for 10: independent hands-on evidence of retention-by-cohort slicing in practice and no community corroboration specifically on retention feature usability.", "evidenceIds": [ "mixpanel-docs-3", "mixpanel-docs-16", "mixpanel-docs-39", "mixpanel-docs-43" ] }, { "productId": "mixpanel", "storyId": "session-replay-capture", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Mixpanel's Session Replay lets PMs watch real user session recordings and even use AI to summarize/playlist replays (mixpanel-docs-4, mixpanel-docs-27, mixpanel-docs-42), but the evidence pack contains no documentation confirming that sensitive input fields (e.g., passwords, form data) are masked by default in these recordings. Privacy docs cover GDPR/EU data residency (mixpanel-docs-12, mixpanel-docs-38) but not replay-specific input masking. Missing for 10: explicit documentation of default input/text masking or redaction controls within Session Replay, and any independent confirmation that masking works as claimed.", "evidenceIds": [ "mixpanel-docs-4", "mixpanel-docs-27", "mixpanel-docs-42", "mixpanel-docs-12", "mixpanel-docs-38" ] }, { "productId": "mixpanel", "storyId": "sql-query-access", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Mixpanel doesn't offer SQL, but its JQL (JavaScript Query Language) lets developers write custom code queries against raw event data directly in the platform, and the Query API can filter/break down events and return activity streams — a functional but not syntactic SQL-equivalent. missing for 10: no actual SQL support, no evidence JQL/Query API is positioned as a full SQL-equivalent, and no independent/hands-on corroboration of its expressiveness for raw-event querying.", "evidenceIds": [ "mixpanel-docs-37", "mixpanel-docs-40", "mixpanel-docs-39", "mixpanel-docs-43" ] }, { "productId": "mixpanel", "storyId": "user-paths", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Mixpanel's 'Flows' report is explicitly designed to visualize the paths users take before/after a given event, and this is corroborated by independent hands-on community praise ('gets to the heart of most of the information you need on user flows') as well as the docs' framing around discovering drop-off points and calibrating user journeys. However, community feedback also flags real friction (hover-based UI hard to use, requests to unify page-view vs custom events) and no probe confirms path exploration works flawlessly today. missing for 10: recent independent corroboration of UI usability, explicit documentation of 'before event' path exploration (most docs emphasize funnels/retention/after-event flows).", "evidenceIds": [ "mixpanel-docs-45", "mixpanel-docs-19", "mixpanel-comm-15", "mixpanel-comm-16", "mixpanel-comm-17" ] }, { "productId": "mixpanel", "storyId": "warehouse-sync", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Mixpanel's Warehouse Connectors let you sync data from Snowflake, BigQuery, Databricks, Redshift, and Postgres into Mixpanel, and the Raw Event Export API (JSONL, recommended for high-volume) lets you pull events out for warehouse loading, satisfying both directions of the story at a basic level. However, continuous/automated export to a warehouse is not natively built-in as a managed reverse-sync pipeline—Mixpanel's own partner directory lists third-party Reverse ETL tools like Census for pushing Mixpanel data to warehouses, implying continuous outbound sync relies on partner tooling rather than a first-party feature. Missing for 10: evidence of a native continuous/scheduled export-to-warehouse pipeline (not just an on-demand API), and independent/hands-on confirmation that Warehouse Connectors and export API work reliably at scale.", "evidenceIds": [ "mixpanel-docs-6", "mixpanel-docs-24", "mixpanel-docs-34", "mixpanel-docs-7", "mixpanel-docs-9", "mixpanel-docs-17", "mixpanel-docs-25", "mixpanel-intdir-2", "mixpanel-intdir-5", "mixpanel-intdir-6" ] }, { "productId": "plausible", "storyId": "ab-experiments", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Plausible is a privacy-focused web analytics tool offering goals, funnels, and traffic stats, but there is no evidence of any A/B testing/experimentation feature, variant assignment, or statistical significance reporting on results.", "evidenceIds": [] }, { "productId": "plausible", "storyId": "agent-answers-growth-questions", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Plausible has a documented Stats API with a /api/v2/query endpoint supporting complex stats queries via API keys, which an agent could call to answer growth questions from live data. However, there is no evidence of an official MCP server, no OpenAPI/machine-readable spec (probes show 404s for llms.txt, docs.md, openapi.json), and no agent-specific tooling or SDK documented. Missing for 10: MCP server integration, machine-readable API schema/discovery, and any agent-oriented documentation or examples.", "evidenceIds": [ "plausible-docs-4", "plausible-docs-16", "plausible-docs-17", "plausible-probe-1", "plausible-probe-2", "plausible-probe-3" ] }, { "productId": "plausible", "storyId": "agent-creates-dashboard", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Plausible exposes a Stats API (/api/v2/query) that returns JSON metrics and could be used by an agent to assemble a custom dashboard, but there is no evidence of an MCP server, agent-oriented SDK, or documented workflow for programmatic dashboard creation end-to-end. Probes confirm no llms.txt, docs.md, or OpenAPI spec, suggesting the API isn't packaged for agent discovery/consumption. missing for 10: official MCP server, agent-facing API discovery (OpenAPI/llms.txt), end-to-end example of an agent building a dashboard via API.", "evidenceIds": [ "plausible-docs-4", "plausible-docs-16", "plausible-docs-17", "plausible-probe-1", "plausible-probe-2", "plausible-probe-3" ] }, { "productId": "plausible", "storyId": "agent-manages-feature-flags", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Plausible Analytics is a privacy-focused web analytics tool with no concept of feature flags or experimentation targeting; this axis belongs to feature-flag/experimentation platforms, not analytics products.", "evidenceIds": [] }, { "productId": "plausible", "storyId": "agentic-agent-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Probes explicitly show llms.txt, docs.md, and OpenAPI endpoints all return 404, indicating no agent-oriented documentation is exposed at expected locations, and no other evidence mentions llms.txt or agent-readable docs.", "evidenceIds": [ "plausible-probe-1", "plausible-probe-2", "plausible-probe-3" ] }, { "productId": "plausible", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of AI-generated insights, natural-language querying, or automated suggestions within the Plausible dashboard. It offers AI-traffic tracking (which AI tools send traffic) but not AI-generated analysis of the data itself; funnels, comparisons, and filters are manual analysis tools, not AI-driven insights.", "evidenceIds": [] }, { "productId": "plausible", "storyId": "agentic-autonomous-automation", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of any automation, scheduling, workflow, or agentic background-task capability beyond scheduled email/Slack reports, which is not autonomous automation setup; there's no workflow builder, webhook triggers, or agent framework. Missing for 10: any automation/workflow builder, trigger-based actions, agent/autonomous task execution, or scripting interface for background automations.", "evidenceIds": [ "plausible-docs-48", "plausible-docs-49", "plausible-docs-4", "plausible-docs-16" ] }, { "productId": "plausible", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of any built-in AI assistant feature in Plausible; it only tracks AI referral traffic (docs-20/58), which is unrelated to delegating tasks to an assistant within the product. Missing for 10: any built-in AI assistant, agentic chat interface, or task-delegation feature.", "evidenceIds": [] }, { "productId": "plausible", "storyId": "agentic-headless", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Plausible offers a documented Events API for server-side/programmatic tracking and a Stats API (including a JSON query endpoint with scoped API keys) that can be scripted or called from CI/automation pipelines without a UI. However, there is no explicit CLI, official CI/CD integration guide, or documented automation examples, and probes show no llms.txt/OpenAPI spec for programmatic discovery. Missing for 10: dedicated CLI/CI documentation, OpenAPI/machine-readable spec, and concrete automation/CI examples.", "evidenceIds": [ "plausible-docs-1", "plausible-docs-15", "plausible-docs-4", "plausible-docs-16", "plausible-docs-17", "plausible-probe-3" ] }, { "productId": "plausible", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Plausible is an analytics SaaS product, not an AI agent; there is no evidence it acts as an MCP client that plugs in external MCP servers to use their tools. This story is about agent-role capability, which is a category error for an analytics dashboard product.", "evidenceIds": [] }, { "productId": "plausible", "storyId": "agentic-mcp-server", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Plausible is a SaaS analytics product, so an official MCP server is a plausible ecosystem offering, but no evidence pack item mentions MCP, an agent integration, or any AI-agent connector; probes confirm no llms.txt/openapi discoverability either.", "evidenceIds": [ "plausible-probe-1", "plausible-probe-2", "plausible-probe-3" ] }, { "productId": "plausible", "storyId": "agentic-nl-commands", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of any natural-language command interface, chatbot, AI assistant, or agentic control layer for Plausible; only a Stats/Events API requiring structured requests, and probes for llms.txt/docs.md/openapi all 404. missing for 10: any NL command interface, AI assistant/chat feature, or agent-facing API discovery mechanism.", "evidenceIds": [ "plausible-docs-4", "plausible-docs-16", "plausible-probe-1", "plausible-probe-2", "plausible-probe-3" ] }, { "productId": "plausible", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of an official CLI tool for Plausible; the product only offers REST APIs (Events/Stats) and dashboard/scripts, with no mention of a CLI in docs or GitHub, and probes for llms.txt/docs.md/openapi found nothing CLI-related either.", "evidenceIds": [ "plausible-docs-1", "plausible-docs-4", "plausible-docs-16", "plausible-probe-1", "plausible-probe-2", "plausible-probe-3" ] }, { "productId": "plausible", "storyId": "agentic-public-api", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Plausible documents both a Stats API (read-only query endpoint with API keys, JSON responses) and an Events API for recording pageviews/custom events server-side, giving programmatic access to core functionality. However missing for 10: no formal OpenAPI/Swagger spec (probes show 404s), no llms.txt/machine-readable docs for AI agents, and no evidence of official SDKs or independent hands-on API integration reports.", "evidenceIds": [ "plausible-docs-1", "plausible-docs-4", "plausible-docs-16", "plausible-docs-17", "plausible-probe-3" ] }, { "productId": "plausible", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Plausible's Stats API keys are explicitly scoped to a single team (determining which sites can be queried) and the Stats API itself is read-only, giving some least-privilege boundary useful for an agent with read access; the Events API is separate for write-side tracking. However there's no evidence of finer-grained scopes (e.g., per-site, read/write toggles, expiring/revocable tokens, or agent-specific credential issuance) beyond team-level scoping. Missing for 10: granular per-site or per-scope permissions, credential expiry/rotation controls, and any agent-specific credential management docs.", "evidenceIds": [ "plausible-docs-17", "plausible-docs-4", "plausible-docs-16" ] }, { "productId": "plausible", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Plausible offers Events API and Stats API but there is no evidence of official client SDKs/libraries (e.g., for Python, JS, Node, etc.) for developers to build against beyond raw HTTP APIs; probes show no OpenAPI spec or llms.txt either. missing for 10: official SDK packages, language client libraries, OpenAPI/machine-readable spec, llms.txt.", "evidenceIds": [ "plausible-docs-1", "plausible-docs-4", "plausible-docs-16", "plausible-probe-1", "plausible-probe-2", "plausible-probe-3" ] }, { "productId": "plausible", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Plausible offers an Events API (inbound, for recording events) and a Stats API (read-only query), but no evidence of outbound webhooks that let a subscriber be notified of events happening. No documentation mentions webhooks or push-based event subscriptions.", "evidenceIds": [ "plausible-docs-1", "plausible-docs-4", "plausible-docs-16" ] }, { "productId": "plausible", "storyId": "analytics-query-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Plausible documents a Stats API (/api/v2/query) that accepts simple and complex stats queries and returns JSON, plus an Events API for recording events, and API keys scoped for querying sites — this covers programmatic querying of aggregated stats. However, this is not a SQL/warehouse-style raw event export API; there's no evidence of raw event-level data export or a documented OpenAPI spec (probes confirm openapi endpoints 404), and no independent hands-on corroboration of the query API's depth or limitations. Missing for 10: raw event-level data export/warehouse-SQL-style querying, OpenAPI/schema documentation, and independent developer corroboration of the query API's capabilities.", "evidenceIds": [ "plausible-docs-4", "plausible-docs-16", "plausible-docs-17", "plausible-docs-1", "plausible-probe-3" ] }, { "productId": "plausible", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "There is documentation for the Stats API and Events API, but no evidence of an interactive API reference with runnable examples (e.g., Swagger/OpenAPI explorer or embedded try-it console). Probes explicitly confirm no OpenAPI spec, llms.txt, or docs.md is available, indicating no interactive/agent-friendly API reference exists. Missing for 10: interactive API explorer, runnable code examples, OpenAPI/machine-readable spec.", "evidenceIds": [ "plausible-docs-4", "plausible-docs-16", "plausible-docs-17", "plausible-probe-1", "plausible-probe-2", "plausible-probe-3" ] }, { "productId": "plausible", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Plausible documents its Stats API and Events API in prose, but explicit probes for an OpenAPI/Swagger spec (openapi.json, swagger.json, well-known paths) and llms.txt/docs.md all returned 404, with no other evidence of a machine-readable API spec.", "evidenceIds": [ "plausible-probe-3", "plausible-probe-1", "plausible-probe-2", "plausible-docs-4", "plausible-docs-16" ] }, { "productId": "plausible", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence of any sandbox/test environment, staging API, or mock mode for Plausible; only production Events/Stats API docs and self-hosting options are documented, none of which describe an isolated sandbox for AI-native testing.", "evidenceIds": [] }, { "productId": "plausible", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "There's documentation for the Events API and Stats API (including a v2 query endpoint), but no evidence of API versioning scheme or a documented deprecation policy; probes for OpenAPI spec/docs.md all returned 404s. missing for 10: explicit API version numbers/labels, changelog of breaking changes, documented deprecation timeline or sunset policy.", "evidenceIds": [ "plausible-docs-4", "plausible-docs-16", "plausible-probe-3" ] }, { "productId": "plausible", "storyId": "autocapture-events", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Plausible auto-tracks pageviews and scroll depth with no code, and can automatically turn pages into goals for file downloads, form completions and outbound link clicks without custom event code (plausible-docs-40, plausible-docs-54, plausible-docs-21, plausible-docs-37). However, dedicated 'click tracking' beyond outbound links, and richer form-interaction capture, generally require custom events/Events API or tag manager setup rather than being fully automatic, and one community comment notes founders avoid deeper user-flow tracking for privacy reasons (plausible-comm-5). Missing for 10: comprehensive automatic click tracking (not just outbound links), evidence of fully automatic form field-level interaction capture, and independent hands-on confirmation of zero-code setup across all these event types.", "evidenceIds": [ "plausible-docs-40", "plausible-docs-54", "plausible-docs-21", "plausible-docs-37", "plausible-comm-5" ] }, { "productId": "plausible", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence shows only single-item APIs (Events API for recording one event/pageview, Stats API for querying stats) and CSV import/export/consolidated dashboards, none of which describe bulk create/update/delete operations across many items (e.g., batch site/goal/user management). No documentation or community evidence indicates a bulk-operations capability suited to AI-native automation.", "evidenceIds": [ "plausible-docs-1", "plausible-docs-16", "plausible-docs-50", "plausible-docs-51", "plausible-docs-35" ] }, { "productId": "plausible", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Plausible offers notifications (traffic spikes, email/Slack reports) and read/write APIs, but there is no evidence of a rules engine or automation feature that lets users define conditional triggers to take actions on events (e.g., webhooks, if-this-then-that logic). Missing for 10: rule definition UI/API, conditional trigger logic, automated action execution beyond static notifications.", "evidenceIds": [ "plausible-docs-45", "plausible-docs-48", "plausible-docs-49", "plausible-docs-1" ] }, { "productId": "plausible", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of any scheduling, workflow automation, or recurring job capability for AI-native users; Plausible only offers scheduled email/Slack reports for humans, not programmable recurring jobs/workflows. Missing for 10: any scheduler API, cron-like job configuration, or workflow automation tooling for AI agents.", "evidenceIds": [] }, { "productId": "plausible", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Plausible is an analytics product with no automation/workflow feature to version, review, or roll back; this axis concerns AI agent automation building, which is outside its category.", "evidenceIds": [] }, { "productId": "plausible", "storyId": "cdp-destinations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only covers Plausible's Events API (ingest custom events into Plausible) and Stats API (read-only query of Plausible's own stats), plus GTM/webhook-free tracking setup — there is no mention of streaming events onward to CRMs, ad platforms, or external webhooks. No outbound integration/export-to-third-party pipeline is documented anywhere in the pack.", "evidenceIds": [ "plausible-docs-1", "plausible-docs-15", "plausible-docs-16", "plausible-docs-17", "plausible-docs-3", "plausible-docs-28" ] }, { "productId": "plausible", "storyId": "client-server-sdks", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Plausible offers a generic Events API for recording pageviews/custom events usable from mobile or server-side apps, plus a script tag for web, but there's no evidence of official SDKs/libraries for specific languages (e.g., Swift, Kotlin, Node, Python) beyond raw HTTP API and script snippet. missing for 10: official mobile SDKs (iOS/Android), official backend language SDKs, community/hands-on confirmation of SDK usage.", "evidenceIds": [ "plausible-docs-1", "plausible-docs-15", "plausible-docs-26" ] }, { "productId": "plausible", "storyId": "cookieless-tracking", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Plausible's docs explicitly state it collects no personal data, uses no cookies, and requires no consent banner, which is corroborated by community reports of GDPR-compliant, cookie-free operation and no IP storage. missing for 10: independent legal/regulatory audit confirming zero consent-banner requirement across all jurisdictions.", "evidenceIds": [ "plausible-docs-23", "plausible-docs-29", "plausible-docs-59", "plausible-comm-10", "plausible-comm-11" ] }, { "productId": "plausible", "storyId": "dashboards-api", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Plausible's Stats API only provides read-only query access to stats and the Events API only records events; there is no evidence of an API for creating/managing dashboards or saved insights programmatically. Missing for 10: any API endpoint for creating/editing dashboards, saving insights, or managing report configurations.", "evidenceIds": [ "plausible-docs-4", "plausible-docs-16", "plausible-docs-17" ] }, { "productId": "plausible", "storyId": "dashboards-builder", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Plausible offers dashboard sharing (shared links, embed dashboard, consolidated multi-site view) and team collaboration via SSO/roles, letting a PM share the existing analytics dashboard with a team. However there is no evidence of a 'saved insights' concept or a dashboard builder that lets a PM compose multiple saved reports/segments into a custom shareable dashboard — Plausible's dashboard is a fixed, single-page view per site (or consolidated across sites), not an assembled collection of saved insights. Missing for 10: saved/insight-level bookmarking, drag-and-drop or composable dashboard layout, ability to mix multiple custom reports into one shareable view.", "evidenceIds": [ "plausible-docs-9", "plausible-docs-24", "plausible-docs-47", "plausible-docs-46", "plausible-gh-3", "plausible-docs-35" ] }, { "productId": "plausible", "storyId": "feature-flags-targeting", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Plausible is a privacy-focused web analytics product, not a feature-flag/experimentation platform; feature flagging with targeting by user properties, cohorts, and percentage rollouts is outside its category and product scope.", "evidenceIds": [] }, { "productId": "plausible", "storyId": "founder-self-serve-answers", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Plausible has a dedicated, documented Funnel Analysis feature (sequential or strict-order steps, drop-off measurement) plus a User Journeys view showing paths visitors take, and the dashboard is explicitly designed for no-training self-serve use — directly matching the 'find where activation drops off myself' story. However, independent community feedback flags real limits: one HN commenter notes Plausible 'lacks much of the information from Matomo (like after viewing /foo, visitors tend to view...)', suggesting the flow/transition analysis is shallower than competitors, and another older comment claims the founders resisted user-flow tracking on privacy grounds. missing for 10: independent hands-on validation that funnel drop-off analysis works well in practice for real activation funnels, and resolution of the comparative gap noted vs Matomo's flow reports.", "evidenceIds": [ "plausible-docs-5", "plausible-docs-41", "plausible-docs-42", "plausible-docs-56", "plausible-docs-55", "plausible-docs-60", "plausible-comm-6", "plausible-comm-5" ] }, { "productId": "plausible", "storyId": "funnel-analysis", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Plausible's docs explicitly describe a funnel-analysis feature that lets users define a sequence of steps, choose sequential or strict-order funnels, and see drop-off at each step, complemented by filters/segments (docs-32, docs-44) and a 'user journeys' report showing paths visitors take (docs-55, docs-41, docs-42, docs-56). This directly matches the story's ask for multi-step funnels with filters and breakdowns to find drop-off points. Missing for 10: independent/hands-on validation of funnel usability in practice, and an old community comment (plausible-comm-5) noting the founders once resisted user-flow tracking, which slightly tempers confidence though it predates the now-documented feature.", "evidenceIds": [ "plausible-docs-5", "plausible-docs-41", "plausible-docs-42", "plausible-docs-55", "plausible-docs-56", "plausible-docs-32", "plausible-docs-44", "plausible-comm-5", "plausible-comm-6" ] }, { "productId": "plausible", "storyId": "group-account-analytics", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Plausible is designed around anonymous, privacy-first visitor-level metrics (no personal data, no user profiles/fingerprinting) and there is no evidence of any account-based or company-level grouping, CRM/CDP integration, or B2B account analytics feature in the docs, GitHub description, or community discussion.", "evidenceIds": [ "plausible-docs-23", "plausible-docs-29", "plausible-comm-11", "plausible-comm-13" ] }, { "productId": "plausible", "storyId": "historical-event-import", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Plausible has documented Google Analytics import (bring in historical data) and CSV import/export features, directly supporting bulk-import of historical events from another tool or data export. Community evidence corroborates users migrating from GA with import as a known feature, though one older comment requested the feature before it existed, suggesting it was a later addition. Missing for 10: independent hands-on verification of import fidelity/limits, and detail on supported source formats beyond GA/CSV.", "evidenceIds": [ "plausible-docs-11", "plausible-docs-22", "plausible-docs-31", "plausible-gh-1", "plausible-gh-4", "plausible-docs-50", "plausible-comm-12" ] }, { "productId": "plausible", "storyId": "identity-resolution", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Plausible's core design explicitly avoids collecting personal data or building user profiles — it uses privacy-preserving daily-salted hashes for unique visitor counts rather than persistent identifiers, and docs/community evidence confirm it does not store IPs or build cross-session/cross-device profiles. There is no mechanism described for identifying users across devices or merging anonymous activity into an identified profile.", "evidenceIds": [ "plausible-docs-23", "plausible-docs-29", "plausible-comm-11", "plausible-comm-13" ] }, { "productId": "plausible", "storyId": "metric-alerts", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Plausible offers scheduled email/Slack reports (subscribe to dashboard) and explicit 'traffic spike notifications' that alert when traffic moves abnormally, directly matching the core of this story. However, evidence only covers traffic volume anomalies, not configurable alerts on other key metrics like conversion rate, revenue, or goal completions, and there's no mention of custom thresholds or anomaly sensitivity settings. Missing for 10: alerting on arbitrary/custom key metrics beyond traffic, threshold configuration, and real-time push alerts (vs. periodic email/Slack digests).", "evidenceIds": [ "plausible-docs-45", "plausible-docs-48", "plausible-docs-49", "plausible-gh-5" ] }, { "productId": "plausible", "storyId": "openness-api-parity", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Plausible ships a read-only Stats API for querying metrics and an Events API for recording pageviews/custom events, letting AI-native users pull dashboard-like data and send events programmatically, but there is no evidence of API coverage for configuration tasks that are core UI features — creating/managing funnels, goals, annotations, shared links, team/SSO settings, email/Slack reports, or site management. No OpenAPI spec is discoverable (probe results all 404), reinforcing that API surface is narrower than the UI. missing for 10: API endpoints for funnel/goal/annotation creation, team & site management, shared-link/report configuration, and a discoverable OpenAPI/spec confirming full parity.", "evidenceIds": [ "plausible-docs-4", "plausible-docs-16", "plausible-docs-17", "plausible-docs-1", "plausible-docs-15", "plausible-docs-5", "plausible-docs-8", "plausible-docs-46", "plausible-docs-47", "plausible-docs-48", "plausible-docs-49", "plausible-probe-3" ] }, { "productId": "plausible", "storyId": "openness-full-export", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Plausible offers explicit data export (CSV export/import), a read-only Stats API for programmatic access, and is fully open-source/self-hostable (Community Edition), meaning users can export data and run their own instance to 'leave' the SaaS entirely without lock-in. Community evidence corroborates the FOSS/self-host value as a real selling point against vendor lock-in. Missing for 10: no evidence of a bulk 'export everything' guarantee covering raw event-level data, no independent hands-on test of export completeness, and no explicit documentation of data portability guarantees beyond CSV/API.", "evidenceIds": [ "plausible-docs-50", "plausible-docs-51", "plausible-docs-4", "plausible-docs-16", "plausible-docs-2", "plausible-comm-14" ] }, { "productId": "plausible", "storyId": "openness-open-license", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Plausible Community Edition is open-source on GitHub, and community evidence corroborates that the FOSS/self-hosted nature is a genuine, trusted selling point (people can maintain the code themselves if the company disappears). Missing for 10: explicit license name/text, and no direct evidence of AI-native machine-readable license documentation.", "evidenceIds": [ "plausible-docs-2", "plausible-comm-14", "plausible-comm-7" ] }, { "productId": "plausible", "storyId": "openness-self-host", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Plausible ships an official open-source Community Edition with self-hosting docs (GitHub repo), and community members confirm running it successfully in production for years, valuing that it's FOSS and can be maintained independently. Missing for 10: independent benchmarking of self-hosted performance/scale, and one community comment notes self-hosting complexity (ClickHouse dependency) as a real friction point.", "evidenceIds": [ "plausible-docs-2", "plausible-comm-7", "plausible-comm-14", "plausible-comm-9" ] }, { "productId": "plausible", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Plausible enables data residency control primarily through self-hosting (Community Edition), letting users run and store data on their own infrastructure/region, corroborated by community reports of long-term self-hosted use. However there is no evidence of an official region-selection option within the hosted Cloud product (e.g., EU vs US data center choice) — missing for 10: explicit cloud region/residency selection docs, compliance certifications tied to specific regions, and independent verification of hosted-data location choices.", "evidenceIds": [ "plausible-docs-2", "plausible-comm-7", "plausible-comm-14", "plausible-comm-9" ] }, { "productId": "plausible", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers Plausible's privacy stance on cookies, consent banners, and personal data collection, but nowhere does it address whether collected data is used to train AI models or how a user could opt out of such use. No policy or documentation snippet specifically discusses AI-training data usage.", "evidenceIds": [] }, { "productId": "plausible", "storyId": "privacy-retention-controls", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Plausible collects no personal data/no cookies and offers CSV export, self-hosting, and data-policy docs implying minimal retention, but there's no explicit documented mechanism for AI-native users to programmatically control retention windows or trigger deletion of specific data via API. missing for 10: explicit retention policy/duration documentation, API or dashboard controls for data deletion/right-to-erasure, and independent confirmation of these controls working.", "evidenceIds": [ "plausible-docs-23", "plausible-docs-29", "plausible-docs-59", "plausible-docs-51", "plausible-docs-50", "plausible-comm-11" ] }, { "productId": "plausible", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence describes Plausible's privacy-friendly approach to tracking *website visitors* (no cookies, no personal data collection, no consent banners) but contains no mention of any telemetry or usage-tracking mechanism in the Plausible product itself (e.g., self-hosted CE phoning home to Plausible Inc.) nor any documented opt-out setting for such telemetry.", "evidenceIds": [] }, { "productId": "plausible", "storyId": "replay-linked-to-events", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Plausible is a privacy-focused, aggregate web analytics tool with no session replay feature; the evidence explicitly notes founders refuse to track detailed user flows for privacy reasons, and there is no error/session-level tracking of individual users. Session replay is fundamentally outside this product's category (privacy-first, no personal data collection), so this axis does not apply.", "evidenceIds": [ "plausible-comm-5", "plausible-docs-23", "plausible-docs-29" ] }, { "productId": "plausible", "storyId": "retention-cohorts", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Plausible's docs describe funnels, goals, and dashboard filters/segments, but there is no evidence of retention curves, cohort definitions, or cohort-based behavioral analysis over time. In fact, Plausible's privacy-first design (no persistent personal identifiers, daily-rotating hashed uniques, no user-flow tracking per plausible-comm-5) architecturally works against cross-session cohort/retention analysis.", "evidenceIds": [ "plausible-docs-5", "plausible-docs-41", "plausible-comm-5", "plausible-comm-6", "plausible-comm-13" ] }, { "productId": "plausible", "storyId": "session-replay-capture", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Plausible Analytics is a privacy-focused, cookieless web analytics tool with no session recording/replay functionality; session replay is entirely outside its product category (it explicitly avoids collecting detailed user-level behavioral data for privacy reasons), making this a wrong-axis question rather than a missing feature.", "evidenceIds": [ "plausible-docs-23", "plausible-comm-5" ] }, { "productId": "plausible", "storyId": "sql-query-access", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Plausible offers a Stats API (JSON query endpoint) and Events API, but there is no evidence of SQL or SQL-equivalent query language access to raw events inside the platform — no warehouse-style query editor, no SQL interface. missing for 10: SQL/SQL-equivalent query language, raw event-level query access, in-platform query editor.", "evidenceIds": [ "plausible-docs-16", "plausible-docs-4" ] }, { "productId": "plausible", "storyId": "user-paths", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Plausible's marketing docs explicitly advertise a 'User journeys' feature to 'explore the paths visitors actually take to discover flows you didn't predict' (docs-41, docs-55), which matches the story closely. However, community hands-on reports directly contradict this: one HN commenter states 'The founders refuse to adopt tracking of user flows through a web site, allegedly for privacy reasons' and another notes Plausible 'lacks much of the information from Matomo (like after viewing /foo, visitors tend to view…)' — suggesting the feature is either new/limited or not as capable as claimed. Missing for 10: independent corroboration that the User Journeys feature actually surfaces pre/post-event path exploration as described, and resolution of the conflicting community reports of it being absent or weak.", "evidenceIds": [ "plausible-docs-41", "plausible-docs-55", "plausible-comm-5", "plausible-comm-6" ] }, { "productId": "plausible", "storyId": "warehouse-sync", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Plausible offers a Stats API for querying its own data and an Events API for sending events in, plus CSV export/import, but there is no evidence of syncing or continuously exporting events to an external data warehouse, nor querying warehouse tables in-product. missing for 10: warehouse export/sync capability (e.g. BigQuery/Snowflake integration), continuous event streaming to external warehouse, in-product warehouse query support.", "evidenceIds": [ "plausible-docs-4", "plausible-docs-16", "plausible-docs-51", "plausible-docs-1" ] }, { "productId": "posthog", "storyId": "ab-experiments", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "PostHog's Experiments docs explicitly describe defining variants, picking goal metrics, randomizing users, tracking exposures, and running statistics (Bayesian or frequentist) to determine significance, which directly matches the story. Missing for 10: independent/hands-on validation of the statistical reporting accuracy and no community corroboration specific to experiments (only analytics/session-replay community feedback exists).", "evidenceIds": [ "posthog-docs-7", "posthog-docs-23", "posthog-docs-27", "posthog-docs-30" ] }, { "productId": "posthog", "storyId": "agent-answers-growth-questions", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "PostHog documents a first-party MCP server and CLI that let agents run HogQL/SQL queries, ship flags, and answer product questions from live data (posthog-docs-2, posthog-docs-19, posthog-docs-24, posthog-probe-4, posthog-docs-8/17 for SQL, posthog-docs-9 for warehouse queries). However, a hands-on community report describes the LLM/query integration failing to create a cohort via a SQL query and causing severe UI slowdowns, directly contradicting the smooth 'agent answers growth questions via query' promise (posthog-comm-8). missing for 10: independent corroboration that MCP-driven SQL queries reliably work at scale, and resolution of the reported failure/performance bug.", "evidenceIds": [ "posthog-docs-2", "posthog-docs-19", "posthog-docs-24", "posthog-probe-4", "posthog-docs-9", "posthog-docs-17", "posthog-comm-8" ] }, { "productId": "posthog", "storyId": "agent-creates-dashboard", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "PostHog has a documented official MCP server and public API (openapi.json) that let an agent run trends, funnels, retention, and SQL queries, ship feature flags, and inspect stack traces from an editor/agent (posthog-docs-2, 12, 19, 24, 31, posthog-probe-4, posthog-probe-3). Dashboard assembly, sharing, and alerting are documented product capabilities (posthog-docs-3, 25), but the evidence never explicitly shows the MCP/API path being used to assemble and publish a full dashboard end-to-end, and a hands-on community report describes the AI/LLM integration failing on a related task (cohort creation) and causing severe UI slowdown (posthog-comm-8), raising doubts about reliability of agent-driven workflows. Missing for 10: explicit documentation/demo of an agent creating and saving a dashboard object (not just insights/queries) via MCP or API, and independent corroboration that this specific end-to-end flow works reliably.", "evidenceIds": [ "posthog-docs-2", "posthog-docs-3", "posthog-docs-12", "posthog-docs-19", "posthog-docs-24", "posthog-docs-31", "posthog-probe-3", "posthog-probe-4", "posthog-comm-8" ] }, { "productId": "posthog", "storyId": "agent-manages-feature-flags", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "PostHog has an official MCP server (posthog-probe-4) and docs explicitly state that 'with just plain text questions your agents can ship a feature flag from a prompt' (posthog-docs-19, posthog-docs-24, posthog-docs-31), showing agent-driven flag creation via MCP. An OpenAPI spec exists (posthog-probe-3) implying REST API access to flags, but no evidence explicitly documents targeting rules or toggling flags via API/MCP endpoints specifically. missing for 10: explicit documentation of flag targeting (cohorts/properties) and toggle actions via MCP/API, and independent hands-on confirmation that agent-driven flag creation works reliably.", "evidenceIds": [ "posthog-docs-19", "posthog-docs-24", "posthog-docs-31", "posthog-probe-4", "posthog-probe-3" ] }, { "productId": "posthog", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "PostHog serves a working llms.txt (probe confirms HTTP 200 with structured content describing the platform) and extensive agent-oriented docs like the MCP integration page (posthog-docs-12, 19, 24, 31) explicitly designed for AI agents/editors to consume. Individual docs pages also expose `.md` variants as seen in citations (e.g., product-analytics.md, session-replay.md), showing a broader agent-readable docs pattern. Missing for 10: a root docs.md aggregate returned 404, so machine-readable coverage isn't perfectly uniform across all docs.", "evidenceIds": [ "posthog-probe-1", "posthog-docs-12", "posthog-docs-19", "posthog-docs-24", "posthog-probe-4", "posthog-probe-2" ] }, { "productId": "posthog", "storyId": "agentic-ai-insights", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "PostHog markets in-product AI ('PostHog AI', 'self-driving products') that can suggest code fixes from stack traces, auto-write Hog functions, and turn product signals into researched reports/PRs (posthog-docs-12, posthog-docs-28, posthog-gh-1, posthog-probe-1). However a hands-on community report describes the in-app LLM feature failing a basic task (creating a SQL-based cohort) and severely degrading device performance, directly contradicting the polished 'insights on demand' claim (posthog-comm-8). missing for 10: independent corroboration that AI-generated insights/suggestions work reliably at scale, no rebuttal or fix documented for the reported failure.", "evidenceIds": [ "posthog-docs-12", "posthog-docs-28", "posthog-gh-1", "posthog-probe-1", "posthog-comm-8" ] }, { "productId": "posthog", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "PostHog's CDP pipelines and workflows can run automatically in realtime or on a schedule (posthog-docs-18, posthog-gh-7), with Hog functions optionally written by PostHog AI (posthog-docs-28), and alerts/feature-flag automation exist (posthog-docs-3, posthog-docs-5). However, evidence centers on data-pipeline/workflow automation rather than an AI-native agent explicitly setting up and running autonomous background jobs end-to-end, and one community report notes friction with AI-driven query/cohort creation (posthog-comm-8). Missing for 10: first-party documentation of an AI agent configuring and monitoring a fully autonomous background automation, independent success reports of such automations running unattended.", "evidenceIds": [ "posthog-docs-18", "posthog-gh-7", "posthog-docs-28", "posthog-docs-3", "posthog-comm-8" ] }, { "productId": "posthog", "storyId": "agentic-builtin-assistant", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "PostHog documents an in-product AI assistant/agent (\"PostHog AI\") that can write Hog functions, turn product signals into researched reports and PRs, and be steered from Slack/web/desktop/editor (posthog-docs-28, posthog-gh-1, posthog-gh-2), which matches the story of delegating tasks to a built-in assistant. However, a hands-on community report describes the LLM integration failing to perform a requested task (creating a cohort via SQL) and causing severe performance degradation, directly contradicting smooth delegation (posthog-comm-8). Missing for 10: first-party detail on the assistant's task scope/reliability, more independent corroboration of successful delegation, and resolution of the reported failure.", "evidenceIds": [ "posthog-docs-28", "posthog-gh-1", "posthog-gh-2", "posthog-comm-8" ] }, { "productId": "posthog", "storyId": "agentic-headless", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "PostHog's CLI is explicitly documented for use in 'local scripts, and CI/CD pipelines' alongside terminal and coding agent use, plus REST/HTTP APIs and OpenAPI spec enable headless automation of data capture, queries, and flag management. missing for 10: independent/hands-on confirmation of CI/CD usage, and no explicit CI example/config shown (e.g., GitHub Actions snippet).", "evidenceIds": [ "posthog-docs-20", "posthog-probe-5", "posthog-docs-11", "posthog-probe-3", "posthog-docs-13" ] }, { "productId": "posthog", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "All MCP-related evidence describes PostHog exposing its own MCP server so external AI clients/editors (Claude, Cursor, etc.) can query PostHog's data and tools — the reverse direction of this story. There is no evidence that PostHog itself acts as an MCP client that can plug in external MCP servers to use their tools.", "evidenceIds": [ "posthog-docs-2", "posthog-docs-12", "posthog-docs-19", "posthog-docs-24", "posthog-docs-31", "posthog-probe-4", "posthog-gh-2" ] }, { "productId": "posthog", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "PostHog documents an official MCP server (docs/model-context-protocol) enabling agents to run trends, funnels, HogQL queries, ship feature flags, and pull stack traces directly from MCP clients like Claude, corroborated by the GitHub README describing steering via MCP and a dedicated probe confirming the docs page exists. Missing for 10: independent third-party hands-on validation of the MCP server (only vendor docs/GitHub and one unrelated community complaint about a different AI assistant feature).", "evidenceIds": [ "posthog-docs-12", "posthog-docs-19", "posthog-docs-24", "posthog-docs-31", "posthog-gh-2", "posthog-probe-4" ] }, { "productId": "posthog", "storyId": "agentic-nl-commands", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "PostHog documents natural-language operation via its MCP integration—shipping feature flags, running HogQL queries, and debugging stack traces 'from a prompt' or 'plain text questions' (posthog-docs-12, posthog-docs-19, posthog-docs-24, posthog-docs-31, posthog-probe-4). However, a hands-on community report directly contradicts this: a user tried the LLM integration to create a cohort via SQL and it failed with an incorrect response, plus caused severe performance degradation (posthog-comm-8). missing for 10: independent corroboration that the failure was fixed, broader hands-on validation of natural-language commands across other product areas (analytics, experiments, session replay) beyond MCP docs.", "evidenceIds": [ "posthog-docs-12", "posthog-docs-19", "posthog-docs-24", "posthog-docs-31", "posthog-probe-4", "posthog-comm-8" ] }, { "productId": "posthog", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "PostHog documents an official CLI usable from terminal, coding agents, local scripts, and CI/CD pipelines, with concrete features like source-map injection/upload for error tracking, and this is corroborated by a dedicated docs page and a probe confirming the CLI documentation is live. missing for 10: independent hands-on community review of the CLI itself (community evidence only covers MCP/LLM features, not the CLI specifically).", "evidenceIds": [ "posthog-docs-20", "posthog-docs-13", "posthog-probe-5" ] }, { "productId": "posthog", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "PostHog has a documented public API (posthog.com/docs/api.md, HTTP capture from any language), a live OpenAPI spec confirmed by probe (openapi.json), an official CLI for terminal/CI/CD use, and a documented MCP server enabling AI agents to run queries, ship flags, and inspect stack traces via plain text — all first-party documented and probe-verified. Missing for 10: independent hands-on corroboration of the API/OpenAPI usage beyond vendor docs and probes.", "evidenceIds": [ "posthog-docs-11", "posthog-probe-3", "posthog-probe-4", "posthog-probe-5", "posthog-docs-19", "posthog-docs-20", "posthog-docs-12" ] }, { "productId": "posthog", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence describes scoped or least-privilege API key/credential issuance for agents — the API docs mention basic HTTP event capture and an OpenAPI spec exists, but nothing about permission scoping, token minting with restricted access, or credential management for AI agents specifically.", "evidenceIds": [] }, { "productId": "posthog", "storyId": "agentic-sdks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "PostHog documents official SDKs/APIs (HTTP capture from any language, OpenAPI spec, CLI, MCP server) that let AI-native builders integrate programmatically, including AI-specific tooling like LLM observability SDKs and MCP-based agent workflows (posthog-docs-11, posthog-probe-3, posthog-docs-20/24, posthog-gh-6/9). A hands-on community comment corroborates smooth SDK integration for a Swift iOS app (posthog-comm-3), though another notes friction with LLM/AI query features (posthog-comm-8), suggesting some rough edges. Missing for 10: an explicit enumerated list of officially maintained language SDKs and stronger independent third-party validation of SDK completeness/versioning.", "evidenceIds": [ "posthog-docs-11", "posthog-probe-3", "posthog-docs-20", "posthog-docs-24", "posthog-gh-6", "posthog-gh-9", "posthog-comm-3", "posthog-comm-8" ] }, { "productId": "posthog", "storyId": "agentic-webhooks", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "PostHog's CDP pipelines can send event data to Slack, PagerDuty, and other SaaS destinations, and can filter/reshape events and route them 'wherever it needs to go' in realtime, which implies webhook-style outbound delivery — but no evidence pack item explicitly names 'webhook' as a subscription mechanism or ties this to AI-agent consumption of the event stream. missing for 10: explicit webhook endpoint/subscription documentation, confirmation that arbitrary event streams can be pushed to a caller-defined webhook URL, and any AI-native usage example of subscribing via webhook.", "evidenceIds": [ "posthog-docs-10", "posthog-docs-18", "posthog-intdir-1", "posthog-intdir-2", "posthog-gh-7" ] }, { "productId": "posthog", "storyId": "analytics-query-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "PostHog documents a full HogQL SQL query interface, a data warehouse that lets you query events/persons alongside external sources, a public OpenAPI spec (confirmed live at /openapi.json), and API-based event capture/export, plus MCP support for running queries programmatically. This directly satisfies running analytical queries and pulling raw event data via a documented API. missing for 10: independent/hands-on confirmation of raw event export via REST API specifically (evidence is mostly first-party docs), and no third-party validation of API rate limits/completeness for large-scale export.", "evidenceIds": [ "posthog-docs-8", "posthog-docs-17", "posthog-docs-9", "posthog-docs-11", "posthog-probe-3", "posthog-docs-24" ] }, { "productId": "posthog", "storyId": "api-interactive-docs", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The evidence shows PostHog publishes a machine-readable OpenAPI spec (openapi.json) and API docs referencing HTTP-based event capture, which implies some API reference exists, but there is no direct evidence of an interactive reference page with runnable/try-it examples. missing for 10: confirmation of an interactive API console, runnable code snippets, or live 'try it' functionality in the docs.", "evidenceIds": [ "posthog-probe-3", "posthog-docs-11" ] }, { "productId": "posthog", "storyId": "api-machine-spec", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "A probe confirms PostHog serves a machine-readable OpenAPI spec at https://posthog.com/openapi.json (HTTP 200, contains 'openapi' key), and the API docs reference programmatic access via HTTP. Missing for 10: no explicit first-party docs page describing the OpenAPI spec's usage/versioning or independent community confirmation of its completeness.", "evidenceIds": [ "posthog-probe-3", "posthog-docs-11" ] }, { "productId": "posthog", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers self-hosting (Docker Compose/MIT license), MCP integration, feature flags, and experiments, but nothing describes a dedicated sandbox/staging environment or test-mode data isolation from production. Self-hosting is a deployment option, not a sandbox testing feature, so it does not satisfy this story.", "evidenceIds": [] }, { "productId": "posthog", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "There is an OpenAPI spec probe (posthog-probe-3) confirming an API exists, but no evidence anywhere in the pack of API versioning scheme or a documented deprecation policy for that API.", "evidenceIds": [ "posthog-probe-3" ] }, { "productId": "posthog", "storyId": "autocapture-events", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack discusses capturing custom events via SDKs/HTTP requests and building insights, but nowhere states that PostHog automatically captures clicks, pageviews, or form interactions without instrumenting code (no mention of autocapture). Missing for 10: any documented autocapture feature, evidence of automatic click/pageview/form tracking, or an example of a PM enabling capture with zero code.", "evidenceIds": [ "posthog-docs-1", "posthog-docs-11" ] }, { "productId": "posthog", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "PostHog offers SQL for querying/modifying data at scale, a CDP pipeline that can reshape/filter events in bulk, and a CLI/API for scripting operations, which could underpin bulk actions, but there is no explicit documentation of a 'bulk update/delete/tag N items' feature, and a hands-on report shows the AI/SQL interface failing on a basic bulk task (creating a cohort via SQL). missing for 10: explicit bulk-operation API/UI (e.g., mass edit flags, batch delete events), documented bulk-scale limits, and independent confirmation that SQL/CLI bulk actions work reliably for AI-driven workflows.", "evidenceIds": [ "posthog-docs-8", "posthog-docs-9", "posthog-docs-10", "posthog-docs-20", "posthog-comm-8" ] }, { "productId": "posthog", "storyId": "automation-rules-engine", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "PostHog's CDP/Pipelines ('destinations') and alerting system let users define trigger conditions on events (errors, rage clicks, property changes, etc.) that automatically fire actions like Slack alerts, CRM syncs, PagerDuty notifications, or custom Hog functions, and workflows can automate actions/messages to users. missing for 10: no independent/hands-on verification of complex multi-condition automation rules, and no detail on latency/reliability of the automation engine at scale.", "evidenceIds": [ "posthog-docs-10", "posthog-docs-18", "posthog-docs-28", "posthog-intdir-1", "posthog-intdir-2", "posthog-gh-7" ] }, { "productId": "posthog", "storyId": "automation-scheduled-jobs", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "PostHog's CDP/pipelines can run 'in realtime or on a schedule' (posthog-docs-18) and GitHub notes 'workflows that automate actions or send messages to your users' (posthog-gh-7), suggesting some recurring automation capability, but there's no documentation of a general-purpose recurring job/workflow scheduler accessible via API, CLI, or MCP for AI-native orchestration. Missing for 10: explicit docs on scheduling arbitrary jobs/workflows via API or agent tooling, cron-like configuration details, and independent confirmation that scheduled pipelines work reliably.", "evidenceIds": [ "posthog-docs-18", "posthog-gh-7", "posthog-docs-28" ] }, { "productId": "posthog", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack items describe version history, review workflows, or rollback capabilities for automations (feature flags, CDP pipelines, or workflows); feature flags can be toggled off but there's no mention of versioning, diffing, or rollback to a previous version. Missing for 10: version history for flags/pipelines, review/approval workflow, rollback mechanism, audit trail of changes.", "evidenceIds": [] }, { "productId": "posthog", "storyId": "cdp-destinations", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "PostHog's CDP explicitly supports streaming events to third-party destinations including CRMs (Salesforce, HubSpot), ad platforms, webhooks, and SaaS tools (Braze, Customer.io, Slack, PagerDuty), with real-time or scheduled delivery via configurable Hog functions. missing for 10: independent/hands-on confirmation of destination reliability or throughput, and a full list of supported destinations beyond the named examples.", "evidenceIds": [ "posthog-docs-10", "posthog-docs-18", "posthog-docs-28", "posthog-docs-32", "posthog-intdir-1", "posthog-intdir-2" ] }, { "productId": "posthog", "storyId": "client-server-sdks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs confirm event capture from \"any SDK\" and from any HTTP-capable language, and community evidence corroborates real-world use of the iOS SDK integrating smoothly, supporting broad SDK coverage across platforms. Missing for 10: an explicit enumerated list of official SDKs per language/platform (web, Android, Node, Python, etc.) and independent confirmation for backend/server SDKs beyond the HTTP API and iOS mention.", "evidenceIds": [ "posthog-docs-1", "posthog-docs-11", "posthog-comm-3", "posthog-comm-5" ] }, { "productId": "posthog", "storyId": "cookieless-tracking", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no mention of cookie-based vs cookieless tracking, consent banners, or privacy-compliant capture configuration—only general analytics, session replay, and SDK/API capture details. Without any documentation addressing cookieless tracking or consent-banner avoidance, this axis has no supporting evidence.", "evidenceIds": [] }, { "productId": "posthog", "storyId": "dashboards-api", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "The evidence confirms PostHog exposes a general REST API (openapi.json spec) and documents building/sharing dashboards and insights with alerts, implying programmatic access is plausible, but no citation explicitly documents dashboard/insight CRUD API endpoints or SDK examples for managing them programmatically. missing for 10: explicit API docs/endpoints for creating/updating dashboards and insights, code examples for programmatic dashboard management, independent confirmation of this specific API surface.", "evidenceIds": [ "posthog-probe-3", "posthog-docs-3", "posthog-docs-25" ] }, { "productId": "posthog", "storyId": "dashboards-builder", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Docs explicitly state users can build insights, save them to dashboards, share them, and set alerts (posthog-docs-3, posthog-docs-25), directly matching the story of composing saved insights into shareable dashboards. Missing for 10: no independent/hands-on corroboration of the dashboard-sharing workflow specifically, and no detail on team-level permissions or collaboration features around shared dashboards.", "evidenceIds": [ "posthog-docs-3", "posthog-docs-25", "posthog-docs-15" ] }, { "productId": "posthog", "storyId": "feature-flags-targeting", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "PostHog docs explicitly describe flagging changes and rolling out to a percentage of users, and targeting by person property, cohort, or group, which directly matches the story. missing for 10: no independent/hands-on corroboration of targeting accuracy or percentage rollout behavior beyond vendor docs.", "evidenceIds": [ "posthog-docs-5", "posthog-docs-6" ] }, { "productId": "posthog", "storyId": "founder-self-serve-answers", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "PostHog's product analytics explicitly builds funnels, retention, trends, paths, stickiness, and lifecycle insights on captured events, with dashboards and alerts, all self-serve without requiring an analyst (posthog-docs-15, posthog-docs-25, posthog-docs-1). This directly supports self-service drop-off/activation analysis via funnels. missing for 10: no independent/hands-on evidence of a founder successfully using funnels to diagnose activation drop-off, and no case study demonstrating non-analyst usability in practice.", "evidenceIds": [ "posthog-docs-15", "posthog-docs-25", "posthog-docs-1", "posthog-docs-3" ] }, { "productId": "posthog", "storyId": "funnel-analysis", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "PostHog docs explicitly state funnels are a first-class insight type built on captured events ('Build trends, funnels, retention, paths, stickiness...') alongside dashboards, alerts, and saved insights, which covers the core funnel-building use case for a PM. However, the evidence pack doesn't explicitly detail funnel-specific filter/breakdown UI mechanics beyond the general mention. Missing for 10: explicit documentation or screenshots of step-level filters and breakdown-by-property configuration within funnels, and independent user validation of funnel usability.", "evidenceIds": [ "posthog-docs-15", "posthog-docs-25", "posthog-docs-1", "posthog-docs-3" ] }, { "productId": "posthog", "storyId": "group-account-analytics", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "PostHog documents a group concept for targeting (person property, cohort, or group) used in feature flag rollouts, implying group-level segmentation exists, but the evidence pack never explicitly ties 'groups' to trends/funnels/retention insights or shows account-level rollups in analytics dashboards. missing for 10: explicit docs on group-level funnels/retention insights, account/company entity definition and configuration, examples of B2B account analytics dashboards.", "evidenceIds": [ "posthog-docs-6", "posthog-docs-15", "posthog-docs-25" ] }, { "productId": "posthog", "storyId": "historical-event-import", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "There's no first-party documentation of a dedicated bulk-import tool for historical events from another analytics tool, but the data warehouse/CDP features allow syncing external data sources into PostHog for unified querying, and a community comment notes Mixpanel-library compatibility making switching 'really easy.' missing for 10: explicit import API/tool for historical event data, documented data-export ingestion pipeline, and first-party confirmation of the Mixpanel-compatibility claim.", "evidenceIds": [ "posthog-docs-9", "posthog-docs-10", "posthog-comm-5" ] }, { "productId": "posthog", "storyId": "identity-resolution", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers event capture, analytics, session replay, feature flags, experiments, and CDP integrations, but nowhere mentions identify()/alias() calls, distinct_id merging, or cross-device/anonymous-to-identified user reconciliation. This is a standard axis for an event-ingestion/analytics product, but no evidence supports it here. Missing for 10: any mention of identify/alias APIs, anonymous-to-known user merging, or cross-device identity resolution documentation.", "evidenceIds": [] }, { "productId": "posthog", "storyId": "metric-alerts", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "PostHog docs explicitly state dashboards/insights can be saved, shared, and users can 'get alerted when they move' (posthog-docs-3, posthog-docs-25), and alerts can be routed to Slack/PagerDuty (posthog-intdir-2), directly matching the PM story of dashboard subscriptions with abnormal-metric alerts. Missing for 10: no independent/hands-on confirmation of the alerting UX, no detail on anomaly-detection thresholds, and no explicit mention of subscribing to an entire dashboard vs individual insights.", "evidenceIds": [ "posthog-docs-3", "posthog-docs-25", "posthog-intdir-2" ] }, { "productId": "posthog", "storyId": "openness-api-parity", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "PostHog exposes a full OpenAPI-documented REST API (posthog-probe-3, posthog-docs-11), an MCP server enabling agents to run queries, ship flags, and inspect errors (posthog-docs-2, -12, -19, -24, probe-4), a CLI for terminal/CI use (posthog-docs-20, probe-5), and SQL access throughout the product (posthog-docs-8, -17), collectively covering analytics, flags, experiments, error tracking, and data warehouse operations programmatically. missing for 10: an explicit vendor statement guaranteeing 1:1 API/UI parity, and independent verification that every UI workflow (e.g. survey building, dashboard sharing) is fully replicable via API without gaps.", "evidenceIds": [ "posthog-docs-2", "posthog-docs-11", "posthog-docs-12", "posthog-docs-19", "posthog-docs-20", "posthog-docs-8", "posthog-probe-3", "posthog-probe-4", "posthog-probe-5" ] }, { "productId": "posthog", "storyId": "openness-full-export", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "PostHog is open-source (MIT-licensed self-host Docker Compose) and exposes data via SQL, an HTTP API, and data-warehouse sync, which together give strong data portability and self-hosting options implying users can retain full control of their data. However, there is no explicit documented 'export all your data' feature or open-format bulk export tool, and no independent evidence of someone successfully exporting/migrating a full production dataset. Missing for 10: explicit bulk-export/download-all-data documentation, confirmation of open standard export formats (e.g., CSV/Parquet dumps), and independent hands-on evidence of a full data migration/exit.", "evidenceIds": [ "posthog-docs-21", "posthog-docs-14", "posthog-docs-8", "posthog-docs-17", "posthog-docs-9", "posthog-docs-11", "posthog-probe-3" ] }, { "productId": "posthog", "storyId": "openness-open-license", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "PostHog explicitly documents its open-source nature and MIT-licensed self-host deployment (Docker Compose), with the actual source hosted publicly on GitHub, satisfying the ability to read the product's source under an open license. Missing for 10: no explicit confirmation of the license covering the entire codebase (vs just the self-host component) or independent audit of license terms.", "evidenceIds": [ "posthog-docs-14", "posthog-docs-21" ] }, { "productId": "posthog", "storyId": "openness-self-host", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "PostHog's docs explicitly state the core product is open-source and offers a free MIT-licensed Docker Compose self-host deployment (posthog-docs-14/21), which is a clear ai-native/openness claim. However, a hands-on community report describes the production Dockerfile/posthog:latest image as 'busted' and failing to create a database, requiring significant debugging effort — a concrete contradiction of the self-host claim working out of the box. Missing for 10: independent confirmation the current self-host path works reliably, resolution or acknowledgment of the reported Docker issue, and details on feature parity between self-hosted and cloud versions.", "evidenceIds": [ "posthog-docs-14", "posthog-docs-21", "posthog-comm-4" ] }, { "productId": "posthog", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack has self-hosting/open-source docs (posthog-docs-14, posthog-docs-21) which relate to deployment flexibility but no mention of data residency/region selection (e.g., EU vs US cloud region choice) for PostHog Cloud. Missing for 10: explicit region/residency selection options, data storage location controls, compliance documentation on where data resides.", "evidenceIds": [ "posthog-docs-14", "posthog-docs-21" ] }, { "productId": "posthog", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses AI-training data opt-out or any privacy control preventing customer data from being used to train AI models; nothing in the docs or community items mentions this capability.", "evidenceIds": [] }, { "productId": "posthog", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack contains no mention of data retention settings, deletion APIs, GDPR-style data deletion, or retention policy controls anywhere in the docs, CLI, API, or CDP descriptions. This is a fair and expected axis for an analytics/data platform handling user event data, so absence of evidence means the story is unmet rather than inapplicable.", "evidenceIds": [] }, { "productId": "posthog", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers PostHog's product analytics, session replay, feature flags, MCP/CLI integrations, and self-hosting, but contains no mention of an opt-out setting for PostHog's own telemetry/usage tracking (e.g., CLI or SDK phone-home data collection). No documentation or community evidence addresses this capability.", "evidenceIds": [] }, { "productId": "posthog", "storyId": "replay-linked-to-events", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "PostHog documents linking session replays directly to flag rollouts and to exceptions/errors ('see the session replays, events, and exceptions from the people who got the flag'), and session replay's synced DevTools panel shows errors at the moment they happened, supporting a PM-style jump from error/event to replay. However, the evidence never explicitly documents jumping from a funnel drop-off step to session replays, only from feature-flag cohorts/exceptions. missing for 10: explicit funnel-to-replay linking documentation, independent/hands-on confirmation of this specific workflow.", "evidenceIds": [ "posthog-docs-6", "posthog-docs-26", "posthog-docs-16", "posthog-docs-29" ] }, { "productId": "posthog", "storyId": "retention-cohorts", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "PostHog explicitly ships retention as a core insight type (posthog-docs-15, posthog-docs-25) alongside cohort support elsewhere in the product (posthog-docs-6 references rolling out and analyzing by cohort, posthog-docs-8/17 SQL for custom cohort queries). This directly matches the PM story of measuring retention and slicing by behavioral cohorts, though the evidence doesn't show a dedicated doc excerpt showing retention insight breakdown-by-cohort UI directly. missing for 10: an explicit doc/screenshot showing retention insight with cohort breakdown control, and independent hands-on confirmation of this specific workflow.", "evidenceIds": [ "posthog-docs-15", "posthog-docs-25", "posthog-docs-6", "posthog-docs-8", "posthog-docs-17" ] }, { "productId": "posthog", "storyId": "session-replay-capture", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence confirms PostHog's session replay lets PMs watch DVR-style recordings with synced DevTools panel (posthog-docs-26, posthog-docs-29, posthog-gh-8), but nothing in the pack mentions input masking, privacy controls, or sensitive-data redaction being applied by default. missing for 10: any documentation of input masking/privacy defaults, evidence of what is masked or how, and confirmation it's on by default rather than opt-in.", "evidenceIds": [ "posthog-docs-26", "posthog-docs-29", "posthog-gh-8" ] }, { "productId": "posthog", "storyId": "sql-query-access", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "PostHog documents HogQL, a SQL-like query language, that lets developers directly query raw events with SELECT/FROM/JOIN/WHERE/GROUP BY, and also join across data warehouse sources alongside events/persons/sessions in one SQL query, plus MCP/agent access to run HogQL queries. This is well-documented first-party capability with clear syntax examples. missing for 10: independent hands-on verification of SQL query correctness/performance at scale (one community comment notes LLM confusion with cohorts, not the SQL feature itself).", "evidenceIds": [ "posthog-docs-8", "posthog-docs-17", "posthog-docs-9", "posthog-docs-19", "posthog-docs-24" ] }, { "productId": "posthog", "storyId": "user-paths", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "PostHog explicitly documents 'paths' insights alongside trends, funnels, retention, stickiness, and lifecycle built on captured events (posthog-docs-15, posthog-docs-25), directly matching the path-analysis story. Missing for 10: no dedicated docs excerpt detailing path-specific configuration (e.g., start/end event anchoring, wildcard grouping) or independent/hands-on validation of the paths feature specifically.", "evidenceIds": [ "posthog-docs-15", "posthog-docs-25", "posthog-docs-1" ] }, { "productId": "posthog", "storyId": "warehouse-sync", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "PostHog's data warehouse feature lets developers sync external sources (Stripe, Postgres, Salesforce, HubSpot, etc.) and query them alongside native events/persons/sessions in one SQL query, and the CDP allows continuous export of events to external destinations in realtime or on a schedule, satisfying both directions of the sync story. Missing for 10: independent/hands-on verification of the warehouse sync reliability or performance at scale, and clearer documentation of specific export destinations (data warehouses) beyond CRM/Slack examples.", "evidenceIds": [ "posthog-docs-9", "posthog-docs-8", "posthog-docs-17", "posthog-docs-18", "posthog-docs-28", "posthog-docs-22" ] } ]