[ { "productId": "chargebee", "storyId": "agent-manages-subscriptions", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Chargebee has a full REST API (with Basic Auth, SDKs) that supports subscription creation, plan changes, and credits, plus an official MCP server that exposes 'selected capabilities' to AI clients like Cursor/Claude/ChatGPT, and custom MCP servers that scope which tools are exposed. This directly supports agent-driven billing operations with some permission scoping, but evidence doesn't confirm granular scoped-permission models (e.g., per-action API keys) for agents beyond curated tool subsets, nor independent/hands-on validation of agent-driven write operations. missing for 10: documented fine-grained scoped API-key/OAuth permission model for agent use, independent/hands-on evidence of an agent successfully executing subscription/credit/plan-change operations via MCP or API.", "evidenceIds": [ "chargebee-docs-1", "chargebee-docs-4", "chargebee-docs-7", "chargebee-docs-8", "chargebee-docs-16", "chargebee-probe-4" ] }, { "productId": "chargebee", "storyId": "agent-reads-billing-state", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Chargebee offers a REST API for querying subscriptions, invoices, and account data (chargebee-docs-1) plus official MCP servers letting AI clients like Cursor, Claude, ChatGPT and Codex 'securely access selected capabilities' including subscription/invoice lookups (chargebee-docs-7, chargebee-docs-8, chargebee-probe-4), directly enabling agent-driven billing queries. Missing for 10: explicit documented support for MRR/delinquent-account aggregate queries via MCP or API, and independent/hands-on confirmation that the MCP server reliably answers such billing-state questions.", "evidenceIds": [ "chargebee-docs-1", "chargebee-docs-7", "chargebee-docs-8", "chargebee-docs-16", "chargebee-probe-4" ] }, { "productId": "chargebee", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "A live probe confirms llms.txt returns HTTP 200 with a proper agent-oriented summary of the API reference, and docs are also served in .md format for direct agent consumption, alongside first-party MCP server docs for agent tool access. Missing for 10: no independent/community confirmation of agents actually using llms.txt in practice.", "evidenceIds": [ "chargebee-probe-1", "chargebee-docs-1", "chargebee-docs-7" ] }, { "productId": "chargebee", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Chargebee docs describe an in-product AI assistant ('Use AI inside the Chargebee Billing UI to answer questions and complete tasks') and AI that can draft plans/pricing from documents, which are in-product AI-driven capabilities, but there's no evidence of proactive AI-generated insights or suggestions derived from the customer's own billing/subscription data (e.g., churn risk, revenue anomalies) — the described features skew toward Q&A and configuration drafting rather than data-driven insight generation. Missing for 10: evidence of analytics-style insights/suggestions surfaced from account data, and independent/hands-on validation of the AI assistant's usefulness.", "evidenceIds": [ "chargebee-docs-16", "chargebee-docs-14", "chargebee-docs-7" ] }, { "productId": "chargebee", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Chargebee offers webhooks, dunning/smart retry, and MCP servers that let AI clients access billing capabilities, which could underpin background automations, but there's no direct evidence of a scheduler or agent-orchestration feature that runs autonomous, unattended workflows over time. missing for 10: explicit autonomous/background automation or workflow engine documentation, evidence of scheduled/triggered agentic tasks, independent confirmation of unattended operation.", "evidenceIds": [ "chargebee-docs-5", "chargebee-docs-7", "chargebee-docs-8", "chargebee-docs-11", "chargebee-docs-12", "chargebee-probe-4" ] }, { "productId": "chargebee", "storyId": "agentic-builtin-assistant", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Chargebee documents a built-in AI assistant inside the Billing UI that can 'answer questions and complete tasks' and draft pricing/product configurations from a description, showing real in-product task delegation (chargebee-docs-16, chargebee-docs-14). However, the primary AI-native evidence emphasizes MCP servers for external AI clients (Cursor, Claude, ChatGPT) rather than a deeply documented native assistant experience, and there is no independent/hands-on corroboration of the in-app assistant's task scope or reliability. missing for 10: independent verification of the built-in assistant's capabilities, detail on task types beyond drafting/answering, and community evidence confirming real-world use.", "evidenceIds": [ "chargebee-docs-16", "chargebee-docs-14", "chargebee-docs-7" ] }, { "productId": "chargebee", "storyId": "agentic-headless", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Chargebee exposes a full REST API (Basic Auth, JSON) plus SDKs like Node.js, which can be scripted and called from CI pipelines without a UI, and an API Explorer confirms programmatic testing is possible outside the UI. However, there is no explicit CI/CD documentation, no official CLI, and no example pipelines or automation recipes demonstrating headless CI usage. missing for 10: CI/CD-specific docs or examples, an official CLI tool, and evidence of automated pipeline integration beyond generic API/SDK access.", "evidenceIds": [ "chargebee-docs-1", "chargebee-docs-2", "chargebee-docs-9" ] }, { "productId": "chargebee", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only shows Chargebee exposes MCP servers for external AI clients (Cursor, Claude, ChatGPT) to consume Chargebee's own tools, i.e. Chargebee acts as the MCP server, not as a client plugging in third-party MCP servers to gain their tools. There is no documentation or example of Chargebee's AI features consuming external MCP servers. Missing for 10: any evidence of Chargebee acting as an MCP client, configuration for adding external MCP server endpoints, or first-party support for extending its AI assistant with third-party MCP tool servers.", "evidenceIds": [ "chargebee-docs-7", "chargebee-docs-8", "chargebee-docs-16", "chargebee-probe-4" ] }, { "productId": "chargebee", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Chargebee documents official MCP Servers that let AI clients (Cursor, Claude, ChatGPT, Codex, etc.) connect and access selected Chargebee capabilities, plus a custom MCP server builder for tailored tool sets, confirmed via docs and a probe hit. Missing for 10: independent/hands-on third-party corroboration of the MCP server working in practice.", "evidenceIds": [ "chargebee-docs-7", "chargebee-docs-8", "chargebee-probe-4" ] }, { "productId": "chargebee", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Chargebee documents an in-product AI assistant that lets users 'answer questions and complete tasks' via natural language inside the Billing UI, plus AI-driven drafting of pricing/plans from a description, and MCP servers that let external AI clients (Cursor, Claude, ChatGPT) operate Chargebee capabilities conversationally. This is first-party doc evidence only, with no independent/hands-on corroboration of how robust or complete the natural-language command coverage actually is. Missing for 10: independent/hands-on validation of the AI assistant's command coverage, and detail on which actions can be fully executed vs. only queried via natural language.", "evidenceIds": [ "chargebee-docs-16", "chargebee-docs-14", "chargebee-docs-7", "chargebee-docs-8", "chargebee-probe-4" ] }, { "productId": "chargebee", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence shows SDKs (Node.js), REST API, and MCP servers, but there is no mention of an official CLI tool for Chargebee. Missing for 10: any documentation or evidence of an official CLI, install instructions, or command-line usage.", "evidenceIds": [] }, { "productId": "chargebee", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Chargebee documents a full REST API with auth, JSON responses, SDKs, an API Explorer for live testing, and an llms.txt reference confirming API v2 coverage — this is a clearly documented, drivable public API. Missing for 10: publicly hosted OpenAPI/swagger spec (probe found 404s) and independent third-party corroboration of API robustness.", "evidenceIds": [ "chargebee-docs-1", "chargebee-docs-2", "chargebee-docs-9", "chargebee-probe-1", "chargebee-probe-3" ] }, { "productId": "chargebee", "storyId": "agentic-scoped-keys", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Chargebee's API uses HTTP Basic Auth with API keys and offers custom MCP servers that can be scoped to a subset of the tool catalog (e.g., invoice-only), suggesting some ability to limit an agent's surface area, but there is no documented support for scoped/least-privilege API key roles, granular permission scopes, or agent-specific credential issuance in the core REST API. missing for 10: explicit API key roles/permission scopes documentation, least-privilege credential issuance workflow for agents, independent verification that custom MCP server scoping enforces true least-privilege access control.", "evidenceIds": [ "chargebee-docs-1", "chargebee-docs-8", "chargebee-probe-4" ] }, { "productId": "chargebee", "storyId": "agentic-sdks", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Chargebee documents official SDKs (e.g., Node.js via npm) and a REST API with docs, satisfying the core 'build against official SDKs' ask, but evidence only lists one SDK explicitly and lacks broader SDK language coverage, versioning, or independent developer corroboration; OpenAPI spec discovery also failed in probes. Missing for 10: multi-language SDK list/documentation depth, independent hands-on validation, discoverable OpenAPI/schema artifacts.", "evidenceIds": [ "chargebee-docs-2", "chargebee-docs-1", "chargebee-probe-3" ] }, { "productId": "chargebee", "storyId": "agentic-webhooks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Chargebee documents event-driven webhooks (events trigger configured webhooks) and community evidence confirms multiple webhook URLs can be configured for different environments, showing real-world subscription-to-events usage. missing for 10: no detailed webhook payload/signature-verification docs, no explicit AI-agent-oriented subscription workflow beyond generic webhook config, and no independent hands-on verification beyond the one HN mention.", "evidenceIds": [ "chargebee-docs-5", "chargebee-comm-2" ] }, { "productId": "chargebee", "storyId": "api-interactive-docs", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Chargebee documents an API Explorer that lets users make live API calls in a browser with a guided interface to test endpoints without writing code, which matches the interactive/runnable-example story. However, no OpenAPI spec was discoverable via probe (all candidate paths 404), and there's no independent/hands-on corroboration of the explorer's depth or coverage. Missing for 10: discoverable OpenAPI/Swagger spec, independent/hands-on validation of the API Explorer's runnable examples across endpoints.", "evidenceIds": [ "chargebee-docs-9", "chargebee-docs-1", "chargebee-probe-3" ] }, { "productId": "chargebee", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Direct probes for standard OpenAPI/Swagger spec locations all returned 404, and no documentation page claims a downloadable OpenAPI/machine-readable spec exists; only an llms.txt index and human-readable markdown docs were found.", "evidenceIds": [ "chargebee-probe-3", "chargebee-probe-1", "chargebee-probe-2" ] }, { "productId": "chargebee", "storyId": "api-sandbox", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Evidence shows Chargebee supports separate dev/staging environments (webhooks routed independently per environment) and an API Explorer for testing endpoints without code, implying some sandbox-like testing capability, but there is no explicit documentation of a dedicated sandbox/test site that guarantees production data isolation for AI-driven testing. Missing for 10: explicit sandbox/test-mode documentation, confirmation that API keys or MCP tools can target a non-production environment, and first-party guidance on avoiding production data during AI testing.", "evidenceIds": [ "chargebee-docs-9", "chargebee-comm-2" ] }, { "productId": "chargebee", "storyId": "api-versioning-policy", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Evidence shows Chargebee's API is versioned (API v2, Product Catalog) per the llms.txt probe, but there is no documentation of a deprecation policy, sunset timeline, or migration guidance for older API versions anywhere in the pack. Missing for 10: explicit deprecation/sunset policy docs, versioning changelog, migration guide for deprecated endpoints, independent confirmation of long-term API stability.", "evidenceIds": [ "chargebee-probe-1", "chargebee-docs-1" ] }, { "productId": "chargebee", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Chargebee's REST API, SDKs, and Export resource allow programmatic access to many records at once (e.g. bulk export jobs), and its MCP servers let AI clients invoke these API operations, but the evidence never documents a dedicated bulk/batch-update endpoint for mutating many items in a single call — automation would rely on looping single-item API calls. missing for 10: explicit bulk-create/bulk-update API endpoints, documented batch size limits, and independent confirmation of bulk operations working via MCP/AI clients.", "evidenceIds": [ "chargebee-docs-6", "chargebee-docs-1", "chargebee-docs-2", "chargebee-docs-7", "chargebee-docs-8", "chargebee-probe-4" ] }, { "productId": "chargebee", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Chargebee supports event-driven automation via webhooks (events trigger webhooks, multiple URLs supported) and has built-in automated actions like Dunning email reminders and Smart Retry logic that act on transaction event types, but there's no evidence of a general-purpose, user-configurable rules/workflow engine where an AI-native user can define arbitrary 'if event X then action Y' rules beyond these pre-built billing automations. Missing for 10: a documented rules/workflow builder or automation engine, evidence of custom trigger-action definitions beyond webhooks/dunning/retries, and independent confirmation of flexibility.", "evidenceIds": [ "chargebee-docs-5", "chargebee-comm-2", "chargebee-docs-11", "chargebee-docs-12" ] }, { "productId": "chargebee", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Chargebee's evidence covers recurring billing cycles, dunning retry schedules, and async export jobs, but these are internal billing mechanics, not a user- or AI-schedulable job/workflow automation feature (e.g., no API endpoint or MCP tool to create/schedule arbitrary recurring workflows). The MCP server evidence describes connecting AI clients to Chargebee's data/tools, not job scheduling.", "evidenceIds": [ "chargebee-docs-11", "chargebee-docs-12", "chargebee-docs-15", "chargebee-docs-6", "chargebee-docs-7" ] }, { "productId": "chargebee", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of any versioning, review, or rollback capability for automations (e.g., workflow/automation revision history or undo) in Chargebee's docs — only export jobs, webhooks, and MCP AI access are mentioned, none of which describe automation change tracking or rollback.", "evidenceIds": [] }, { "productId": "chargebee", "storyId": "billing-cycle-flexibility", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no documentation on billing anchor dates, calendar-vs-anniversary billing choice, cross-subscription date alignment, or custom invoice date control — only generic multi-frequency billing (chargebee-docs-15) which doesn't address anchor/alignment mechanics. Missing for 10: docs on billing anchor/alignment settings, calendar vs anniversary billing options, custom invoice date configuration.", "evidenceIds": [ "chargebee-docs-15" ] }, { "productId": "chargebee", "storyId": "compliant-invoice-generation", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack contains no documentation of invoice/credit note/receipt generation, sequential numbering, custom fields, or memos — only tangential billing topics like dunning, revenue recognition, and API basics. missing for 10: invoice generation docs, credit note/receipt automation, sequential numbering compliance, custom fields/memos support.", "evidenceIds": [] }, { "productId": "chargebee", "storyId": "dunning-comms-automation", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm Chargebee's Dunning feature sends email reminders and schedules smart payment retries, covering core automated dunning emails and retry logic, but there is no evidence of in-app banners, configurable grace periods, or explicit cancel-vs-unpaid final-state rule configuration. missing for 10: in-app banner support, grace period configuration, final-state (cancel vs unpaid) rule documentation, branding/customization details for dunning emails.", "evidenceIds": [ "chargebee-docs-11", "chargebee-docs-12" ] }, { "productId": "chargebee", "storyId": "full-billing-export", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Chargebee's docs confirm a generic Export API resource that creates export jobs and provides a download URL (chargebee-docs-6), implying some data-export capability, but there is no evidence specifying export of subscriptions, invoices, and usage records specifically, nor any mention of porting stored payment methods to another provider. missing for 10: explicit documentation of exporting subscriptions/invoices/usage data, any payment-method portability/export mechanism, and independent confirmation of successful migration workflows.", "evidenceIds": [ "chargebee-docs-6" ] }, { "productId": "chargebee", "storyId": "gl-erp-sync", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The pack shows Chargebee has RevRec (ASC 606) and generic Export/API/webhook capabilities, but no evidence of any named QuickBooks, Xero, or NetSuite connector or GL/ERP sync workflow — missing for 10: named ERP/accounting integrations, sync mechanism details, and evidence that invoices/payments/recognition entries flow automatically without manual export/import.", "evidenceIds": [ "chargebee-docs-13", "chargebee-docs-6", "chargebee-docs-1" ] }, { "productId": "chargebee", "storyId": "hybrid-pricing-models", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Evidence confirms usage-event recording for usage-based billing and multi-frequency subscription billing, which supports basic usage-plus-subscription models, but there is no documentation of tiered, volume, graduated, per-unit, or hybrid pricing structures, nor of credits, commitments, or overage handling. missing for 10: explicit tiered/volume/graduated/per-unit pricing model docs, credit/commitment/overage handling documentation, no-code configuration evidence for these pricing types.", "evidenceIds": [ "chargebee-docs-3", "chargebee-docs-15", "chargebee-docs-4" ] }, { "productId": "chargebee", "storyId": "lifecycle-event-stream", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Chargebee's Events API triggers configured webhooks (chargebee-docs-5), and community evidence confirms multiple webhook URLs can be configured for different environments, corroborating reliable event delivery (chargebee-comm-2). Missing for 10: explicit documentation of the full set of lifecycle event types (renewal, upgrade, payment failure, cancellation) and delivery-retry/reliability guarantees for webhook events.", "evidenceIds": [ "chargebee-docs-5", "chargebee-comm-2" ] }, { "productId": "chargebee", "storyId": "multi-currency-billing", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Chargebee docs mention item prices tied to specific currencies (implying per-currency price points exist), but there is no evidence describing consolidated reporting back into a home currency for finance leads. Missing for 10: explicit multi-currency pricing catalog documentation, currency conversion/consolidation reporting features, and any finance-lead-facing reporting tool showing home-currency rollups.", "evidenceIds": [ "chargebee-docs-4" ] }, { "productId": "chargebee", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Chargebee's REST API documentation covers a broad range of core UI-configurable features (subscriptions, usage events, exports, webhooks, entitlements) suggesting strong API/UI parity, and an API Explorer lets users test all documented endpoints. However, there's no explicit vendor claim of full UI-API parity, and features like AI drafting (chargebee-docs-14) or dunning setup are only described as UI features without confirmation they're also fully API-driven. Missing for 10: explicit parity statement, evidence that every UI action (e.g., AI drafting, dunning configuration) has an API equivalent, independent/hands-on confirmation of parity.", "evidenceIds": [ "chargebee-docs-1", "chargebee-docs-3", "chargebee-docs-4", "chargebee-docs-5", "chargebee-docs-6", "chargebee-docs-9", "chargebee-docs-10", "chargebee-docs-11", "chargebee-docs-14" ] }, { "productId": "chargebee", "storyId": "openness-full-export", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Chargebee documents an Export resource that produces job status and a download URL (chargebee-docs-6), and its REST API returns JSON that could be used to pull data programmatically (chargebee-docs-1), giving a route to extract data in a machine-readable format. However there is no explicit documentation of bulk/full-account data export, no mention of standard open formats like CSV/JSON dumps of all account data, and no evidence of a clear 'leave with your data' workflow. missing for 10: explicit full-account bulk export documentation, confirmation of open/standard export formats (CSV/JSON) for all entities, and independent confirmation that this satisfies a complete data portability use case.", "evidenceIds": [ "chargebee-docs-6", "chargebee-docs-1" ] }, { "productId": "chargebee", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "chargebee", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "chargebee", "storyId": "payment-method-recovery", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Chargebee documents dunning email reminders and smart retry logic for recovering failed payments, which supports part of the dunning-recovery workflow, but the evidence pack contains no explicit mention of network account updater, backup payment methods, or hosted update-payment pages. missing for 10: explicit documentation of network account updater (card auto-refresh via card networks), backup/secondary payment method fallback, and hosted payment page update flows for customers.", "evidenceIds": [ "chargebee-docs-11", "chargebee-docs-12" ] }, { "productId": "chargebee", "storyId": "plan-entitlements-api", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Chargebee documents an entitlements/Features model attached to plans, including a UI to manage customer privileges and AI-assisted entitlement creation, and separately documents a general REST API and Node.js SDK. However, no evidence explicitly shows an API/SDK endpoint or method for checking a customer's entitlements at runtime. missing for 10: explicit entitlements-check API endpoint or SDK method, code sample showing runtime lookup, independent/hands-on confirmation of this specific capability.", "evidenceIds": [ "chargebee-docs-10", "chargebee-docs-14", "chargebee-docs-1", "chargebee-docs-2" ] }, { "productId": "chargebee", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack discusses data residency, regional data storage options, or hosting location controls for Chargebee; the docs cover API usage, billing features, and MCP integration but nothing about choosing storage regions. Missing for 10: any mention of data residency/region selection, compliance certifications tied to geographic storage, or customer-facing controls for data location.", "evidenceIds": [] }, { "productId": "chargebee", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses AI training data opt-out, data usage policies for model training, or any privacy controls specific to preventing data from being used to train AI models; evidence only covers billing API/MCP features unrelated to this privacy axis.", "evidenceIds": [] }, { "productId": "chargebee", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack discusses data retention policies, deletion controls, or privacy/data lifecycle management for AI-native users; the pack covers billing APIs, MCP integrations, and dunning/revenue features but nothing about controlling or deleting stored data.", "evidenceIds": [] }, { "productId": "chargebee", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses telemetry/usage-tracking opt-out settings for Chargebee's product, MCP servers, or SDKs; this is an applicable privacy-posture axis for any SaaS/API product but no documentation or community evidence confirms such a control exists.", "evidenceIds": [] }, { "productId": "chargebee", "storyId": "proration-correctness", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack contains only generic API/billing feature docs (usage events, item price compatibility, dunning, RevRec) but nothing about proration mechanics, credit calculations, partial-period invoice line items, or auditable math for upgrades/downgrades. Missing for 10: proration policy docs, credit note reconciliation examples, partial-period invoice line item breakdowns, and any audit-trail evidence.", "evidenceIds": [] }, { "productId": "chargebee", "storyId": "realtime-usage-visibility", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Chargebee's usage_events API supports recording usage for usage-based billing and webhooks can propagate events, but there is no documented dashboard for real-time cycle usage/cost visibility to customers and teams, nor any explicit threshold/alert feature for pre-invoice warnings. Missing for 10: real-time usage/cost dashboard for customers, configurable usage thresholds, proactive alerting before invoicing.", "evidenceIds": [ "chargebee-docs-3", "chargebee-docs-5" ] }, { "productId": "chargebee", "storyId": "revrec-asc606-reports", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Chargebee's RevRec product is explicitly documented as ASC 606 compliant, covering contract changes like renewal/modification/cancellation, directly matching the finance-lead's revenue recognition need. However, evidence lacks detail on IFRS 15 specifically, deferred revenue schedule specifics, audit-ready report generation/export workflows, or independent/community corroboration of accuracy in practice. Missing for 10: IFRS 15 explicit coverage, deferred revenue schedule detail, audit trail/report examples, and third-party validation of RevRec accuracy.", "evidenceIds": [ "chargebee-docs-13" ] }, { "productId": "chargebee", "storyId": "self-serve-plan-changes", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers Chargebee's API, dunning, entitlements admin UI, and MCP/AI tooling, but nothing describes a hosted customer portal or embeddable self-service components letting end customers upgrade/downgrade/cancel on their own. missing for 10: hosted customer portal documentation, embeddable self-service widget docs, evidence of ticket-free self-service flow.", "evidenceIds": [] }, { "productId": "chargebee", "storyId": "smart-retries", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Chargebee's docs confirm scheduled dunning retries and \"Smart Retry\" logic that varies intervals dynamically based on gateway decline type, directly supporting automated, optimized retry cadences aimed at recovering failed renewal payments. However, missing for 10: no quantified recovery-rate data, no explicit claim of ML-based optimization (only 'dynamic intervals based on error type'), and no independent/customer evidence measuring actual involuntary-churn recovery impact.", "evidenceIds": [ "chargebee-docs-11", "chargebee-docs-12" ] }, { "productId": "chargebee", "storyId": "subscription-analytics", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence shows RevRec for ASC 606 compliance and generic data Export jobs, but nothing documents native MRR, churn, cohort retention, or LTV analytics computed within Chargebee itself — the only analytics-adjacent capability shown (Exports) is exactly the 'bolted-on export' pattern the story asks to avoid.", "evidenceIds": [ "chargebee-docs-6", "chargebee-docs-13" ] }, { "productId": "chargebee", "storyId": "subscription-crud-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Chargebee's REST API and Subscriptions resource are documented (chargebee-docs-1, chargebee-docs-4), and it supports form-encoded requests, JSON responses, and even a live API Explorer for testing endpoints (chargebee-docs-9), consistent with full CRUD-style subscription management. However, the evidence pack never explicitly cites 'create', 'update', 'pause', or 'cancel' subscription endpoints or their request/response schemas, so the full lifecycle claim is inferred rather than directly documented. missing for 10: explicit endpoint-level documentation for pause and cancel operations, and independent developer confirmation of using these endpoints end-to-end.", "evidenceIds": [ "chargebee-docs-1", "chargebee-docs-4", "chargebee-docs-9" ] }, { "productId": "chargebee", "storyId": "subscription-import-migration", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence of any migration tooling, bulk import for live subscriptions, PCI-compliant card data transfer/tokenization import, or zero-downtime cutover capability is present in the evidence pack. Only general API/export/dunning/revrec docs are shown, none addressing subscription migration from spreadsheets or competitor platforms.", "evidenceIds": [] }, { "productId": "chargebee", "storyId": "tax-on-invoices", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence in the pack shows Chargebee performing native or integrated sales tax/VAT/GST calculation, registrations, or rate management on invoices. The only related community comment explicitly states dissatisfaction with having to 'handle VAT' themselves, implying no seamless native solution, and no docs reference tax engines like Avalara/TaxJar or VAT/GST features.", "evidenceIds": [ "chargebee-comm-1" ] }, { "productId": "chargebee", "storyId": "trials-coupons-promos", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers dunning/smart retry, entitlements, RevRec, and multi-frequency billing, but contains no mention of free trials, coupon/promotion code functionality, automatic trial-to-paid conversion, or trial-end notification requirements — the specific subscription-lifecycle capabilities named in the story are entirely undocumented here.", "evidenceIds": [] }, { "productId": "chargebee", "storyId": "usage-event-ingestion", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Chargebee's API docs confirm a usage_events resource for recording usage-based billing events, but the evidence pack contains no detail on idempotent ingestion, backdating, or high-volume streaming capability. Missing for 10: idempotency key support, backdating/late event handling, batch/streaming ingestion guarantees, and any performance/scale documentation.", "evidenceIds": [ "chargebee-docs-3" ] }, { "productId": "chargebee", "storyId": "versioned-plan-migrations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Chargebee's docs describe item prices, subscriptions, and entitlements, but none of the evidence mentions versioned plans, grandfathering existing subscribers, or bulk cohort migration tools for pricing changes. Missing for 10: explicit versioned-plan mechanism, grandfathering documentation, and a bulk subscription migration tool.", "evidenceIds": [] }, { "productId": "lago", "storyId": "agent-manages-subscriptions", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Lago exposes a full REST API (every endpoint has a generated command) plus an OpenAPI 3.1 schema explicitly positioned for exposing billing actions to AI agents, and ships a documented MCP server giving AI assistants direct access to billing data, with a working example (Dunning Agent) reading account/invoice data. Subscription assignment and plan upgrade/downgrade are documented API-driven flows, supporting the 'create a subscription, change a plan' part of the story. Missing for 10: no explicit documentation of scoped/permissioned API keys or credit-application endpoints for agents, and no independent hands-on report of an agent actually performing these billing operations end-to-end.", "evidenceIds": [ "lago-docs-1", "lago-docs-4", "lago-docs-9", "lago-docs-10", "lago-docs-16", "lago-probe-3", "lago-probe-4" ] }, { "productId": "lago", "storyId": "agent-reads-billing-state", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Lago ships an official MCP server explicitly described as giving AI assistants 'direct access to Lago's billing data' (lago-docs-9, lago-probe-4), plus a documented OpenAPI schema explicitly positioned for 'exposing billing actions to an AI agent' (lago-docs-16), and a Dunning Agent example that reads a customer's invoices, payments and disputes (lago-docs-10) — directly matching the story of an agent querying billing state like delinquency and invoices. Missing for 10: explicit worked examples/queries for MRR aggregation and upcoming-renewal lookups via the MCP server, and independent (non-vendor) confirmation that these agent queries work in practice.", "evidenceIds": [ "lago-docs-9", "lago-docs-10", "lago-docs-16", "lago-probe-4", "lago-probe-3" ] }, { "productId": "lago", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Lago has a confirmed live llms.txt at the root domain (HTTP 200) plus a docs-specific llms.txt index and .md-formatted doc pages, explicitly designed for agent consumption, alongside an OpenAPI schema and MCP server for programmatic access. This directly satisfies pointing an agent at agent-oriented docs, with only independent third-party confirmation of agent usage missing for a perfect score. Missing for 10: independent/hands-on verification that an agent successfully consumes these files end-to-end.", "evidenceIds": [ "lago-probe-1", "lago-probe-2", "lago-probe-3", "lago-probe-4", "lago-docs-16" ] }, { "productId": "lago", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Lago documents specific AI-agent features—an MCP server that gives assistants like Claude direct access to billing data, and a 'Dunning Agent' that reads 12 months of invoices/payments/disputes to inform actions—showing some in-product AI analysis of data. However, this is narrow (payment collection) rather than a general AI-insights/suggestions layer across revenue analytics, plans, or usage data. missing for 10: evidence of AI-generated insights/suggestions surfaced in the main product UI (e.g., revenue analytics, plan optimization), independent/hands-on confirmation the Dunning Agent's outputs are used, and broader scope beyond dunning/collections.", "evidenceIds": [ "lago-docs-9", "lago-docs-10", "lago-probe-4" ] }, { "productId": "lago", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Lago documents a Dunning Agent that autonomously reads account/invoice/payment history and presumably acts on overdue accounts, plus an MCP server exposing billing data to AI agents like Claude, showing some autonomous background automation capability. However, evidence doesn't show a general-purpose automation/agent-builder framework, scheduling, or independent confirmation of the Dunning Agent running unattended in production. missing for 10: broader autonomous automation framework beyond dunning agent, independent/hands-on verification of autonomous background operation, details on triggering/scheduling/monitoring these agents.", "evidenceIds": [ "lago-docs-9", "lago-docs-10", "lago-probe-4" ] }, { "productId": "lago", "storyId": "agentic-builtin-assistant", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Lago ships an 'AI agents' documentation section describing a built-in Dunning Agent that autonomously reads invoices, disputes, and payment provider status to act on overdue accounts, and an MCP server that gives external AI assistants (e.g., Claude) direct access to billing data. This shows some built-in agentic delegation for a specific task (dunning/collections), but there's no evidence of a general-purpose in-product AI assistant users can converse with or delegate arbitrary tasks to. missing for 10: evidence of a general conversational assistant UI, breadth of tasks beyond dunning, independent/hands-on confirmation of the Dunning Agent actually operating in production.", "evidenceIds": [ "lago-docs-9", "lago-docs-10", "lago-probe-4" ] }, { "productId": "lago", "storyId": "agentic-headless", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Lago is self-hostable via Docker (docs-11), fully API/CLI-driven with a generated command for every endpoint (docs-1) and an OpenAPI 3.1 schema for scripting/client generation (docs-16), and the GitHub repo contains a script that spins up a disposable local Lago instance and runs automated pricing scenarios (gh-1), all consistent with headless/CI automation. However there is no explicit CI/headless-mode documentation, example CI pipeline config, or independent report of running Lago fully headless in a pipeline. Missing for 10: dedicated CI/headless-mode docs, a documented CI pipeline example, and independent confirmation of headless automated runs.", "evidenceIds": [ "lago-docs-11", "lago-docs-1", "lago-docs-16", "lago-gh-1" ] }, { "productId": "lago", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence shows Lago exposes its own MCP server so external AI assistants can access Lago's billing data (lago-docs-9, lago-probe-4) and has AI agents (Dunning Agent) that read Lago's own account data (lago-docs-10) — this is Lago acting as an MCP server/tool provider, not as a client that plugs in external MCP servers to gain new tools. No evidence shows Lago's agents or platform consuming/connecting to third-party MCP servers.", "evidenceIds": [ "lago-docs-9", "lago-docs-10", "lago-probe-4" ] }, { "productId": "lago", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Lago is not itself an AI agent but a billing platform, so publishing an official MCP server is a fair axis; docs explicitly confirm a dedicated MCP server giving AI assistants like Claude direct access to Lago's billing data, plus a probe confirming the page exists. Missing for 10: independent/hands-on third-party verification of the MCP server working in practice.", "evidenceIds": [ "lago-docs-9", "lago-probe-4" ] }, { "productId": "lago", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Lago documents an official MCP server that gives AI assistants like Claude direct access to billing data, plus specialized AI agents (e.g., Dunning Agent) that read account data and OpenAPI schema exposure for AI agents to invoke billing actions — enabling natural-language operation via connected AI assistants. However, this is mediated through external assistants/MCP rather than a built-in NL command interface, and there's no independent/hands-on evidence of actual NL command execution or the breadth of supported commands. Missing for 10: hands-on/independent verification of NL command execution, a native in-product chat/command interface, and documentation of the full range of commands executable via natural language.", "evidenceIds": [ "lago-docs-9", "lago-docs-10", "lago-docs-16", "lago-probe-4" ] }, { "productId": "lago", "storyId": "agentic-official-cli", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs mention that 'every endpoint in this reference has a generated command,' implying an official CLI exists, but there is no dedicated CLI documentation, install instructions, or AI-native framing (e.g., LLM-friendly usage, agent integration) beyond this single line. Missing for 10: explicit CLI install/setup docs, AI-native use-case description, independent confirmation of CLI usage.", "evidenceIds": [ "lago-docs-1" ] }, { "productId": "lago", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Lago publishes a full public API reference with generated commands per endpoint, an OpenAPI 3.1 schema (verified live at /openapi.json), explicit versioning/stability guarantees, and docs/llms.txt aimed at AI-native consumption; it even documents an MCP server exposing billing data to AI agents. This is strong first-party and probe-verified evidence of a documented, machine-consumable public API. Missing for 10: independent third-party developer testimonials specifically about API usage/integration experience (community evidence is mostly about billing product value, not API DX).", "evidenceIds": [ "lago-docs-1", "lago-docs-8", "lago-docs-16", "lago-probe-1", "lago-probe-2", "lago-probe-3", "lago-docs-9", "lago-probe-4" ] }, { "productId": "lago", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Lago's docs describe an MCP server and API access for AI agents but provide no evidence of scoped or least-privilege API key/credential issuance (e.g., role-based keys, permission scopes) for agents; existing docs only mention general API keys and OpenAPI schema access without granular scoping controls.", "evidenceIds": [ "lago-docs-9", "lago-docs-16", "lago-probe-4" ] }, { "productId": "lago", "storyId": "agentic-sdks", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Lago publishes an OpenAPI 3.1 spec and API reference with per-endpoint generated commands, explicitly positioned for generating clients or exposing actions to AI agents, but the evidence never names concrete official first-party SDKs (e.g., language libraries) beyond spec-generated clients. missing for 10: named official SDK packages/libraries, independent developer confirmation of SDK usage, versioning/maintenance details for SDKs.", "evidenceIds": [ "lago-docs-16", "lago-docs-1", "lago-probe-3", "lago-docs-8" ] }, { "productId": "lago", "storyId": "agentic-webhooks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Lago documents a webhook system with HMAC signing options (lago-docs-7) and explicit API versioning guarantees for webhook event types (lago-docs-8), meaning any consumer—including an AI agent—can subscribe to billing events via webhooks. missing for 10: a full catalog/list of supported webhook event types, and independent/hands-on confirmation of webhook reliability in production use.", "evidenceIds": [ "lago-docs-7", "lago-docs-8" ] }, { "productId": "lago", "storyId": "api-interactive-docs", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Docs confirm a generated command for every endpoint and an OpenAPI 3.1 schema (lago-docs-1, lago-docs-16, lago-probe-3), suggesting a structured, code-snippet-rich API reference, but there is no evidence of an in-browser 'try it out' or runnable execution feature. Missing for 10: explicit interactive 'run this request' UI, hands-on confirmation of executability, and independent corroboration of the reference's interactivity.", "evidenceIds": [ "lago-docs-1", "lago-docs-16", "lago-probe-3", "lago-probe-2" ] }, { "productId": "lago", "storyId": "api-machine-spec", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Lago publishes an OpenAPI 3.1 schema (openapi.json returns 200 with an 'openapi' key) and docs explicitly state it can be used to generate clients, inspect typed operations, or expose actions to an AI agent. missing for 10: no independent third-party corroboration of the spec's completeness or usage beyond vendor docs/probe.", "evidenceIds": [ "lago-probe-3", "lago-docs-16" ] }, { "productId": "lago", "storyId": "api-sandbox", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Lago's self-hosted Docker setup and the GitHub quick-start script that spins up a 'disposable local organization' for testing suggest a way to try Lago without touching production data, but there is no documented dedicated sandbox/test-mode with separate test API keys as many billing platforms provide. Missing for 10: explicit sandbox/test-mode documentation, test vs. live API key separation, and hands-on confirmation that test data is fully isolated from production billing.", "evidenceIds": [ "lago-gh-1", "lago-docs-11" ] }, { "productId": "lago", "storyId": "api-versioning-policy", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Lago documents an explicit API versioning policy (v1) specifying that clients should tolerate additive changes like new fields, enums, endpoints, and webhook event types, indicating a stable versioned API contract with deprecation-safe evolution rules, and backs this with a published OpenAPI 3.1 schema. Missing for 10: an explicit deprecation/sunset timeline policy (e.g., how long old versions are supported) and independent/community confirmation of version stability in practice.", "evidenceIds": [ "lago-docs-8", "lago-docs-16", "lago-probe-3" ] }, { "productId": "lago", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence describes bulk/batch endpoints or batch operations (e.g., bulk invoice creation, bulk customer updates) in Lago's API or UI; documentation only mentions per-endpoint commands and general API access. Missing for 10: any mention of batch/bulk API endpoints, batch job status, or UI batch-select actions.", "evidenceIds": [ "lago-docs-1", "lago-docs-16" ] }, { "productId": "lago", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Lago documents webhooks that fire on billing events (lago-docs-7) and a dedicated 'Dunning Agent' that reads account state (invoices, payments, disputes) and presumably acts on overdue-payment events (lago-docs-10), which together resemble event-triggered automation. However, there is no documented general-purpose rule/condition engine letting users define arbitrary 'if event X then action Y' logic beyond these specific built-in agents/webhooks. Missing for 10: a configurable rules/conditions UI or API for arbitrary triggers, and evidence of multiple event types beyond dunning/webhooks.", "evidenceIds": [ "lago-docs-7", "lago-docs-10", "lago-docs-9" ] }, { "productId": "lago", "storyId": "automation-scheduled-jobs", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Lago's core billing engine inherently runs recurring cycles (monthly/yearly subscription billing) and the Dunning Agent appears to run periodic checks over invoices/payments, which is a recurring automated workflow, but there is no documented feature letting an AI-native user define or schedule arbitrary recurring jobs/workflows via API or agent. missing for 10: explicit job/workflow scheduler, cron-like API, user-configurable recurrence for AI agent tasks beyond built-in billing cycles.", "evidenceIds": [ "lago-docs-6", "lago-docs-10", "lago-docs-9" ] }, { "productId": "lago", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Lago is a billing platform, not an automation/workflow builder; there is no concept of 'automations' with versioning, review, or rollback in the evidence pack. This axis is a category error for this product.", "evidenceIds": [] }, { "productId": "lago", "storyId": "billing-cycle-flexibility", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack shows general plan/subscription docs (yearly billing, upgrades/downgrades) but never addresses calendar vs anniversary billing anchors, cross-subscription date alignment, or custom invoice date overrides — the specific billing-math controls this story asks about.", "evidenceIds": [ "lago-docs-6", "lago-docs-3", "lago-docs-4" ] }, { "productId": "lago", "storyId": "compliant-invoice-generation", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Lago's docs confirm core invoicing functionality (downloadable invoices, customizable invoice templates) and cash-collection tracking of overdue invoices, showing invoicing is a first-class feature, but the evidence pack contains no mention of credit notes, receipts, sequential numbering, custom fields, or memos, nor confirmation of fully automated delivery to customers. missing for 10: credit note support, receipt generation, sequential invoice numbering, custom fields/memos on invoices, automatic delivery mechanism confirmation.", "evidenceIds": [ "lago-docs-5", "lago-docs-13" ] }, { "productId": "lago", "storyId": "dunning-comms-automation", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Lago documents dunning-adjacent capabilities: a 'cash-collection' feature to spot overdue/unpaid invoices via filters (lago-docs-13) and a 'Dunning Agent' that reads invoices, payments, disputes, and provider status (lago-docs-10). However, there is no evidence of configurable branded emails, in-app banners, grace periods, or explicit cancel-vs-unpaid final-state rules. Missing for 10: branded email templates, in-app banner UI, grace period configuration, explicit final-state (cancel/unpaid) rule settings, and independent confirmation of the dunning workflow in practice.", "evidenceIds": [ "lago-docs-13", "lago-docs-10" ] }, { "productId": "lago", "storyId": "full-billing-export", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Lago's API/OpenAPI schema and CSV export for analytics (lago-docs-15, lago-docs-16, lago-docs-1) mean a finance lead could theoretically pull subscription, invoice, and usage data out via API, and self-hosting (lago-docs-11) avoids lock-in of infrastructure. However, there is no documented bulk-export or migration tool, no explicit invoice/subscription/usage-record export workflow, and payment methods are held by external providers (Stripe etc.) rather than Lago itself, so 'porting stored payment methods' isn't addressed at all. Missing for 10: dedicated data-export/migration documentation, explicit payment-method portability mechanism, and any hands-on evidence of a successful migration off Lago.", "evidenceIds": [ "lago-docs-15", "lago-docs-16", "lago-docs-1", "lago-docs-5", "lago-docs-11" ] }, { "productId": "lago", "storyId": "gl-erp-sync", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack shows Lago's API, webhooks, invoicing, and OpenAPI schema, but there is no mention of any native or partner integration with QuickBooks, Xero, or NetSuite, nor any revenue-recognition export/sync to a GL/ERP system. Finance-lead close automation is a fair axis for a billing platform, but nothing in the pack demonstrates it.", "evidenceIds": [] }, { "productId": "lago", "storyId": "hybrid-pricing-models", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Lago's own positioning ('usage-based, subscription-based, and hybrid pricing models') and the plan-model/usage-metering docs confirm broad subscription+usage flexibility, and community reviews praise it for handling complex billing without custom code, but the evidence pack never explicitly documents tiered, volume, graduated, or per-unit charge models, nor credits, commitments, or overage handling as distinct configurable features. missing for 10: explicit docs on tiered/volume/graduated/per-unit charge model configuration, credits and commitment mechanics, overage billing rules, and independent confirmation these work without custom code.", "evidenceIds": [ "lago-probe-1", "lago-docs-6", "lago-docs-14", "lago-docs-13", "lago-comm-1", "lago-comm-4" ] }, { "productId": "lago", "storyId": "lifecycle-event-stream", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Lago documents a webhook system with signing (HMAC) and explicitly treats 'webhook event types' as a versioned API surface, implying granular event types exist, but the evidence never enumerates specific subscription lifecycle events (renewal, upgrade, payment failure, cancellation) or demonstrates delivery reliability guarantees like retries/idempotency for webhooks specifically. missing for 10: explicit list/documentation of subscription lifecycle event types, evidence of webhook delivery reliability (retries, dead-letter, idempotency) beyond event-ingestion dedup which is a different feature.", "evidenceIds": [ "lago-docs-7", "lago-docs-8" ] }, { "productId": "lago", "storyId": "multi-currency-billing", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses multi-currency billing, per-currency price points, or consolidated home-currency reporting; docs cover plans, invoicing, dunning, and revenue analytics but never mention currency handling or FX consolidation. Missing for 10: any documentation of multi-currency price points, currency conversion/consolidation logic, and reporting rollups in a base currency.", "evidenceIds": [] }, { "productId": "lago", "storyId": "openness-api-parity", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Lago's docs show an API-first design: every endpoint has a generated CLI/code sample (lago-docs-1), a full OpenAPI 3.1 schema is available (lago-docs-16, lago-probe-3), and specific UI actions like usage-charge updates are explicitly available via both API and no-code UI (lago-docs-14). This implies strong API/UI parity but there is no explicit vendor claim or independent audit confirming 100% parity across every UI workflow (e.g., dunning, revenue analytics dashboards). missing for 10: an explicit 'everything in the UI is available via API' statement, and independent/community confirmation of full parity across all UI features.", "evidenceIds": [ "lago-docs-1", "lago-docs-14", "lago-docs-16", "lago-probe-3" ] }, { "productId": "lago", "storyId": "openness-full-export", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Lago's open-source/self-hosted nature (Docker deployment), open API (OpenAPI 3.1 schema, full REST reference), and CSV export of analytics/invoices give users real data-portability levers, and self-hosting means the underlying database is always in the user's own control. However there is no documented single 'export all account data' or account-closure/migration feature — only piecemeal exports (invoices, revenue analytics CSV, API access to individual objects). Missing for 10: a dedicated full-account data export/backup tool, explicit data-portability/migration documentation, and independent confirmation of a clean 'leave' workflow.", "evidenceIds": [ "lago-docs-11", "lago-docs-5", "lago-docs-15", "lago-docs-16", "lago-probe-3" ] }, { "productId": "lago", "storyId": "openness-open-license", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Lago's docs and GitHub explicitly describe it as an open-source billing platform with a public source repository (getlago/lago), and the llms.txt probe confirms this framing, satisfying the 'read source under open license' story. Missing for 10: explicit citation of the specific license name/file (e.g., AGPL) and independent verification of license terms beyond the 'open-source' label.", "evidenceIds": [ "lago-probe-1", "lago-gh-1" ] }, { "productId": "lago", "storyId": "openness-self-host", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs explicitly cover self-hosting via Docker with a dedicated guide, and community comments confirm self-hosting is a real, used option ('self-hosting is not an issue', 'a self-hosted option fits our culture'). Missing for 10: independent hands-on report of running the full self-hosted stack in production, and more detail on self-hosted feature parity/limitations vs cloud.", "evidenceIds": [ "lago-docs-11", "lago-comm-5", "lago-comm-10" ] }, { "productId": "lago", "storyId": "payment-method-recovery", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions network account updater, backup payment methods, or hosted update-payment pages; Lago's dunning docs focus on invoice/dispute tracking and an AI dunning agent, not card-updating mechanics.", "evidenceIds": [] }, { "productId": "lago", "storyId": "plan-entitlements-api", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Lago has a dedicated entitlements platform page combining billing and feature flags, confirming the concept exists, but the evidence pack contains no detail on how entitlements are attached to plans nor concrete API/SDK examples for runtime entitlement checks. missing for 10: API reference for entitlement endpoints, SDK code samples for checking a customer's entitlements at runtime, plan-to-entitlement modeling details.", "evidenceIds": [ "lago-docs-12" ] }, { "productId": "lago", "storyId": "privacy-data-residency", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Lago's self-hosted Docker deployment (lago-docs-11) implicitly lets a user control where their data resides by choosing their own infrastructure/region, but there is no explicit documentation of a region-selection feature, data-residency guarantees, or compliance certifications for the managed/cloud offering. missing for 10: explicit data-residency/region-selection feature docs, compliance certifications (e.g., GDPR/SOC2 region controls), managed-cloud region options.", "evidenceIds": [ "lago-docs-11" ] }, { "productId": "lago", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Lago is a billing platform; evidence covers billing, invoicing, self-hosting, and AI agent integrations (MCP server, dunning agent) but nowhere addresses data-use-for-AI-training opt-outs or privacy controls over model training. Missing for 10: any privacy policy statement, opt-out mechanism, or data-training exclusion clause.", "evidenceIds": [] }, { "productId": "lago", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses data retention policies, deletion of customer/billing data, or GDPR-style controls; Lago's docs focus on billing, invoicing, and AI agent features but never mention data retention/deletion controls. Missing for 10: documentation on data retention windows, customer data deletion/export mechanisms, and any privacy/GDPR compliance controls.", "evidenceIds": [] }, { "productId": "lago", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses telemetry, usage tracking, or opt-out settings for Lago itself (self-hosted or cloud). The docs cover billing features, AI agents, and self-hosting setup, but nothing about product analytics/telemetry collection or a way to disable it. Missing for 10: any mention of telemetry collection, an opt-out flag/config, or privacy documentation addressing data sent back to Lago.", "evidenceIds": [] }, { "productId": "lago", "storyId": "proration-correctness", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Lago documents that customers can upgrade/downgrade plans at any time (lago-docs-4) and has dedicated subscription/plan docs, implying some proration handling, but no evidence shows documented proration formulas, credit line-item detail, or penny-level invoice reconciliation. A community question about upgrade/downgrade tracking (lago-comm-3) goes unanswered in the evidence pack. Missing for 10: explicit proration algorithm documentation, sample invoice line items showing credits/partial periods, and any audit or reconciliation guarantee.", "evidenceIds": [ "lago-docs-4", "lago-docs-3", "lago-comm-3" ] }, { "productId": "lago", "storyId": "realtime-usage-visibility", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Lago documents usage-metering and customer-facing analytics dashboards (lago-docs-14, lago-docs-15) and webhooks for event notifications (lago-docs-7), which could support real-time usage visibility, but there is no explicit documentation of configurable usage thresholds or proactive alerting before invoice generation. missing for 10: explicit threshold/alert configuration feature, evidence of real-time cost accrual view distinct from dashboards, confirmation that customers (not just ops) see live usage/cost.", "evidenceIds": [ "lago-docs-14", "lago-docs-15", "lago-docs-7" ] }, { "productId": "lago", "storyId": "revrec-asc606-reports", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence describes ASC 606/IFRS 15 deferred revenue schedules or audit-ready recognition reports; the closest material (revenue-analytics dashboards/CSV export) is generic reporting, not recognition accounting. A community comment even explicitly asks whether Lago handles revenue recognition, with no documented answer.", "evidenceIds": [ "lago-docs-15", "lago-comm-3" ] }, { "productId": "lago", "storyId": "self-serve-plan-changes", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence confirms upgrade/downgrade functionality exists (lago-docs-4, lago-docs-3) but only via API or Lago's internal admin UI — there is no mention anywhere in the pack of a customer-facing hosted portal or embeddable self-serve components that end customers could use directly. Community comments even question whether Lago handles full self-serve lifecycle (lago-comm-3), and nothing confirms a portal exists.", "evidenceIds": [ "lago-docs-4", "lago-docs-3", "lago-comm-3" ] }, { "productId": "lago", "storyId": "smart-retries", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Lago documents cash-collection tooling to flag overdue/unpaid invoices and an AI 'Dunning Agent' that reads account history (invoices, payments, disputes, provider status), implying some automated dunning capability, but there is no documented ML- or schedule-optimized retry cadence, no explicit retry logic, and no metrics on churn recovery. Missing for 10: explicit description of automated retry scheduling/cadence optimization, evidence of ML-driven timing, and quantified recovery/churn-reduction outcomes.", "evidenceIds": [ "lago-docs-10", "lago-docs-13" ] }, { "productId": "lago", "storyId": "subscription-analytics", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Lago's docs mention a generic 'revenue-analytics' page with custom dashboards, filters, and CSV export (lago-docs-15), which is closer to the bolted-on BI export the story explicitly wants to avoid, not native MRR/churn/cohort-retention/LTV computation. No evidence pack item shows Lago computing these specific metrics natively from billing data, and a community question in lago-comm-3 explicitly asks whether Lago handles revenue recognition, left unanswered.", "evidenceIds": [ "lago-docs-15", "lago-comm-3" ] }, { "productId": "lago", "storyId": "subscription-crud-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Lago's docs cover API-driven subscription creation (assign-plan), upgrades/downgrades, and a full OpenAPI 3.1 schema with generated commands per endpoint, supporting a documented API lifecycle. However, the evidence pack never explicitly documents pause or cancel subscription endpoints, and most subscription guides (e.g., assign-plan) describe dashboard clicks rather than pure API-only flows. Missing for 10: explicit pause/cancel endpoint documentation, and hands-on/independent confirmation that the entire lifecycle can be done via API without the dashboard.", "evidenceIds": [ "lago-docs-1", "lago-docs-3", "lago-docs-4", "lago-docs-16", "lago-probe-3" ] }, { "productId": "lago", "storyId": "subscription-import-migration", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack describes any migration tooling, bulk-import from spreadsheets, or PCI-compliant card-data transfer capability; the API/docs cover plan assignment and event ingestion but nothing about importing live subscriptions from another platform. Community comments even suggest Lago does not itself handle payment/PCI processing (lago-comm-8), reinforcing the absence of this capability.", "evidenceIds": [ "lago-comm-8" ] }, { "productId": "lago", "storyId": "tax-on-invoices", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item mentions tax calculation, VAT/GST/sales tax handling, tax registrations, tax rates, or integrations with tax engines (e.g., Avalara, Stripe Tax, TaxJar). The docs cover invoicing, plans, subscriptions, and dunning but never address tax compliance. Missing for 10: any mention of native tax calculation, tax rate/registration management, or a first-class tax-engine integration.", "evidenceIds": [] }, { "productId": "lago", "storyId": "trials-coupons-promos", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack items mention free trials, coupons, promotion codes, automatic trial-to-paid conversion, or trial-end notifications required by card-network rules; evidence covers plans, subscriptions, invoicing, webhooks, and AI agents but not this trial/coupon lifecycle. missing for 10: docs on free trials, coupons/promo codes, automatic conversion to paid, and trial-end notification compliance.", "evidenceIds": [] }, { "productId": "lago", "storyId": "usage-event-ingestion", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Lago's docs confirm idempotent event ingestion (dedup on repeated event IDs) and a usage-metering pipeline supporting API-driven event updates, which covers the core ingestion/idempotency claims. However, there is no explicit evidence of backdating support (e.g., specifying a past timestamp for events) or of high-volume streaming architecture/throughput guarantees for API calls, tokens, or compute-seconds use cases. missing for 10: explicit backdating/timestamp-override documentation, evidence of high-volume/streaming ingestion architecture or throughput benchmarks, independent verification of dedup behavior at scale.", "evidenceIds": [ "lago-docs-2", "lago-docs-14" ] }, { "productId": "lago", "storyId": "versioned-plan-migrations", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Docs show basic plan/subscription management (assign plan, upgrade/downgrade) but no explicit mention of plan versioning, grandfathering existing subscribers, or bulk cohort migration tools. missing for 10: explicit versioned-plan mechanism, grandfathering documentation, bulk/cohort migration tooling or API endpoints.", "evidenceIds": [ "lago-docs-3", "lago-docs-4", "lago-docs-6" ] }, { "productId": "metronome", "storyId": "agent-manages-subscriptions", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Metronome exposes a programmatic API (OpenAPI spec, bearer-token auth) that can manage subscription lifecycle changes, seat/plan changes, and credits, which an agent could call. However there is no evidence of scoped/fine-grained permissions (e.g., per-action or per-resource API keys) or of an agent-oriented interface (like an MCP server or agent SDK) — only generic bearer tokens are documented. missing for 10: scoped/least-privilege permission model for API keys, explicit agent/automation integration (MCP or similar), and hands-on evidence of an agent successfully executing these billing operations end-to-end.", "evidenceIds": [ "metronome-docs-3", "metronome-docs-6", "metronome-docs-7", "metronome-docs-14", "metronome-probe-2" ] }, { "productId": "metronome", "storyId": "agent-reads-billing-state", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Metronome exposes a documented REST API (bearer-token auth, full OpenAPI spec) covering subscriptions, invoices, revenue data, and data warehouse export, which a custom agent could technically query for billing state. However, there is no evidence of a purpose-built agent/AI interface, natural-language query layer, or MCP server exposing MRR, delinquent accounts, or renewal status directly. Missing for 10: agent-facing query tools/MCP server, explicit MRR/delinquency/renewal endpoints, and any hands-on demonstration of an agent querying billing state.", "evidenceIds": [ "metronome-docs-3", "metronome-docs-6", "metronome-docs-7", "metronome-docs-9", "metronome-docs-10", "metronome-probe-2" ] }, { "productId": "metronome", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "A direct probe confirms Metronome serves a valid llms.txt at docs.metronome.com/llms.txt (HTTP 200) listing structured doc links, and an OpenAPI spec is also available, both of which an agent could be pointed at for agent-oriented consumption. Missing for 10: no explicit vendor documentation/blog announcing or explaining the llms.txt file's purpose or independent commentary confirming agents use it successfully.", "evidenceIds": [ "metronome-probe-1", "metronome-probe-2" ] }, { "productId": "metronome", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers usage-based billing, invoicing, revenue recognition, data export and dashboards, but nothing describes AI-generated insights or suggestions surfaced within the product. Missing for 10: any mention of AI/ML-driven analytics, anomaly detection, or automated recommendations derived from billing/usage data.", "evidenceIds": [] }, { "productId": "metronome", "storyId": "agentic-autonomous-automation", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Metronome is a usage-based billing platform; evidence covers ingestion, invoicing, webhooks, and subscription management, but there is no mention of autonomous background automations or agentic workflows that run independently for an AI-native user.", "evidenceIds": [] }, { "productId": "metronome", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of a built-in AI assistant within Metronome; the pack only covers billing, usage ingestion, invoicing, and reporting features.", "evidenceIds": [] }, { "productId": "metronome", "storyId": "agentic-headless", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Metronome exposes a REST/API surface (bearer-token auth, OpenAPI spec, ingest endpoint, webhooks) that could be scripted headlessly in CI for usage ingestion or billing automation, but there's no documented CLI, SDK for CI pipelines, or explicit guidance/examples for running Metronome workflows in automated/headless CI contexts. missing for 10: explicit CI/headless automation documentation, official CLI or SDK examples for scripted/automated pipelines, and independent evidence of real-world CI usage.", "evidenceIds": [ "metronome-docs-3", "metronome-probe-2", "metronome-docs-1", "metronome-docs-4" ] }, { "productId": "metronome", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Metronome is a usage-based billing platform; no evidence anywhere in the pack mentions MCP servers or plugging tool integrations into an AI agent context. This is an applicable axis for a SaaS platform (it could ship an MCP server), but there is no evidence of one, so it is 'none' rather than 'na'.", "evidenceIds": [] }, { "productId": "metronome", "storyId": "agentic-mcp-server", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of an official MCP server for Metronome; docs cover REST API, webhooks, integrations, and llms.txt/openapi probes but nothing about MCP support.", "evidenceIds": [ "metronome-probe-1", "metronome-probe-2" ] }, { "productId": "metronome", "storyId": "agentic-nl-commands", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence Metronome supports natural-language command interfaces, chat-based operation, or an AI assistant layer for managing billing; evidence only shows a REST API, webhooks, and dashboards. Missing for 10: any NL command interface, chat/AI assistant, or agentic control surface.", "evidenceIds": [] }, { "productId": "metronome", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of an official Metronome CLI; evidence shows API, webhooks, and integrations but nothing about a command-line tool for AI-native workflows.", "evidenceIds": [] }, { "productId": "metronome", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Metronome exposes a documented public API (bearer-token auth, ingest endpoints, subscription/webhook management) plus a verified OpenAPI spec and llms.txt for machine discoverability, confirming programmatic drivability. Missing for 10: no explicit AI-agent/SDK-for-agents examples or independent third-party confirmation of agentic usage.", "evidenceIds": [ "metronome-docs-1", "metronome-docs-3", "metronome-docs-4", "metronome-docs-6", "metronome-probe-1", "metronome-probe-2" ] }, { "productId": "metronome", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Metronome's docs confirm bearer-token API authentication but give no evidence of scoped, least-privilege, or agent-specific credential issuance — no mention of API key scopes, permission granularity, or agent-oriented tokens.", "evidenceIds": [ "metronome-docs-3" ] }, { "productId": "metronome", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows Metronome has a REST API, bearer-token auth, and an OpenAPI spec, but nowhere mentions official client SDKs (e.g., Node, Python, Go libraries) for developers to build against. missing for 10: explicit official SDK documentation, language-specific client libraries, SDK versioning/support info.", "evidenceIds": [ "metronome-docs-3", "metronome-probe-2" ] }, { "productId": "metronome", "storyId": "agentic-webhooks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Metronome documents outbound webhooks that POST events (contract creation, threshold reached, invoice finalized) to a configured URL, directly matching the subscribe-to-events story. Missing for 10: no evidence of granular event-type filtering/selection, retry/signature verification details, or independent/hands-on confirmation of webhook reliability.", "evidenceIds": [ "metronome-docs-4" ] }, { "productId": "metronome", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Metronome has an OpenAPI spec and auth docs, but nothing indicates an interactive API reference with runnable/try-it examples; the only 'explorer' mentioned is for event pipeline data, not API docs.", "evidenceIds": [ "metronome-probe-2", "metronome-docs-3", "metronome-docs-12" ] }, { "productId": "metronome", "storyId": "api-machine-spec", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "A direct probe confirms https://docs.metronome.com/openapi.json returns HTTP 200 with a valid 'openapi' key, i.e. a downloadable machine-readable OpenAPI spec, supplemented by an llms.txt for AI-native discovery. Missing for 10: no independent/community corroboration of using this spec in practice.", "evidenceIds": [ "metronome-probe-2", "metronome-probe-1" ] }, { "productId": "metronome", "storyId": "api-sandbox", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs confirm a distinct sandbox environment exists (data can be sent 'from sandbox and production' separately), implying isolated testing is possible, but there is no dedicated guide on switching to/using sandbox mode, sandbox API keys, or seeding test data without affecting production. missing for 10: explicit sandbox setup/switching docs, sandbox API key management, independent confirmation that sandbox testing never touches production billing/customer data.", "evidenceIds": [ "metronome-docs-10" ] }, { "productId": "metronome", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows an OpenAPI spec and bearer-token auth docs, but nothing about API versioning scheme or a documented deprecation policy. Missing for 10: explicit API version headers/paths, changelog, deprecation/sunset policy documentation.", "evidenceIds": [ "metronome-docs-3", "metronome-probe-2" ] }, { "productId": "metronome", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Metronome supports high-throughput event ingestion (up to 110k events/sec) and bulk import of existing invoices/contracts, and exposes a full REST API (openapi.json) that could be scripted for bulk operations. However there is no explicit documentation of a batch/bulk endpoint for operations like updating many customers, contracts, or seats at once beyond single-entity API calls. Missing for 10: dedicated bulk/batch API endpoints, documented bulk update/delete operations across many resources, and any AI-native tooling (SDK helpers, scripts) demonstrating bulk operations in practice.", "evidenceIds": [ "metronome-docs-2", "metronome-docs-8", "metronome-docs-7", "metronome-probe-2" ] }, { "productId": "metronome", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Metronome supports webhooks that fire automatically on system events (contract created, threshold reached, invoice finalized), which is a form of event-triggered automation, but there's no evidence of a general user-defined rules engine (custom conditions/actions) beyond these fixed trigger types. Missing for 10: user-configurable rule/condition builder, evidence of custom action chaining beyond webhook notification, and any UI for defining arbitrary automation logic.", "evidenceIds": [ "metronome-docs-4" ] }, { "productId": "metronome", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "metronome", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Metronome is a usage-based billing/invoicing platform, not an automation/workflow-building tool; there is no concept of 'automations' to version, review, or roll back in this product category.", "evidenceIds": [] }, { "productId": "metronome", "storyId": "billing-cycle-flexibility", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence covers subscription lifecycle, seats, invoicing integrations, and revenue recognition, but nothing addresses billing anchor control, calendar vs anniversary billing alignment, or custom invoice date setting. This is a plausible axis for a billing platform, but no evidence demonstrates the specific capability.", "evidenceIds": [] }, { "productId": "metronome", "storyId": "compliant-invoice-generation", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Metronome documents invoice generation (via Stripe integration) and importing/managing invoices, showing it can produce and deliver invoices automatically, but there is no evidence of sequential invoice numbering, credit notes, receipts, custom fields, or memos as compliance features. missing for 10: sequential numbering, credit notes, receipts, custom fields, memos.", "evidenceIds": [ "metronome-docs-5", "metronome-docs-8" ] }, { "productId": "metronome", "storyId": "dunning-comms-automation", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers billing/invoicing, Stripe integration, revenue recognition, and webhooks, but nothing addresses dunning sequences, branded emails, in-app banners, grace periods, or cancel-vs-unpaid final states. Missing for 10: any mention of dunning workflows, email/banner templates, grace period configuration, or automated account state transitions.", "evidenceIds": [ "metronome-docs-5", "metronome-docs-6" ] }, { "productId": "metronome", "storyId": "full-billing-export", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Metronome docs show data warehouse export of usage/invoice/contract data and an import path for existing contracts/invoices, suggesting some data portability, but there is no evidence of a dedicated 'export everything' migration tool or of exporting/porting stored payment methods (which live in Stripe, not Metronome) to another provider. Missing for 10: explicit payment-method export/portability feature, and a documented full-account export/migration workflow for finance leads leaving the platform.", "evidenceIds": [ "metronome-docs-8", "metronome-docs-10", "metronome-docs-5" ] }, { "productId": "metronome", "storyId": "gl-erp-sync", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Metronome can export data to a data warehouse and has a native Stripe integration, and stores revenue-recognition data, but there is no mention of any QuickBooks, Xero, NetSuite, or general ERP/GL sync capability. Missing for 10: any documented QuickBooks/Xero/NetSuite connector or GL export mapping, evidence of automated journal-entry sync, and confirmation that finance teams avoid manual CSV reconciliation.", "evidenceIds": [ "metronome-docs-9", "metronome-docs-10", "metronome-docs-5" ] }, { "productId": "metronome", "storyId": "hybrid-pricing-models", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs show support for enterprise deal structures (commitments, credits, one-time charges), subscription lifecycle management, and seat-based billing, all configurable without custom code (metronome-docs-6, -13, -14, -7). However, no explicit documentation confirms specific pricing structures like tiered, volume, graduated, or per-unit rate cards, or how overage is billed as distinct from commitments.\n\nmissing for 10: explicit doc evidence of tiered/volume/graduated/per-unit pricing structures, explicit overage billing mechanics beyond commitments, independent corroboration of no-code configurability", "evidenceIds": [ "metronome-docs-6", "metronome-docs-13", "metronome-docs-14", "metronome-docs-7" ] }, { "productId": "metronome", "storyId": "lifecycle-event-stream", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Metronome does support webhooks triggered by events like contract creation, threshold reached, and invoice finalized (metronome-docs-4), and separately supports subscription lifecycle management for upgrades/downgrades/trials (metronome-docs-6) and Stripe invoicing for payment (metronome-docs-5). However, the evidence never confirms webhook events specifically for renewal, upgrade, payment failure, or cancellation as named in the story — only a partial overlapping set of triggers is documented. Missing for 10: explicit webhook event types for renewal/upgrade/payment-failure/cancellation, reliability guarantees (retries, delivery guarantees), and independent confirmation of webhook coverage.", "evidenceIds": [ "metronome-docs-4", "metronome-docs-6", "metronome-docs-5", "metronome-docs-7" ] }, { "productId": "metronome", "storyId": "multi-currency-billing", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack contains no mention of multi-currency support, per-currency price points, FX conversion, or consolidated home-currency reporting anywhere in Metronome's docs, integrations, or reporting features. Billing platforms commonly support multi-currency, so this axis applies, but no evidence confirms the capability. Missing for 10: any documentation of currency configuration, FX rate handling, or currency-consolidated financial reporting.", "evidenceIds": [] }, { "productId": "metronome", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Metronome documents extensive API coverage for core billing operations (usage ingestion, subscriptions, seats, invoices, webhooks, revenue recognition, data export) and has a public OpenAPI spec, suggesting broad API-UI parity. However, docs explicitly call out UI-only features like the 'event explorer' for pipeline inspection, and no evidence confirms parity for UI-specific customization like the in-product billing dashboard. Missing for 10: explicit statement or evidence of full API parity for the event explorer and dashboard customization features, and independent/hands-on confirmation of parity claims.", "evidenceIds": [ "metronome-docs-6", "metronome-docs-7", "metronome-docs-9", "metronome-docs-10", "metronome-docs-12", "metronome-probe-2" ] }, { "productId": "metronome", "storyId": "openness-full-export", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Metronome documents exporting usage/billing data to a customer's own data warehouse and provides an OpenAPI-described API for programmatic access to raw event and invoice data, giving a path to open-format export. However there is no documented one-click 'export everything and leave' capability, no explicit open-format (CSV/JSON) account-wide export guarantee, and no mention of full account/data portability upon offboarding. missing for 10: explicit full-account export/offboarding docs, named open export formats, independent confirmation of complete data portability.", "evidenceIds": [ "metronome-docs-10", "metronome-docs-9", "metronome-probe-2" ] }, { "productId": "metronome", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "metronome", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Metronome is a hosted SaaS billing platform with no evidence of a self-hostable core product, on-prem deployment package, or open-source release; all evidence pack items describe cloud API/docs usage only.", "evidenceIds": [] }, { "productId": "metronome", "storyId": "payment-method-recovery", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Metronome delegates payment collection to Stripe (metronome-docs-5) but no evidence describes card updater networks, backup payment methods, or hosted update-payment pages being managed by Metronome itself; this is a Stripe-level dunning feature not documented here.", "evidenceIds": [ "metronome-docs-5" ] }, { "productId": "metronome", "storyId": "plan-entitlements-api", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers usage ingestion, invoicing, subscription lifecycle, and revenue recognition, but nothing describes entitlements modeling attached to plans or a runtime entitlement-check API/SDK. This is a fair question for a usage-based billing/pricing platform, but no evidence supports it.", "evidenceIds": [] }, { "productId": "metronome", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack items mention data residency, region selection, or data storage location controls for Metronome; all citations concern billing, ingestion, webhooks, and integrations. missing for 10: any mention of data residency options, regional storage, or compliance certifications tied to geography.", "evidenceIds": [] }, { "productId": "metronome", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Metronome is a usage-based billing platform, not an AI model or data-processing tool whose training-data use would be a relevant axis; none of the evidence relates to AI training data opt-out.", "evidenceIds": [] }, { "productId": "metronome", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Metronome is a usage-based billing platform; evidence covers ingestion, invoicing, webhooks, revenue recognition, etc., but nothing addresses data retention policies, deletion controls, or privacy/data lifecycle management for AI-native users. Missing for 10: any documentation on data retention periods, deletion/erasure APIs, or privacy controls.", "evidenceIds": [] }, { "productId": "metronome", "storyId": "privacy-telemetry-optout", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Metronome is a usage-based billing platform for its customers' end-users; the story concerns opting out of the vendor's own product telemetry/usage tracking, which is a category mismatch for a billing/metering product whose core function is customer usage tracking. No evidence pack item addresses a telemetry opt-out for the product itself.", "evidenceIds": [] }, { "productId": "metronome", "storyId": "proration-correctness", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm subscription lifecycle management (upgrades/downgrades), seat-change balance tracking with edit history, and revenue-recognition data storage, which together imply proration handling, but none of the evidence documents the actual proration formulas, credit line-item breakdowns, or penny-level reconciliation examples a finance lead would need for audit. Missing for 10: explicit proration/credit calculation documentation, sample invoice line-item reconciliation, and independent/finance-user validation that math ties out to the penny.", "evidenceIds": [ "metronome-docs-6", "metronome-docs-7", "metronome-docs-9", "metronome-docs-5" ] }, { "productId": "metronome", "storyId": "realtime-usage-visibility", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Metronome supports webhooks for threshold-reached events, an in-product billing dashboard for customer-facing transparency, and an event explorer for internal inspection of usage data, which together cover real-time visibility and alerting infrastructure. However, there's no explicit documentation of a unified 'current-cycle usage and accrued cost' real-time dashboard for internal ops teams (vs customer-facing), nor detail on configuring specific cost thresholds/alert rules beyond generic webhook triggers. Missing for 10: dedicated ops-facing real-time cost/usage dashboard, granular threshold-configuration UI/API details, and independent/hands-on confirmation that alerts fire reliably before invoicing.", "evidenceIds": [ "metronome-docs-4", "metronome-docs-11", "metronome-docs-12" ] }, { "productId": "metronome", "storyId": "revrec-asc606-reports", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Metronome has a dedicated revenue-recognition doc claiming it stores raw data needed to manage complex revenue recognition for usage-based models, and supports data export to warehouses for building custom reports, but the evidence never explicitly mentions ASC 606/IFRS 15 compliance, deferred revenue schedules, or audit-ready recognition reports as a built-in output. missing for 10: explicit ASC 606/IFRS 15 framing, deferred revenue schedule generation details, audit-ready report examples/screenshots, independent corroboration from finance teams.", "evidenceIds": [ "metronome-docs-9", "metronome-docs-10" ] }, { "productId": "metronome", "storyId": "self-serve-plan-changes", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Metronome exposes APIs to manage subscription lifecycle (upgrades, downgrades, trials, seat changes) and offers a customizable in-product billing dashboard, implying some embeddable UI capability, but there is no explicit evidence of a hosted customer self-service portal or turnkey embeddable components for customers to upgrade/downgrade/cancel themselves without engineering work or a support ticket. missing for 10: dedicated hosted customer portal, prebuilt embeddable upgrade/downgrade/cancel UI components, evidence of self-service cancellation flow.", "evidenceIds": [ "metronome-docs-6", "metronome-docs-11", "metronome-docs-7" ] }, { "productId": "metronome", "storyId": "smart-retries", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence shows Metronome handles invoicing (including Stripe integration for payment collection), subscription lifecycle, and revenue reporting, but there is no mention of automated dunning/retry logic, ML- or schedule-optimized payment retry cadences, or churn recovery metrics. This is a distinct dunning-recovery capability that would need explicit documentation to credit.", "evidenceIds": [ "metronome-docs-5", "metronome-docs-6" ] }, { "productId": "metronome", "storyId": "subscription-analytics", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Metronome stores raw billing/revenue-recognition data and can export data to a warehouse for building reports with 'your favorite data tools' — i.e., analytics are explicitly bolted-on via external BI, not computed natively as MRR/churn/cohort/LTV dashboards. No evidence of native MRR, churn, cohort retention, or LTV metrics computed inside Metronome itself.", "evidenceIds": [ "metronome-docs-9", "metronome-docs-10" ] }, { "productId": "metronome", "storyId": "subscription-crud-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm an API-driven subscription lifecycle (pricing changes, trials, upgrades/downgrades, seat changes) and a published OpenAPI spec, supporting programmatic management without a dashboard. However, explicit create/pause/cancel endpoint documentation and hands-on confirmation of pause/cancel specifically are not shown. missing for 10: explicit docs/examples for 'pause' and 'cancel' subscription actions, independent developer confirmation of full lifecycle via API alone.", "evidenceIds": [ "metronome-docs-6", "metronome-docs-7", "metronome-probe-2" ] }, { "productId": "metronome", "storyId": "subscription-import-migration", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Metronome docs show it can import existing contracts/invoices from outside systems and manage subscription lifecycle/seats, giving a foundation for migrating billing data, and its Stripe integration handles payment processing which is PCI-compliant by nature of using Stripe. However, there is no explicit documentation of a spreadsheet-import workflow, no dedicated migration tooling, and no mention of transferring stored card data (vaulted PANs) between platforms or ensuring seamless/no-notice cutover for live subscriptions. Missing for 10: dedicated spreadsheet/CSV migration tooling, explicit PCI card-data transfer/vaulting process, documented zero-downtime cutover guarantees, and independent case studies of live migrations.", "evidenceIds": [ "metronome-docs-8", "metronome-docs-6", "metronome-docs-7", "metronome-docs-5" ] }, { "productId": "metronome", "storyId": "tax-on-invoices", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of native tax calculation or a first-class tax integration (e.g., Avalara, Stripe Tax, TaxJar) anywhere in the evidence pack; only Stripe invoicing integration for payment collection is mentioned, not tax logic.", "evidenceIds": [] }, { "productId": "metronome", "storyId": "trials-coupons-promos", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Docs confirm Metronome supports managing free trials as part of subscription lifecycle management, but there is no evidence of coupons, promotion codes, automatic conversion mechanics, or card-network-mandated trial-end notifications. missing for 10: coupons/promo code support, explicit automatic trial-to-paid conversion flow, and trial-end notification/reminder functionality.", "evidenceIds": [ "metronome-docs-6" ] }, { "productId": "metronome", "storyId": "usage-event-ingestion", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm a dedicated /ingest endpoint capable of very high throughput (110k events/sec) plus an event explorer for pipeline inspection, covering the 'stream high-volume usage events' part of the story. However, the evidence pack contains no mention of idempotent ingestion (e.g., idempotency keys/dedup) or backdating support for events. Missing for 10: explicit documentation of idempotent event ingestion and backdated event timestamp support.", "evidenceIds": [ "metronome-docs-1", "metronome-docs-2", "metronome-docs-12" ] }, { "productId": "metronome", "storyId": "versioned-plan-migrations", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs show subscription lifecycle management including pricing changes, upgrades/downgrades, and seat changes, which implies some ability to alter pricing for individual subscribers without full evidence of plan versioning or grandfathering mechanics. There's no explicit mention of bulk migration tooling for moving entire cohorts to new pricing. Missing for 10: versioned plan definitions, grandfathering logic for existing subscribers, and bulk/cohort migration tooling.", "evidenceIds": [ "metronome-docs-6", "metronome-docs-7" ] }, { "productId": "orb", "storyId": "agent-manages-subscriptions", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Orb documents a full programmatic surface for the core actions an agent would need: creating subscriptions via API (orb-docs-8), editing subscription pricing/config (orb-docs-10, orb-docs-11), and plan changes with automatic proration credit (orb-docs-16). However, there is no evidence of a scoped-permission or API-key-role model that would let an agent operate with restricted access rather than full account credentials — the error-handling doc (orb-docs-15) only covers error codes, not auth scopes, and the OpenAPI spec probe returned 404s (orb-probe-2), leaving the permission-scoping mechanism unverified.\nmissing for 10: documented scoped API keys/OAuth roles for agent-level permissions, an official agent/MCP-style integration, independent confirmation that these endpoints support least-privilege access.", "evidenceIds": [ "orb-docs-8", "orb-docs-10", "orb-docs-11", "orb-docs-16", "orb-probe-2" ] }, { "productId": "orb", "storyId": "agent-reads-billing-state", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers billing mechanics (dunning, revenue recognition, invoice issuance, exports) but nothing shows an agent-facing query interface for aggregate billing state like MRR, delinquent-account lists, or renewal schedules — no MCP server, natural-language API, or reporting endpoint is documented, and the openapi probe returned 404s, undermining even basic API discoverability for an autonomous agent.", "evidenceIds": [ "orb-docs-19", "orb-docs-21", "orb-docs-22", "orb-probe-2" ] }, { "productId": "orb", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "A live probe confirms Orb serves a proper llms.txt at docs.withorb.com/llms.txt returning HTTP 200 with structured links to docs, directly satisfying the agent-oriented docs discovery story. Missing for 10: no independent third-party corroboration of agent usage/adoption beyond the probe itself.", "evidenceIds": [ "orb-probe-1" ] }, { "productId": "orb", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Orb is a billing/usage-metering platform; evidence covers forecasting, revenue recognition, invoicing, and pricing analytics, but none describes AI-generated insights or suggestions surfaced within the product itself. No mention of AI/ML features, copilots, or generative recommendations anywhere in the docs pack.", "evidenceIds": [] }, { "productId": "orb", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Orb has several autonomous background processes (auto dunning retries, automatic invoice issuance after grace periods, usage-threshold alerts, and real-time webhooks) that run without manual intervention, which partially satisfies 'automations that run in the background.' However there is no evidence of an AI-native automation builder, agent-configurable workflows, or general-purpose autonomous task orchestration beyond these fixed billing behaviors. Missing for 10: user-configurable automation/workflow engine, AI-agent-triggered automation setup, evidence of arbitrary background task scheduling beyond billing-specific triggers.", "evidenceIds": [ "orb-docs-19", "orb-docs-21", "orb-docs-6", "orb-docs-23" ] }, { "productId": "orb", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "orb", "storyId": "agentic-headless", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Orb is API-first for core actions (e.g., subscription creation, event ingestion, webhooks) which implies operations can be automated without the UI, but there is no evidence of a CLI, SDK, or explicit CI/pipeline integration story — the evidence never discusses running Orb 'headlessly' in an automation/CI context. missing for 10: CLI or SDK for automation, explicit CI/pipeline integration examples, documentation framing API use as headless/agentic automation.", "evidenceIds": [ "orb-docs-8", "orb-docs-17", "orb-docs-23" ] }, { "productId": "orb", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Orb is a billing/revenue platform, not an AI agent or MCP-serving product; the evidence covers billing, invoicing, revenue recognition and webhooks with no mention of MCP servers or agent tool integration, so this axis is a category error for this product.", "evidenceIds": [] }, { "productId": "orb", "storyId": "agentic-mcp-server", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "orb", "storyId": "agentic-nl-commands", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack of a natural-language interface, chat/AI assistant, or NL-driven command execution for Orb; all documented interactions are via UI or REST/SQL APIs.", "evidenceIds": [ "orb-docs-8", "orb-docs-5", "orb-probe-2" ] }, { "productId": "orb", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Orb is a billing platform; the evidence pack shows no official CLI for AI-native or agentic workflows anywhere in its docs, only API/UI/webhooks and data export features. This is a fair axis (many SaaS platforms ship CLIs), so absence of evidence yields 'none' rather than 'na'.", "evidenceIds": [] }, { "productId": "orb", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Orb's docs explicitly reference creating subscriptions via API, an api-reference error-responses page, per-event idempotency through the API, and a real-time webhooks system, indicating a documented public API spanning core billing operations. Missing for 10: a discoverable machine-readable OpenAPI/swagger spec (probe returned 404 on all candidates) and independent third-party corroboration of API usage.", "evidenceIds": [ "orb-docs-8", "orb-docs-15", "orb-docs-17", "orb-docs-23", "orb-probe-2" ] }, { "productId": "orb", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "orb", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows Orb has a REST API (endpoints, error responses, webhooks) but contains no mention of official language SDKs (e.g., Python, Node, Java client libraries), and the OpenAPI probe returned 404s, suggesting no public spec/SDK generation surface. Missing for 10: any documentation of official SDKs, client libraries, or language-specific packages for building AI-native integrations.", "evidenceIds": [ "orb-docs-8", "orb-docs-15", "orb-probe-2" ] }, { "productId": "orb", "storyId": "agentic-webhooks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Orb documents a real-time webhooks system that issues calls to an endpoint for account events, directly enabling event subscription. Missing for 10: details on event payload schema, list of supported event types, signature verification/security, and independent/hands-on confirmation beyond docs.", "evidenceIds": [ "orb-docs-23" ] }, { "productId": "orb", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only shows standard API reference docs (e.g., error-responses.md) but no mention of an interactive explorer or runnable code examples; the openapi spec probe returned 404s, suggesting no discoverable machine-readable spec to power such interactivity.", "evidenceIds": [ "orb-docs-15", "orb-probe-2" ] }, { "productId": "orb", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows explicit probes for a machine-readable OpenAPI spec (openapi.json, swagger.json, etc.) all returning 404, and no docs page offers a downloadable API spec — only an llms.txt index and general API reference mentions. Missing for 10: any working OpenAPI/Swagger endpoint, a documented spec download link, or third-party confirmation of spec availability.", "evidenceIds": [ "orb-probe-2", "orb-probe-1", "orb-docs-15" ] }, { "productId": "orb", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The axis applies since billing/API platforms commonly offer sandbox/test environments, but the evidence pack never mentions a distinct sandbox or test-mode environment isolated from production data — only 'testing pricing changes against historical usage' (orb-docs-1), which is backtesting, not a sandbox. missing for 10: explicit sandbox/test-mode environment, API keys or docs distinguishing test vs live data, confirmation that test data never touches production.", "evidenceIds": [ "orb-docs-1" ] }, { "productId": "orb", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item documents API versioning scheme or a deprecation policy; only error-response handling and general docs are covered. Missing for 10: versioning scheme (e.g., date-based API versions), deprecation/sunset policy, changelog or migration guides for breaking changes.", "evidenceIds": [] }, { "productId": "orb", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Orb documents high-throughput event ingestion (1M+ events/sec) and backfilling historical usage events, which are forms of bulk data operations, but there is no explicit documentation of a bulk/batch API for operations like mass-editing subscriptions, invoices, or other resources. missing for 10: explicit bulk/batch API endpoints for subscriptions, invoices, or other catalog items; UI or API support for multi-item batch edits.", "evidenceIds": [ "orb-docs-3", "orb-docs-14", "orb-docs-17", "orb-docs-12" ] }, { "productId": "orb", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Orb offers usage-threshold alerts and a real-time webhooks system that fires on defined events, which functions as event-triggered automation, and dunning/invoice issuance include automatic rule-based actions (auto-retry, auto-issue). However, these are fixed, product-specific automations rather than a general user-defined rules/actions engine (no evidence of custom conditional logic, multi-step workflows, or arbitrary action definitions). Missing for 10: a general-purpose rule builder, evidence of custom conditional logic beyond thresholds, and support for arbitrary automated actions beyond alerts/webhooks/dunning.", "evidenceIds": [ "orb-docs-6", "orb-docs-23", "orb-docs-19", "orb-docs-21" ] }, { "productId": "orb", "storyId": "automation-scheduled-jobs", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Orb offers some built-in recurring automated processes—daily scheduled data exports, automatic dunning retries on fixed day intervals, and automatic invoice issuance after grace periods—which are recurring, schedule-driven jobs, but these are fixed billing-domain behaviors rather than a general-purpose scheduler or workflow engine that an AI-native user could configure for arbitrary recurring tasks. Missing for 10: a documented job/workflow scheduling API or UI, support for custom recurrence rules or triggers beyond billing events, and any evidence of user-defined automation workflows.", "evidenceIds": [ "orb-docs-12", "orb-docs-19", "orb-docs-21", "orb-docs-23" ] }, { "productId": "orb", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Orb is a usage-based billing and revenue platform, not an automation-building tool; it has no concept of 'automations' to version, review, or roll back. The evidence pack covers billing metrics, subscriptions, invoicing, and revenue recognition, none of which relate to versioning/rollback of automations, so this axis is a category mismatch for Orb's product type.", "evidenceIds": [] }, { "productId": "orb", "storyId": "billing-cycle-flexibility", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm subscriptions can be aligned to the creation date (anniversary billing) and that billing cycle alignment is configurable, but there's no explicit evidence of calendar-vs-anniversary toggle, cross-subscription cycle alignment for a single customer, or custom invoice date overrides. Missing for 10: explicit calendar billing option, multi-subscription alignment mechanics, custom invoice date configuration.", "evidenceIds": [ "orb-docs-9", "orb-docs-8", "orb-docs-10" ] }, { "productId": "orb", "storyId": "compliant-invoice-generation", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Orb documents automatic invoice issuance, proration/minimums/maximums calculations, and revenue recognition, indicating invoices are generated and delivered automatically, but there is no explicit evidence of sequential invoice numbering, credit note generation, receipts, or custom fields/memos on invoices. missing for 10: sequential numbering guarantees, credit note issuance, receipt generation, custom fields/memo support on invoices, and independent confirmation of compliance formatting.", "evidenceIds": [ "orb-docs-20", "orb-docs-21", "orb-docs-22", "orb-docs-16" ] }, { "productId": "orb", "storyId": "dunning-comms-automation", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Orb docs confirm basic dunning mechanics — automatic payment retries at 1/4/9 days (orb-docs-19) and automatic vs manual invoice issuance after grace periods (orb-docs-21) — but there is no evidence of branded/customizable dunning emails, in-app banners, or configurable final-state rules like cancel vs unpaid. missing for 10: branded email templates, in-app banner/notification configuration, customizable grace period settings beyond fixed retry schedule, and explicit cancel-vs-unpaid final state logic.", "evidenceIds": [ "orb-docs-19", "orb-docs-21" ] }, { "productId": "orb", "storyId": "full-billing-export", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Orb documents managed daily data exports of resource types to any destination (orb-docs-12) and revenue/financial exports (orb-docs-7), which could cover subscriptions/invoices/usage, but there is no evidence describing exporting stored payment methods or any outbound migration path — the only migration content covers importing customers *into* Orb from Stripe (orb-docs-13, orb-docs-14), not leaving Orb for another provider. Missing for 10: explicit payment-method export/portability, confirmation that exports cover all three record types comprehensively, and documentation of an outbound/offboarding migration process.", "evidenceIds": [ "orb-docs-12", "orb-docs-7", "orb-docs-13", "orb-docs-14" ] }, { "productId": "orb", "storyId": "gl-erp-sync", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Orb documents ASC606-compliant revenue recognition tracking and generic data exports 'for financial systems' to 'a data destination of your choice,' which could feed a GL, but no evidence names a native QuickBooks, Xero, or NetSuite connector or a no-CSV sync path — buyers would need to build the export pipeline themselves. Missing for 10: named ERP/accounting-system integrations, automated invoice/payment sync (not just data exports), and confirmation that no manual CSV mapping is required.", "evidenceIds": [ "orb-docs-7", "orb-docs-12", "orb-docs-22" ] }, { "productId": "orb", "storyId": "hybrid-pricing-models", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Orb's docs cover billable metrics with flexible aggregation/SQL (orb-docs-5, orb-docs-18), subscription edits and plan changes with proration (orb-docs-10, orb-docs-16), minimums/maximums for commitments and overage (orb-docs-20), and subscription/plan configuration via UI or API — collectively supporting tiered, volume, graduated, per-unit, and hybrid subscription+usage pricing without custom code. missing for 10: explicit named documentation pages for each pricing model (tiered/volume/graduated) and credits/prepaid commitment mechanics, plus independent (non-vendor) confirmation of these pricing structures in production use.", "evidenceIds": [ "orb-docs-5", "orb-docs-18", "orb-docs-10", "orb-docs-11", "orb-docs-16", "orb-docs-20", "orb-docs-8", "orb-docs-9" ] }, { "productId": "orb", "storyId": "lifecycle-event-stream", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Orb documents a real-time webhooks system for triggering calls on event occurrence, and related lifecycle mechanics (dunning retries, proration on plan change, invoice issuance) exist, implying underlying state changes are trackable. However, the evidence never enumerates which specific lifecycle events (renewal, upgrade, cancellation, payment failure) fire webhooks, nor details delivery guarantees like retries/signing for reliability. Missing for 10: explicit list of webhook event types covering renewal/upgrade/cancellation/payment failure, documentation of delivery reliability guarantees (retries, at-least-once delivery, signature verification), and independent/hands-on confirmation of webhook reliability.", "evidenceIds": [ "orb-docs-23", "orb-docs-19", "orb-docs-16", "orb-docs-21" ] }, { "productId": "orb", "storyId": "multi-currency-billing", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions multi-currency support, per-currency price points, or currency-consolidated reporting; Orb's docs cover proration, revenue recognition, dunning, invoicing but nothing about local currency billing or FX consolidation.", "evidenceIds": [] }, { "productId": "orb", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs show many UI actions—creating subscriptions, editing subscriptions, configuring billing metrics, exports, webhooks—are explicitly available via API endpoints (orb-docs-8, orb-docs-10, orb-docs-12, orb-doces-18, orb-docs-23), suggesting strong UI/API parity for core billing workflows. However there is no explicit blanket claim that 'everything in the UI is possible via API,' and a probe for a public OpenAPI/swagger spec returned 404s, weakening confidence in full documented API surface coverage. missing for 10: an explicit parity statement, a discoverable full API reference/OpenAPI spec, and confirmation that dashboard-only features (e.g., invoice approval UI, analytics dashboards) have full API equivalents.", "evidenceIds": [ "orb-docs-8", "orb-docs-10", "orb-docs-12", "orb-docs-18", "orb-docs-23", "orb-probe-2" ] }, { "productId": "orb", "storyId": "openness-full-export", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Orb supports managed daily exports of resource types to a destination of choice and revenue/financial exports, which gives data portability, but this is billing-data export for downstream systems rather than a full account-level open-format export-and-leave capability aimed at an AI-native user. missing for 10: explicit full account/data export (all entities, configs, historical events) in open formats, documented data portability/deletion workflow for leaving the platform, and independent confirmation of export completeness.", "evidenceIds": [ "orb-docs-12", "orb-docs-7" ] }, { "productId": "orb", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "orb", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "orb", "storyId": "payment-method-recovery", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers dunning retry scheduling (orb-docs-19) but there is no mention of network account updater, backup payment methods, or hosted update-payment pages anywhere in the pack, so the specific card-self-update capability is unevidenced.", "evidenceIds": [ "orb-docs-19" ] }, { "productId": "orb", "storyId": "plan-entitlements-api", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers subscriptions, plans, usage metrics, invoicing, and revenue recognition, but nowhere mentions entitlements, feature-level access modeling, or an API/SDK for checking a customer's entitlements at runtime. Since billing platforms plausibly could offer such a feature, the axis applies, but there is no supporting evidence.", "evidenceIds": [] }, { "productId": "orb", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "orb", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Orb is a billing/revenue infrastructure platform; AI-model-training data usage opt-out is not a relevant axis for this product category, and none of the evidence touches AI training data policy.", "evidenceIds": [] }, { "productId": "orb", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers billing, invoicing, revenue recognition, and data exports, but contains no mention of customer data retention policies, deletion APIs, or privacy controls for AI-native users. This is an applicable axis for any SaaS handling customer data, but there is no evidence Orb provides retention/deletion controls.", "evidenceIds": [] }, { "productId": "orb", "storyId": "privacy-telemetry-optout", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Orb is a billing/revenue platform, not an AI agent or tool with telemetry collected from AI-native usage; the evidence pack contains no telemetry/analytics-opt-out feature relevant to an 'AI-native user's own tool usage tracking. This axis is a category mismatch for a billing infrastructure product.", "evidenceIds": [] }, { "productId": "orb", "storyId": "proration-correctness", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Orb documents day-based proration on plan changes, credits for unused time, a preview of billing/invoice impact before saving edits, and minimum/maximum charge handling — supporting the core proration and invoice line-item math. However, there is no explicit documentation of penny-level reconciliation guarantees, detailed audit trails, or worked examples showing exact credit/debit math across partial periods. missing for 10: worked/auditable proration examples, explicit 'reconciles to the penny' guarantee, audit-log/traceability documentation for finance review.", "evidenceIds": [ "orb-docs-16", "orb-docs-10", "orb-docs-11", "orb-docs-20" ] }, { "productId": "orb", "storyId": "realtime-usage-visibility", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Orb docs confirm accrued cost/revenue tracking before invoicing (orb-docs-4), configurable usage-threshold alerts for product-led growth (orb-docs-6), and real-time webhooks for event notifications (orb-docs-23), covering the core alerting/real-time-visibility ask. missing for 10: explicit evidence of a customer-facing self-serve usage dashboard and independent/hands-on confirmation that alerts reliably fire before invoice generation.", "evidenceIds": [ "orb-docs-4", "orb-docs-6", "orb-docs-23" ] }, { "productId": "orb", "storyId": "revrec-asc606-reports", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Orb explicitly automates ASC606-compliant recognized/deferred revenue calculation (orb-docs-22), tracks accrued revenue before invoicing (orb-docs-4), and supports exporting recognized/deferred revenue data for financial systems (orb-docs-7), plus general data exports to downstream destinations (orb-docs-12). This covers the core finance-lead need of ASC606 deferred revenue schedules and exportable recognition data. missing for 10: no explicit mention of IFRS 15 compliance, no detail on audit-ready report formatting/controls, and no independent/hands-on corroboration beyond vendor docs.", "evidenceIds": [ "orb-docs-22", "orb-docs-4", "orb-docs-7", "orb-docs-12" ] }, { "productId": "orb", "storyId": "self-serve-plan-changes", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack details subscription creation, edits, plan changes, and proration via the Orb UI or API, but nowhere mentions a hosted customer-facing portal or embeddable self-serve components that end customers could use to upgrade/downgrade/cancel without a support ticket. Missing for 10: any mention of a hosted billing portal, embeddable UI components, or self-serve customer-facing flows.", "evidenceIds": [ "orb-docs-8", "orb-docs-9", "orb-docs-10", "orb-docs-11", "orb-docs-16" ] }, { "productId": "orb", "storyId": "smart-retries", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Orb documents automatic dunning retries on a fixed schedule (1, 4, and 9 days after failed payment) via its 'essential-dunning' feature, which addresses the schedule-optimized retry cadence for involuntary churn recovery. However, there is no evidence of ML-based optimization of retry timing, nor any documented metrics or case studies showing measurable churn recovery impact. Missing for 10: ML-driven retry optimization, evidence of measured recovery rates/churn reduction, configurability of retry cadence per customer segment.", "evidenceIds": [ "orb-docs-19", "orb-docs-21" ] }, { "productId": "orb", "storyId": "subscription-analytics", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Orb documents revenue-related metrics computed from billing data itself (accrued revenue, recognized/deferred revenue per ASC606, expansion forecasting) rather than a bolted-on BI export, and supports data exports/SQL for custom analytics. However, there is no explicit mention of MRR, churn, cohort retention, or LTV dashboards as native analytics features — these appear to require building via SQL/export rather than being out-of-the-box computed metrics. missing for 10: explicit MRR/churn/cohort retention/LTV dashboards or reports, evidence these are native computed analytics rather than DIY SQL queries, independent confirmation of accuracy/usability.", "evidenceIds": [ "orb-docs-4", "orb-docs-5", "orb-docs-7", "orb-docs-22", "orb-docs-12" ] }, { "productId": "orb", "storyId": "subscription-crud-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm subscription creation via API (orb-docs-8), billing cycle alignment (orb-docs-9), and editing subscription pricing with preview of impact (orb-docs-10, orb-docs-11), plus plan-change proration logic (orb-docs-16) — covering create/update well. However, no explicit documentation of 'pause' or 'cancel' subscription API endpoints appears in the evidence pack, and no OpenAPI/spec confirmation was found (orb-probe-2 shows 404s for spec files), so full lifecycle coverage is unverified via API alone. Missing for 10: explicit pause/cancel endpoint documentation, confirmed OpenAPI spec listing all lifecycle operations, independent/hands-on verification of API-only workflow.", "evidenceIds": [ "orb-docs-8", "orb-docs-9", "orb-docs-10", "orb-docs-11", "orb-docs-16", "orb-probe-2" ] }, { "productId": "orb", "storyId": "subscription-import-migration", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Orb documents migration from Stripe Billing (including flat-rate, usage-based, seat pricing) and backfilling historical usage events for accurate invoicing, which supports competing-platform migration. However there is no evidence of spreadsheet-based import tooling, PCI-compliant card/payment-method data transfer, or explicit assurances that migration is seamless enough that customers won't notice. Missing for 10: spreadsheet import support, PCI-compliant card data transfer mechanism, and documentation/testimony of zero customer disruption during migration.", "evidenceIds": [ "orb-docs-13", "orb-docs-14", "orb-docs-8" ] }, { "productId": "orb", "storyId": "tax-on-invoices", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers billing, metering, revenue recognition, dunning, and invoicing calculations but contains no mention of sales tax, VAT, GST calculation, tax registrations, or a first-class tax integration (e.g., Avalara, Stripe Tax). This is a fair axis for a billing/invoicing platform, so absent evidence yields 'none' rather than 'na'.", "evidenceIds": [] }, { "productId": "orb", "storyId": "trials-coupons-promos", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers subscriptions, proration, dunning, invoicing, and revenue recognition, but contains no mention of free trials, coupon/promotion codes, automatic trial-to-paid conversion, or trial-end notifications required by card-network rules. Since Orb is a subscription billing platform, this axis is a fair question, but there is no supporting evidence for these specific capabilities.", "evidenceIds": [] }, { "productId": "orb", "storyId": "usage-event-ingestion", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Orb's docs directly support high-volume streaming ingestion (1M+ events/sec), per-event idempotency guarantees, and backdating/backfill support for historical usage recalculation and invoice adjustment. missing for 10: independent/hands-on benchmarks corroborating the 1M+ events/sec claim, and more detail on the exact idempotency key mechanism/grace period edge cases.", "evidenceIds": [ "orb-docs-3", "orb-docs-17", "orb-docs-2", "orb-docs-14", "orb-docs-1" ] }, { "productId": "orb", "storyId": "versioned-plan-migrations", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Orb documents subscription-level pricing edits with pre-save impact previews, proration on plan changes, and a Stripe migration path with usage backfill — enabling controlled repricing without breaking existing invoices. But there is no explicit documentation of plan versioning, formal grandfathering mechanisms, or a bulk cohort-migration tool for moving many subscribers to new prices at once. missing for 10: explicit plan-versioning/grandfathering feature, bulk cohort migration tooling beyond one-off Stripe import, evidence of moving existing subscriber cohorts to a new plan version at scale.", "evidenceIds": [ "orb-docs-10", "orb-docs-11", "orb-docs-16", "orb-docs-13", "orb-docs-14", "orb-docs-1" ] }, { "productId": "recurly", "storyId": "agent-manages-subscriptions", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Recurly exposes a full REST API and client libraries covering subscription changes, credits, and invoices (recurly-docs-1, -2, -9, -13), which could technically be driven by an agent. However, the only AI-specific surface documented, Recurly Compass MCP, is explicitly a documentation/knowledge connector for AI dev tools, not an operational interface for executing billing actions (recurly-docs-12, recurly-probe-2), and there is no evidence of scoped/restricted API keys or permission models designed for autonomous agent use. Missing for 10: an agent-oriented action interface (e.g., MCP server or agent SDK) that can actually invoke billing operations, and documentation of scoped/least-privilege API credentials for agent access.", "evidenceIds": [ "recurly-docs-1", "recurly-docs-2", "recurly-docs-9", "recurly-docs-13", "recurly-docs-12", "recurly-probe-2" ] }, { "productId": "recurly", "storyId": "agent-reads-billing-state", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Recurly exposes a REST API and official client libraries that could let an agent fetch account/invoice/subscription data, and entitlements are described as checkable in real time via the API. However, the only AI-agent integration evidenced (Compass MCP) is explicitly scoped to documentation and API knowledge, not live billing-state queries, and there is no evidence of dedicated endpoints/reports for MRR, delinquent-account lists, or renewal forecasts that an agent could query directly. missing for 10: evidence of an MCP or agent-facing endpoint returning MRR/delinquency/renewal data, documentation of a reporting/analytics API for these metrics, independent confirmation of agent-driven billing queries working in practice.", "evidenceIds": [ "recurly-docs-12", "recurly-docs-8", "recurly-docs-13", "recurly-docs-10", "recurly-probe-2" ] }, { "productId": "recurly", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "A live llms.txt is confirmed via direct probe (HTTP 200) at docs.recurly.com/llms.txt, and Recurly also documents a public MCP server (Compass) that connects AI dev tools directly to its documentation and API knowledge, giving agents a first-party structured path to Recurly's docs. missing for 10: independent/community confirmation of agents actually using these endpoints successfully.", "evidenceIds": [ "recurly-probe-1", "recurly-probe-2", "recurly-docs-12" ] }, { "productId": "recurly", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence Recurly generates AI-driven insights or suggestions from a merchant's own subscription/billing data; the only AI-related feature (Compass MCP) merely connects AI dev tools to documentation/API knowledge, not to account data analytics.", "evidenceIds": [ "recurly-docs-12", "recurly-probe-2" ] }, { "productId": "recurly", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Recurly ships built-in autonomous background processes—dunning campaigns (up to 50 targeted campaigns), automatic backup payment method fallback, and involuntary churn recovery tools—that run without manual intervention once configured. However, there is no evidence of a general-purpose automation/workflow builder, triggers, or agent-configurable rules engine that an AI-native user could set up beyond these fixed billing-retention features. Missing for 10: a configurable automation/rules engine, evidence of user-defined triggers/actions beyond dunning, and any AI-agent-specific automation tooling.", "evidenceIds": [ "recurly-docs-5", "recurly-docs-6", "recurly-docs-7" ] }, { "productId": "recurly", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Recurly is a subscription billing platform; evidence shows an MCP server for connecting AI dev tools to documentation/API knowledge, but nothing indicates a built-in AI assistant within the product that users can delegate tasks to.", "evidenceIds": [ "recurly-docs-12", "recurly-probe-2" ] }, { "productId": "recurly", "storyId": "agentic-headless", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Recurly offers a full REST API and official client libraries (Ruby, Node, Python, .NET, Java, PHP, Go) which could be scripted headlessly in CI, but there is no explicit documentation of a CLI, CI/CD examples, or automation-focused tooling for headless/agentic workflows. missing for 10: explicit CI/headless usage examples, a documented CLI tool, evidence of non-interactive/automation-first design beyond generic API/SDK availability.", "evidenceIds": [ "recurly-docs-13", "recurly-docs-14" ] }, { "productId": "recurly", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence only shows Recurly publishing its own MCP server (Compass) so external AI tools can query Recurly's docs/API — this is Recurly acting as an MCP server provider, not as a client that can plug in and use other MCP servers' tools. No evidence Recurly itself can consume/plug in external MCP servers.", "evidenceIds": [ "recurly-docs-12", "recurly-probe-2" ] }, { "productId": "recurly", "storyId": "agentic-mcp-server", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Recurly ships an official 'Compass' MCP server that connects AI dev tools directly to Recurly's documentation and API knowledge, confirmed both in docs and via a live probe of the docs page. However, evidence only describes it as a documentation/knowledge connector, not a full agentic interface for performing billing actions (e.g., managing subscriptions, invoices) via MCP. Missing for 10: evidence of MCP tool-calling for actual account/subscription operations, independent hands-on validation of the MCP server in use.", "evidenceIds": [ "recurly-docs-12", "recurly-probe-2" ] }, { "productId": "recurly", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Recurly's Compass MCP server lets AI dev tools query Recurly's documentation and API knowledge, which is a step toward AI-native/agentic interaction, but the evidence describes it as a knowledge/docs connector rather than a natural-language interface for actually executing billing operations (e.g., changing subscriptions, issuing credits) end-to-end. Missing for 10: evidence of natural-language command execution for actual account/subscription actions, hands-on proof of the MCP server performing operations (not just doc lookup), and any client-facing chat/NL interface.", "evidenceIds": [ "recurly-docs-12", "recurly-probe-2" ] }, { "productId": "recurly", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of an official Recurly CLI; documentation instead references client libraries (Ruby, Node.js, Python, etc.), mobile SDKs, and an MCP server for AI tools, but nothing about a command-line interface.", "evidenceIds": [ "recurly-docs-13", "recurly-docs-14", "recurly-docs-12" ] }, { "productId": "recurly", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Recurly documents a full public API with official client libraries in seven languages, entitlements checkable via API, and even an MCP server (Compass) exposing API/doc knowledge to AI tools, indicating strong API-first, AI-accessible design. missing for 10: independent developer corroboration of API robustness/rate limits and no evidence of API versioning or webhooks documentation depth.", "evidenceIds": [ "recurly-docs-8", "recurly-docs-12", "recurly-docs-13", "recurly-probe-1", "recurly-probe-2" ] }, { "productId": "recurly", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of scoped or least-privilege API key/credential issuance for agents; the MCP server (recurly-docs-12, recurly-probe-2) only connects AI tools to documentation, not scoped API credentials, and no API key management or permission-scoping docs are present.", "evidenceIds": [ "recurly-docs-12", "recurly-probe-2" ] }, { "productId": "recurly", "storyId": "agentic-sdks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Recurly documents official client libraries across seven languages (Ruby, Node.js, Python, .NET, Java, PHP, Go) plus native mobile SDKs for iOS/Android, giving AI-native developers a solid multi-language SDK surface, and an official Compass MCP server ties AI tools into its API docs. Missing for 10: independent hands-on developer confirmation of SDK quality/reliability and no explicit mention of AI-specific SDK tooling beyond the MCP docs connector.", "evidenceIds": [ "recurly-docs-13", "recurly-docs-14", "recurly-docs-12", "recurly-probe-2" ] }, { "productId": "recurly", "storyId": "agentic-webhooks", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "The only mention of webhooks in the evidence pack is an indirect community comparison noting Recurly automates retention 'without needing to implement webhooks' for dunning, implying webhook functionality exists, but no first-party documentation, API reference, or event-list evidence describes how to subscribe to or configure webhooks. Missing for 10: official webhook API/docs, list of subscribable events, setup instructions, and any AI-agent-specific webhook integration guidance.", "evidenceIds": [ "recurly-comm-8" ] }, { "productId": "recurly", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows client libraries, SDKs, an MCP server for docs/API knowledge, and an llms.txt file, but there is no mention of an interactive API reference with runnable/live-executable examples (e.g., a try-it-now console). Missing for 10: interactive API console, runnable code examples, in-browser execution/testing of API calls.", "evidenceIds": [ "recurly-docs-12", "recurly-docs-13", "recurly-probe-1", "recurly-probe-2" ] }, { "productId": "recurly", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows official client libraries, mobile SDKs, an llms.txt file, and an MCP server, but no mention of a downloadable OpenAPI/Swagger spec or any machine-readable API schema file. Since Recurly is an API-based product, this axis clearly applies, but there is no evidence of the specific capability requested.", "evidenceIds": [ "recurly-docs-13", "recurly-probe-1", "recurly-probe-2" ] }, { "productId": "recurly", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item mentions a sandbox, test/dev environment, or test API keys for Recurly; documentation covers subscription changes, billing, and MCP tooling but not isolated test environments. Missing for 10: any mention of sandbox/test mode, test credentials, or separation from production data.", "evidenceIds": [] }, { "productId": "recurly", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers subscription features, client libraries, and an MCP server, but contains no documentation of API versioning schemes or a deprecation policy. Missing for 10: any mention of API version headers/URLs, changelog, or a documented deprecation/sunset policy.", "evidenceIds": [] }, { "productId": "recurly", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Recurly's docs mention bulk data import/export to populate accounts and analyze records, which implies some bulk operation support, and API client libraries could theoretically be used to script batch changes. However, there is no explicit documentation of a bulk-update/bulk-cancel/bulk-modify API for subscriptions, invoices, or entitlements, nor any AI-native tooling for orchestrating such bulk actions. missing for 10: explicit bulk subscription/invoice update API, bulk operation endpoints or SDK examples, evidence of AI-driven or agentic bulk workflows, independent confirmation of bulk import scale/performance.", "evidenceIds": [ "recurly-docs-11", "recurly-docs-15", "recurly-docs-13" ] }, { "productId": "recurly", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Recurly ships built-in automated responses to specific billing events—dunning campaigns triggered by failed payments (up to 50 targeted campaigns) and automatic fallback to backup payment methods—but these are pre-defined behaviors within fixed billing workflows, not a general-purpose rule engine where users define arbitrary event→action logic. Missing for 10: a configurable trigger/condition/action framework (e.g., custom webhooks-driven rules, if-this-then-that automation builder) and evidence of user-defined (not just Recurly-provided) automation logic.", "evidenceIds": [ "recurly-docs-5", "recurly-docs-6", "recurly-docs-7", "recurly-comm-8" ] }, { "productId": "recurly", "storyId": "automation-scheduled-jobs", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Recurly's core product is built around recurring billing cycles that run automatically (subscription renewals, usage-based billing at cycle end, dunning email campaigns), which functions as a domain-specific recurring workflow engine, but it offers no general-purpose job/task scheduler an AI-native user could configure for arbitrary recurring workflows. Missing for 10: a generic scheduling/automation API or cron-like mechanism for arbitrary AI-driven tasks, and any evidence of AI-agent-triggered scheduling beyond built-in billing cycles.", "evidenceIds": [ "recurly-docs-3", "recurly-docs-5", "recurly-docs-6", "recurly-docs-2" ] }, { "productId": "recurly", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of version control, review workflows, or rollback for automations/workflows in Recurly; the docs cover subscription/billing config changes, dunning campaigns, and an MCP server for documentation access, but none describe versioning, audit/review, or rollback of automations themselves.", "evidenceIds": [] }, { "productId": "recurly", "storyId": "billing-cycle-flexibility", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Evidence only shows post-issuance invoice detail edits (due dates, PO numbers, terms) via recurly-docs-10, which loosely touches 'custom invoice dates' but doesn't address billing anchor/cycle configuration. No documentation covers calendar-vs-anniversary billing modes or aligning billing cycles across a customer's multiple subscriptions. Missing for 10: explicit billing anchor/cycle-start configuration, calendar vs anniversary billing toggle, cross-subscription date alignment tooling, and any independent/hands-on confirmation of these mechanics.", "evidenceIds": [ "recurly-docs-10" ] }, { "productId": "recurly", "storyId": "compliant-invoice-generation", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Docs confirm invoice modification (notes, due dates, PO numbers, terms), separate credit invoices, and account/record-keeping features, implying automated invoice generation and delivery, but there is no explicit evidence of sequential/compliant numbering rules, custom field configuration on invoices, memo fields, or receipt generation/delivery workflows. missing for 10: explicit documentation of sequential invoice numbering compliance, custom fields on invoices, memo support, automatic receipt generation/delivery, and independent confirmation of these features working as described.", "evidenceIds": [ "recurly-docs-9", "recurly-docs-10", "recurly-docs-11" ] }, { "productId": "recurly", "storyId": "dunning-comms-automation", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm dunning campaigns (up to 50 targeted campaigns) and proactive involuntary-churn tools like backup payment methods, and community evidence corroborates automated dunning emails as a differentiator vs competitors. However, there's no documentation of branded email customization, in-app banners, configurable grace periods, or final-state rules distinguishing cancel vs unpaid outcomes. missing for 10: branded email templates/customization, in-app banner support, grace period configuration, and cancel-vs-unpaid final-state rule settings.", "evidenceIds": [ "recurly-docs-5", "recurly-docs-6", "recurly-docs-7", "recurly-comm-8", "recurly-comm-9" ] }, { "productId": "recurly", "storyId": "full-billing-export", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Recurly docs mention a general 'Data Imports and Exports' feature for populating accounts and keeping compliance records, and a community comment specifically praises Recurly's gateway for supporting 'credit card portability' to another provider. However, there is no vendor documentation detailing bulk export of subscriptions, invoices, or usage records, nor an official description of the payment-method portability mechanism (it's only referenced anecdotally by a user, not documented by Recurly). Missing for 10: explicit docs on invoice/usage-record export formats and scope, official documentation of stored payment-method portability/migration tooling, and any independent verification that a full data export actually contains all subscription/invoice/usage history.", "evidenceIds": [ "recurly-docs-11", "recurly-docs-15", "recurly-comm-7" ] }, { "productId": "recurly", "storyId": "gl-erp-sync", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows generic data import/export capability but no mention of specific GL/ERP integrations (QuickBooks, Xero, NetSuite) or revenue recognition sync. Missing for 10: any documentation of accounting/ERP connectors, revenue recognition automation, or close-process integration.", "evidenceIds": [ "recurly-docs-11", "recurly-docs-15" ] }, { "productId": "recurly", "storyId": "hybrid-pricing-models", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Recurly's docs confirm usage-based billing with measured units and flexible subscription/add-on changes (recurly-docs-1, recurly-docs-3, recurly-docs-4), which supports basic per-unit and subscription-plus-usage hybrid billing without code. However, the evidence never explicitly documents tiered, volume, or graduated pricing structures, nor named 'commitment' or 'overage' billing mechanics as configurable models. missing for 10: explicit tiered/volume/graduated pricing tier configuration docs, explicit commitment and overage billing mechanics, independent confirmation these complex pricing combos work without custom code.", "evidenceIds": [ "recurly-docs-1", "recurly-docs-3", "recurly-docs-4", "recurly-docs-9" ] }, { "productId": "recurly", "storyId": "lifecycle-event-stream", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no documentation of Recurly's webhook or event system for subscription lifecycle changes (renewal, upgrade, payment failure, cancellation). The only indirect mention is a community comment noting dunning emails work 'without needing to implement webhooks,' which implies webhooks exist as an option but provides no detail on reliability, event types, or delivery guarantees.", "evidenceIds": [ "recurly-comm-8" ] }, { "productId": "recurly", "storyId": "multi-currency-billing", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "None of the evidence items mention multi-currency pricing, per-currency price points, or consolidated home-currency reporting — the pack covers subscription changes, usage billing, dunning, entitlements, invoices, and general community commentary, none of which address currency handling. This is a fair capability to expect from a billing platform, so absence of evidence yields 'none' rather than 'na'.", "evidenceIds": [] }, { "productId": "recurly", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs show broad API coverage of core subscription, invoicing, entitlements, and data import/export features (recurly-docs-1,8,10,11) alongside official client libraries (recurly-docs-13), suggesting most UI actions are API-accessible. However, there is no explicit statement of full UI/API parity, and some UI-only features like dunning campaign configuration (recurly-docs-5) aren't confirmed as API-manageable. Missing for 10: an explicit parity guarantee or comprehensive API reference mapping every UI action, and independent confirmation that all UI-configurable settings (e.g., dunning campaigns) are exposed via API.", "evidenceIds": [ "recurly-docs-1", "recurly-docs-8", "recurly-docs-10", "recurly-docs-11", "recurly-docs-13", "recurly-docs-5" ] }, { "productId": "recurly", "storyId": "openness-full-export", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Recurly documents a data-imports-and-exports feature for populating accounts and keeping records, and community mentions note credit-card portability, suggesting some data can be extracted, but there is no evidence of a comprehensive, open-format (CSV/JSON) full-account export or an official data-portability/exit process. missing for 10: explicit open-format export documentation, full account data export scope, and confirmation of easy migration/exit workflow.", "evidenceIds": [ "recurly-docs-11", "recurly-docs-15", "recurly-comm-7" ] }, { "productId": "recurly", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "recurly", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "recurly", "storyId": "payment-method-recovery", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm backup payment methods and general 'tools to proactively catch and resolve' involuntary churn from expired cards/failed payments, but the evidence pack never explicitly names or documents a network account updater feature or hosted update-payment pages. missing for 10: explicit documentation of account updater integration, explicit hosted update-payment page feature, and any hands-on confirmation these self-heal expiring cards.", "evidenceIds": [ "recurly-docs-6", "recurly-docs-7" ] }, { "productId": "recurly", "storyId": "plan-entitlements-api", "verdict": "full", "quality": 6, "confidence": "medium", "rationale": "Recurly's docs explicitly describe an entitlements feature that can be checked in real time via the Recurly API across platforms, matching the core of the story. Missing for 10: detailed documentation of how entitlements are attached to specific plans, explicit SDK-level (not just API) entitlement-check examples, and independent/hands-on confirmation of this specific feature.", "evidenceIds": [ "recurly-docs-8", "recurly-docs-13" ] }, { "productId": "recurly", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions data residency, regional storage options, or geographic data control for Recurly; this is a fair axis for a SaaS billing platform handling customer/payment data, but absence of evidence means 'none'.", "evidenceIds": [] }, { "productId": "recurly", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Recurly is a subscription billing/payments platform, not an AI model or AI product that trains on user data; the axis of preventing data use for AI training does not apply to its category.", "evidenceIds": [] }, { "productId": "recurly", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers subscription management, billing, invoicing, imports/exports and an MCP docs connector, but nothing addresses data retention policies, deletion requests, or privacy/GDPR-style controls for AI-native users. missing for 10: data retention policy documentation, data deletion/right-to-erasure mechanisms, privacy compliance controls (GDPR/CCPA), evidence of user-initiated data deletion API or workflow.", "evidenceIds": [] }, { "productId": "recurly", "storyId": "privacy-telemetry-optout", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Recurly is a subscription billing platform, not an AI agent/tool whose telemetry or usage-tracking opt-out would be a meaningful axis for AI-native users; this consumer-privacy-style story doesn't map to its product category.", "evidenceIds": [] }, { "productId": "recurly", "storyId": "proration-correctness", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Recurly's docs confirm mid-cycle changes are billed as prorated differences and that credits get their own separate, dedicated invoices, plus invoices can be amended post-issuance for record accuracy — supporting the core proration/credit reconciliation need. However, there's no documented breakdown of the proration math formula, no explicit audit trail feature, and no evidence (first-party or independent) that invoice reconciliation is verified to the penny in practice. Missing for 10: detailed proration calculation documentation, explicit audit-log/traceability feature, and independent verification of penny-accurate reconciliation.", "evidenceIds": [ "recurly-docs-2", "recurly-docs-9", "recurly-docs-10" ] }, { "productId": "recurly", "storyId": "realtime-usage-visibility", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Recurly's docs mention usage-based billing that charges at the end of the billing cycle and 'measured units' for reporting, but there is no evidence of real-time current-cycle usage/cost dashboards, customer-facing usage views, or threshold/alert mechanisms to warn before an invoice is generated.", "evidenceIds": [ "recurly-docs-3", "recurly-docs-4" ] }, { "productId": "recurly", "storyId": "revrec-asc606-reports", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers subscriptions, invoicing, dunning, entitlements, and data import/export for general compliance, but nothing addresses ASC 606/IFRS 15 deferred revenue schedules or audit-ready recognition reporting specifically. Absence of evidence for this applicable billing-platform capability means it can't be credited as delivered.", "evidenceIds": [] }, { "productId": "recurly", "storyId": "self-serve-plan-changes", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm customers can change plans (upgrade/downgrade, add-ons, proration) via Recurly's subscription APIs, implying self-service capability, but no direct evidence of a hosted customer portal or embeddable UI components for self-cancel/upgrade, nor explicit mention of eliminating support tickets. missing for 10: explicit hosted portal documentation, embeddable component/widget evidence, self-service cancellation flow.", "evidenceIds": [ "recurly-docs-1", "recurly-docs-2" ] }, { "productId": "recurly", "storyId": "smart-retries", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Recurly documents dunning management and involuntary-churn tools like targeted campaigns and backup payment methods, and community commentary notes Recurly's automated retention/dunning emails as a differentiator, but there is no evidence of ML-optimized or schedule-optimized retry cadence logic, nor any quantified recovery-rate/churn-reduction metrics. missing for 10: documentation of the actual retry scheduling/ML logic behind payment retries, evidence of adaptive or optimized cadence (vs simple campaign rules), and third-party or first-party data on measurable involuntary churn recovery.", "evidenceIds": [ "recurly-docs-5", "recurly-docs-6", "recurly-docs-7", "recurly-comm-8", "recurly-comm-9" ] }, { "productId": "recurly", "storyId": "subscription-analytics", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers subscription changes, dunning, entitlements, invoicing, and data import/export, but contains no mention of MRR, churn, cohort retention, or LTV analytics/reporting dashboards computed from billing data. Missing for 10: any documentation of built-in revenue/retention analytics, cohort or LTV reporting, or dashboards distinct from raw data export.", "evidenceIds": [ "recurly-docs-11", "recurly-docs-15" ] }, { "productId": "recurly", "storyId": "subscription-crud-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm a documented API for changing subscriptions (upgrade/downgrade, quantity/price, add-ons) with official client libraries in 7 languages, supporting programmatic subscription management. However, the evidence never explicitly documents 'create' or 'pause'/'cancel' endpoints—only mid-cycle change operations are detailed, so full lifecycle coverage is inferred rather than shown directly. Missing for 10: explicit documentation of subscription creation and cancellation/pause API calls, and any hands-on developer confirmation of the full lifecycle working end-to-end via API alone.", "evidenceIds": [ "recurly-docs-1", "recurly-docs-2", "recurly-docs-13" ] }, { "productId": "recurly", "storyId": "subscription-import-migration", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Recurly's docs mention generic data import/export tools (recurly-docs-11, recurly-docs-15) and a community comment notes Recurly's gateway offers 'credit card portability' (recurly-comm-7), suggesting some support for migrating billing data and card credentials. However, there is no documentation of a structured migration workflow, PCI-compliant card tokenization/transfer process, or assurance that live subscriptions migrate without customer disruption. missing for 10: explicit migration/onboarding documentation, PCI-compliant card data transfer mechanics, evidence of seamless customer experience during migration.", "evidenceIds": [ "recurly-docs-11", "recurly-docs-15", "recurly-comm-7" ] }, { "productId": "recurly", "storyId": "tax-on-invoices", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack contains no mention of tax calculation, VAT/GST handling, tax registrations, or any first-party or third-party tax integration (e.g., Avalara, TaxJar) — invoicing/credit/dunning features are covered but tax compliance is entirely absent. missing for 10: any documentation of sales tax/VAT/GST calculation, tax registration management, or tax engine integration.", "evidenceIds": [] }, { "productId": "recurly", "storyId": "trials-coupons-promos", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no documentation of free trials, coupons/promotion codes, automatic trial-to-paid conversion, or card-network-mandated trial-end notifications. Related dunning/involuntary-churn features are covered, but none address trial or coupon lifecycle specifically. missing for 10: docs on free trial setup, coupon/promo code creation, automatic trial conversion, and trial-end notification compliance.", "evidenceIds": [] }, { "productId": "recurly", "storyId": "usage-event-ingestion", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Recurly documents usage-based billing where usage-based add-ons are reported and billed at the end of a cycle (recurly-docs-3, recurly-docs-4), showing basic support for metering usage into billing, but there is no evidence of a high-volume streaming ingestion API, idempotency keys/idempotent ingestion, or explicit backdating support for usage events. Missing for 10: evidence of a streaming/bulk usage ingestion endpoint, documented idempotency guarantees for usage events, and backdating/timestamp-adjustment support for late-arriving usage records.", "evidenceIds": [ "recurly-docs-3", "recurly-docs-4" ] }, { "productId": "recurly", "storyId": "versioned-plan-migrations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Docs only describe changing an individual subscription's plan/price/add-ons (recurly-docs-1, recurly-docs-2); there is no mention of versioned plan definitions, grandfathering existing subscribers on old pricing, or bulk/cohort migration tooling for moving groups of subscribers to new pricing.", "evidenceIds": [ "recurly-docs-1", "recurly-docs-2" ] }, { "productId": "revenuecat", "storyId": "agent-manages-subscriptions", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "RevenueCat ships an MCP server explicitly for AI assistants to manage subscription apps, Products, Entitlements, and Offerings (revenuecat-docs-9), backed by a REST API authenticated via scoped API keys (revenuecat-docs-5), and SDK/dashboard features let you grant promotional subscriptions and view customer transaction histories (revenuecat-gh-4). However, the evidence centers on catalog/config management (products, entitlements, offerings) rather than explicit end-user billing actions like 'create a subscription', 'apply a credit', or 'change a plan' for a specific customer, and there's no documentation of granular per-action permission scoping beyond generic API keys. Missing for 10: explicit agent-driven customer-level actions (create subscription, apply credit, change plan) and detailed scoped-permission/role model for agent access.", "evidenceIds": [ "revenuecat-docs-9", "revenuecat-docs-5", "revenuecat-gh-4", "revenuecat-docs-8" ] }, { "productId": "revenuecat", "storyId": "agent-reads-billing-state", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "RevenueCat's Rico assistant explicitly lets users ask plain-language billing questions like 'why did MRR drop,' and the MCP server/CustomerInfo API expose subscription and customer purchase data programmatically, showing agent-queryable billing state. However, there's no explicit evidence for querying delinquent accounts, upcoming renewals, or per-customer invoice histories via an agent interface. Missing for 10: documented support for delinquency/renewal queries, invoice-level querying, and independent confirmation of Rico's accuracy/scope beyond MRR examples.", "evidenceIds": [ "revenuecat-docs-10", "revenuecat-docs-9", "revenuecat-docs-3", "revenuecat-docs-6", "revenuecat-gh-4" ] }, { "productId": "revenuecat", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "RevenueCat serves a live llms.txt (HTTP 200) describing itself in agent-friendly terms, and docs pages throughout the evidence pack use the .md suffix convention (agent-oriented docs format) confirming machine-readable documentation exists. missing for 10: no root docs.md index (404) and no OpenAPI spec discoverable, which would round out full agentic documentation coverage.", "evidenceIds": [ "revenuecat-probe-1", "revenuecat-probe-2", "revenuecat-probe-3", "revenuecat-docs-9" ] }, { "productId": "revenuecat", "storyId": "agentic-ai-insights", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "RevenueCat's Rico feature explicitly lets users ask plain-language questions about their data (e.g. \"Why did MRR drop in Germany last week?\") and get AI-driven insights, directly matching the story; charts/metrics dashboard provides the underlying data surface. Missing for 10: independent/hands-on validation of Rico's insight quality and broader coverage beyond the single documented example.", "evidenceIds": [ "revenuecat-docs-10", "revenuecat-docs-6" ] }, { "productId": "revenuecat", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "RevenueCat offers webhooks for real-time event notifications and scheduled data exports that run on a set cadence, which are background automations, but these are infrastructure features rather than AI-driven autonomous agentic workflows. The MCP server and AI toolkit let AI assistants configure or query RevenueCat interactively, not run persistent background automations. Missing for 10: evidence of AI agents autonomously triggering actions or workflows in the background without human initiation, scheduling/orchestration of agentic tasks, or any first-party 'automation builder' feature.", "evidenceIds": [ "revenuecat-gh-3", "revenuecat-docs-7", "revenuecat-docs-9" ] }, { "productId": "revenuecat", "storyId": "agentic-builtin-assistant", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "RevenueCat documents \"Rico\", a built-in AI assistant in the dashboard that lets users ask plain-language questions about their data instead of digging through charts (revenuecat-docs-10). This shows a built-in assistant capability, but evidence only covers Q&A/insight generation, not broader task delegation or action-taking within the product, and there's no independent/community corroboration of its real-world reliability. Missing for 10: evidence of Rico performing actions/tasks beyond answering questions, and independent user reports validating its usefulness.", "evidenceIds": [ "revenuecat-docs-10" ] }, { "productId": "revenuecat", "storyId": "agentic-headless", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "RevenueCat exposes a REST API authenticated via API keys and supports server-side data import/export (revenuecat-docs-5, revenuecat-docs-4, revenuecat-docs-7), which enables non-interactive, programmatic use suitable for automation pipelines. However, there is no explicit documentation of CI/CD integration, headless test runners, or scripted deployment workflows. Missing for 10: explicit CI/CD examples, headless SDK testing guidance, and confirmation of non-interactive automation use cases.", "evidenceIds": [ "revenuecat-docs-5", "revenuecat-docs-4", "revenuecat-docs-7" ] }, { "productId": "revenuecat", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "RevenueCat is a subscription/monetization backend platform, not an AI agent or assistant that a user interacts with and extends via plugged-in MCP servers. The evidence shows RevenueCat ships its OWN MCP server (revenuecat-docs-9) so that external AI assistants can call RevenueCat's tools — this is the server role, the opposite of the client-side 'plug servers into this product' story, so the axis is a category mismatch here.", "evidenceIds": [ "revenuecat-docs-9" ] }, { "productId": "revenuecat", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "RevenueCat documents an official MCP server that lets AI assistants manage subscription apps, Products, Entitlements, etc. directly, without the dashboard, which directly satisfies the story. Missing for 10: independent/hands-on third-party corroboration of the MCP server working, and technical setup/auth details beyond the docs summary.", "evidenceIds": [ "revenuecat-docs-9" ] }, { "productId": "revenuecat", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "RevenueCat offers an MCP server letting AI assistants manage products/entitlements via natural language, and Rico allows plain-language analytics queries — both first-party docs confirm agentic/NL control of the dashboard. However, this covers admin/analytics operations, not the core end-user subscription flows, and there's no independent/hands-on corroboration of these NL features working reliably. Missing for 10: third-party validation of MCP/Rico usage, broader coverage of NL control across all product surfaces (not just admin tasks).", "evidenceIds": [ "revenuecat-docs-9", "revenuecat-docs-10", "revenuecat-docs-8" ] }, { "productId": "revenuecat", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item describes a dedicated official CLI tool; the closest items are an MCP server (revenuecat-docs-9), an AI toolkit doc (revenuecat-docs-8), and a natural-language analytics assistant 'Rico' (revenuecat-docs-10), none of which are described as a command-line interface. Missing for 10: explicit CLI product/documentation, installation instructions, command reference, or community confirmation of CLI usage.", "evidenceIds": [ "revenuecat-docs-8", "revenuecat-docs-9", "revenuecat-docs-10" ] }, { "productId": "revenuecat", "storyId": "agentic-public-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "RevenueCat confirms a REST API secured by API keys (revenuecat-docs-5) and even offers an MCP server letting AI assistants manage subscriptions programmatically (revenuecat-docs-9) and an AI toolkit (revenuecat-docs-8), showing real support for AI-driven API access. However, probes for a public OpenAPI/swagger spec or docs.md all 404 (revenuecat-probe-2, revenuecat-probe-3), so a fully documented, discoverable public API reference isn't evidenced. Missing for 10: a published OpenAPI/API reference spec, broader REST endpoint documentation beyond auth, and independent confirmation of programmatic API usage outside RevenueCat's own docs.", "evidenceIds": [ "revenuecat-docs-5", "revenuecat-docs-9", "revenuecat-docs-8", "revenuecat-probe-2", "revenuecat-probe-3" ] }, { "productId": "revenuecat", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows RevenueCat uses API keys for authentication (revenuecat-docs-5) and has an MCP server/AI toolkit for agents (revenuecat-docs-8, revenuecat-docs-9), but nothing describes scoped, granular, or least-privilege API key permissions for agents — no mention of read-only keys, role-based scopes, or key restriction settings.", "evidenceIds": [ "revenuecat-docs-5", "revenuecat-docs-9" ] }, { "productId": "revenuecat", "storyId": "agentic-sdks", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "RevenueCat ships official native SDKs (iOS, and implied Android/cross-platform per GitHub repo) with well-documented capabilities—purchases, entitlements, customer info, authentication—that developers, including AI-native ones, can build against directly, and this is corroborated by both first-party GitHub repos and extensive docs. Missing for 10: independent hands-on developer corroboration specifically praising SDK integration ergonomics, and explicit multi-platform SDK breadth beyond iOS in the evidence pack.", "evidenceIds": [ "revenuecat-gh-1", "revenuecat-docs-1", "revenuecat-docs-3", "revenuecat-docs-5", "revenuecat-docs-9" ] }, { "productId": "revenuecat", "storyId": "agentic-webhooks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "RevenueCat explicitly documents webhooks for real-time event notifications, letting external systems (including AI agents) subscribe to subscription/purchase events. Missing for 10: no first-party docs on webhook payload schema/customization or independent hands-on confirmation of reliability, and no mention of AI-specific webhook consumption patterns.", "evidenceIds": [ "revenuecat-gh-3", "revenuecat-docs-5" ] }, { "productId": "revenuecat", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of an interactive API reference or runnable examples; probes for openapi/swagger specs and docs.md return 404, and no documentation item describes a try-it-out console or embedded runnable code samples.", "evidenceIds": [ "revenuecat-probe-2", "revenuecat-probe-3" ] }, { "productId": "revenuecat", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "RevenueCat exposes a REST API (per revenuecat-docs-5) but no evidence of a downloadable OpenAPI/Swagger spec; the probe explicitly found all standard OpenAPI spec paths return 404, and no docs reference a machine-readable API spec.", "evidenceIds": [ "revenuecat-probe-3", "revenuecat-docs-5" ] }, { "productId": "revenuecat", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack contains no mention of a sandbox/test mode, test API keys, or any mechanism to validate purchases/subscriptions without affecting production data—only production-oriented SDK, dashboard, and AI-toolkit/MCP features are described.", "evidenceIds": [] }, { "productId": "revenuecat", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of API versioning scheme or a documented deprecation policy; OpenAPI spec probes returned 404s and no docs mention version lifecycle or deprecation commitments.", "evidenceIds": [ "revenuecat-probe-3", "revenuecat-probe-2" ] }, { "productId": "revenuecat", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence shows RevenueCat's MCP server and AI toolkit let an AI assistant manage individual Products, Entitlements, and Offerings, and server-side import of existing purchase data exists, but nothing describes batch/bulk operations across many items at once (e.g., bulk-editing many products, mass customer updates, or batch API calls) for an AI-native workflow. Missing for full credit: explicit bulk/batch API endpoints, documentation of bulk operations via MCP/AI toolkit, and any example of operating on many items simultaneously.", "evidenceIds": [ "revenuecat-docs-9", "revenuecat-docs-8", "revenuecat-docs-4" ] }, { "productId": "revenuecat", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "RevenueCat offers webhooks for real-time event notifications (revenuecat-gh-3) that can be wired to trigger external actions, but there's no built-in rule engine to define conditional logic or automated actions natively within RevenueCat itself. A community feature request explicitly asks for 'timed hooks' like trial-expiry reminders, indicating this kind of automation isn't natively supported (revenuecat-comm-6). Missing for 10: native rule/trigger builder, conditional logic definitions, in-product automated actions beyond raw webhook delivery.", "evidenceIds": [ "revenuecat-gh-3", "revenuecat-comm-6" ] }, { "productId": "revenuecat", "storyId": "automation-scheduled-jobs", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "RevenueCat supports scheduled recurring data exports to cloud storage/email, which is a narrow instance of a recurring job, but there is no evidence of a general-purpose scheduler for recurring workflows or automations (e.g., via MCP/AI toolkit) that an AI-native user could configure. missing for 10: evidence of configurable recurring workflow/job scheduling beyond data exports, integration of scheduling with the MCP/AI toolkit or Rico assistant, and any cron-like automation API.", "evidenceIds": [ "revenuecat-docs-7", "revenuecat-docs-9", "revenuecat-docs-10" ] }, { "productId": "revenuecat", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "RevenueCat is a subscription/monetization backend, not an automation/workflow tool; there's no concept of 'automations' with version history or rollback to speak of. This axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "revenuecat", "storyId": "billing-cycle-flexibility", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack items mention billing anchor/cycle controls, calendar vs anniversary billing alignment, or custom invoice dates — RevenueCat's documented features focus on entitlements, purchases, offerings, and analytics, not invoice/billing-cycle configuration.", "evidenceIds": [] }, { "productId": "revenuecat", "storyId": "compliant-invoice-generation", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence RevenueCat generates compliant invoices, credit notes, or receipts with sequential numbering, custom fields, or memos; evidence covers subscription management, entitlements, paywalls, and analytics only.", "evidenceIds": [] }, { "productId": "revenuecat", "storyId": "dunning-comms-automation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows RevenueCat's core subscription infrastructure (entitlements, offerings, paywalls, webhooks, exports) but contains no mention of dunning-specific features like branded recovery emails, in-app banners, configurable grace periods, or cancel-vs-unpaid final-state rules. A community comment even requests exactly this kind of timed-notification/dunning hook as a missing feature, reinforcing the absence of evidence for it.", "evidenceIds": [ "revenuecat-comm-6", "revenuecat-docs-11", "revenuecat-docs-8" ] }, { "productId": "revenuecat", "storyId": "full-billing-export", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "RevenueCat offers scheduled data exports to cloud storage/email and REST API access to customer/subscription data (docs-7, docs-5, docs-3), which covers some of the 'export subscriptions/customer records' need. However, there is no evidence of invoice-level or usage-record export, and no mention anywhere of porting stored payment methods to another provider — RevenueCat itself doesn't store card details (they live with Stripe/Apple/Google), so this key part of the story is unaddressed. Community discussion even flags that migrating subscriptions between providers is 'hard to make smooth' (revenuecat-comm-8), reinforcing the gap. Missing for 10: documented invoice/usage export, explicit payment-method portability/export mechanism, and confirmation of smooth cross-provider migration.", "evidenceIds": [ "revenuecat-docs-7", "revenuecat-docs-5", "revenuecat-docs-3", "revenuecat-comm-8" ] }, { "productId": "revenuecat", "storyId": "gl-erp-sync", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows RevenueCat offers scheduled data exports to cloud storage/email and webhooks, but nothing about direct sync of invoices, payments, or revenue recognition entries into QuickBooks, Xero, or NetSuite ERPs.", "evidenceIds": [] }, { "productId": "revenuecat", "storyId": "hybrid-pricing-models", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "RevenueCat's evidence covers mobile subscription entitlements, offerings, paywalls, and analytics, but there is no mention of tiered/volume/graduated/per-unit pricing, usage-based billing, credits, commitments, or overage handling — the core usage-metering pricing models in the story are entirely absent.", "evidenceIds": [] }, { "productId": "revenuecat", "storyId": "lifecycle-event-stream", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "RevenueCat explicitly advertises real-time webhook events (revenuecat-gh-3) and dashboard/customer-info data on subscription state (revenuecat-docs-3), supporting the general claim that lifecycle changes are exposed as events. However, the evidence pack lacks detail on which specific lifecycle events (renewal, upgrade, payment failure, cancellation) are covered, delivery guarantees/retries, or reliability at scale, and a community request for additional timed hooks (revenuecat-comm-6) suggests some lifecycle notifications are still missing today. Missing for 10: documented list of webhook event types, delivery/retry guarantees, and evidence of reliability under real-world load.", "evidenceIds": [ "revenuecat-gh-3", "revenuecat-docs-3", "revenuecat-comm-6" ] }, { "productId": "revenuecat", "storyId": "multi-currency-billing", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence describes local-currency price points, per-currency pricing configuration, or consolidated home-currency reporting; the evidence pack covers entitlements, paywalls, SDKs, MCP tools, and dashboards but nothing about multi-currency billing or currency-consolidated financial reports.", "evidenceIds": [] }, { "productId": "revenuecat", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "RevenueCat documents a REST API/authentication (revenuecat-docs-5), an MCP server that lets AI assistants manage apps, Products, Entitlements 'without using the RevenueCat dashboard' (revenuecat-docs-9), and an AI toolkit for setting up subscriptions, offerings, and packages (revenuecat-docs-8), showing strong API parity for core configuration tasks. However, no OpenAPI spec was discoverable (revenuecat-probe-3) and UI-centric features like paywall design, charts, and dashboard-and-metrics exploration appear tied to the dashboard or scheduled exports rather than full API control (revenuecat-docs-6, revenuecat-docs-7, revenuecat-docs-11). Missing for 10: publicly documented OpenAPI/full REST API reference, evidence that paywall building/analytics dashboards are fully API-accessible, and independent confirmation of complete UI-API parity.", "evidenceIds": [ "revenuecat-docs-9", "revenuecat-docs-8", "revenuecat-docs-5", "revenuecat-docs-6", "revenuecat-docs-11", "revenuecat-probe-3" ] }, { "productId": "revenuecat", "storyId": "openness-full-export", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "RevenueCat does support scheduled data exports to cloud storage/email and a REST API for programmatic access to customer/subscription data, giving some data portability. However, there's no documented comprehensive 'export all your data and leave' feature, no explicit open-format full account export, and no OpenAPI spec found (probe returned 404s). Missing for 10: a documented full-account data export/deletion workflow, explicit open-format guarantees, and confirmation of a public OpenAPI/schema for complete data extraction.", "evidenceIds": [ "revenuecat-docs-7", "revenuecat-docs-5", "revenuecat-probe-3" ] }, { "productId": "revenuecat", "storyId": "openness-open-license", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "RevenueCat publishes open-source client SDKs on GitHub (e.g., purchases-ios) showing feature code, but the evidence never states a specific open-source license nor shows the core backend/platform source, which remains a closed SaaS product. missing for 10: explicit license statement, evidence of core platform/service source being open, independent confirmation of license terms.", "evidenceIds": [ "revenuecat-gh-1", "revenuecat-gh-2", "revenuecat-gh-3", "revenuecat-gh-4" ] }, { "productId": "revenuecat", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "RevenueCat is delivered purely as a hosted SaaS platform (dashboard, cloud APIs, webhooks, MCP server) with no mention of an on-premises or self-hosted deployment option anywhere in the evidence pack; community discussion even treats it as an external service to 'switch to' rather than something to run locally. Missing for 10: any documentation of self-hosted/on-prem deployment, open-source core, or Docker/Kubernetes packaging.", "evidenceIds": [ "revenuecat-probe-1", "revenuecat-docs-5", "revenuecat-docs-9", "revenuecat-comm-1" ] }, { "productId": "revenuecat", "storyId": "payment-method-recovery", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "RevenueCat's evidence covers subscription management, entitlements, paywalls, MCP/AI tools, and analytics, but nothing addresses card-level dunning recovery mechanics like network account updater, backup payment methods, or hosted update-payment pages — these are Stripe/payment-processor-level features that RevenueCat, as an app-store/subscription management layer, does not document handling itself.", "evidenceIds": [] }, { "productId": "revenuecat", "storyId": "plan-entitlements-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs confirm entitlements model access based on purchases without hardcoding product identifiers, and CustomerInfo (via SDK/API, authenticated by API keys) exposes purchase/entitlement data at runtime. This directly matches the story of entitlements attached to plans checked at runtime via SDK/API. Missing for 10: explicit code snippet showing an entitlement check call and independent/hands-on developer confirmation of the runtime check pattern.", "evidenceIds": [ "revenuecat-docs-1", "revenuecat-docs-3", "revenuecat-docs-5", "revenuecat-gh-1" ] }, { "productId": "revenuecat", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of data residency/region selection controls in RevenueCat's docs, GitHub, or community sources; nothing addresses where customer/subscription data is stored or region choice.", "evidenceIds": [] }, { "productId": "revenuecat", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "RevenueCat is a subscription/monetization infrastructure product, not an AI model or AI product with training-data policies; opting out of AI training data usage is not a relevant axis for this category of product.", "evidenceIds": [] }, { "productId": "revenuecat", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence describes data retention policies, deletion controls, or GDPR/CCPA-style data management features for RevenueCat customers or end-users; evidence covers subscription management, entitlements, dashboards, and AI tooling only.", "evidenceIds": [] }, { "productId": "revenuecat", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item addresses telemetry/usage-tracking opt-out settings for RevenueCat itself; the axis is applicable since RevenueCat is a SaaS platform/SDK that could plausibly document such controls, but nothing here does.", "evidenceIds": [] }, { "productId": "revenuecat", "storyId": "proration-correctness", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence addresses proration math, credits, partial-period calculations, or invoice line-item reconciliation; documentation covers entitlements, SDKs, paywalls, and dashboards but nothing about auditable billing math for upgrades/downgrades.", "evidenceIds": [] }, { "productId": "revenuecat", "storyId": "realtime-usage-visibility", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "revenuecat", "storyId": "revrec-asc606-reports", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers subscription SDKs, entitlements, charts, and data exports, but nothing addresses ASC 606/IFRS 15 deferred revenue schedules or audit-ready recognition reporting. Missing for 10: any mention of deferred revenue calculation, revenue recognition compliance, or audit-ready financial reports.", "evidenceIds": [] }, { "productId": "revenuecat", "storyId": "self-serve-plan-changes", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence covers RevenueCat's SDKs, entitlements, paywalls, and dashboard analytics, but nothing describes a hosted customer portal or embeddable UI components letting end customers upgrade/downgrade/cancel without a support ticket. Community threads even discuss the difficulty of managing cross-platform (iOS/Android/Stripe) subscription moves, but no resolution or feature is cited that gives self-service cancellation/upgrade capability.", "evidenceIds": [ "revenuecat-docs-11", "revenuecat-comm-8", "revenuecat-comm-6" ] }, { "productId": "revenuecat", "storyId": "smart-retries", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions failed payment retry logic, dunning, billing recovery, or ML/schedule-optimized retry cadences; RevenueCat evidence covers entitlements, paywalls, SDKs, analytics, and AI tools but nothing about involuntary churn recovery.", "evidenceIds": [] }, { "productId": "revenuecat", "storyId": "subscription-analytics", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "RevenueCat's dashboard/charts and Rico assistant explicitly surface MRR and LTV computed directly from subscription/billing events (revenuecat-docs-6, revenuecat-docs-10, revenuecat-gh-4), and scheduled exports let founders pull this data (revenuecat-docs-7), all sourced from the billing data rather than a bolted-on BI tool. However, the evidence pack never explicitly mentions churn metrics or cohort retention analysis by name, so the story's full metric set isn't documented. Missing for 10: explicit churn-rate analytics, explicit cohort retention analytics/reporting.", "evidenceIds": [ "revenuecat-docs-6", "revenuecat-docs-10", "revenuecat-gh-4", "revenuecat-docs-7" ] }, { "productId": "revenuecat", "storyId": "subscription-crud-api", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "RevenueCat exposes API-based mechanisms for parts of subscription management (granting promotional subscriptions via API/dashboard, managing Products/Entitlements/Offerings via the AI toolkit and MCP server 'without using the dashboard'), and confirms REST API authentication exists. However, there's no direct evidence of documented endpoints to create, update, pause, or cancel a live subscription's lifecycle end-to-end, and a probe for OpenAPI/swagger docs returned 404s, suggesting the full API surface isn't publicly documented in this pack. missing for 10: explicit API endpoints for pause/cancel/update, a discoverable OpenAPI spec, and confirmation that full lifecycle actions (not just entitlement grants/catalog config) can be done purely via API.", "evidenceIds": [ "revenuecat-docs-5", "revenuecat-docs-8", "revenuecat-docs-9", "revenuecat-gh-4", "revenuecat-probe-3" ] }, { "productId": "revenuecat", "storyId": "subscription-import-migration", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "RevenueCat explicitly documents importing existing purchase/subscription data 'server-side or client-side' when migrating to the platform, which supports the core migration story, but there is no evidence describing spreadsheet-based import, PCI-compliant card data transfer, or explicit guarantees that migration is invisible to end customers. Community comments (revenuecat-comm-8, revenuecat-comm-3) show users actively wondering whether smooth cross-platform migration (App Store/Stripe) is even possible, indicating this remains an open question rather than a proven capability. missing for 10: PCI-compliant card data transfer mechanics, spreadsheet import tooling, documented zero-customer-disruption guarantee, and independent confirmation of smooth migration.", "evidenceIds": [ "revenuecat-docs-4", "revenuecat-comm-8", "revenuecat-comm-3" ] }, { "productId": "revenuecat", "storyId": "tax-on-invoices", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence RevenueCat calculates or manages sales tax, VAT, or GST, nor any mention of tax registrations, rates, or a tax integration; RevenueCat's role for App Store/Play purchases means Apple/Google handle tax, and no tax-handling capability is documented for its own billing (e.g., Stripe-side) invoicing.", "evidenceIds": [] }, { "productId": "revenuecat", "storyId": "trials-coupons-promos", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "RevenueCat supports free-trial eligibility checks, entitlements, and granting promotional subscriptions, and webhooks could be used to build custom trial-reminder logic, but there is no documented native coupon/promo-code management or built-in trial-end notification feature — in fact a community post explicitly requests this as a missing feature ('handling timed hooks to send notifications... reminding users before a trial expires'). missing for 10: native coupon/promo-code support, built-in trial-end notification delivery, card-network compliance documentation.", "evidenceIds": [ "revenuecat-docs-2", "revenuecat-gh-4", "revenuecat-gh-3", "revenuecat-comm-6" ] }, { "productId": "revenuecat", "storyId": "usage-event-ingestion", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "RevenueCat is a subscription/purchase management platform focused on entitlements, offerings, and billing data — nothing in the evidence describes streaming high-volume usage events (API calls, tokens, compute-seconds) with idempotent ingestion or backdating support, which is a usage-based metering capability outside RevenueCat's documented scope.", "evidenceIds": [ "revenuecat-docs-3", "revenuecat-docs-4", "revenuecat-docs-6", "revenuecat-probe-1" ] }, { "productId": "revenuecat", "storyId": "versioned-plan-migrations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "While RevenueCat's Offerings/Packages system (revenuecat-docs-8) implies some product versioning, there is no evidence of price-specific grandfathering, versioned pricing plans, or bulk cohort migration tools; the only 'migration' documentation (revenuecat-docs-4) concerns importing existing purchase data when onboarding to RevenueCat, not moving cohorts between price tiers post-launch. This is a fair capability to expect from a subscription billing platform, so the axis applies, but no concrete evidence supports it.", "evidenceIds": [ "revenuecat-docs-8", "revenuecat-docs-4" ] }, { "productId": "stripe-billing", "storyId": "agent-manages-subscriptions", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Stripe Billing exposes a full programmatic Subscriptions API (create, change plan, proration preview) and an official MCP server letting AI agents interact with the Stripe API, which supports the 'agent operates billing programmatically' story. However, the evidence never documents scoped/restricted permission controls for agent access, nor explicit 'apply a credit' functionality, so the scoped-permission and credit-specific parts of the story are unevidenced. Missing for 10: documented scoped/restricted API key or permission model for agent use, explicit credit-application API evidence, and independent corroboration of the MCP server's agent workflows.", "evidenceIds": [ "stripe-billing-docs-2", "stripe-billing-docs-4", "stripe-billing-docs-18", "stripe-billing-probe-4" ] }, { "productId": "stripe-billing", "storyId": "agent-reads-billing-state", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Stripe ships an official MCP server explicitly for AI agents to interact with the Stripe API (docs-18, probe-4), which would let an agent query subscriptions, invoices, and customer records directly; docs also show subscription/invoice/analytics APIs (docs-2, docs-8, docs-11) that could back such queries. However, there's no direct evidence the MCP/API surfaces higher-level billing-state metrics like MRR or delinquency status as first-class queryable objects, and analytics is described as dashboard-facing rather than agent-facing. Missing for 10: explicit MCP tool/API documentation for MRR aggregation, delinquent-account flags, and renewal-forecast queries, plus any hands-on agent demo confirming these questions can be answered end-to-end.", "evidenceIds": [ "stripe-billing-docs-18", "stripe-billing-probe-4", "stripe-billing-docs-2", "stripe-billing-docs-8", "stripe-billing-docs-11" ] }, { "productId": "stripe-billing", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Stripe hosts an llms.txt at docs.stripe.com/llms.txt (HTTP 200) and markdown-formatted docs (e.g. billing.md), confirming agent-friendly documentation is directly accessible. Missing for 10: independent/third-party confirmation that agents successfully consume these files in practice beyond the probe check.", "evidenceIds": [ "stripe-billing-probe-1", "stripe-billing-probe-2" ] }, { "productId": "stripe-billing", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence shows only standard analytics dashboards (docs-11) and an MCP server for external agents to call the API (docs-18), but nothing indicating the product itself generates AI-driven insights or suggestions from a merchant's billing data. missing for 10: any AI/ML-generated insights or recommendation features, in-product analysis narratives, or evidence of generative AI surfacing suggestions from billing/subscription data.", "evidenceIds": [ "stripe-billing-docs-11", "stripe-billing-docs-18" ] }, { "productId": "stripe-billing", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Stripe Billing ships genuine background automations that run autonomously without manual intervention — recurring subscription billing, smart retries for failed payments, and webhook-driven event automation — and it exposes an official MCP server letting AI agents interact with the Billing API, aligning with the AI-native automation story. However, there is no evidence of an AI-native automation/rules builder (e.g., agent-authored workflows, scheduled agent tasks) beyond the MCP connector and the native recurring-billing engine. Missing for 10: dedicated AI-agent workflow/automation builder, evidence of agents autonomously configuring or modifying billing automations, independent hands-on validation of MCP-driven autonomous operation.", "evidenceIds": [ "stripe-billing-docs-5", "stripe-billing-docs-6", "stripe-billing-docs-16", "stripe-billing-docs-18", "stripe-billing-probe-4" ] }, { "productId": "stripe-billing", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "stripe-billing", "storyId": "agentic-headless", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Stripe Billing is fully accessible via API and has a documented CLI for building, testing, and managing integrations from the command line, both of which support headless/CI automation, and webhooks enable event-driven automation flows. Missing for 10: explicit CI pipeline examples (e.g., GitHub Actions) or documented headless-mode flags for the CLI, and independent confirmation of CI usage.", "evidenceIds": [ "stripe-billing-docs-19", "stripe-billing-probe-5", "stripe-billing-docs-16", "stripe-billing-docs-2" ] }, { "productId": "stripe-billing", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "medium", "rationale": "Stripe Billing is a SaaS billing/subscriptions platform, not an AI agent or assistant that would consume external MCP servers as tools; the evidence only shows Stripe exposing an MCP server for other agents to call into Stripe (docs-18, probe-4), which is the reverse direction. There is no product surface in Stripe Billing designed to plug in and use third-party MCP tools, making this axis a category error for this product type.", "evidenceIds": [ "stripe-billing-docs-18", "stripe-billing-probe-4" ] }, { "productId": "stripe-billing", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Stripe (parent product of Stripe Billing) documents an official MCP server that lets AI agents interact with the Stripe API, confirmed by both docs and probe evidence. Missing for 10: no independent/community hands-on validation of the MCP server specifically, and no detail on which Billing-specific endpoints/tools are exposed via MCP.", "evidenceIds": [ "stripe-billing-docs-18", "stripe-billing-probe-4" ] }, { "productId": "stripe-billing", "storyId": "agentic-nl-commands", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Stripe documents an official MCP server that lets AI agents interact with the Stripe API (which includes Billing endpoints for subscriptions, invoices, etc.), enabling natural-language-driven operation via an AI agent layer. However, this is evidenced only via first-party docs at the general Stripe API level, not Billing-specific NL commands, and there's no independent/hands-on confirmation of it working well for Billing tasks. Missing for 10: Billing-specific natural-language examples, independent/hands-on validation of MCP-driven billing operations, and documented limitations/edge cases.", "evidenceIds": [ "stripe-billing-docs-18", "stripe-billing-probe-4" ] }, { "productId": "stripe-billing", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Stripe documents an official Stripe CLI for building, testing, and managing Stripe integrations from the command line, confirmed by both docs and probe evidence. Missing for 10: independent hands-on community reports specifically about CLI usage/quality (community evidence here focuses on billing fees, not the CLI).", "evidenceIds": [ "stripe-billing-docs-19", "stripe-billing-probe-5" ] }, { "productId": "stripe-billing", "storyId": "agentic-public-api", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Stripe Billing is fully drivable via the well-documented Subscriptions/Invoicing/Usage-based/Prorations APIs, with webhooks for events and even an official CLI and MCP server for agent-driven access. Missing for 10: no published OpenAPI spec discoverable at standard locations (404s on probe) and no independent hands-on confirmation of full API parity for AI-agent workflows.", "evidenceIds": [ "stripe-billing-docs-2", "stripe-billing-docs-16", "stripe-billing-docs-18", "stripe-billing-docs-19", "stripe-billing-probe-2", "stripe-billing-probe-3", "stripe-billing-probe-4", "stripe-billing-probe-5" ] }, { "productId": "stripe-billing", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows Stripe's MCP server and CLI for agent integration, but nothing documents scoped or least-privilege API credential issuance for agents (e.g., restricted API keys with granular permissions) — this specific capability is absent from the pack. Missing for 10: documentation of restricted/scoped API key creation, permission granularity controls, and any agent-specific credential-scoping workflow.", "evidenceIds": [ "stripe-billing-docs-18", "stripe-billing-probe-4", "stripe-billing-docs-19" ] }, { "productId": "stripe-billing", "storyId": "agentic-sdks", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "The pack shows Stripe exposes a public API, an official CLI, and an MCP server for agent access, and the llms.txt note implies npm-published SDK packages, but there is no explicit documentation of official language SDKs (Node/Python/Ruby/etc.) or SDK-specific guides in the evidence pack. Missing for 10: explicit SDK reference docs/links, language coverage list, and any independent corroboration of SDK quality or agent usage with them.", "evidenceIds": [ "stripe-billing-probe-1", "stripe-billing-docs-19", "stripe-billing-docs-18", "stripe-billing-probe-4", "stripe-billing-probe-5" ] }, { "productId": "stripe-billing", "storyId": "agentic-webhooks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Stripe's docs explicitly describe webhooks pushing real-time event data when things happen in the account, which is the standard mechanism for subscribing to billing-related events (e.g., subscription/invoice lifecycle). Missing for 10: no specifics on which billing event types are supported, no documentation of webhook endpoint management via API, and no agent-specific webhook subscription flow (e.g., via MCP) tying it to the 'AI-native' framing.", "evidenceIds": [ "stripe-billing-docs-16" ] }, { "productId": "stripe-billing", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no mention of an interactive API reference with runnable/executable code examples for Stripe Billing; it only covers docs, CLI, MCP server, and general billing features. Absence of evidence for this applicable capability yields 'none'.", "evidenceIds": [] }, { "productId": "stripe-billing", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows an explicit probe for OpenAPI/Swagger spec files at all standard paths returning 404, and no docs or community item mentions a downloadable OpenAPI/equivalent spec for Stripe Billing's API; only llms.txt and markdown docs endpoints are confirmed, which are not API specs.", "evidenceIds": [ "stripe-billing-probe-3", "stripe-billing-probe-1", "stripe-billing-probe-2" ] }, { "productId": "stripe-billing", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack never mentions a Stripe test mode, sandbox environment, or test API keys — only general docs on billing features, CLI (\"build, test, and manage... from the command line\"), and MCP server access, none of which document an explicit non-production sandbox for safely testing AI-driven billing workflows. Since sandbox/test-mode support is a mainstream, applicable expectation for a billing platform, its absence here yields 'none' rather than 'na'.", "evidenceIds": [ "stripe-billing-docs-19", "stripe-billing-probe-5" ] }, { "productId": "stripe-billing", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers Billing features, pricing, fees, and community reactions but contains no mention of API versioning scheme, version pinning, or a documented deprecation policy for the Stripe API. Missing for 10: any documentation of API version headers, changelog/versioning policy, or deprecation timelines.", "evidenceIds": [] }, { "productId": "stripe-billing", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Stripe Billing exposes a full REST API, CLI, and MCP server that could be scripted for bulk actions, and docs mention importing many existing subscriptions from third-party systems in bulk. However, there is no explicit documentation of batch/bulk endpoints, rate-limit-aware bulk workflows, or AI-agent-specific bulk operation support beyond general API access. Missing for 10: explicit bulk/batch API endpoints, documented bulk operation patterns via CLI/MCP, and independent evidence of successful large-scale bulk usage.", "evidenceIds": [ "stripe-billing-docs-10", "stripe-billing-docs-19", "stripe-billing-docs-18", "stripe-billing-probe-4", "stripe-billing-probe-5" ] }, { "productId": "stripe-billing", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Stripe Billing ships built-in automations tied to events (Smart Retries triggers on failed payment events, entitlements auto grant/revoke access) and exposes webhooks so developers can build custom event-triggered actions, satisfying the spirit of 'rules trigger actions on events.' However, there's no documented no-code/rule-builder interface for defining arbitrary trigger-condition-action rules — webhooks require custom code to consume events. Missing for 10: a dedicated rules/automation engine or UI for defining conditional triggers, and evidence of AI-native/agent-configurable rule definition beyond raw webhooks.", "evidenceIds": [ "stripe-billing-docs-5", "stripe-billing-docs-6", "stripe-billing-docs-9", "stripe-billing-docs-16" ] }, { "productId": "stripe-billing", "storyId": "automation-scheduled-jobs", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Stripe Billing's core function is automating recurring billing workflows (subscriptions, trials, retries, prorations) and it exposes an API, CLI, and MCP server that let an AI-native user programmatically create/manage these recurring billing schedules. However, this is scoped to billing/subscription recurrence, not general-purpose job/workflow scheduling, and there's no explicit documentation of a 'subscription schedule' or cron-like workflow orchestration primitive beyond subscriptions/retries. missing for 10: dedicated subscription-schedule/workflow-scheduling docs, independent hands-on confirmation of AI-agent-driven recurring job setup via MCP/CLI.", "evidenceIds": [ "stripe-billing-docs-1", "stripe-billing-docs-2", "stripe-billing-docs-5", "stripe-billing-docs-6", "stripe-billing-docs-14", "stripe-billing-docs-18", "stripe-billing-docs-19", "stripe-billing-probe-4", "stripe-billing-probe-5" ] }, { "productId": "stripe-billing", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Stripe Billing is a billing/subscription management product, not an automation/workflow-building tool with version control of 'automations'; there is no concept of versioning, reviewing, or rolling back automations in this product's domain. This axis is a category error for a billing platform.", "evidenceIds": [] }, { "productId": "stripe-billing", "storyId": "billing-cycle-flexibility", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers subscriptions, usage-based pricing, prorations, trials, and invoicing generally, but nothing documents billing_cycle_anchor controls, calendar-vs-anniversary billing options, cross-subscription alignment, or custom invoice date setting. Missing for 10: any doc/community mention of billing cycle anchor parameters, alignment across subscriptions, or custom invoice date configuration.", "evidenceIds": [] }, { "productId": "stripe-billing", "storyId": "compliant-invoice-generation", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Docs confirm Stripe Billing/Invoicing can create, customize, and automatically send hosted invoices from the Dashboard (stripe-billing-docs-8), but the evidence never explicitly documents sequential invoice numbering, credit notes, receipts, or memo/custom-field compliance details, and a community comment even claims Stripe historically lacked easy 'email invoicing' compared to competitors (stripe-billing-comm-6). missing for 10: explicit documentation of sequential numbering, credit note generation, receipt issuance, and memo/custom field support with independent corroboration that these are compliant and automated end-to-end.", "evidenceIds": [ "stripe-billing-docs-8", "stripe-billing-comm-6", "stripe-billing-comm-9" ] }, { "productId": "stripe-billing", "storyId": "dunning-comms-automation", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Stripe Billing docs confirm automated Smart Retries to recover failed payments, which is the core of dunning automation, but the evidence pack lacks any mention of branded dunning emails, in-app banners, configurable grace periods, or explicit final-state rules (cancel vs. mark-unpaid) that the story requires. missing for 10: branded email templates, in-app banner configuration, grace period settings, explicit cancel-vs-unpaid final state rules.", "evidenceIds": [ "stripe-billing-docs-5", "stripe-billing-docs-6" ] }, { "productId": "stripe-billing", "storyId": "full-billing-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only documents importing subscriptions *into* Stripe from other providers (Zuora, Recurly, etc.) and migrating customer/payment data *to* Stripe — the reverse direction (exporting subscriptions/invoices/usage and porting stored payment methods to another provider) is never documented. Community evidence even shows a team having to 'write code to move from stripe to recurly/adyen' themselves, implying no built-in export/portability tool exists.", "evidenceIds": [ "stripe-billing-docs-10", "stripe-billing-docs-17", "stripe-billing-comm-6" ] }, { "productId": "stripe-billing", "storyId": "gl-erp-sync", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows Stripe Billing's revenue recognition module (docs-7) and webhooks (docs-16) but contains no mention of native or documented sync to QuickBooks, Xero, NetSuite, or any ERP/GL system — only generic webhook/API primitives that a team would need to build custom integration on top of. Missing for 10: documented native connectors or partner integrations for QuickBooks/Xero/NetSuite, evidence of automated invoice/payment/rev-rec entry sync to a GL, and any customer account of eliminating manual CSV reconciliation.", "evidenceIds": [ "stripe-billing-docs-7", "stripe-billing-docs-16", "stripe-billing-docs-17" ] }, { "productId": "stripe-billing", "storyId": "hybrid-pricing-models", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs confirm Stripe Billing supports usage-based subscription pricing and custom pricing plans without code (stripe-billing-docs-1, stripe-billing-docs-3), and community corroborates per-user/per-usage/per-tier models (stripe-billing-comm-1). However, the evidence never explicitly documents tiered, volume, graduated, or per-unit pricing structures, nor credits, commitments, or overage billing mechanics as named capabilities. Missing for 10: explicit documentation of tiered/volume/graduated/per-unit price types, credits and commitment/overage billing constructs, and any hands-on confirmation these can be combined without custom code.", "evidenceIds": [ "stripe-billing-docs-1", "stripe-billing-docs-3", "stripe-billing-comm-1" ] }, { "productId": "stripe-billing", "storyId": "lifecycle-event-stream", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Docs confirm Stripe pushes real-time webhook events for account changes (stripe-billing-docs-16) and describe automated retry logic for failed payments (stripe-billing-docs-5/6), supporting reliable notification of key lifecycle changes like renewals and payment failures. However, the pack lacks explicit documentation enumerating specific event types for upgrades/downgrades/cancellations or delivery-guarantee details (retries, signing, idempotency) tied directly to subscription state transitions. Missing for 10: explicit list of subscription-lifecycle event types (e.g., customer.subscription.updated/deleted), and evidence of webhook delivery reliability guarantees.", "evidenceIds": [ "stripe-billing-docs-16", "stripe-billing-docs-5", "stripe-billing-docs-6" ] }, { "productId": "stripe-billing", "storyId": "multi-currency-billing", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Stripe Billing supports subscriptions and analytics, but a Stripe PM confirmed that multi-currency pricing currently requires creating a separate plan per currency rather than a single price point with per-currency variants, and reporting/analytics is mentioned only generically without evidence of home-currency consolidation across currencies. missing for 10: native single-price multi-currency price points (not per-currency plan duplication), documented consolidated/rollup reporting in home currency across multiple billing currencies, independent confirmation of this workflow.", "evidenceIds": [ "stripe-billing-comm-2", "stripe-billing-docs-11", "stripe-billing-docs-1" ] }, { "productId": "stripe-billing", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Stripe's API covers most Billing operations (subscriptions, usage-based pricing, prorations, trials, entitlements, invoicing) and there's also a CLI and MCP server for programmatic/agent access, but some UI-only features are explicitly dashboard-driven with no-code framing (e.g., hosted invoice creation, customer portal branding/activation), and community comments note gaps like easy plan management/analytics parity versus competitors. missing for 10: confirmation that customer portal setup/branding and dashboard-only analytics views are fully API-configurable, independent verification of full UI/API parity, and evidence addressing feature gaps raised by users (e.g., portal customization, analytics).", "evidenceIds": [ "stripe-billing-docs-2", "stripe-billing-docs-3", "stripe-billing-docs-4", "stripe-billing-docs-9", "stripe-billing-docs-13", "stripe-billing-docs-18", "stripe-billing-docs-19", "stripe-billing-comm-6", "stripe-billing-comm-9" ] }, { "productId": "stripe-billing", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers importing/migrating data INTO Stripe Billing (from Zuora, Recurly, etc.) but nothing about exporting a merchant's own subscription/invoice/customer data out in open formats to leave the platform. No documentation of a data export or portability tool for departing customers.", "evidenceIds": [ "stripe-billing-docs-10", "stripe-billing-docs-17" ] }, { "productId": "stripe-billing", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Stripe Billing is a closed-source SaaS product; there is no evidence of any open-licensed source code being available for inspection. Documentation, CLI, and MCP server are all provided, but none imply open-source availability of the underlying product code.", "evidenceIds": [] }, { "productId": "stripe-billing", "storyId": "openness-self-host", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "stripe-billing", "storyId": "payment-method-recovery", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Stripe's customer portal (docs-12, docs-13) provides hosted pages where customers can update payment methods, and Smart Retries (docs-5, docs-6) addresses failed payment recovery, but the evidence pack never mentions a network account updater service or backup/secondary payment methods functionality explicitly. missing for 10: explicit documentation of card network account updater, backup payment method fallback logic, and independent confirmation these auto-update cards without customer action.", "evidenceIds": [ "stripe-billing-docs-12", "stripe-billing-docs-13", "stripe-billing-docs-5", "stripe-billing-docs-6" ] }, { "productId": "stripe-billing", "storyId": "plan-entitlements-api", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Stripe explicitly documents an Entitlements feature for granting/revoking product feature access to customers, matching the story's core idea of feature entitlements tied to products/plans, but the evidence pack gives only a single high-level doc link with no detail on the runtime API/SDK check flow or independent corroboration. missing for 10: specifics on the Entitlements API/SDK call for runtime checks, code examples, and community or hands-on validation of the feature.", "evidenceIds": [ "stripe-billing-docs-9" ] }, { "productId": "stripe-billing", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack discusses data residency, regional storage options, or data sovereignty controls for Stripe Billing; the axis applies to a payments/data platform like Stripe but is unaddressed here.", "evidenceIds": [] }, { "productId": "stripe-billing", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Stripe Billing is a subscription/invoicing management product, not an AI model provider or data-processing service that trains AI models on user data; controlling AI training data usage is not an applicable axis for this product category.", "evidenceIds": [] }, { "productId": "stripe-billing", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers subscriptions, invoicing, tax, portal, MCP/CLI tooling, and pricing complaints, but nothing addresses data retention policies, deletion controls, or privacy/data lifecycle management for AI-native users. Missing for 10: any documentation on data retention settings, deletion/erasure APIs, or privacy controls for customer/usage data.", "evidenceIds": [] }, { "productId": "stripe-billing", "storyId": "privacy-telemetry-optout", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Stripe Billing is a payments/subscription platform, not a developer tool with telemetry collection concerns, but no evidence addresses any telemetry/usage-tracking opt-out mechanism for AI-native users interacting with its API, CLI, or MCP server; the evidence pack contains no mention of telemetry at all.", "evidenceIds": [] }, { "productId": "stripe-billing", "storyId": "proration-correctness", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Stripe docs confirm proration is supported with a preview capability before applying changes, giving finance teams visibility into charges before invoicing, and invoices/line items are generated automatically. However, there is no documented detail on the exact proration math methodology (day-count conventions, rounding rules), no auditable reconciliation reports, and no independent/community evidence confirming penny-level accuracy in practice. missing for 10: detailed documented proration formula/rounding rules, audit trail/reconciliation reporting for finance teams, independent verification of penny-accurate reconciliation.", "evidenceIds": [ "stripe-billing-docs-4", "stripe-billing-docs-8" ] }, { "productId": "stripe-billing", "storyId": "realtime-usage-visibility", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers usage-based pricing setup, proration previews, and subscription analytics, but there is no mention of real-time usage dashboards for customers/team, configurable usage thresholds, or proactive alerts before invoicing. missing for 10: real-time usage/cost visibility for customers and team, threshold configuration, proactive alerting before invoice generation.", "evidenceIds": [ "stripe-billing-docs-3", "stripe-billing-docs-11" ] }, { "productId": "stripe-billing", "storyId": "revrec-asc606-reports", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Stripe has a dedicated Revenue Recognition product that automates deferred revenue schedules per global accounting standards, directly matching the story's core intent, but the evidence pack gives only a single thin doc snippet with no mention of ASC 606/IFRS 15 specifically, no detail on audit-ready reporting outputs, and no independent/community corroboration of its use for finance audits. missing for 10: explicit ASC606/IFRS15 standard mapping, description of audit-ready report generation, independent or hands-on validation of the recognition reports.", "evidenceIds": [ "stripe-billing-docs-7" ] }, { "productId": "stripe-billing", "storyId": "self-serve-plan-changes", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Stripe docs explicitly describe a hosted Customer Portal that lets customers manage their own accounts (upgrade/downgrade/cancel) with Dashboard-based activation and branding, directly addressing self-service without support tickets. Community evidence corroborates the portal exists though notes some confusion over what's included and general pricing/product complaints unrelated to portal functionality itself. Missing for 10: independent hands-on report specifically confirming self-serve upgrade/downgrade/cancel flows work smoothly in production, and no mention of embeddable pricing-table/components for the same use case.", "evidenceIds": [ "stripe-billing-docs-12", "stripe-billing-docs-13", "stripe-billing-comm-9" ] }, { "productId": "stripe-billing", "storyId": "smart-retries", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Stripe Billing documents Smart Retries, an automated system that retries failed subscription/invoice payments to reduce involuntary churn, directly matching the story's core ask (docs-5, docs-6). However, evidence doesn't detail ML- or schedule-optimization mechanics, retry cadence customization, or quantified recovery metrics, and community evidence focuses on pricing/fees rather than corroborating recovery effectiveness. Missing for 10: independent/hands-on evidence of measurable churn recovery impact, specifics on the ML/schedule optimization logic, and customer-facing proof of recovered revenue rates.", "evidenceIds": [ "stripe-billing-docs-5", "stripe-billing-docs-6" ] }, { "productId": "stripe-billing", "storyId": "subscription-analytics", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Stripe's docs only vaguely reference subscription analytics ('Monitor overall performance at a glance') with no explicit mention of MRR, churn, cohort retention, or LTV computation, and community evidence directly contradicts the claim that Stripe provides adequate built-in analytics, with one team citing lack of analytics as a reason for migrating away from Stripe Billing to Recurly. missing for 10: explicit documentation of MRR/churn/cohort/LTV dashboards, independent confirmation that these metrics are computed natively rather than requiring external BI tools.", "evidenceIds": [ "stripe-billing-docs-11", "stripe-billing-comm-6" ] }, { "productId": "stripe-billing", "storyId": "subscription-crud-api", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Docs confirm subscriptions can be created and updated via the Subscriptions API (including proration previews for changes) and trials/usage-based pricing are API-configurable, plus CLI/webhooks support programmatic workflows without the dashboard. However, explicit documentation of 'pause' and 'cancel' endpoints isn't present in the evidence pack, only implied by general subscription management docs. missing for 10: explicit doc citation for pause-subscription and cancel-subscription API endpoints, independent hands-on confirmation of full lifecycle via API only.", "evidenceIds": [ "stripe-billing-docs-2", "stripe-billing-docs-4", "stripe-billing-docs-14", "stripe-billing-docs-3", "stripe-billing-probe-2", "stripe-billing-probe-5" ] }, { "productId": "stripe-billing", "storyId": "subscription-import-migration", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs explicitly cover importing subscriptions from competing platforms like Zuora, Recurly, Chargify, and Chargebee (docs-10) and a general 'multi-step' customer/payments data migration guide (docs-17), which supports the platform-migration part of the story. However there is no evidence of spreadsheet-based import, no detail on PCI-compliant card data transfer mechanics, and no confirmation that migrated subscriptions continue billing seamlessly without customer-visible disruption. Missing for 10: spreadsheet import support, explicit PCI-compliant card token transfer process, and evidence/testimony that migration is invisible to end customers.", "evidenceIds": [ "stripe-billing-docs-10", "stripe-billing-docs-17" ] }, { "productId": "stripe-billing", "storyId": "tax-on-invoices", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Stripe Tax is documented as a native, first-class feature that automates sales tax, VAT, and GST compliance directly within the Stripe ecosystem (stripe-billing-docs-15), aligning with Stripe Billing's invoicing flow. Missing for 10: independent/hands-on corroboration of tax calculation accuracy and detail on how registrations/rates are managed across multiple jurisdictions beyond the single doc reference.", "evidenceIds": [ "stripe-billing-docs-15", "stripe-billing-docs-8" ] }, { "productId": "stripe-billing", "storyId": "trials-coupons-promos", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Docs confirm a Trial Offer API for free/paid trial periods and subscription automation (stripe-billing-docs-14, stripe-billing-docs-2), and community notes that Stripe emails (e.g., card update notices) are off by default and merchants can use webhooks instead (stripe-billing-comm-7), implying trial-end reminders aren't automatic out-of-the-box. No evidence pack items mention coupons or promotion codes specifically, nor explicit trial-end notification features required by card-network rules. missing for 10: coupon/promotion-code documentation, explicit trial-end notification/reminder feature docs, evidence of automatic trial-to-paid conversion flow, and confirmation of card-network compliance requirements being met.", "evidenceIds": [ "stripe-billing-docs-14", "stripe-billing-docs-2", "stripe-billing-comm-7" ] }, { "productId": "stripe-billing", "storyId": "usage-event-ingestion", "verdict": "partial", "quality": 2, "confidence": "low", "rationale": "The evidence only shows generic usage-based subscription pricing support (stripe-billing-docs-3) with no documentation of a dedicated meter/usage-events API, idempotency keys, backdating of events, or high-volume streaming ingestion mechanics. Missing for 10: explicit meter events API docs, idempotent ingestion mechanism details, backdating/timestamp support evidence, high-volume throughput claims or benchmarks.", "evidenceIds": [ "stripe-billing-docs-3" ] }, { "productId": "stripe-billing", "storyId": "versioned-plan-migrations", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Stripe docs show proration previews for subscription changes and a tool for importing subscriptions from third-party billing systems, which touch on the mechanics of adjusting pricing without immediately breaking billing, but there is no documentation of explicit 'versioned prices' grandfathering existing subscribers, nor a dedicated bulk cohort-migration tool for moving groups of subscribers to new prices. Missing for 10: explicit grandfathering/versioned-plan documentation, a bulk cohort price-migration API/tool, and independent evidence that ops teams use it this way.", "evidenceIds": [ "stripe-billing-docs-4", "stripe-billing-docs-10", "stripe-billing-docs-3", "stripe-billing-docs-9" ] } ]