[ { "productId": "forter", "storyId": "account-abuse-coverage", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Forter's marketing copy explicitly claims to 'Stop ATO, block fake accounts, protect account integrity' alongside its checkout/fraud-decision APIs, indicating account-takeover and fake-account protection live in the same platform as order-level fraud scoring. However, there is no documentation of promo abuse or policy abuse detection/management, and no detail on how ATO/fake-account signals are scored or surfaced to analysts beyond a single marketing line. Missing for 10: dedicated docs/API for promo abuse and policy abuse detection, analyst-facing workflow/management details for ATO and fake-account cases beyond marketing claims.", "evidenceIds": [ "forter-docs-23", "forter-docs-1", "forter-docs-21" ] }, { "productId": "forter", "storyId": "agent-buyer-detection", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Forter's agentic-orders docs show a dedicated orderType=AI_AGENT field that flags agent-driven purchases so Forter's risk engine can treat them distinctly rather than blanket-blocking, which directly targets this story. However, the evidence is entirely first-party documentation with no independent testing or detail on the underlying bot-vs-legitimate-agent classification logic. Missing for 10: independent/hands-on validation of accuracy, technical detail on how malicious bots are distinguished from legitimate agents beyond a merchant-set flag.", "evidenceIds": [ "forter-docs-11", "forter-docs-19", "forter-probe-1" ] }, { "productId": "forter", "storyId": "agent-drives-rules", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Forter documents an official MCP server that lets an AI system 'review decisions, manage disputes, or update policies through natural language commands' and can connect to Claude Desktop for conversational workflows, which supports agentic read/manage access to fraud posture. However, there is no explicit documentation of proposing a velocity rule, updating a blocklist, or any human-in-the-loop approval gate mechanism for these programmatic changes. Missing for 10: explicit velocity-rule/blocklist management examples, documented approval-gate workflow, and independent/hands-on corroboration of MCP write actions.", "evidenceIds": [ "forter-docs-9", "forter-docs-20", "forter-probe-2" ] }, { "productId": "forter", "storyId": "agent-triages-reviews", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Forter's MCP server explicitly lets an AI agent 'perform real actions—such as reviewing decisions, managing disputes, or updating policies through natural language commands' via Claude Desktop, showing agent access to review/decision workflows. However, there is no explicit documentation of pulling a flagged-case queue, summarizing evidence, or producing a recommendation for human confirmation as a distinct workflow. missing for 10: explicit queue-pulling UI/API for flagged cases, evidence-summarization output, human-confirm approval step.", "evidenceIds": [ "forter-docs-9", "forter-docs-20", "forter-probe-2" ] }, { "productId": "forter", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Forter has a confirmed live llms.txt at docs.forter.com/llms.txt (HTTP 200, forter-probe-1) listing agent-oriented doc overviews, plus explicit agent-facing docs like the MCP getting-started guide for connecting to Claude Desktop and agentic-orders docs — all markdown-served for agent consumption. Missing for 10: independent third-party confirmation that agents successfully consume the llms.txt in practice beyond the probe check.", "evidenceIds": [ "forter-probe-1", "forter-docs-20", "forter-docs-9" ] }, { "productId": "forter", "storyId": "agentic-ai-insights", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Forter provides an AI-driven recommendation engine that suggests additional evidence to bolster dispute win rates and a 'team of Forter Agents' concept, plus MCP-enabled natural language interactions for reviewing decisions and managing disputes. This shows some AI-generated insight/suggestion capability embedded in the product, but evidence is thin on broader analytics/insights (e.g., fraud trend explanations, risk scoring rationale) beyond the dispute-evidence use case. Missing for 10: broader in-product AI insights beyond dispute evidence suggestions, independent/hands-on corroboration of the recommendation engine's quality, and detail on what other 'Forter Agents' actually surface to users.", "evidenceIds": [ "forter-docs-10", "forter-docs-15", "forter-docs-12", "forter-docs-9", "forter-docs-20" ] }, { "productId": "forter", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Forter's fraud automation (webhooks, chargeback forwarding, evidence recommendation, MCP for reviewing decisions/managing disputes) provides background automation of fraud/dispute workflows, and the MCP server lets AI agents trigger actions, but this is not framed as user-configurable 'set up automations that run autonomously' in the sense of scheduled/triggered workflows the AI-native user builds themselves. missing for 10: explicit user-facing automation builder/scheduler, evidence of autonomous background runs initiated by the user rather than reactive webhook/event responses, and independent confirmation of real-world autonomous operation.", "evidenceIds": [ "forter-docs-3", "forter-docs-8", "forter-docs-9", "forter-docs-15", "forter-docs-20", "forter-probe-2" ] }, { "productId": "forter", "storyId": "agentic-builtin-assistant", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Forter's marketing mentions a 'team of Forter Agents' suggesting built-in AI-driven automation, but the evidence pack gives no detail on how a user interacts with or delegates tasks to such an assistant inside the product UI. The only detailed AI-interaction evidence (MCP) is about connecting external assistants like Claude Desktop to Forter, not a built-in assistant. Missing for 10: documentation of an in-product conversational assistant UI, examples of task delegation, and how 'Forter Agents' are invoked/configured.", "evidenceIds": [ "forter-docs-12", "forter-docs-9" ] }, { "productId": "forter", "storyId": "agentic-headless", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Forter is a REST/webhook-based fraud API (order submission, status updates, webhooks, sandbox/test environment) which can technically be called from scripts or CI pipelines, and it offers an MCP server for programmatic/agentic access. However, there is no evidence of a CLI tool, CI/CD integration examples, headless testing harness, or automation-focused tooling explicitly designed for CI pipelines. missing for 10: dedicated CLI, CI/CD pipeline examples or GitHub Actions integration, headless test automation docs, explicit 'run in CI' guidance beyond generic sandbox API testing.", "evidenceIds": [ "forter-docs-4", "forter-docs-13", "forter-docs-1", "forter-probe-2", "forter-docs-9" ] }, { "productId": "forter", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "medium", "rationale": "Forter's MCP evidence shows it exposing its own MCP server so external AI agents (e.g., Claude Desktop) can call Forter's fraud/dispute tools, not Forter itself acting as an MCP client that consumes and uses other servers' tools. As a fraud-decisioning SaaS platform, not an agent/orchestrator, the client-side 'plug in MCP servers' story is the wrong axis for this product's role.", "evidenceIds": [ "forter-docs-9", "forter-docs-20", "forter-probe-2" ] }, { "productId": "forter", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Forter documents an official MCP server enabling AI agents to review decisions, manage disputes, and update policies via natural language, with a getting-started guide for connecting to Claude Desktop. missing for 10: independent/third-party corroboration of real-world MCP usage and broader client compatibility beyond Claude Desktop.", "evidenceIds": [ "forter-docs-9", "forter-docs-20", "forter-probe-2" ] }, { "productId": "forter", "storyId": "agentic-nl-commands", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Forter documents a first-party MCP server enabling natural-language commands to review decisions, manage disputes, and update policies, with a getting-started guide for connecting to Claude Desktop for conversational workflows. Missing for 10: independent/hands-on corroboration beyond vendor docs, and broader coverage of which actions are NL-operable vs API-only.", "evidenceIds": [ "forter-docs-9", "forter-docs-20", "forter-probe-2", "forter-probe-1" ] }, { "productId": "forter", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Forter offers REST APIs, webhooks, and an MCP server for conversational AI workflows, but no official CLI tool is documented anywhere in the evidence pack.", "evidenceIds": [ "forter-docs-9", "forter-docs-20", "forter-probe-2" ] }, { "productId": "forter", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Forter publishes a documented public REST API (Order v3, Order Status, Evidence API, webhooks) with sandbox/production environments, giving AI-native users a clear path to drive the product programmatically; this is reinforced by an official MCP server enabling natural-language/agentic control of decisions and disputes. Missing for 10: independent/hands-on corroboration beyond vendor docs, and more detail on auth/rate limits or SDK coverage.", "evidenceIds": [ "forter-docs-1", "forter-docs-2", "forter-docs-3", "forter-docs-4", "forter-docs-9", "forter-docs-20", "forter-probe-1", "forter-probe-2" ] }, { "productId": "forter", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Forter has API/webhook docs and even an MCP server for agent-driven actions, but no evidence describes issuing scoped or least-privilege API credentials/tokens for an agent identity. Missing for 10: any mention of API key/token scoping, permission levels, or credential issuance workflow for agents.", "evidenceIds": [ "forter-docs-9", "forter-docs-20", "forter-probe-2" ] }, { "productId": "forter", "storyId": "agentic-sdks", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Forter provides extensive REST API references (Order v3, Order Status, Evidence API, webhooks), a sandbox/production environment, and even a documented MCP server for agentic workflows, which together give an AI-native developer solid material to build against. However, none of the evidence explicitly names or links to official client-language SDKs (e.g., Python/Node/Java packages) — only API/reference docs and an MCP integration are shown. Missing for 10: explicit official SDK packages/libraries, versioned SDK release notes, and independent corroboration of SDK usage in production.", "evidenceIds": [ "forter-docs-1", "forter-docs-2", "forter-docs-3", "forter-docs-4", "forter-docs-9", "forter-docs-20", "forter-probe-2" ] }, { "productId": "forter", "storyId": "agentic-webhooks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Forter documents a dedicated webhooks system for real-time event notifications (order status, chargeback forwarding, dispute webhooks) that can be configured to notify external services, directly matching the subscribe-to-events story; this is a well-documented first-party capability. missing for 10: independent/hands-on corroboration of webhook reliability, and a comprehensive list of subscribable event types/payload schema beyond the examples shown.", "evidenceIds": [ "forter-docs-3", "forter-docs-2", "forter-docs-8", "forter-docs-6", "forter-docs-14" ] }, { "productId": "forter", "storyId": "ai-fraud-analyst", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Forter markets 'Forter Agents' and ships an MCP server letting external AI systems (e.g., Claude Desktop) issue natural-language commands to review decisions, manage disputes, or update policies, which gestures at the AI-assistant theme. However, the evidence never describes a native, embedded console assistant with NL querying over fraud data, rule drafting, or investigation summaries — the MCP approach is an integration layer for external AI clients rather than a first-party in-console assistant. Missing for 10: concrete documentation of a built-in console chat/assistant UI, evidence of drafted-rule generation, and investigation-summary generation features.", "evidenceIds": [ "forter-docs-9", "forter-docs-12", "forter-docs-20", "forter-probe-2" ] }, { "productId": "forter", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack shows Forter has API reference documentation (order-v3, order-status, webhooks, environments) but nothing indicates the docs are interactive with runnable/try-it examples—no mention of an embedded API console, code sandbox, or 'try it' functionality typical of interactive API references.", "evidenceIds": [] }, { "productId": "forter", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Forter publishes REST API reference docs (order-v3, order-status, webhooks) but none of the evidence mentions a downloadable OpenAPI/Swagger spec or machine-readable API definition file; the llms.txt probe lists doc pages, not an API spec.", "evidenceIds": [ "forter-docs-1", "forter-docs-2", "forter-probe-1" ] }, { "productId": "forter", "storyId": "api-sandbox", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Forter explicitly documents both production and sandbox (TEST) environments, stating the sandbox's purpose is to verify integration before production deployment, allowing testing without touching production data. Missing for 10: no independent/hands-on corroboration of sandbox behavior or details on sandbox data parity/limitations.", "evidenceIds": [ "forter-docs-4", "forter-docs-13" ] }, { "productId": "forter", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows versioned API endpoints (e.g. order-v3) and sandbox/production environments, but there is no documentation of a versioning scheme, version support lifecycle, or a deprecation policy anywhere in the pack.", "evidenceIds": [ "forter-docs-1", "forter-docs-4", "forter-docs-13" ] }, { "productId": "forter", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Forter's docs describe per-order/per-dispute APIs (Order v3, Evidence API, webhooks, MCP natural-language actions) but no evidence of a bulk/batch endpoint or mechanism for acting across many items simultaneously. The axis is applicable (merchants would plausibly want bulk dispute/evidence submission or bulk order review), but no such capability is documented.", "evidenceIds": [ "forter-docs-1", "forter-docs-16", "forter-docs-9" ] }, { "productId": "forter", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Forter supports event-driven automation via webhooks (real-time notifications on events like disputes/chargebacks) and configurable decision behaviors (e.g., Auto Invoice triggering capture), and its MCP interface lets an AI agent perform actions like reviewing decisions or updating policies via natural language. However, there is no evidence of a user-facing rule-definition engine (e.g., 'if X then Y' conditions) that AI-native users can author themselves. Missing for 10: explicit rule-builder/conditional logic UI or API, examples of user-defined trigger-action pairs beyond fixed webhook events and preset decision toggles.", "evidenceIds": [ "forter-docs-3", "forter-docs-18", "forter-docs-9", "forter-docs-8" ] }, { "productId": "forter", "storyId": "automation-scheduled-jobs", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Forter is a fraud-decisioning platform with webhooks, an MCP integration, and event-driven notifications, but there is no evidence of a scheduler or ability to define recurring/cron-style jobs or workflows within Forter itself.", "evidenceIds": [] }, { "productId": "forter", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of any versioning, review, or rollback mechanism for automations/workflows in Forter's docs; the MCP and agentic-order features cover fraud decisioning and dispute automation, not automation lifecycle management like version history or rollback.", "evidenceIds": [] }, { "productId": "forter", "storyId": "chargeback-guarantee", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack details dispute automation, evidence submission, webhook forwarding, and win-rate optimization tools, but none of it documents an actual chargeback guarantee, reimbursement mechanism, or liability-shift terms — the core of this finance-lead story. Missing for 10: explicit guarantee/reimbursement policy language, coverage terms, or liability-shift contract details.", "evidenceIds": [ "forter-docs-8", "forter-docs-10", "forter-docs-15", "forter-docs-16", "forter-docs-21", "forter-docs-22" ] }, { "productId": "forter", "storyId": "custom-rules-authoring", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers order submission, webhooks, chargeback/dispute automation, and MCP/agentic order tagging, but contains no mention of a custom rules engine, rule authoring UI/API, velocity counters, list matching, or configurable allow/block/review decision logic controlled by risk analysts.", "evidenceIds": [] }, { "productId": "forter", "storyId": "custom-signals-enrichment", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Forter's Order API accepts order data and metadata at checkout (forter-docs-1) and supports custom decision configuration (forter-docs-18), implying some ability to send custom signals, but there is no documented schema or capability for developers to submit their own device fingerprints, custom behavioral signals, or arbitrary metadata that directly influences scoring weights. missing for 10: explicit API fields/documentation for submitting custom device fingerprint or behavioral data, evidence that custom signals are actually weighted in scoring versus Forter's own network-derived signals, and any tuning/configuration interface for business-specific signal weighting.", "evidenceIds": [ "forter-docs-1", "forter-docs-18" ] }, { "productId": "forter", "storyId": "data-residency-controls", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack items address data residency options, regional data storage, or retention/deletion controls; documentation covers order APIs, webhooks, disputes, PCI scope, and MCP/agent features but nothing about where fraud data is stored or how long it is kept.", "evidenceIds": [] }, { "productId": "forter", "storyId": "dispute-evidence-automation", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Forter documents automated evidence workflows: chargebacks are ingested via webhook forwarding, matched to transactions, and a recommendation engine suggests additional evidence, plus an Evidence API to 'automatically send post-order evidence for chargeback disputes' without manual copy-paste of transaction data. However, the evidence is described as 'suggested' by a recommendation engine rather than fully autonomous compilation and submission, and there's no explicit mention of session data being incorporated into evidence packages. Missing for 10: explicit confirmation that session-level data is included in compiled evidence, and independent/hands-on validation that submission is fully automatic without ops review step.", "evidenceIds": [ "forter-docs-8", "forter-docs-10", "forter-docs-15", "forter-docs-16", "forter-docs-17" ] }, { "productId": "forter", "storyId": "fraud-outcome-reporting", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers APIs, webhooks, MCP, and dispute automation but contains no mention of a reporting dashboard, board-level metrics (dispute rate, false-positive rate, approval-rate lift, review workload), or any export/reporting feature aimed at finance leads. Only a vague reference to 'better reporting on agentic traffic' exists, which does not address the specific KPIs in the story.", "evidenceIds": [ "forter-docs-19" ] }, { "productId": "forter", "storyId": "lists-velocity-management", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no mention of allow/block lists (emails, cards, devices, IPs) or velocity limit configuration, either via dashboard or API — only order decisioning, webhooks, disputes, and agentic order APIs are documented. This is a fair capability to expect from a fraud-rules platform, but no evidence supports it.", "evidenceIds": [] }, { "productId": "forter", "storyId": "manual-review-queue", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers order decisioning APIs, webhooks, chargeback/dispute automation, and MCP/agentic order tagging, but there is no mention of a review queue UI showing flagged transactions with customer history, signals, or similar cases for analyst decisioning.", "evidenceIds": [] }, { "productId": "forter", "storyId": "model-performance-visibility", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of model-level metrics (precision/recall), shadow-mode trials, or A/B testing of models/rules before rollout; evidence covers order decisioning, webhooks, disputes, and agentic orders but nothing about model performance measurement or staged rollout tooling for finance leads.", "evidenceIds": [] }, { "productId": "forter", "storyId": "network-intelligence", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers order submission, webhooks, disputes, chargebacks, PCI scope, and agentic/MCP features, but none of it describes a cross-merchant network effect, shared identity/card risk signals, or network-wide fraud intelligence informing decisions. Missing for 10: any documentation or claim about network-level data sharing, identity graph across merchants, or aggregated fraud signals from other businesses.", "evidenceIds": [] }, { "productId": "forter", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Forter exposes core actions via API/webhooks (order decisions, order status, evidence submission, chargeback forwarding) and an MCP server that lets AI agents review decisions, manage disputes, and update policies via natural language, suggesting broad API/agent coverage of UI functions. However, there is no explicit documentation stating full feature parity between the UI dashboard and API/MCP, and some UI-only configuration (e.g., policy setup screens, analytics dashboards) isn't shown as API-accessible. Missing for 10: explicit parity statement, evidence that every UI configuration/reporting feature is also API-exposed, independent confirmation of MCP completeness.", "evidenceIds": [ "forter-docs-9", "forter-docs-11", "forter-docs-16", "forter-docs-1", "forter-docs-2", "forter-docs-3", "forter-probe-2" ] }, { "productId": "forter", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Forter is a fraud-detection/risk B2B API platform; no evidence of any end-user data export feature or open-format data portability for merchants or consumers to 'leave' with their data. Missing for 10: any documented data export tool, open format (CSV/JSON) export capability, or account-closure data portability process.", "evidenceIds": [] }, { "productId": "forter", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Forter is a closed, proprietary fraud-prevention SaaS; no evidence of any open-source license or public source code repository for the core product. Missing for 10: any open-source license grant, public source repository, or licensing terms permitting source inspection/modification.", "evidenceIds": [] }, { "productId": "forter", "storyId": "openness-self-host", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Forter is a SaaS fraud-prevention platform delivered as a cloud API/service; there is no evidence of a self-hostable core product, and self-hosting is not a plausible axis for this category of hosted decisioning service.", "evidenceIds": [] }, { "productId": "forter", "storyId": "platform-integrations", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Evidence confirms a maintained Shopify integration (quickstart-shopify.md) and Stripe dispute webhook support, indicating some pre-built commerce/PSP connectors exist. However, there is no evidence of maintained integrations for Salesforce Commerce, BigCommerce, or other major PSPs (Adyen, Braintise, PayPal, etc.), leaving the breadth of the 'commerce stack' claim largely unsubstantiated. missing for 10: documented Salesforce Commerce Cloud integration, BigCommerce integration, coverage of additional major PSPs beyond Stripe, and any partner/marketplace integration directory confirming maintenance status.", "evidenceIds": [ "forter-docs-7", "forter-docs-6", "forter-docs-17" ] }, { "productId": "forter", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack addresses data residency, regional storage options, or geographic controls over where customer data is held; all evidence concerns fraud decisioning, disputes, webhooks, and agentic order tagging.", "evidenceIds": [] }, { "productId": "forter", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Forter is a fraud-detection/e-commerce risk platform; the evidence pack contains no privacy policy, data-processing agreement, or opt-out mechanism addressing whether user data is used for AI model training. No documentation of a training opt-out or data-use control was found.", "evidenceIds": [] }, { "productId": "forter", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack items address data retention policies, deletion controls, or user-facing privacy/data-management settings; the docs focus on fraud detection, disputes, and MCP integration.", "evidenceIds": [] }, { "productId": "forter", "storyId": "privacy-telemetry-optout", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Forter is a fraud-detection/e-commerce risk platform, not an AI assistant or developer tool with client-side telemetry that an end user could opt out of; the privacy-posture/telemetry-opt-out axis is a category mismatch for this product type.", "evidenceIds": [] }, { "productId": "forter", "storyId": "psd2-sca-optimization", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions SCA, 3DS, PSD2 exemptions, or European payment authentication routing at all; Forter's docs focus on fraud decisioning, disputes, chargebacks, and agentic order tagging, none of which address SCA/3DS logic.", "evidenceIds": [] }, { "productId": "forter", "storyId": "psp-agnostic-coverage", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Forter is payment-agnostic in principle (works off order/checkout data, not tied to a specific processor's rails), and evidence shows Stripe-specific dispute webhook integration plus a Shopify quickstart, implying multi-platform support. However there is no explicit documentation enumerating support across multiple PSPs/gateways beyond Stripe and Shopify, nor guidance for other major processors (Adyen, Braintree, PayPal, etc.). missing for 10: explicit list/documentation of supported PSPs and gateways beyond Stripe/Shopify, evidence of platform-agnostic integration guides for other processors, independent confirmation of multi-PSP deployments.", "evidenceIds": [ "forter-docs-6", "forter-docs-17", "forter-docs-7", "forter-docs-4" ] }, { "productId": "forter", "storyId": "realtime-scoring-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Forter's Order API (order-v3) is documented as a synchronous checkout-time call returning a fraud/abuse decision, and order-status/webhooks show authorization events are tracked separately, implying the decision precedes authorization — consistent with real-time pre-auth scoring. However, the docs describe a categorical 'decision' rather than an explicit numeric ML risk score, and there's no independent/hands-on latency or scoring-format confirmation. Missing for 10: explicit documentation of a numeric/probabilistic risk score field, and independent verification of real-time performance.", "evidenceIds": [ "forter-docs-1", "forter-docs-2", "forter-docs-4" ] }, { "productId": "forter", "storyId": "review-outcomes-feedback", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Forter's docs show data flowing back to Forter post-decision (order status updates including compensation granted, chargeback/dispute webhook ingestion, Evidence API) which implies outcome data reaches Forter's systems, but there is no explicit documentation that a risk analyst's manual review decisions or confirmed-fraud outcomes are used to retrain models or update rules. Missing for 10: explicit description of a feedback/learning loop tying analyst review outcomes to model or rule updates, and any mention of an analyst review queue at all.", "evidenceIds": [ "forter-docs-2", "forter-docs-8", "forter-docs-14", "forter-docs-6", "forter-docs-17", "forter-docs-16" ] }, { "productId": "forter", "storyId": "review-team-workflows", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers order decisioning, webhooks, chargeback/dispute automation, and AI-agent/MCP integrations, but contains no mention of a human team review workflow with case assignment, escalation rules, SLAs, or an approval audit trail for ops reviewers. missing for 10: case assignment/queue management, escalation rules, SLA tracking, and a decision audit trail showing reviewer approvals and rationale.", "evidenceIds": [] }, { "productId": "forter", "storyId": "rules-backtesting", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of any rule backtesting, historical simulation, or cost/impact preview capability in the evidence pack — coverage is about order decisioning, webhooks, disputes, and MCP/agentic orders, none of which address testing rules against historical traffic before deployment.", "evidenceIds": [] }, { "productId": "forter", "storyId": "score-explainability", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence describes score explainability, reason codes, or risk-factor breakdowns accompanying Forter's fraud decisions; the evidence pack covers order submission, webhooks, disputes, and agentic order tagging but nothing about transparency into why a score was assigned.", "evidenceIds": [] }, { "productId": "forter", "storyId": "score-thresholds-actions", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers order decisions, webhooks, disputes, PCI scope, and agentic order tagging, but contains no mention of score-range-to-action mapping, configurable thresholds, or step-up 3DS logic that ops users could tune. Forter's docs reference binary/decision-based order outcomes (fraud/abuse decision) rather than a threshold-tuning console. Missing for 10: any documentation of score bands, adjustable thresholds, or 3DS step-up configuration UI/API.", "evidenceIds": [ "forter-docs-1", "forter-docs-18" ] }, { "productId": "forter", "storyId": "trusted-agent-signals", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Forter's agentic-orders API lets developers set orderType to AI_AGENT so the risk engine treats agent-originated traffic differently and improves decisioning/reporting, which shows agent identity is ingested as a factor. However, the evidence only shows a coarse flag, not verified signed agent tokens, delegated spending scopes, or support for specific agentic-payment protocols (e.g., AP2/Visa/Mastercard agent tokens) as first-class structured signals. missing for 10: schema/fields for signed agent credentials or delegation scopes, explicit protocol support for agentic-payment standards, independent confirmation these fields are cryptographically verified rather than self-declared.", "evidenceIds": [ "forter-docs-11", "forter-docs-19", "forter-docs-9" ] }, { "productId": "riskified", "storyId": "account-abuse-coverage", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Riskified's Login API explicitly scores account-takeover attempts (allow/notify/challenge) alongside its core checkout fraud decisioning (decide/submit/advise) and automated chargeback handling, showing the same system spans multiple fraud surfaces beyond checkout. However, there is no documentation of fake-account-creation detection or promo/coupon abuse scoring as distinct managed surfaces. Missing for 10: explicit fake-account-creation protection, promo/policy abuse scoring, and independent confirmation these are unified in one decisioning system.", "evidenceIds": [ "riskified-docs-9", "riskified-docs-20", "riskified-docs-4", "riskified-docs-16", "riskified-docs-1", "riskified-docs-3" ] }, { "productId": "riskified", "storyId": "agent-buyer-detection", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "riskified", "storyId": "agent-drives-rules", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers order review/decision APIs, notifications, chargeback automation, and Beacon data collection, but there is no mention of programmatic management of fraud rules, velocity rules, or blocklists, nor any agent-facing approval workflow for such changes.", "evidenceIds": [] }, { "productId": "riskified", "storyId": "agent-triages-reviews", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Riskified's evidence describes automated decisioning APIs (/decide, /submit, /advise), notifications, and merchant integrations, but nothing describes an agent-accessible review queue with case context retrieval, evidence summarization, or a human-confirm workflow. No API for pulling flagged cases with context or generating human-reviewable recommendations is documented.", "evidenceIds": [] }, { "productId": "riskified", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "A probe confirms Riskified hosts a working llms.txt at developers.riskified.com/llms.txt returning HTTP 200, and docs explicitly note pages can be fetched as markdown by appending .md — exactly the agent-oriented docs pattern. Missing for 10: no independent/third-party corroboration of an agent successfully consuming it, and no broader agent-specific documentation beyond the llms.txt index.", "evidenceIds": [ "riskified-probe-1" ] }, { "productId": "riskified", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "riskified", "storyId": "agentic-autonomous-automation", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "riskified", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "riskified", "storyId": "agentic-headless", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Riskified is fundamentally an API-first fraud-decision service (decide/submit/advise/chargeback endpoints, SDKs, webhook notifications) that can be called programmatically without a UI, which supports headless/automated integration into checkout pipelines. However, there is no evidence of CI/CD-specific tooling, a CLI, or documented automation-testing workflows — the docs focus on production checkout integration, not build/test automation. Missing for 10: explicit CI/CD examples, a CLI or automation-testing guide, and confirmation of headless operation outside the live transaction flow.", "evidenceIds": [ "riskified-docs-1", "riskified-docs-2", "riskified-docs-3", "riskified-docs-5", "riskified-docs-8" ] }, { "productId": "riskified", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Riskified is a fraud-detection/e-commerce risk platform, not an AI agent or agent-enabling framework; plugging MCP servers into it so it can use external tools is a category mismatch for this product type, and no evidence suggests otherwise.", "evidenceIds": [] }, { "productId": "riskified", "storyId": "agentic-mcp-server", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "riskified", "storyId": "agentic-nl-commands", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "riskified", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "riskified", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Riskified exposes a comprehensive documented REST API (decide, submit, advise, chargeback, notifications, login) with authentication (HMAC), synchronous/asynchronous flows, and optional SDKs, all directly callable via HTTP — clearly a documented public API an AI-native user could drive. The docs site also exposes an llms.txt for AI-friendly consumption, though no machine-readable OpenAPI/Swagger spec was found (404s on standard paths). Missing for 10: a discoverable OpenAPI/swagger schema and explicit rate-limit/versioning documentation.", "evidenceIds": [ "riskified-docs-1", "riskified-docs-2", "riskified-docs-3", "riskified-docs-5", "riskified-docs-7", "riskified-docs-8", "riskified-docs-23", "riskified-probe-1", "riskified-probe-2" ] }, { "productId": "riskified", "storyId": "agentic-scoped-keys", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Riskified is a fraud-decision API/service for merchants, not an agent-facing platform; there is no concept of issuing scoped credentials for an AI agent to act on a user's behalf. Authentication uses a single HMAC token per merchant integration, not agent-scoped credentialing — this is a category mismatch, not a missing feature.", "evidenceIds": [] }, { "productId": "riskified", "storyId": "agentic-sdks", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Riskified's docs explicitly state that SDKs are provided for several languages to simplify API integration, confirming official SDK support (riskified-docs-5). However, the evidence lacks specifics such as which languages are covered, links to SDK repositories, or independent/community corroboration of SDK quality. Missing for 10: list of supported languages, links to SDK repos/package registries, hands-on developer feedback on SDK usability.", "evidenceIds": [ "riskified-docs-5" ] }, { "productId": "riskified", "storyId": "agentic-webhooks", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Riskified documents a webhook-style notification mechanism: decision outcomes are pushed to a merchant-defined endpoint with retries on non-2xx and HMAC-SHA256 signature verification, which functions as an event webhook for fraud decisions and chargebacks. However, this is a fixed set of built-in notification types (order decision, chargeback) rather than a general-purpose subscribable event system with selectable event topics or a dashboard for managing webhook subscriptions. Missing for 10: a documented event catalog/subscription model letting users choose which event types to receive, and any UI/API for creating or managing multiple webhook subscriptions.", "evidenceIds": [ "riskified-docs-8", "riskified-docs-18", "riskified-docs-7", "riskified-docs-2", "riskified-docs-16" ] }, { "productId": "riskified", "storyId": "ai-fraud-analyst", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers API endpoints, webhooks, chargeback automation, Beacon SDK, and integration processes, but nothing describes a built-in AI assistant, natural-language query interface, rule drafting, or investigation summaries in the console.", "evidenceIds": [] }, { "productId": "riskified", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows only static API reference docs (endpoints, parameters, notifications) with no mention of an interactive console, runnable examples, or live API explorer; the openapi.json/swagger.json probe returned 404s, indicating no machine-readable spec to power an interactive reference.", "evidenceIds": [ "riskified-probe-2", "riskified-docs-1", "riskified-docs-23" ] }, { "productId": "riskified", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "A probe explicitly checked common OpenAPI/swagger spec locations and all returned 404, and no evidence pack item references a downloadable machine-readable API spec; only human-readable docs and an llms.txt are present.", "evidenceIds": [ "riskified-probe-2", "riskified-probe-1" ] }, { "productId": "riskified", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence of a sandbox or test environment separate from production; docs describe live API endpoints, integration flows, and Shopify onboarding but never mention a sandbox/test mode for AI-native or developer testing.", "evidenceIds": [] }, { "productId": "riskified", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of API versioning scheme (e.g., v1/v2 paths, version headers) or any documented deprecation policy; OpenAPI spec probes all returned 404. missing for 10: versioned endpoint scheme, deprecation/sunset policy documentation, changelog or migration guides.", "evidenceIds": [ "riskified-probe-2" ] }, { "productId": "riskified", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Riskified's API is order-by-order (decide/submit/advise per transaction), with no evidence of a batch or bulk endpoint for processing many orders/items in a single call, nor any bulk chargeback or data-export operation.", "evidenceIds": [] }, { "productId": "riskified", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Riskified automatically triggers fixed actions on fraud-decision events (webhook notifications, Shopify integration auto-voiding/restocking/capturing on approve/decline) but there is no evidence of a user-configurable rules engine where an AI-native user can define custom conditions/actions themselves — the automation is built-in and non-customizable rather than rule-definable. Missing for 10: a rules/condition-authoring API or UI, examples of custom trigger logic, and any AI-native/agentic rule-configuration workflow.", "evidenceIds": [ "riskified-docs-8", "riskified-docs-18", "riskified-docs-11", "riskified-docs-22" ] }, { "productId": "riskified", "storyId": "automation-scheduled-jobs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Riskified is a fraud-decisioning API/service, not an automation platform or agent framework; scheduling recurring jobs/workflows is not a relevant capability for this product category — it operates via real-time synchronous/asynchronous fraud decisions triggered by order events, not user-defined recurring schedules.", "evidenceIds": [] }, { "productId": "riskified", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Riskified is a fraud-decisioning API/service, not an automation-authoring tool with versionable workflows; there is no concept of user-built 'automations' to version, review, or roll back in this product category.", "evidenceIds": [] }, { "productId": "riskified", "storyId": "chargeback-guarantee", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "The evidence pack documents Riskified's API decisioning, chargeback notification, and chargeback gateway integration (automating chargeback reporting/disputing) but never states explicit financial liability-shift terms, reimbursement guarantees, coverage limits, or exclusions that a finance lead would need to evaluate a chargeback guarantee. Chargeback automation is a workflow/process feature, not proof of a monetary guarantee. missing for 10: explicit guarantee/reimbursement terms, coverage scope and exclusions, contractual liability-shift language, any evidence of actual reimbursement payouts.", "evidenceIds": [ "riskified-docs-4", "riskified-docs-16", "riskified-docs-17", "riskified-docs-8" ] }, { "productId": "riskified", "storyId": "custom-rules-authoring", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Riskified's docs describe API endpoints (decide/submit/advise), notifications, and integrations, but there is no evidence of a user-facing custom rules authoring interface where a risk analyst can combine model scores, velocity counters, list matches, and transaction attributes into allow/block/review logic. The decisions described (allow/notify/challenge, approve/decline) are produced by Riskified's own models, not by analyst-authored rules. Missing for 10: any rule-builder UI, rule syntax/DSL, ability to combine velocity counters/list matches/model scores into custom logic, or analyst-facing rule management docs.", "evidenceIds": [ "riskified-docs-1", "riskified-docs-3", "riskified-docs-9", "riskified-docs-20" ] }, { "productId": "riskified", "storyId": "custom-signals-enrichment", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Riskified's API endpoints (/decide, /submit, /advise) and Beacon.js collect a fixed, vendor-defined set of order/behavioral data required for fraud analysis, and model calibration is explicitly handled by Riskified's own analytics team (docs-12), not the developer. There is no evidence of an API or schema allowing developers to inject custom device fingerprints, arbitrary behavioral signals, or custom metadata fields into the scoring model to tailor it to their own business logic.", "evidenceIds": [ "riskified-docs-6", "riskified-docs-12", "riskified-docs-2", "riskified-docs-3" ] }, { "productId": "riskified", "storyId": "data-residency-controls", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack addresses data residency options, regional data storage, or retention/deletion controls; documentation covers API flows, integrations, and authentication only.", "evidenceIds": [] }, { "productId": "riskified", "storyId": "dispute-evidence-automation", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Riskified's Chargeback Gateway Integration (CGI) is documented as automating 'end-to-end chargeback handling from chargeback reporting to the disputing process' across major gateways, implying dispute evidence submission without manual copy-paste, and Riskified already ingests order, session (Beacon), and decision data through its APIs. However, the docs don't explicitly describe compiling delivery data or detail the specific evidence package sent to issuers during disputes. Missing for 10: explicit description of delivery-data inclusion in chargeback evidence, detail on the dispute evidence package format submitted to issuers, and independent/hands-on confirmation that submission is fully automated end-to-end.", "evidenceIds": [ "riskified-docs-4", "riskified-docs-16", "riskified-docs-17", "riskified-docs-6", "riskified-docs-2" ] }, { "productId": "riskified", "storyId": "fraud-outcome-reporting", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers API/integration mechanics (decide, submit, chargeback automation, notifications) but nothing about a finance/reporting dashboard, KPI metrics (dispute rate, false-positive rate, approval-rate lift, review workload), or board-ready export functionality.", "evidenceIds": [] }, { "productId": "riskified", "storyId": "lists-velocity-management", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers order decisioning APIs, notifications, chargebacks, and integrations, but nothing describes allow/block lists for emails, cards, devices, or IPs, nor velocity limit configuration via dashboard or API.", "evidenceIds": [] }, { "productId": "riskified", "storyId": "manual-review-queue", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers Riskified's API endpoints (/decide, /submit, /advise), notifications, chargeback automation, and Beacon data collection, but nothing describes a human review-queue UI showing analyst context, customer history, or similar cases for manual decisioning — Riskified's documented flow is automated/API-driven decisioning, not analyst-facing case review tooling.", "evidenceIds": [] }, { "productId": "riskified", "storyId": "model-performance-visibility", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack only covers API endpoints, integrations, notifications, and chargeback automation — nothing about model performance metrics (precision/recall), model versioning, or shadow-mode testing of new models/rules before go-live is documented.", "evidenceIds": [] }, { "productId": "riskified", "storyId": "network-intelligence", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack only documents Riskified's API endpoints, SDKs, and integration flows (decide, submit, advise, chargeback, notifications, Beacon, login) — none of it describes a cross-merchant data network, shared fraud signals across Riskified's merchant base, or how identity/card history from other businesses informs a given merchant's score. Without any documentation of the network-effect mechanism, this axis has no supporting evidence.", "evidenceIds": [] }, { "productId": "riskified", "storyId": "openness-api-parity", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack documents Riskified's API endpoints (decide, submit, advise, chargeback, notifications) and SDKs, but never discusses the merchant dashboard/UI or compares what can be done there versus via API, so there's no evidence of feature parity between UI and API.", "evidenceIds": [] }, { "productId": "riskified", "storyId": "openness-full-export", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Riskified is a fraud-decision API/service, not a data-storage or content-authoring product where a user's own data/documents need export in open formats; there's no concept of user-owned data corpus to 'export and leave' — this axis is a category error for this product type.", "evidenceIds": [] }, { "productId": "riskified", "storyId": "openness-open-license", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Riskified is a closed, proprietary fraud-detection SaaS product; there is no source code to read and no open-license claim anywhere in the evidence. Reading source under an open license is a category error for this type of hosted commercial service.", "evidenceIds": [] }, { "productId": "riskified", "storyId": "openness-self-host", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Riskified is a SaaS fraud-decisioning platform delivered via API/notifications with backend model calibration performed by Riskified's own analytics team; self-hosting the core product is not a coherent axis for this category of service.", "evidenceIds": [] }, { "productId": "riskified", "storyId": "platform-integrations", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Riskified documents maintained platform integrations beyond raw API: a dedicated Shopify app with automated order actions (docs-11,12,21,22), a Salesforce Commerce Cloud/Agentforce Commerce fraud cartridge (docs-10), and automated Chargeback Gateway Integration with major PSPs (Braintree, Stripe, Adyen, PayPal) (docs-4,16,17). However, there is no mention of a BigCommerce integration, and the PSP integrations are scoped to chargeback data only rather than full checkout/decisioning flows. Missing for 10: explicit BigCommerce integration, evidence of maintained PSP integrations beyond chargebacks (e.g., payment/checkout-time connectors), independent confirmation of integration quality/maintenance cadence.", "evidenceIds": [ "riskified-docs-4", "riskified-docs-10", "riskified-docs-11", "riskified-docs-12", "riskified-docs-17", "riskified-docs-21", "riskified-docs-22" ] }, { "productId": "riskified", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions data residency, region selection, or storage location options for merchant/customer data; documentation covers API endpoints, integrations, and notifications only.", "evidenceIds": [] }, { "productId": "riskified", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Riskified is a fraud-prevention/e-commerce risk API, not an AI model or AI assistant product; the story about preventing personal data being used for AI model training is not a relevant axis for this kind of product.", "evidenceIds": [] }, { "productId": "riskified", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack items address data retention policies, deletion mechanisms, or user/customer data control tools; the docs focus entirely on fraud-decision APIs and integrations with no privacy-posture or data lifecycle controls mentioned.", "evidenceIds": [] }, { "productId": "riskified", "storyId": "privacy-telemetry-optout", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Riskified is a fraud-prevention/risk API platform, not an AI developer tool or agent with telemetry collected from AI-native usage; opt-out of telemetry/usage tracking is not a relevant axis for this product category.", "evidenceIds": [] }, { "productId": "riskified", "storyId": "psd2-sca-optimization", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Riskified's /advise endpoint explicitly returns CVV/3DS recommendations and adds TRA/SCA exemption recommendations for regulated markets, directly matching the story of intelligently triggering 3DS or requesting exemptions per order. This is first-party documentation with no independent corroboration of real-world accuracy or European-specific behavior. Missing for 10: independent/hands-on validation of SCA exemption success rates, and explicit detail on how 'European traffic' specifically is distinguished/routed versus other regions.", "evidenceIds": [ "riskified-docs-3", "riskified-docs-15" ] }, { "productId": "riskified", "storyId": "psp-agnostic-coverage", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Riskified is gateway/platform-agnostic via its own API (/decide, /submit, /advise), with automated chargeback integrations naming specific gateways (Braintree, Stripe, Adyen, PayPal) and platform cartridges for Shopify and SFCC, indicating it can plug into multiple PSPs/platforms rather than lock a merchant into one processor. However, the evidence doesn't explicitly enumerate broad PSP/gateway compatibility beyond the four named chargeback gateways, nor discuss developer-facing flexibility across arbitrary payment stacks explicitly. missing for 10: a comprehensive list of supported PSPs/gateways beyond chargeback integrations, explicit statements about processor-agnostic architecture, and independent confirmation of multi-PSP interoperability.", "evidenceIds": [ "riskified-docs-4", "riskified-docs-17", "riskified-docs-10", "riskified-docs-11", "riskified-docs-23" ] }, { "productId": "riskified", "storyId": "realtime-scoring-api", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Riskified's documented `/decide` endpoint explicitly returns a synchronous fraud decision before payment authorization completes, matching the real-time pre-auth workflow described in the story, and Riskified's own docs classify this as 'Pre-authorization Synchronous' integration. However, the docs describe a categorical decision (approve/decline) rather than an explicit numeric ML risk score, and no OpenAPI/swagger spec exists (probe returned 404s) to confirm formal API schema. missing for 10: explicit numeric risk-score field documentation, machine-readable OpenAPI spec, independent/hands-on corroboration of latency/real-time behavior.", "evidenceIds": [ "riskified-docs-1", "riskified-docs-13", "riskified-docs-23", "riskified-probe-2" ] }, { "productId": "riskified", "storyId": "review-outcomes-feedback", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes automated decisioning APIs (/decide, /submit, /advise), chargeback notification integrations, and onboarding model calibration by Riskified's own analytics team, but nothing describes an analyst-facing review queue where human review decisions or confirmed fraud outcomes are captured and fed back into the model/rules on an ongoing basis. Missing for 10: evidence of an analyst review UI, a mechanism for analysts to submit manual review/fraud confirmations, and documentation that such decisions retrain or update the fraud model/rules.", "evidenceIds": [ "riskified-docs-12", "riskified-docs-16", "riskified-docs-4" ] }, { "productId": "riskified", "storyId": "review-team-workflows", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers API decisioning, notifications, chargeback automation, and integrations, but nothing describes a human ops review workflow with case assignment, escalation, SLAs, or an audit trail of who approved what and why.", "evidenceIds": [] }, { "productId": "riskified", "storyId": "rules-backtesting", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers Riskified's fraud-decision APIs, notifications, chargeback integrations, and merchant integration flows, but nothing describes a rule backtesting capability against historical traffic showing what would have been blocked/flagged/cost before deployment.", "evidenceIds": [] }, { "productId": "riskified", "storyId": "score-explainability", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence describes decision outcomes (approve/decline/allow/notify/challenge) and notification mechanisms, but nothing indicates that decisions are accompanied by explanatory risk factors or reason codes for analysts. No mention of feature-level explanations, contributing signals, or reason breakdowns anywhere in the docs.", "evidenceIds": [] }, { "productId": "riskified", "storyId": "score-thresholds-actions", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Riskified's docs describe its endpoints (/decide, /submit, /advise, login) returning pre-computed decisions like allow/notify/challenge or approve/decline, but there is no evidence that ops users can view underlying risk scores or configure/tune score-to-action thresholds themselves — decisioning appears to be Riskified's managed model rather than a merchant-tunable scorecard.", "evidenceIds": [ "riskified-docs-1", "riskified-docs-3", "riskified-docs-9", "riskified-docs-20" ] }, { "productId": "riskified", "storyId": "trusted-agent-signals", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions agentic-payment protocols, signed agent tokens, or delegated spending scopes as inputs to Riskified's risk decision; the documented signals are customer/order/payment data, HMAC request auth, and Beacon session IDs, none of which represent verified agent identity as a first-class fraud signal.", "evidenceIds": [] }, { "productId": "sift", "storyId": "account-abuse-coverage", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Docs explicitly cover multiple fraud surfaces beyond checkout — account takeover ('Block unauthorized access in real time'), fake account creation ('Stop fraudulent signups at the door'), and abuse-specific scoring ('fight multiple types of fraud at once... add an abuse-specific risk score'), all managed within the same Workflows/Decisions/Review Queue system. Missing for 10: dedicated documentation or case study specifically on promo/policy abuse handling, and independent hands-on verification of multi-surface scoring in one unified dashboard.", "evidenceIds": [ "sift-docs-13", "sift-docs-14", "sift-docs-9", "sift-docs-6", "sift-docs-12", "sift-docs-7" ] }, { "productId": "sift", "storyId": "agent-buyer-detection", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers general fraud scoring, workflows, and decisions but nothing addresses distinguishing AI shopping agents from malicious bots or agentic-commerce specific handling; missing for 10: any mention of AI agent traffic detection, agent-vs-bot classification, or agentic-commerce policy/whitelisting.", "evidenceIds": [] }, { "productId": "sift", "storyId": "agent-drives-rules", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows REST/Decisions/Workflows APIs for events, scoring, and rules automation, but nothing describes an AI agent reading fraud posture or proposing/updating rules and lists with a human-approval workflow — this is a generic developer API story, not an agent-access pattern. missing for 10: any mention of AI agent integration, agentic rule-proposal workflow, or human-approval gate mechanism tied to programmatic rule/list changes.", "evidenceIds": [ "sift-docs-1", "sift-docs-3", "sift-docs-12" ] }, { "productId": "sift", "storyId": "agent-triages-reviews", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Sift has Review Queues and REST/Decisions APIs for humans to review flagged cases (sift-docs-12, sift-docs-3), and community evidence confirms a human-in-the-loop review model (sift-comm-4), but there is no evidence of an AI agent programmatically pulling flagged cases, summarizing evidence, or recommending decisions for human confirmation.", "evidenceIds": [ "sift-docs-12", "sift-docs-3", "sift-comm-4" ] }, { "productId": "sift", "storyId": "agentic-agent-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of an llms.txt file or agent-oriented documentation format; the docs are standard developer API references with no mention of AI-agent consumption. missing for 10: llms.txt file, agent-readable docs format, any mention of AI-agent-targeted documentation.", "evidenceIds": [] }, { "productId": "sift", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Sift provides risk scores and rule-based Workflows/Decisions derived from ML models, but there is no evidence of AI-generated natural-language insights or suggestions surfaced to users inside a product UI—everything is API-driven scoring and automation rather than generative/agentic insight delivery. Missing for 10: any documented AI-generated narrative insights, recommendations, or conversational/agentic assistant surfaced in-product, and any independent confirmation of such a feature.", "evidenceIds": [ "sift-docs-5", "sift-docs-6", "sift-docs-12" ] }, { "productId": "sift", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Sift's Workflows feature is described as a 'rules automation platform' that runs risk-based decisions autonomously in real time on incoming events, which fits the general notion of background automations. However, this is traditional rules/ML-based fraud automation, not an AI-native/agentic automation-building experience tailored to an 'AI-native user.' Missing for 10: evidence of AI-native automation authoring (e.g., natural-language or agent-driven workflow creation), broader use-case automations beyond fraud decisioning, and independent confirmation of autonomous operation quality.", "evidenceIds": [ "sift-docs-6", "sift-docs-12", "sift-docs-7" ] }, { "productId": "sift", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "sift", "storyId": "agentic-headless", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Sift is fundamentally API-driven (REST Events/Decisions APIs, SDKs) which is inherently 'headless' and can be called from backend or automated systems, but there is no explicit evidence of a CLI, CI pipeline integration, or documented headless automation workflow. Missing for 10: explicit CI/CD examples, CLI tooling, or automation-specific documentation beyond generic API integration.", "evidenceIds": [ "sift-docs-1", "sift-docs-2", "sift-docs-3", "sift-docs-4" ] }, { "productId": "sift", "storyId": "agentic-mcp-client", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Sift is a fraud-detection/risk-scoring platform, not an AI agent or assistant; the story asks whether the product can plug in MCP servers to consume external tools, which is a client-agent capability not applicable to this kind of product.", "evidenceIds": [] }, { "productId": "sift", "storyId": "agentic-mcp-server", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Sift is a fraud-detection SaaS with REST/Decisions APIs and SDKs, but no evidence of an official MCP server for agent connectivity. missing for 10: any mention of MCP, agent integration protocol, or official MCP server endpoint.", "evidenceIds": [ "sift-docs-1" ] }, { "productId": "sift", "storyId": "agentic-nl-commands", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "sift", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Sift's evidence pack covers REST APIs, SDKs, JS snippet, and Decisions/Workflows APIs, but no official CLI tool is mentioned anywhere in docs or community sources.", "evidenceIds": [] }, { "productId": "sift", "storyId": "agentic-public-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Sift documents REST APIs (Events, Decisions, Score, Workflows) that let developers programmatically drive fraud scoring and decisioning, which supports an AI-native integration story. However, this is a traditional fraud-ops API, not one purpose-built for AI agent orchestration, and there's no evidence of agent-specific tooling like an MCP server, function-calling schemas, or SDKs for LLM agents. missing for 10: agent-oriented API framing (e.g., MCP server, function-calling schema), independent hands-on validation of API usability beyond one user noting confusing docs.", "evidenceIds": [ "sift-docs-1", "sift-docs-2", "sift-docs-3", "sift-docs-5", "sift-docs-6", "sift-comm-6" ] }, { "productId": "sift", "storyId": "agentic-scoped-keys", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Sift is a fraud-detection/risk-scoring platform, not an agent-facing product issuing scoped API credentials for AI agents; no evidence pack items address credential scoping or agent-specific access control, and this is a category mismatch rather than a gap in an applicable capability.", "evidenceIds": [] }, { "productId": "sift", "storyId": "agentic-sdks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Sift has REST APIs, JS snippet, and mobile SDKs for iOS/Android, but there's no mention of SDKs oriented toward AI-native development (e.g., LLM/agent SDKs, official Python/Node/AI framework SDKs). This story concerns AI-native developer tooling, which is not addressed anywhere in the pack. missing for 10: any evidence of AI-agent-oriented SDKs, LLM integration libraries, or agent framework support.", "evidenceIds": [] }, { "productId": "sift", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Sift's documented integration model is inbound REST APIs (Events, Decisions, Score, Workflows) for sending data to Sift, not outbound webhooks for subscribing to events from Sift. No evidence pack item mentions webhook subscriptions or event push notifications to external consumers.", "evidenceIds": [] }, { "productId": "sift", "storyId": "ai-fraud-analyst", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers Sift's APIs, scoring, workflows/rules automation, and review queues, but there is no mention of any built-in AI assistant, natural-language query capability, drafted rules generation, or investigation summary generation in the console. Missing for 10: any documentation or mention of a natural-language/AI assistant feature, evidence of NL-to-query capability, evidence of AI-drafted rules or summaries.", "evidenceIds": [ "sift-docs-6", "sift-docs-12" ] }, { "productId": "sift", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence shows REST/Decisions/Events API docs exist but nothing describes an interactive API reference with runnable examples (e.g., embedded sandbox, try-it console). Missing for 10: interactive API explorer, runnable code examples, sandbox/try-it functionality.", "evidenceIds": [] }, { "productId": "sift", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows Sift has REST APIs (Events, Decisions, Score, Workflows) with docs, but no mention of a downloadable OpenAPI spec or other machine-readable API schema anywhere in the pack.", "evidenceIds": [ "sift-docs-1", "sift-docs-2", "sift-docs-3", "sift-docs-6" ] }, { "productId": "sift", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence pack item mentions a sandbox environment, test mode, or any way to test Sift integrations without touching production data; all docs reference live Events/Decisions/Score APIs. Missing for 10: sandbox/test environment docs, test API keys or staging mode, any mention of separating test vs production data.", "evidenceIds": [] }, { "productId": "sift", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence shows Sift has REST APIs, SDKs, and docs but no mention of API versioning scheme or a documented deprecation policy anywhere in the pack. Missing for 10: explicit API versioning documentation, deprecation policy or changelog, migration timelines for breaking changes.", "evidenceIds": [] }, { "productId": "sift", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Sift documents backfilling months of historical data as a bulk-load capability, but there is no evidence of a dedicated bulk API for batch-scoring, batch-deciding, or bulk-updating many items at once (e.g., no batch endpoint documentation). Missing for 10: explicit bulk/batch API documentation, batch decision or batch scoring examples, and any hands-on evidence of bulk workflows being used successfully.", "evidenceIds": [ "sift-docs-4", "sift-docs-2", "sift-docs-3" ] }, { "productId": "sift", "storyId": "automation-rules-engine", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Sift's docs clearly describe a 'Workflows' rules automation platform that triggers real-time Decisions (Ban Account, Cancel Order) based on event risk scores, directly matching the story. However, a Sift co-founder is quoted saying 'we do not have any rules in our product... rules are rather easy for fraudsters to circumvent,' directly contradicting the vendor's later rules-automation framing. missing for 10: reconciliation of this contradiction and independent hands-on confirmation that customer-defined Workflow rules reliably trigger automated actions in production.", "evidenceIds": [ "sift-docs-6", "sift-docs-7", "sift-docs-12", "sift-comm-8" ] }, { "productId": "sift", "storyId": "automation-scheduled-jobs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Sift is a fraud-detection/risk-scoring platform, not a workflow/job scheduling or automation orchestration tool; scheduling recurring jobs is outside its product category and not addressed by any evidence (Workflows here refer to real-time rules-based decisioning, not recurring job scheduling).", "evidenceIds": [] }, { "productId": "sift", "storyId": "automation-versioned-workflows", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Sift is a fraud-detection/risk-scoring platform, not an automation/workflow builder with version control for user-created automations; the evidence pack shows Workflows and Decisions APIs but no versioning, review, or rollback of automations. This is a category mismatch rather than a missing feature.", "evidenceIds": [] }, { "productId": "sift", "storyId": "chargeback-guarantee", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence anywhere in the pack mentions a chargeback guarantee, liability shift, reimbursement for approved-then-disputed orders, or coverage terms — Sift is documented only as a fraud-scoring/risk-decision API, not an insurance-backed guarantee product.", "evidenceIds": [] }, { "productId": "sift", "storyId": "custom-rules-authoring", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Docs describe Workflows as a 'powerful rules automation platform' for real-time decisioning (allow/block/review) using scores and custom logic (sift-docs-6, sift-docs-7, sift-docs-12), suggesting the requested rule authoring capability. However, a Sift co-founder is on record stating 'we do not have any rules in our product' and that rules are easy to circumvent, directly contradicting the existence of a rules engine (sift-comm-8), and no evidence details velocity counters or list-match integration into rule logic. missing for 10: explicit documentation of velocity counters, list matching, and multi-attribute rule composition within Workflows; independent verification reconciling the rules-vs-no-rules discrepancy.", "evidenceIds": [ "sift-docs-6", "sift-docs-7", "sift-docs-12", "sift-comm-8" ] }, { "productId": "sift", "storyId": "custom-signals-enrichment", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Sift's docs clearly support ingesting custom events, custom fields, session/device signals, and business action data via Events/Decisions APIs, and scoring incorporates this custom data plus historical backfill to tailor risk to the business rather than generic network defaults. Missing for 10: independent hands-on verification of custom-field scoring impact and explicit device-fingerprint API documentation beyond general SDK mentions.", "evidenceIds": [ "sift-docs-2", "sift-docs-3", "sift-docs-4", "sift-docs-10", "sift-docs-11", "sift-docs-5" ] }, { "productId": "sift", "storyId": "data-residency-controls", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack items discuss regional data residency, data hosting locations, retention periods, or compliance/privacy review controls; one comment even notes a prospective customer's hesitation about privacy/legal review with no resolution shown. Missing for 10: any documentation of regional hosting/data residency options, retention/configurable deletion policies, or compliance certifications addressing privacy review.", "evidenceIds": [ "sift-comm-3" ] }, { "productId": "sift", "storyId": "dispute-evidence-automation", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "sift", "storyId": "fraud-outcome-reporting", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers APIs, scoring, workflows, and general product sentiment, but there is no mention of finance-lead dashboards, dispute-rate/false-positive/approval-lift metrics, review workload reporting, or board-ready export capability. missing for 10: dispute-rate and false-positive-rate metrics reporting, approval-rate lift analytics, review workload/queue metrics, export or reporting functionality for finance/board consumption.", "evidenceIds": [] }, { "productId": "sift", "storyId": "lists-velocity-management", "verdict": "disputed", "quality": 3, "confidence": "low", "rationale": "Docs describe a general 'Workflows' rules automation platform and Review Queues, but there's no explicit mention of dedicated allow/block lists for emails, cards, devices, or IPs, nor velocity limit configuration in the dashboard or API. Compounding this, a Sift co-founder is quoted stating 'we do not have any rules in our product' as a core design philosophy, directly conflicting with the rules-engine framing implied by the Workflows docs. missing for 10: explicit allow/block list management (emails, cards, devices, IPs), velocity limit configuration, dashboard UI evidence, and resolution of the rules-vs-no-rules contradiction.", "evidenceIds": [ "sift-docs-12", "sift-docs-6", "sift-comm-8", "sift-comm-10" ] }, { "productId": "sift", "storyId": "manual-review-queue", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Sift documents Review Queues as a product feature alongside Workflows, and community evidence confirms human analysts do review flagged users to make final decisions, supporting the general workflow. However, there is no evidence detailing the review queue UI showing customer history, similar cases, or signal context in one view. missing for 10: documentation/screenshots of review queue UI showing customer history and similar cases, evidence of consolidated signal context, independent hands-on confirmation of queue usability.", "evidenceIds": [ "sift-docs-12", "sift-comm-4" ] }, { "productId": "sift", "storyId": "model-performance-visibility", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers scoring, workflows, decisions, and events APIs, but there is no mention of precision/recall metrics on the customer's own traffic, model performance measurement, or shadow-mode trialing of new models/rules before they go live. missing for 10: precision/recall measurement tooling, shadow-mode/challenger model testing, any model transparency reporting for finance/business stakeholders.", "evidenceIds": [] }, { "productId": "sift", "storyId": "network-intelligence", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack describes Sift's own ML scoring, event/decision APIs, and workflows, but nowhere claims that risk signals (e.g., a card or identity) are shared or aggregated across Sift's merchant customer base to inform another business's score. sift-docs-4 only references a business's own historical data improving its own model, not cross-merchant network effects.", "evidenceIds": [] }, { "productId": "sift", "storyId": "openness-api-parity", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Sift's docs show extensive REST APIs covering events, decisions, scoring, and even a Workflows API for automation, indicating broad programmatic access mirroring core UI functions (sift-docs-1,2,3,5,6,12). However, there's no explicit evidence that Review Queues or full workflow configuration UI actions are fully API-equivalent, and no independent confirmation of complete UI/API parity. missing for 10: explicit parity claim for Review Queues and workflow UI configuration, independent verification of full CRUD API coverage matching every console action.", "evidenceIds": [ "sift-docs-1", "sift-docs-2", "sift-docs-3", "sift-docs-5", "sift-docs-6", "sift-docs-12" ] }, { "productId": "sift", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Sift is a fraud-detection API platform; evidence covers sending events/data into Sift via REST API, scoring, workflows, but there is no mention of exporting or bulk-downloading customer data out of Sift in open formats, nor any data-portability/export feature. missing for 10: any documented data export/download capability, open-format export (CSV/JSON dumps), or account data portability tooling.", "evidenceIds": [] }, { "productId": "sift", "storyId": "openness-open-license", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Sift is a proprietary commercial fraud-detection SaaS product; there is no indication of an open-source license or public source code. This is a category error for the axis — closed commercial SaaS products aren't expected to publish source under open license, so it's inapplicable rather than a failure.", "evidenceIds": [] }, { "productId": "sift", "storyId": "openness-self-host", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Sift is a cloud-based fraud detection SaaS delivered via REST APIs and hosted risk scoring; self-hosting the core product is not a fair axis for this kind of managed service, and no evidence suggests otherwise.", "evidenceIds": [] }, { "productId": "sift", "storyId": "platform-integrations", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence shows only generic REST API, JS snippet, and mobile SDKs — no mention of maintained Shopify, Salesforce Commerce, BigCommerce, or PSP-specific integrations. Everything points to a raw API/SDK integration model, not prebuilt commerce-platform connectors.", "evidenceIds": [ "sift-docs-1", "sift-docs-2", "sift-docs-3" ] }, { "productId": "sift", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions data residency, regional storage options, or geographic control over where Sift stores customer data; one community comment even notes hesitation about sending data to Sift without clarity on compliance handling. Missing for 10: any documentation of region selection, data residency options, or compliance certifications addressing storage location.", "evidenceIds": [ "sift-comm-3" ] }, { "productId": "sift", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Sift is a fraud-detection platform whose business model relies on ingesting user behavioral data to train its ML risk models; no evidence pack item mentions any opt-out, data-training-exclusion policy, or AI-training-specific privacy controls for end users. Missing for 10: any documentation of an opt-out mechanism, training-data exclusion policy, or user-facing privacy control preventing model training use.", "evidenceIds": [] }, { "productId": "sift", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Sift is a fraud-detection API/platform, and the evidence pack contains no documentation of data retention controls, deletion APIs, or configurable data lifecycle policies for users. One community comment (sift-comm-3) even flags unresolved privacy/legal review concerns about sending personal data to Sift, but this is not concrete evidence of a retention/deletion mechanism either way. missing for 10: any documentation of data retention settings, a deletion/right-to-be-forgotten API, or data lifecycle/export controls.", "evidenceIds": [ "sift-comm-3" ] }, { "productId": "sift", "storyId": "privacy-telemetry-optout", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Sift is a backend fraud-detection/risk-scoring API whose core function is ingesting customer/user behavioral data for fraud analysis, not a developer tool or AI agent with its own usage-telemetry settings that an 'AI-native user' would opt out of. The evidence pack contains no mention of telemetry collection about API/dashboard usage, making this axis a category mismatch rather than an unmet capability.", "evidenceIds": [] }, { "productId": "sift", "storyId": "psd2-sca-optimization", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "sift", "storyId": "psp-agnostic-coverage", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Sift's architecture (generic REST Events/Decisions API, JS snippet, SDKs) is inherently processor-agnostic — it ingests events from any application regardless of which PSP or gateway processes the payment, rather than being tied to one processor's rails. However, there is no explicit documentation or case evidence naming specific PSPs/gateways/platforms it integrates with or confirming multi-processor deployments in practice. Missing for 10: explicit multi-PSP/gateway integration examples, named partner processors, or customer testimony confirming cross-stack usage.", "evidenceIds": [ "sift-docs-1", "sift-docs-2", "sift-docs-3", "sift-docs-10" ] }, { "productId": "sift", "storyId": "realtime-scoring-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Sift's documented Score API returns a 0-100 ML risk score via REST, and Workflows enable real-time risk-based decisions during key events, supporting synchronous scoring before a decision. However, the evidence doesn't explicitly confirm the Score API is called synchronously pre-authorization (vs. async/webhook-based scoring), and community feedback notes many customers use human review queues rather than pure real-time automated decisions. Missing for 10: explicit documentation of synchronous call/response latency guarantees for pre-authorization use, and independent confirmation of real-time performance under transaction-blocking scenarios.", "evidenceIds": [ "sift-docs-5", "sift-docs-6", "sift-docs-13", "sift-comm-4" ] }, { "productId": "sift", "storyId": "review-outcomes-feedback", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Sift's Decisions API explicitly captures business actions like 'block due to fraud' or 'cancel due to chargeback' and feeds them back to Sift, and docs state Sift 'gets smarter the more data it has,' supporting a feedback loop from confirmed outcomes into the model. Review Queues and Workflows are documented as connected to decisions, and a founder note confirms customers commonly have analysts make final go/no-go calls that presumably feed Decisions API. Missing for 10: explicit documentation that individual analyst review-queue verdicts (not just automated decisions) are looped back per-case to retrain the model, and any detail on how confirmed fraud outcomes update rules/workflows specifically.", "evidenceIds": [ "sift-docs-3", "sift-docs-4", "sift-docs-12", "sift-comm-4" ] }, { "productId": "sift", "storyId": "review-team-workflows", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Sift documents Review Queues and Workflows for human review, plus a Decisions API that logs business actions like approve/ban/cancel tied to real events, which supports a basic audit trail; a founder comment also confirms a human-in-the-loop review pattern ('have a human make a final go/no-go decision'). However, there's no evidence of explicit team-workflow features like assignment routing, escalation paths, or SLA tracking. missing for 10: assignment/routing mechanics, escalation workflows, SLA timers/tracking, explicit 'who approved what and why' UI/reporting beyond raw decision logs.", "evidenceIds": [ "sift-docs-12", "sift-docs-3", "sift-docs-7", "sift-comm-4" ] }, { "productId": "sift", "storyId": "rules-backtesting", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes Workflows as a rules automation platform and mentions real-time decisioning, but nothing in the pack describes a backtest/simulation capability against historical traffic showing what a rule would have blocked/flagged/cost.", "evidenceIds": [ "sift-docs-12", "sift-docs-6" ] }, { "productId": "sift", "storyId": "score-explainability", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack documents Sift's 0–100 risk score, Workflows, and Decisions APIs, but nowhere mentions score explanations, reason codes, or top contributing risk factors accompanying each score. Missing for 10: any documentation of explainability/reason-code output, feature-importance breakdowns, or examples of a score being paired with human-readable risk drivers.", "evidenceIds": [ "sift-docs-5", "sift-docs-6", "sift-docs-12" ] }, { "productId": "sift", "storyId": "score-thresholds-actions", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Sift's docs describe Workflows as a rules automation platform for real-time risk-based decisions, with custom Decisions (e.g., Ban Account, Cancel Order) tied to score ranges (0-100), and Review Queues for human review — directly supporting configurable score-to-action mapping and threshold tuning. However, no evidence explicitly mentions 3DS step-up as an action type, and an older founder comment (sift-comm-8) claims 'no rules' which is in tension with the current Workflows rules-engine framing, though this appears to reflect product evolution rather than a live contradiction. missing for 10: explicit 3DS/step-up action documentation, independent hands-on confirmation of threshold customization in practice.", "evidenceIds": [ "sift-docs-5", "sift-docs-6", "sift-docs-7", "sift-docs-12", "sift-comm-4", "sift-comm-8" ] }, { "productId": "sift", "storyId": "trusted-agent-signals", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence Sift supports ingesting verified agent identity, agentic-payment protocol tokens, signed agent tokens, or delegated spending scopes as risk signals; docs only describe generic user/business event and custom-field ingestion. missing for 10: any mention of agentic-commerce protocols, signed agent tokens, or delegated spending scope support as a risk signal.", "evidenceIds": [] }, { "productId": "signifyd", "storyId": "account-abuse-coverage", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers checkout/payment fraud, chargebacks, returns, and refund protection, but nothing addresses account takeover, fake account creation, or promo/policy abuse scoring within Signifyd's system. Missing for 10: any documentation of account takeover protection, fake account detection, or promo/policy abuse scoring features.", "evidenceIds": [ "signifyd-docs-1", "signifyd-docs-2", "signifyd-docs-3", "signifyd-docs-4", "signifyd-docs-5", "signifyd-docs-13" ] }, { "productId": "signifyd", "storyId": "agent-buyer-detection", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers Signifyd's fraud-decision API, webhooks, device profiling, chargebacks, and refund/return tooling, but nothing addresses distinguishing malicious bots from legitimate AI shopping/purchasing agents or any agentic-commerce-specific fraud logic. Missing for 10: any mention of AI agent identification, agent-vs-bot classification, or policy to avoid blocking legitimate AI buying agents.", "evidenceIds": [] }, { "productId": "signifyd", "storyId": "agent-drives-rules", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence shows a REST API for order/event submission, decisions, chargebacks, and webhooks, but nothing about rules/lists management (e.g., velocity rules, blocklists) or an agentic workflow with human approval gates. No documentation of a rules API or agent-oriented approval mechanism is present.", "evidenceIds": [ "signifyd-docs-1", "signifyd-docs-4", "signifyd-docs-7" ] }, { "productId": "signifyd", "storyId": "agent-triages-reviews", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence shows only standard REST endpoints for sale/checkout events, decision retrieval, chargebacks, and webhooks — nothing about a queue of flagged cases with case context, an agent summarizing evidence, or generating a recommendation for human confirmation. No case-management or review-queue API surface is documented at all.", "evidenceIds": [ "signifyd-docs-1", "signifyd-docs-4", "signifyd-docs-5" ] }, { "productId": "signifyd", "storyId": "agentic-agent-docs", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The llms.txt probe returned a 404, showing no dedicated agent-oriented manifest exists, and while a markdown-rendered docs page was found at one URL, there's no evidence of a systematic agent-oriented docs structure or llms.txt file across the site. missing for 10: a working llms.txt or equivalent agent-discoverable docs index, and any indication docs are structured/announced for AI agent consumption.", "evidenceIds": [ "signifyd-probe-1", "signifyd-probe-2" ] }, { "productId": "signifyd", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows API endpoints, webhooks, SDKs, and marketing blurbs like 'Return Insights' offering 'actionable intelligence,' but nothing describes AI-generated insights, natural-language explanations, or suggestions surfaced to users inside the product. No dashboard, chat, or generative-AI feature is documented.", "evidenceIds": [ "signifyd-docs-15", "signifyd-docs-16" ] }, { "productId": "signifyd", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Signifyd's core product (fraud decisioning, chargeback handling, webhook notifications) runs autonomously in the background once integrated, and webhooks/API endpoints let a merchant configure automated event flows (e.g. createteamwebhook, getdecision, createchargeback) that fire without manual intervention. However, there is no evidence of an AI-native automation builder, scheduler, or agent-configurable workflow system — the 'automation' here is fixed product behavior wired via API/webhooks rather than a user-defined autonomous automation platform. Missing for 10: evidence of a user-configurable automation/rules engine, scheduling or trigger-condition builder, and any AI-specific automation tooling.", "evidenceIds": [ "signifyd-docs-7", "signifyd-docs-8", "signifyd-docs-4", "signifyd-docs-5", "signifyd-docs-1" ] }, { "productId": "signifyd", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The axis applies to this product kind (peer products hold positive or none verdicts on this story), so lack of evidence for an applicable capability is \"none\", never \"na\". (na/none harmonized at arena bring-up — see pipeline/scripts/na-harmonize.ts.)", "evidenceIds": [] }, { "productId": "signifyd", "storyId": "agentic-headless", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Signifyd's core product is a REST API (with SDKs and webhooks) intended for backend/server-side integration rather than a UI-dependent tool, which inherently supports headless/programmatic use in automated pipelines. However, there is no explicit documentation of CI-specific tooling, a CLI, or automated-testing guidance — missing for 10: CI-specific setup docs, CLI/automation tooling, service-account/API-key guidance for pipeline use, and independent confirmation of headless CI usage.", "evidenceIds": [ "signifyd-docs-1", "signifyd-docs-2", "signifyd-docs-3", "signifyd-docs-7", "signifyd-docs-12" ] }, { "productId": "signifyd", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of an official MCP server or any MCP integration for Signifyd; the evidence only covers its REST API, webhooks, and SDKs. As a fraud-protection SaaS, this axis is fair to expect but there is no documentation or claim of MCP server support.", "evidenceIds": [ "signifyd-docs-1", "signifyd-probe-1" ] }, { "productId": "signifyd", "storyId": "agentic-mcp-server", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Signifyd is a fraud-protection SaaS with a REST API and webhooks, but there is no evidence of an official MCP server for agent connectivity; the llms.txt probe even 404s. missing for 10: any official MCP server, documentation, or first-party endpoint enabling agent connectivity.", "evidenceIds": [ "signifyd-probe-1" ] }, { "productId": "signifyd", "storyId": "agentic-nl-commands", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Signifyd's evidence pack only shows a REST API and webhook integration for fraud/chargeback management; there is no mention of natural-language command interfaces, chat-based control, or NL-driven operation of the product. missing for 10: any NL command interface, chatbot/assistant control surface, or documented natural-language API layer.", "evidenceIds": [ "signifyd-docs-1", "signifyd-docs-2", "signifyd-docs-3" ] }, { "productId": "signifyd", "storyId": "agentic-official-cli", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of any official CLI tool; Signifyd only documents REST API endpoints, SDKs for mobile apps, and webhooks, with no mention of a command-line interface for AI-native workflows.", "evidenceIds": [] }, { "productId": "signifyd", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Signifyd provides a comprehensive, well-documented REST API covering authentication, sale/checkout events, decisions, chargebacks, webhooks, and SDKs, confirmed by both docs and a live probe returning 200 with actual API content. Missing for 10: no llms.txt or explicit AI-agent-oriented documentation, and no independent third-party corroboration of API usability.", "evidenceIds": [ "signifyd-docs-1", "signifyd-docs-2", "signifyd-docs-3", "signifyd-docs-4", "signifyd-docs-7", "signifyd-probe-2" ] }, { "productId": "signifyd", "storyId": "agentic-scoped-keys", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence shows only general REST API authentication docs and endpoints; there is no mention of scoped, least-privilege, or agent-specific API credentials/tokens, OAuth scopes, or granular permissioning for API keys.", "evidenceIds": [ "signifyd-docs-1" ] }, { "productId": "signifyd", "storyId": "agentic-sdks", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Signifyd documents official mobile SDKs with instructions, examples, and project files, plus a REST API reference for direct integration, but the evidence lacks concrete language-specific SDKs (Python, Node, Java, etc.), GitHub repos, or versioning/release info typical of AI-native SDK-driven workflows. missing for 10: language/server-side SDK documentation, GitHub repo links, code samples showing SDK usage beyond mobile.", "evidenceIds": [ "signifyd-docs-12", "signifyd-docs-1" ] }, { "productId": "signifyd", "storyId": "agentic-webhooks", "verdict": "full", "quality": 7, "confidence": "medium", "rationale": "Signifyd has documented webhook creation (createteamwebhook) and configuration docs describing real-time notifications for guarantee decisions, directly supporting event subscription via webhooks. Missing for 10: independent/hands-on corroboration of webhook reliability, full event-type catalog, and payload schema documentation.", "evidenceIds": [ "signifyd-docs-7", "signifyd-docs-8" ] }, { "productId": "signifyd", "storyId": "ai-fraud-analyst", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers only REST API endpoints, webhooks, SDKs, and documentation for fraud data/order management — nothing about a built-in AI assistant, natural-language query interface, rule drafting, or investigation summaries in the console. missing for 10: any mention of an AI/NLP assistant feature, natural-language query capability, AI-drafted rules, or AI-generated investigation summaries.", "evidenceIds": [] }, { "productId": "signifyd", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence shows Signifyd has an extensive REST API reference (endpoints for Sale, Checkout, Decision, Chargeback, Webhooks, etc.) but nothing in the pack indicates an interactive console, runnable code samples, or 'try it now' functionality typical of an AI-native API reference. Missing for 10: any mention of interactive/runnable request builder, live sandbox execution, or SDK-embedded runnable snippets.", "evidenceIds": [ "signifyd-docs-1", "signifyd-docs-2", "signifyd-docs-3", "signifyd-docs-4" ] }, { "productId": "signifyd", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack shows Signifyd has REST API reference docs (built on a docs platform) but no mention of a downloadable OpenAPI/Swagger spec file or machine-readable schema anywhere in the pack, and a direct probe for llms.txt returned 404 with no OpenAPI equivalent found.", "evidenceIds": [ "signifyd-docs-1", "signifyd-probe-1", "signifyd-probe-2" ] }, { "productId": "signifyd", "storyId": "api-sandbox", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence pack item mentions a sandbox, test environment, or test mode for Signifyd's API; all evidence describes production-oriented endpoints (Sale, Checkout, chargeback, webhooks) with no mention of a separate testing environment. Missing for 10: any sandbox/test API docs, test credentials or environment flags, or independent confirmation of a non-production testing mode.", "evidenceIds": [ "signifyd-docs-1", "signifyd-docs-2", "signifyd-docs-3", "signifyd-docs-11" ] }, { "productId": "signifyd", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows a REST API with numerous documented endpoints, but nothing indicates API versioning scheme or a documented deprecation policy anywhere in the docs pack. missing for 10: explicit API version numbering, changelog, deprecation notice/policy, sunset timelines.", "evidenceIds": [] }, { "productId": "signifyd", "storyId": "automation-bulk-operations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence shows only per-order/per-event API endpoints (Sale, Checkout, Chargeback, etc.) with no batch/bulk endpoints or documentation of processing multiple items in a single call.", "evidenceIds": [] }, { "productId": "signifyd", "storyId": "automation-rules-engine", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Signifyd's API/webhook docs support event ingestion and notifications, but there's no evidence of a user-facing rules engine where AI-native users can define custom trigger-condition-action automations. missing for 10: rule-definition interface, conditional trigger/action configuration, evidence of automation builder.", "evidenceIds": [ "signifyd-docs-7", "signifyd-docs-8" ] }, { "productId": "signifyd", "storyId": "automation-scheduled-jobs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Signifyd is a fraud-protection/API platform for order and chargeback data, not a workflow-automation or scheduling tool; the evidence describes REST endpoints and webhooks for event-driven integration, not recurring job scheduling. Scheduling recurring jobs is a category error for this product type.", "evidenceIds": [] }, { "productId": "signifyd", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of any versioning, review, or rollback mechanism for automations/rules/workflows in Signifyd; the docs only cover API endpoints for fraud decisioning, webhooks, and chargebacks. Missing for 10: version history UI, audit trail for rule/automation changes, and rollback capability.", "evidenceIds": [] }, { "productId": "signifyd", "storyId": "chargeback-guarantee", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Signifyd's docs describe the core chargeback-guarantee mechanics: 'Guaranteed Fraud Protection' on integrated orders, a Chargeback Event API to record disputes, Representment Outcome tracking, and explicit guarantee-cancellation rules tied to returns/refunds, plus a dedicated 'Chargeback Recovery' offering. This directly matches the liability-shift/reimbursement story with documented workflow and terms touchpoints (cancellation conditions, representment outcomes). Missing for 10: explicit contractual coverage terms/limits (e.g., reimbursement caps, eligibility exclusions) and independent/customer confirmation that claims are actually paid out as promised.", "evidenceIds": [ "signifyd-docs-11", "signifyd-docs-5", "signifyd-docs-6", "signifyd-docs-13", "signifyd-docs-14" ] }, { "productId": "signifyd", "storyId": "custom-rules-authoring", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers API endpoints for sale/checkout events, webhooks, decisions, chargebacks, and device profiling, but contains no mention of a rules engine or capability for analysts to author custom rules combining model scores, velocity counters, list matches, and transaction attributes into allow/block/review decisions.", "evidenceIds": [] }, { "productId": "signifyd", "storyId": "custom-signals-enrichment", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Signifyd's Sale/Checkout API endpoints let developers submit order and event data, and device-profiling script captures fingerprints, but these are Signifyd's own structured schemas and proprietary device fingerprinting rather than a documented mechanism for injecting arbitrary custom behavioral signals or metadata into scoring. Missing for 10: explicit API fields for custom/behavioral metadata, documentation on how custom signals influence the risk score, and independent confirmation that scoring reflects merchant-specific inputs beyond standard order fields.", "evidenceIds": [ "signifyd-docs-2", "signifyd-docs-3", "signifyd-docs-10", "signifyd-docs-9" ] }, { "productId": "signifyd", "storyId": "data-residency-controls", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers API endpoints, webhooks, SDKs, and a sensitive-data protection layer, but nothing addresses regional data residency options or configurable retention/deletion policies that would survive a privacy review. Missing for 10: documented data residency regions, retention period controls, data deletion/export tooling, and any compliance certification (e.g., GDPR/SOC2) mapping to residency.", "evidenceIds": [ "signifyd-docs-9" ] }, { "productId": "signifyd", "storyId": "dispute-evidence-automation", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Signifyd's API includes explicit Chargeback and Representment Outcome endpoints (createchargeback, representmentoutcome) and a dedicated 'Chargeback Recovery' product line, indicating dispute handling automation exists, but the evidence never details the automated compilation of order/delivery/session evidence or direct submission to issuers without manual intervention. missing for 10: documentation of automatic evidence bundling from order/delivery/session data, proof of direct issuer submission, and any hands-on/independent confirmation of end-to-end automation.", "evidenceIds": [ "signifyd-docs-5", "signifyd-docs-6", "signifyd-docs-14" ] }, { "productId": "signifyd", "storyId": "fraud-outcome-reporting", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers API endpoints for sales, checkouts, chargebacks, webhooks, and device profiling, but nothing mentions a finance/board-facing dashboard, KPI reporting (dispute rate, false-positive rate, approval-rate lift, review workload), or export functionality for reporting purposes. Missing for 10: any documentation of analytics/reporting dashboards, defined KPI metrics, or export/board-reporting features.", "evidenceIds": [ "signifyd-docs-14", "signifyd-docs-15", "signifyd-docs-16" ] }, { "productId": "signifyd", "storyId": "lists-velocity-management", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers Signifyd's REST API for sale/checkout events, decisions, chargebacks, webhooks, and device profiling, but contains no mention of allow/block lists, velocity limits, or list management endpoints/dashboard features for emails, cards, devices, or IPs.", "evidenceIds": [] }, { "productId": "signifyd", "storyId": "manual-review-queue", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence pack covers Signifyd's API endpoints (Sale, Checkout, chargeback, webhooks, device profiling) but contains no mention of a review queue UI, case management dashboard, or consolidated view showing customer history, signals, and similar cases for analysts. No evidence supports the analyst-facing review workflow described in the story.", "evidenceIds": [] }, { "productId": "signifyd", "storyId": "model-performance-visibility", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers API integration, webhooks, chargeback/return workflows, and device profiling, but there is no mention of model performance metrics (precision/recall), shadow-mode testing, or A/B evaluation of decisioning models exposed to customers. Missing for 10: precision/recall reporting on customer traffic, shadow-mode/challenger model trials, any model-transparency or evaluation tooling.", "evidenceIds": [] }, { "productId": "signifyd", "storyId": "network-intelligence", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers Signifyd's API endpoints, webhooks, device profiling, chargebacks, and guarantees, but contains no mention of a cross-merchant network effect, shared consortium data, or how data from other merchants informs a given merchant's risk score. missing for 10: any documentation of network-wide identity/card matching, consortium data sharing, or cross-merchant signal aggregation.", "evidenceIds": [] }, { "productId": "signifyd", "storyId": "openness-api-parity", "verdict": "partial", "quality": 5, "confidence": "low", "rationale": "Signifyd's API covers core fraud-protection workflows (sale/checkout events, decisions, chargebacks, webhooks, device profiling) that mirror much of what merchants would configure in the UI, suggesting broad API/UI parity for core fraud operations. However, there is no evidence confirming that dashboard-only features like Return Insights, Instant Refunds management, or Chargeback Recovery workflows are fully API-accessible, nor any explicit statement of full UI/API parity. missing for 10: explicit parity statement, API coverage for analytics/reporting features (Return Insights, Instant Refunds), evidence of account/team management via API.", "evidenceIds": [ "signifyd-docs-2", "signifyd-docs-3", "signifyd-docs-4", "signifyd-docs-5", "signifyd-docs-6", "signifyd-docs-7", "signifyd-docs-13", "signifyd-docs-15", "signifyd-docs-16" ] }, { "productId": "signifyd", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence shows a REST API for submitting order/checkout/chargeback data into Signifyd and retrieving decisions, but nothing indicates a full data-export capability or open-format bulk export/portability feature for users to take all their data and leave. No mention of GDPR-style export, data portability tooling, or account deletion/export flows. missing for 10: bulk/full data export feature, open-format (CSV/JSON) export documentation, account closure/data portability guarantee.", "evidenceIds": [ "signifyd-docs-1", "signifyd-docs-4" ] }, { "productId": "signifyd", "storyId": "openness-open-license", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Signifyd is a closed commercial fraud-protection SaaS with a proprietary API; there is no indication it is an open-source project, so 'reading source under an open license' is a category error rather than a missing feature.", "evidenceIds": [] }, { "productId": "signifyd", "storyId": "openness-self-host", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Signifyd is a SaaS fraud-protection platform delivered via cloud API/webhooks with no self-hosted deployment offering; self-hosting the core product is not a fair axis for this category of managed service.", "evidenceIds": [] }, { "productId": "signifyd", "storyId": "platform-integrations", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence only covers Signifyd's raw REST API, webhooks, SDKs, and device-profiling script — there is no mention of maintained platform-specific integrations (Shopify app, Salesforce Commerce Cloud cartridge, BigCommerce app, or PSP-specific connectors). No evidence names or links any prebuilt/maintained integration for a specific commerce platform or PSP.", "evidenceIds": [ "signifyd-docs-1", "signifyd-docs-11", "signifyd-docs-12" ] }, { "productId": "signifyd", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack mentions data residency, regional storage options, or any configurability of where data is stored; only API/webhook/fraud-decision docs are present. missing for 10: any mention of data residency/region selection, hosting locations, or compliance controls for data storage.", "evidenceIds": [] }, { "productId": "signifyd", "storyId": "privacy-no-training", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence addresses AI-model-training data usage or opt-out controls; the pack only covers fraud-protection APIs and sensitive-data rejection at the API layer, which is unrelated to AI training data policy.", "evidenceIds": [] }, { "productId": "signifyd", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "Evidence only covers API endpoints, webhooks, and sensitive-data rejection at the API layer; there is no documentation of data retention policies, user-initiated deletion controls, or data lifecycle management. Missing for 10: retention policy documentation, deletion/export APIs or controls, data lifecycle configuration options.", "evidenceIds": [] }, { "productId": "signifyd", "storyId": "privacy-telemetry-optout", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Signifyd is a fraud-protection/e-commerce API platform, not an AI agent or developer tool with telemetry/usage-tracking of AI-native workflows; opting out of telemetry is not a relevant axis for this product category, and no evidence pack content relates to it.", "evidenceIds": [] }, { "productId": "signifyd", "storyId": "psd2-sca-optimization", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "The evidence pack covers Signifyd's fraud/guarantee API (Sale, Checkout, chargebacks, webhooks, device profiling) but contains no mention of SCA, 3DS, PSD2 exemptions, or any European payment-authentication routing logic. Nothing indicates the product decides when to trigger 3DS versus request an exemption. missing for 10: any documentation of SCA/3DS handling, exemption request logic, or PSD2-specific routing features.", "evidenceIds": [] }, { "productId": "signifyd", "storyId": "psp-agnostic-coverage", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Signifyd's API is generic (Checkout/Sale events, webhooks) and explicitly designed to sit outside the payment flow — the Checkout event is called 'before calling your Payment Gateway' — implying gateway/PSP-agnostic architecture rather than lock-in to one processor's rails. However, the evidence pack never explicitly lists supported PSPs, gateways, or platforms, nor confirms interoperability across multiple stacks simultaneously. missing for 10: explicit documentation naming supported PSPs/gateways/platforms, and independent confirmation of multi-PSP usage in production.", "evidenceIds": [ "signifyd-docs-2", "signifyd-docs-3", "signifyd-docs-11", "signifyd-docs-1" ] }, { "productId": "signifyd", "storyId": "realtime-scoring-api", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Signifyd's Checkout endpoint is documented to be called before the payment gateway in a Pre-Auth flow, aligning with the 'before authorization' requirement, and there is a documented GetDecision endpoint to retrieve the fraud decision. However, the evidence does not show the Checkout call synchronously returning an ML risk score in its response; decision retrieval appears to depend on a separate GetDecision call or webhook, suggesting an asynchronous decisioning pattern rather than an inline synchronous score. Missing for 10: explicit documentation that the Checkout API response includes a real-time ML score synchronously, and confirmation that no polling/webhook wait is required before authorization completes.", "evidenceIds": [ "signifyd-docs-3", "signifyd-docs-4", "signifyd-docs-8" ] }, { "productId": "signifyd", "storyId": "review-outcomes-feedback", "verdict": "partial", "quality": 3, "confidence": "low", "rationale": "Signifyd's API lets merchants record Chargeback and Representment Outcome events, which feed confirmed fraud outcomes into the system, but the evidence pack never states that analyst review-queue decisions or these outcomes are used to retrain the model or update rules. Missing for 10: explicit documentation of a review-queue/analyst decision feedback loop, evidence of model retraining or rule updates from confirmed outcomes, and any hands-on/independent confirmation that this loop improves detection.", "evidenceIds": [ "signifyd-docs-5", "signifyd-docs-6", "signifyd-docs-4" ] }, { "productId": "signifyd", "storyId": "review-team-workflows", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers API endpoints for fraud decisions, chargebacks, webhooks, and device profiling, but nothing about a human review-queue workflow with assignment, escalation, SLAs, or a decision audit trail showing who approved what and why. Missing for 10: assignment/queue management docs, escalation workflows, SLA tracking, human-approval audit trail features.", "evidenceIds": [ "signifyd-docs-4", "signifyd-docs-5", "signifyd-docs-8" ] }, { "productId": "signifyd", "storyId": "rules-backtesting", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack items mention rule backtesting, simulation against historical traffic, or previewing block/flag/cost outcomes before deployment; the docs cover API integration, webhooks, chargebacks, and device profiling only.", "evidenceIds": [] }, { "productId": "signifyd", "storyId": "score-explainability", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers API endpoints for submitting orders, retrieving decisions, webhooks, chargebacks, and device profiling, but none of it describes the decision/score payload including risk factor breakdowns or explanations behind a fraud score. The getdecision endpoint doc only says it 'retrieves the latest fraud decision' with no mention of contributing risk factors or explainability.", "evidenceIds": [ "signifyd-docs-4" ] }, { "productId": "signifyd", "storyId": "score-thresholds-actions", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Evidence covers Signifyd's API endpoints for orders, decisions, chargebacks, and webhooks, but nothing describes configurable score-to-action mapping (allow/review/block/step-up 3DS) or threshold tuning for ops users; the getdecision endpoint only retrieves a decision, not a configurable rule engine.", "evidenceIds": [ "signifyd-docs-4", "signifyd-docs-8" ] }, { "productId": "signifyd", "storyId": "trusted-agent-signals", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence in the pack mentions agentic-payment protocols, signed agent tokens, or delegated spending scopes being ingested as risk signals; the API docs cover standard order/checkout/sale/chargeback events and device profiling only.", "evidenceIds": [] }, { "productId": "stripe-radar", "storyId": "account-abuse-coverage", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Radar Pro docs explicitly extend beyond checkout fraud to 'multi-account, free trial, and pay-as-you-go abuse,' covering fake-account and promo/policy abuse in the same platform, but there is no evidence of dedicated account-takeover detection or scoring — the docs focus on payment/charge risk, reviews, and disputes rather than login/session anomaly detection typical of ATO protection. Missing for 10: explicit account-takeover detection/scoring capability, unified dashboard evidence showing ATO alongside promo-abuse cases, and independent confirmation these abuse types are actually managed in one system rather than just marketed together.", "evidenceIds": [ "stripe-radar-docs-16", "stripe-radar-docs-6", "stripe-radar-docs-9" ] }, { "productId": "stripe-radar", "storyId": "agent-buyer-detection", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence describes Radar's general fraud rules, risk scoring, reviews, and lists, but nothing addresses distinguishing legitimate AI purchasing agents from malicious bots — an axis specific to agentic commerce that is plausible for a fraud-prevention product but unevidenced here. Community feedback even shows false positives blocking legitimate low-risk customers, with no mention of agent-specific allowlisting or detection.", "evidenceIds": [ "stripe-radar-docs-6", "stripe-radar-docs-13", "stripe-radar-comm-2", "stripe-radar-comm-3" ] }, { "productId": "stripe-radar", "storyId": "agent-drives-rules", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Stripe exposes APIs for value lists (blocklists/allowlists) and review approval (a human-approval gate: POST /v1/reviews/:id/approve), plus a documented MCP server, which together could let an agent read fraud data and manage lists with approval steps. However, rule creation/velocity-rule authoring is Dashboard-centric with no documented rules-create API, and community evidence directly contradicts the 'agent manages rules programmatically' premise: users report 'the lack of easy programmatic control is an issue for us' and that Allow Rules are disabled by default for newer accounts requiring a manual support request to enable. Missing for 10: a documented API/MCP tool to create or propose new velocity/fraud rules, and independent confirmation that programmatic rule/list management works smoothly without support intervention.", "evidenceIds": [ "stripe-radar-docs-8", "stripe-radar-docs-10", "stripe-radar-docs-14", "stripe-radar-probe-4", "stripe-radar-comm-2", "stripe-radar-comm-7" ] }, { "productId": "stripe-radar", "storyId": "agent-triages-reviews", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Radar exposes a Reviews API (list/retrieve/approve reviews), risk-insights showing related-payment networks, and early-fraud-warning data that an agent could pull as 'flagged case context,' and Stripe documents an official MCP server that could expose these APIs to an agent. However there's no first-party feature for automated evidence summarization or a recommend-then-human-confirm workflow — that logic would have to be built by the integrator, and community reports note UI/override friction around review decisions. Missing for 10: a documented agent/summarization workflow for reviews, evidence the MCP server actually exposes the reviews/early-fraud-warning endpoints, and independent confirmation of an agent successfully working the queue end-to-end.", "evidenceIds": [ "stripe-radar-docs-8", "stripe-radar-docs-11", "stripe-radar-docs-12", "stripe-radar-docs-9", "stripe-radar-probe-4", "stripe-radar-comm-3" ] }, { "productId": "stripe-radar", "storyId": "agentic-agent-docs", "verdict": "full", "quality": 9, "confidence": "high", "rationale": "Probes confirm a live llms.txt at docs.stripe.com/llms.txt (HTTP 200) and markdown-rendered agent-friendly docs pages (e.g. radar.md), directly enabling an agent to be pointed at agent-oriented docs. Missing for 10: no independent/community confirmation of agents actually consuming these docs successfully in practice.", "evidenceIds": [ "stripe-radar-probe-1", "stripe-radar-probe-2" ] }, { "productId": "stripe-radar", "storyId": "agentic-ai-insights", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Radar's docs describe rule-based fraud controls, risk scoring, and dashboard analytics/visualizations (docs-4, docs-12), but there is no evidence of AI-generated natural-language insights or suggestions (e.g., an assistant summarizing fraud trends or recommending rule changes) surfaced inside the product.", "evidenceIds": [] }, { "productId": "stripe-radar", "storyId": "agentic-autonomous-automation", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Radar rules engine runs autonomously in the background on every transaction (auto 3DS requests, auto-pause payouts, custom rules, auto-allow trusted customers, risk settings) per docs-1/2/3/13/14, which is genuine unattended automation. However community feedback notes limited programmatic control over these automations and friction when trying to override or fine-tune them (stripe-radar-comm-2, stripe-radar-comm-3), suggesting the autonomy is somewhat constrained/dashboard-centric rather than fully agent-friendly. missing for 10: evidence of API-driven/programmatic rule creation or agent-triggered automation workflows, and resolution of the 'lack of easy programmatic control' complaint.", "evidenceIds": [ "stripe-radar-docs-1", "stripe-radar-docs-2", "stripe-radar-docs-3", "stripe-radar-docs-13", "stripe-radar-docs-14", "stripe-radar-comm-2", "stripe-radar-comm-3" ] }, { "productId": "stripe-radar", "storyId": "agentic-builtin-assistant", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of a built-in AI assistant within Stripe Radar to which users can delegate tasks; the product offers rules, lists, reviews, and analytics but no conversational/agentic assistant feature is documented.", "evidenceIds": [] }, { "productId": "stripe-radar", "storyId": "agentic-headless", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Radar exposes REST API endpoints (rules, value lists, reviews approve/decline) and has dedicated testing docs with test card numbers for automated fraud-rule verification, and Stripe ships an official CLI — all of which support headless/CI use. However, there's no explicit CI/automation guide, and a community report explicitly flags 'lack of easy programmatic control' as a pain point for adjusting Radar decisions. missing for 10: dedicated CI/headless workflow documentation, explicit automation examples, and resolution of the programmatic-control complaint.", "evidenceIds": [ "stripe-radar-docs-8", "stripe-radar-docs-5", "stripe-radar-probe-5", "stripe-radar-comm-2" ] }, { "productId": "stripe-radar", "storyId": "agentic-mcp-client", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence only shows Stripe publishes an official MCP *server* (docs.stripe.com/mcp) exposing its own tools to external agents, not that Radar itself can act as an MCP client consuming other servers' tools. No documentation or community evidence shows Radar being configured with external MCP servers to extend its own functionality.", "evidenceIds": [ "stripe-radar-probe-4" ] }, { "productId": "stripe-radar", "storyId": "agentic-mcp-server", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Stripe documents an official MCP server (docs.stripe.com/mcp) that lets AI agents connect to Stripe, which as a platform encompasses Radar functionality via its API. Missing for 10: Radar-specific MCP tool examples/independent hands-on corroboration of agent use.", "evidenceIds": [ "stripe-radar-probe-4" ] }, { "productId": "stripe-radar", "storyId": "agentic-nl-commands", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Radar's rule configuration is a structured DSL (attributes/expressions) rather than natural-language commands, and while Stripe has a generic MCP server (stripe-radar-probe-4), there is no evidence it exposes Radar-specific fraud rule management or that Radar can be operated via free-form NL instructions.", "evidenceIds": [ "stripe-radar-docs-3", "stripe-radar-docs-10", "stripe-radar-probe-4" ] }, { "productId": "stripe-radar", "storyId": "agentic-official-cli", "verdict": "full", "quality": 8, "confidence": "medium", "rationale": "Stripe ships an official Stripe CLI (docs.stripe.com/stripe-cli) that covers Radar-related API/webhook workflows, giving AI-native users a scriptable interface. Missing for 10: no direct evidence the CLI has Radar-specific commands or independent hands-on confirmation of its use in agentic workflows.", "evidenceIds": [ "stripe-radar-probe-5" ] }, { "productId": "stripe-radar", "storyId": "agentic-public-api", "verdict": "full", "quality": 8, "confidence": "high", "rationale": "Stripe Radar exposes documented REST API endpoints (e.g. reviews, early_fraud_warnings, value_lists) and is part of Stripe's broader public API with an official CLI and MCP server, confirming programmatic/agentic access. missing for 10: no discoverable OpenAPI/swagger spec file was found (probe 404s) and no independent hands-on report of an AI agent driving Radar specifically via the API.", "evidenceIds": [ "stripe-radar-docs-8", "stripe-radar-docs-9", "stripe-radar-docs-10", "stripe-radar-probe-2", "stripe-radar-probe-4", "stripe-radar-probe-5", "stripe-radar-probe-3" ] }, { "productId": "stripe-radar", "storyId": "agentic-scoped-keys", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Stripe Radar is a fraud-detection/rules product, not an identity/access-management system; scoped API credential issuance for agents is outside its product category (that's a Stripe platform/API-keys concern, not Radar specifically). No evidence pack items address credential scoping in Radar.", "evidenceIds": [] }, { "productId": "stripe-radar", "storyId": "agentic-sdks", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "The evidence shows official API endpoints (e.g., POST /v1/reviews/:id/approve, early_fraud_warnings API) and an official CLI, implying SDK-compatible API access, but there is no explicit documentation pack entry naming or linking official language SDKs (e.g., stripe-node, stripe-python) for Radar-specific features. missing for 10: explicit SDK documentation/references, code samples showing SDK usage for Radar rules/reviews, independent developer confirmation of SDK coverage.", "evidenceIds": [ "stripe-radar-docs-8", "stripe-radar-docs-9", "stripe-radar-probe-5", "stripe-radar-probe-2" ] }, { "productId": "stripe-radar", "storyId": "agentic-webhooks", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack documents Radar's REST APIs (reviews, early fraud warnings, value lists) but never mentions webhook event subscriptions for these Radar events, so there's no evidence of an AI-native webhook subscription capability despite this being a plausible axis for an API-driven fraud product.", "evidenceIds": [] }, { "productId": "stripe-radar", "storyId": "ai-fraud-analyst", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of a built-in AI assistant in the Radar console for natural-language queries, drafted rules, or investigation summaries; documentation covers rules engine, reviews, analytics, and lists but nothing about an AI/NLP assistant feature.", "evidenceIds": [] }, { "productId": "stripe-radar", "storyId": "api-interactive-docs", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of an interactive API reference with runnable examples; the OpenAPI probe returned 404s and docs listed are static markdown pages describing endpoints without runnable/interactive playground features. missing for 10: interactive API explorer/playground, runnable code examples, OpenAPI spec availability.", "evidenceIds": [ "stripe-radar-probe-3" ] }, { "productId": "stripe-radar", "storyId": "api-machine-spec", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack includes a direct probe for OpenAPI/swagger spec files at common Stripe docs paths, all returning 404, and no other citation shows a downloadable machine-readable spec for Radar's API. Only docs pages and llms.txt-style markdown are confirmed, not an OpenAPI/Swagger file.", "evidenceIds": [ "stripe-radar-probe-3" ] }, { "productId": "stripe-radar", "storyId": "api-sandbox", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Stripe Radar's testing docs explicitly provide test card numbers (e.g., 4000000000004954) that simulate specific risk levels, enabling developers to validate fraud rules and review logic without using real transactions or production data. However, the evidence doesn't detail a full sandbox environment (e.g., test-mode API key isolation, sandbox dashboards) beyond these test cards, and there's no independent/hands-on confirmation of sandbox fidelity for AI-native workflows. Missing for 10: explicit documentation of test-mode/live-mode key separation for Radar, broader sandbox environment description, and independent corroboration of safe non-production testing.", "evidenceIds": [ "stripe-radar-docs-5", "stripe-radar-probe-2" ] }, { "productId": "stripe-radar", "storyId": "api-versioning-policy", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no mention of Stripe API versioning scheme, version pinning, or a documented deprecation policy for Radar's API endpoints; only generic docs and community pricing/false-positive discussions are present.", "evidenceIds": [] }, { "productId": "stripe-radar", "storyId": "automation-bulk-operations", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Radar rules, value lists, and targeted transaction-review lists let a rule or list apply automatically across many transactions/customers at once (stripe-radar-docs-1,2,3,10,11,14), which is a form of bulk automation, but the documented Review API only exposes per-item actions (POST /v1/reviews/:id/approve) with no bulk/batch endpoint or explicit multi-item API call shown. missing for 10: evidence of a bulk API endpoint for approving/declining multiple reviews or disputes at once, and any AI-native tooling for programmatic multi-item operations.", "evidenceIds": [ "stripe-radar-docs-1", "stripe-radar-docs-3", "stripe-radar-docs-10", "stripe-radar-docs-11", "stripe-radar-docs-14", "stripe-radar-docs-8" ] }, { "productId": "stripe-radar", "storyId": "automation-rules-engine", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Radar's rule engine clearly supports defining conditional rules that trigger automatic actions (3DS challenge, block, review, allow, pause payouts) based on transaction attributes, and value lists let rules be parameterized (docs-1, docs-2, docs-3, docs-10, docs-13, docs-14). However, rule authoring is Dashboard-centric with no documented rules-creation API, and community feedback explicitly flags a 'lack of easy programmatic control' plus restricted access to Allow Rules for newer accounts (comm-2, comm-7), limiting fit for an AI-native/automated workflow. Missing for 10: a documented API/SDK for programmatically creating or updating rules, and evidence of unrestricted automation access for all account types.", "evidenceIds": [ "stripe-radar-docs-1", "stripe-radar-docs-2", "stripe-radar-docs-3", "stripe-radar-docs-10", "stripe-radar-docs-13", "stripe-radar-docs-14", "stripe-radar-comm-2", "stripe-radar-comm-7" ] }, { "productId": "stripe-radar", "storyId": "automation-scheduled-jobs", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Stripe Radar is a fraud-detection/risk-rules engine, not a workflow/job scheduler; scheduling recurring automation jobs is outside its product scope and no evidence suggests such capability.", "evidenceIds": [] }, { "productId": "stripe-radar", "storyId": "automation-versioned-workflows", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence of version history, rule change review/audit trail, or rollback capability for Radar rules/automations; rules are managed via dashboard/API but no versioning or rollback mechanism is documented. Missing for 10: rule version history, diff/review UI, rollback/restore functionality.", "evidenceIds": [] }, { "productId": "stripe-radar", "storyId": "chargeback-guarantee", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "The evidence pack covers Radar's fraud-scoring, rules, reviews, and dispute-rate monitoring features, but contains no mention of a chargeback guarantee, reimbursement for approved-then-disputed orders, or liability shift terms — that is a distinct Stripe product (Chargeback Protection), not documented here as part of Radar.", "evidenceIds": [] }, { "productId": "stripe-radar", "storyId": "custom-rules-authoring", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Stripe docs show a rich rules engine — custom rules using risk score, transaction attributes, value lists (for list matches), and allow/block/review outcomes (docs-1,3,10,13,14) — which covers most of the story. However, community evidence shows secondary caveats: Allow Rules are disabled by default for newer businesses and require manual support enablement (comm-7), and users report inability to override block decisions via UI plus false positives (comm-2,3), indicating real friction in the rule-authoring/decisioning workflow. Missing for 10: explicit documentation/example of velocity-counter attributes in rule syntax, and independent confirmation that all rule types (allow/block/review) work reliably without support intervention.", "evidenceIds": [ "stripe-radar-docs-1", "stripe-radar-docs-3", "stripe-radar-docs-10", "stripe-radar-docs-13", "stripe-radar-docs-14", "stripe-radar-comm-7", "stripe-radar-comm-3", "stripe-radar-comm-2" ] }, { "productId": "stripe-radar", "storyId": "custom-signals-enrichment", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Radar lets developers write custom rules against 'supported attributes' and value lists (docs-3, docs-10, docs-14), and Radar Session captures device/browser signals for fraud evaluation without full tokenization (docs-15) — this shows some capacity to incorporate custom signals. However, there's no evidence that arbitrary custom metadata or behavioral data actually feeds into or retrains Radar's core ML risk score itself (rather than just triggering rule-based overrides), and community commentary notes a 'lack of easy programmatic control' over scoring (comm-2). missing for 10: explicit documentation that custom metadata/behavioral inputs alter the underlying risk score model, first-party guidance on feeding proprietary signals into scoring, and independent confirmation this works as intended.", "evidenceIds": [ "stripe-radar-docs-3", "stripe-radar-docs-10", "stripe-radar-docs-14", "stripe-radar-docs-15", "stripe-radar-comm-2" ] }, { "productId": "stripe-radar", "storyId": "data-residency-controls", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence pack item addresses data residency options, regional storage location controls, or configurable retention periods for fraud/Radar data; the docs cover rules, reviews, lists, and testing but nothing about compliance/residency controls.", "evidenceIds": [] }, { "productId": "stripe-radar", "storyId": "dispute-evidence-automation", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Radar's documented capabilities are fraud scoring, rules, reviews, and dispute-rate monitoring/analytics (docs-2, docs-4, docs-7, docs-9) — none of the evidence describes compiling order/delivery/session evidence and auto-submitting it to card issuers for chargeback responses. This is a distinct dispute-evidence-automation capability that the evidence pack simply does not show Radar performing.", "evidenceIds": [ "stripe-radar-docs-7", "stripe-radar-docs-9", "stripe-radar-docs-2" ] }, { "productId": "stripe-radar", "storyId": "fraud-outcome-reporting", "verdict": "partial", "quality": 4, "confidence": "medium", "rationale": "Docs show dispute-rate calculation on the Radar dashboard (stripe-radar-docs-7) and fraud-rate/volume trend visualizations (stripe-radar-docs-4), plus reviewable queues (stripe-radar-docs-11, docs-8) that imply review workload tracking. However there is no evidence of a false-positive-rate metric, approval-rate lift measurement, or any export/reporting feature for board consumption. missing for 10: false-positive rate metric, approval-rate lift metric, CSV/board export capability, independent confirmation of dashboard completeness.", "evidenceIds": [ "stripe-radar-docs-7", "stripe-radar-docs-4", "stripe-radar-docs-11", "stripe-radar-docs-8" ] }, { "productId": "stripe-radar", "storyId": "lists-velocity-management", "verdict": "disputed", "quality": 5, "confidence": "medium", "rationale": "Stripe documents value lists for allow/block lists (customers, cards, emails, IPs) manageable via API and dashboard (stripe-radar-docs-10, docs-14) plus rules for velocity/custom attributes (docs-3, docs-1), giving vendor-side coverage of the story. However hands-on community reports contradict smooth programmatic control: one user cites 'lack of easy programmatic control' causing false-positive grief, and another found the dashboard's block-override button non-functional, requiring a support email to whitelist transactions; Stripe itself confirms Allow Rules are gated behind manual support enablement for newer accounts, not self-serve. Missing for 10: clear documentation of dedicated device/IP allow-and-block management (only cards/customers/emails are explicit), and no evidence the programmatic control gap or override bug has been resolved.", "evidenceIds": [ "stripe-radar-docs-10", "stripe-radar-docs-14", "stripe-radar-docs-3", "stripe-radar-docs-1", "stripe-radar-comm-2", "stripe-radar-comm-3", "stripe-radar-comm-7" ] }, { "productId": "stripe-radar", "storyId": "manual-review-queue", "verdict": "partial", "quality": 7, "confidence": "medium", "rationale": "Stripe's docs show a genuine review queue workflow: targeted transaction reviews (docs-11), an approve/reject API (docs-8), and a 'risk insights' view showing related payments across customer ID, IP, or card number for spotting similar cases (docs-12). However, evidence doesn't explicitly confirm a unified single screen combining full customer history + fraud signals + similar-case surfacing in one glance, and community feedback notes UI friction when analysts try to act on flagged transactions (comm-3, unclear override controls). missing for 10: explicit documentation of a consolidated 'customer history' panel within the review UI, and independent/hands-on confirmation that the queue enables fast, consistent decisions without friction.", "evidenceIds": [ "stripe-radar-docs-11", "stripe-radar-docs-8", "stripe-radar-docs-12", "stripe-radar-comm-3" ] }, { "productId": "stripe-radar", "storyId": "model-performance-visibility", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "Docs mention fraud-rate analytics and dispute measurement (fraud insights, risk settings) but nowhere describe precision/recall metrics on the merchant's own traffic or a shadow-mode mechanism to trial new models/rules before they affect decisions. Rules can be created and reviewed, but there's no evidence of a non-blocking 'test' or 'shadow' deployment mode for models.", "evidenceIds": [ "stripe-radar-docs-4", "stripe-radar-docs-7", "stripe-radar-docs-13" ] }, { "productId": "stripe-radar", "storyId": "network-intelligence", "verdict": "partial", "quality": 5, "confidence": "medium", "rationale": "Docs show Radar surfaces a 'network of related payments' across customer ID, IP, or card number (docs-12) and ingests card-issuer signals via Early Fraud Warnings (docs-9), implying some cross-account risk signal, and docs-6 describes 'out-of-the-box' pre-trained fraud detection. However, none of the evidence explicitly states the model is trained on or scores against Stripe's full cross-merchant network of thousands of businesses. Missing for 10: explicit documentation of the network-wide ML training/scoring claim, and independent confirmation that cross-merchant signals (not just same-business history) drive individual risk scores.", "evidenceIds": [ "stripe-radar-docs-12", "stripe-radar-docs-9", "stripe-radar-docs-6" ] }, { "productId": "stripe-radar", "storyId": "openness-api-parity", "verdict": "disputed", "quality": 4, "confidence": "medium", "rationale": "Stripe exposes API endpoints for some Radar objects (reviews approve/reject, early fraud warnings, value lists) but the core capability of authoring/editing Radar Rules and risk settings is documented only via Dashboard-oriented docs (stripe-radar-docs-3, docs-13) with no corresponding rules-API endpoint evidenced. Community feedback explicitly calls out this gap ('The lack of easy programmatic control is an issue for us', stripe-radar-comm-2) and notes Allow Rules require manual support intervention rather than self-service API access (stripe-radar-comm-7), directly contradicting full API/UI parity. missing for 10: documented API endpoints for creating/editing Radar rules, programmatic risk-settings control, independent confirmation that all dashboard actions have API equivalents.", "evidenceIds": [ "stripe-radar-docs-3", "stripe-radar-docs-13", "stripe-radar-docs-8", "stripe-radar-docs-9", "stripe-radar-docs-10", "stripe-radar-comm-2", "stripe-radar-comm-7" ] }, { "productId": "stripe-radar", "storyId": "openness-full-export", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "No evidence of any data export feature or open-format data portability for Radar; docs cover rules, reviews, analytics, and APIs but nothing about exporting all account/fraud data to leave the platform. missing for 10: bulk/full data export tooling, open-format (CSV/JSON) export docs, any account-closure/data-portability guarantee.", "evidenceIds": [] }, { "productId": "stripe-radar", "storyId": "openness-open-license", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Stripe Radar is a closed, proprietary SaaS fraud service; no evidence of any open-source license or public source repository for Radar itself. Documentation, pricing, and CLI/MCP references exist, but nothing indicates the product's source is available under an open license.", "evidenceIds": [] }, { "productId": "stripe-radar", "storyId": "openness-self-host", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Stripe Radar is a hosted SaaS fraud-detection service tightly integrated with Stripe's payment infrastructure; self-hosting the core product is not a plausible axis for this category of product.", "evidenceIds": [] }, { "productId": "stripe-radar", "storyId": "platform-integrations", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "Evidence covers Radar's rules, reviews, risk settings, and API/CLI/MCP tooling, but nothing mentions maintained integrations or plugins for commerce platforms like Shopify, Salesforce Commerce, or BigCommerce, or PSP-specific integrations beyond Stripe's own API. Missing for 10: any documentation of Shopify/BigCommerce/Salesforce Commerce app integrations, partner PSP integrations, or an integrations marketplace listing.", "evidenceIds": [ "stripe-radar-docs-6", "stripe-radar-docs-13", "stripe-radar-probe-2" ] }, { "productId": "stripe-radar", "storyId": "privacy-data-residency", "verdict": "none", "quality": 0, "confidence": "low", "rationale": "No evidence in the pack addresses data residency, regional storage options, or data location controls for Stripe Radar; the documentation covers fraud rules, reviews, and risk settings but not where data is stored.", "evidenceIds": [] }, { "productId": "stripe-radar", "storyId": "privacy-no-training", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Stripe Radar is a fraud-detection tool for payments, not an AI model/data-training product; controlling AI training data usage is a wrong-axis question for this product category.", "evidenceIds": [] }, { "productId": "stripe-radar", "storyId": "privacy-retention-controls", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack contains no mention of data retention policies, deletion controls, or privacy/data lifecycle management features for Radar; documentation covers fraud rules, reviews, testing, and pricing but nothing about controlling how long data is kept or deleting it.", "evidenceIds": [] }, { "productId": "stripe-radar", "storyId": "privacy-telemetry-optout", "verdict": "na", "quality": 0, "confidence": "high", "rationale": "Stripe Radar is a fraud-detection service for payments processing, not an AI assistant/agent tool collecting telemetry from AI-native usage; the notion of opting out of 'AI telemetry/usage tracking' is a category error for this product type, unrelated to its fraud-review data collection.", "evidenceIds": [] }, { "productId": "stripe-radar", "storyId": "psd2-sca-optimization", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Radar rules docs show it can trigger 3DS for specific conditions (e.g., new customers) via custom rules, but there is no evidence of intelligent SCA-wide routing that automatically requests exemptions (TRA, low-value, etc.) to preserve conversion — the exemption side of the story is unaddressed and this Radar-specific capability differs from Stripe's core SCA/Payment Intents engine. missing for 10: evidence of automated exemption requests (TRA/low-value/trusted-beneficiary), evidence of end-to-end SCA compliance logic beyond manual rule authoring, and independent confirmation of conversion impact.", "evidenceIds": [ "stripe-radar-docs-1", "stripe-radar-docs-3" ] }, { "productId": "stripe-radar", "storyId": "psp-agnostic-coverage", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "All evidence describes Radar as a feature built directly into Stripe's own payments processing (rules, reviews, risk settings, session tokenization, testing tied to Stripe test cards) with no mention of usable integration with other PSPs, gateways, or platforms. Fraud-detection tools in general could plausibly support multi-processor use, but nothing in the pack shows Radar operating outside Stripe's own rails — it is described purely as a Stripe-native capability.", "evidenceIds": [ "stripe-radar-docs-6", "stripe-radar-docs-15", "stripe-radar-docs-5", "stripe-radar-probe-2" ] }, { "productId": "stripe-radar", "storyId": "realtime-scoring-api", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Docs confirm Radar attaches a risk level/score to each charge (e.g. test card 4000000000004954 'Results in a charge with a risk level of highest') and that rules/reviews act on this score, implying the score is computed as part of normal payment processing and exposed on the charge object via the API. However, the pack never explicitly documents the exact API field (e.g. charge.outcome.risk_score) or explicitly states the score is available synchronously before authorization completes, and no independent/hands-on confirmation of real-time synchronous scoring is present. missing for 10: explicit API reference/schema for risk_score/risk_level field, explicit documentation stating the score is returned before/at authorization time, independent corroboration of real-time synchronous behavior.", "evidenceIds": [ "stripe-radar-docs-5", "stripe-radar-docs-6", "stripe-radar-docs-1", "stripe-radar-docs-8" ] }, { "productId": "stripe-radar", "storyId": "review-outcomes-feedback", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Radar lets analysts approve/decline reviews (docs-8) and maintain allow/block value lists that then drive future rule evaluation (docs-10, docs-14), which is a manual feedback mechanism, and early fraud warnings feed dispute-rate risk signals (docs-9, docs-7). However there is no documented evidence that confirmed fraud outcomes or review decisions automatically retrain Radar's underlying ML model — the docs describe rules/lists as merchant-configured, not an automated learning loop tied to case outcomes. Missing for 10: explicit documentation of the ML model being retrained from analyst decisions/fraud confirmations, and independent/hands-on confirmation that outcomes measurably change future scoring.", "evidenceIds": [ "stripe-radar-docs-8", "stripe-radar-docs-10", "stripe-radar-docs-14", "stripe-radar-docs-9", "stripe-radar-docs-7" ] }, { "productId": "stripe-radar", "storyId": "review-team-workflows", "verdict": "partial", "quality": 3, "confidence": "medium", "rationale": "Radar's reviews API supports approve/decline actions on flagged payments and lets teams build custom review queues, which implies some decision logging, but there is no documented support for assignment to specific reviewers, escalation paths, or SLA tracking, and no explicit 'who approved what and why' audit trail beyond the approve/reject call itself. missing for 10: reviewer assignment, escalation workflow, SLA tracking, structured decision-rationale audit trail.", "evidenceIds": [ "stripe-radar-docs-8", "stripe-radar-docs-11", "stripe-radar-docs-12" ] }, { "productId": "stripe-radar", "storyId": "rules-backtesting", "verdict": "none", "quality": 0, "confidence": "medium", "rationale": "The evidence pack covers rule creation, value lists, risk settings, reviews, and analytics dashboards, but no documentation or community evidence describes a backtesting/simulation feature that shows what a new rule would have blocked/flagged/cost against historical traffic before deployment.", "evidenceIds": [ "stripe-radar-docs-3", "stripe-radar-docs-4", "stripe-radar-docs-13" ] }, { "productId": "stripe-radar", "storyId": "score-explainability", "verdict": "partial", "quality": 4, "confidence": "low", "rationale": "Radar's Reviews/Risk Insights docs show a 'network of related payments' (same customer ID, IP, card) and analytics trends, suggesting some contextual signals behind a score, but the evidence never describes an explicit list of 'top risk factors' or feature-level explanation attached to each transaction's score. Missing for 10: explicit per-transaction factor breakdown/SHAP-style explanation, quantified factor weighting, independent confirmation that risk-insights actually enumerates specific risk drivers rather than just related-payment context.", "evidenceIds": [ "stripe-radar-docs-12", "stripe-radar-docs-4", "stripe-radar-docs-13" ] }, { "productId": "stripe-radar", "storyId": "score-thresholds-actions", "verdict": "partial", "quality": 6, "confidence": "medium", "rationale": "Stripe Radar's docs clearly support mapping risk levels to actions (block, review, request 3DS, allow via value lists) and tuning via risk-settings/rules with custom attributes, directly matching the story. However, community evidence shows real friction: allow rules are gated behind manual support enablement for newer accounts (comm-7), an ops user hit a 'block' override button that silently did nothing requiring a support ticket (comm-3), and another reports 'lack of easy programmatic control' over false positives (comm-2), indicating the threshold-tuning experience isn't as self-service as docs imply. Missing for 10: independent verification that threshold-to-action mapping is fully self-serve without support intervention, and resolution of the UI override bug.", "evidenceIds": [ "stripe-radar-docs-1", "stripe-radar-docs-13", "stripe-radar-docs-14", "stripe-radar-docs-3", "stripe-radar-comm-2", "stripe-radar-comm-3", "stripe-radar-comm-7" ] }, { "productId": "stripe-radar", "storyId": "trusted-agent-signals", "verdict": "none", "quality": 0, "confidence": "high", "rationale": "No evidence that Radar accepts verified agent identity, agentic-payment protocol tokens, signed agent tokens, or delegated spending scopes as first-class risk inputs; Radar's documented signals are card, customer, IP, and rule/list based, not agent-identity based.", "evidenceIds": [ "stripe-radar-docs-3", "stripe-radar-docs-10", "stripe-radar-docs-13" ] } ]